琐碎-hadoop1.X和2.X的区别

1. jobtracker做了分离，分成了resourceManager和nodemanager；

2. MR变成了和HBase和Hive等一样的yarn上面的一个应用；

3. 1.x的默认块大小为64M，2.x的默认块大小为128M；

4. 在2.x中除了datanode要向namenode报告status，nodemanager也要向ResourceManager报告status

5.MR API差别

旧的WordCount

 1 package org.apache.hadoop.maPRed; 2  3 ... ... 4  5 public class WordCount extends Configured implements Tool { 6    7   public static class MapClass extends MapReduceBase 8     implements Mapper<LongWritable, Text, Text, IntWritable> { 9     10     ... ...11     12     public void map(LongWritable key, Text value, 13                     OutputCollector<Text, IntWritable> output, 14                     Reporter reporter) throws IOException {15       ... ...16     }17   }18 19   public static class Reduce extends MapReduceBase20     implements Reducer<Text, IntWritable, Text, IntWritable> {21     22     public void reduce(Text key, Iterator<IntWritable> values,23                        OutputCollector<Text, IntWritable> output, 24                        Reporter reporter) throws IOException {25      ... ...26     }27   }28   29   static int printUsage() {30     System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");31     ToolRunner.printGenericCommandUsage(System.out);32     return -1;33   }34    35   public int run(String[] args) throws Exception {36     ... ...37     return 0;38   }39   public static void main(String[] args) throws Exception {40     int res = ToolRunner.run(new Configuration(), new WordCount(), args);41     System.exit(res);42   }43 44 }

新的WordCount

 1 package org.apache.hadoop.examples; 2  3 ... ... 4  5 public class WordCount { 6  7   public static class TokenizerMapper  8        extends Mapper<Object, Text, Text, IntWritable>{ 9     10     ... ... 11       12     public void map(Object key, Text value, Context context13                     ) throws IOException, InterruptedException {14       ... ...15     }16   }17   18   public static class IntSumReducer 19        extends Reducer<Text,IntWritable,Text,IntWritable> {20     private IntWritable result = new IntWritable();21 22     public void reduce(Text key, Iterable<IntWritable> values, 23                        Context context24                        ) throws IOException, InterruptedException {25       ... ...26     }27   }28 29   public static void main(String[] args) throws Exception {30     ... ...31     System.exit(job.waitForCompletion(true) ? 0 : 1);32   }33 }

6.