MR基础案例（三）去重

逃离我推掉我的手 2024-04-18 12:53 54阅读 0赞

**利用shuffle，整合相同的key**

数据：

北京-天津	2019-09-03
    北京-天津	2019-09-03
    北京-烟台	2019-09-04
    北京-哈尔滨	2019-09-03
    北京-成都	2019-09-03
    北京-成都	2019-09-03
    北京-三亚	2019-09-04

结果集：

北京-天津	2019-09-03
    北京-烟台	2019-09-04
    北京-哈尔滨	2019-09-03
    北京-成都	2019-09-03
    北京-三亚	2019-09-04

package MR;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    import java.io.IOException;
    
    public class QuChong {
        
        public static void main(String[] args) {
        
            Configuration conf=new Configuration();
            try {
        
                Job job=Job.getInstance(conf);
                job.setJarByClass(Score.class);
                job.setMapperClass(MyMapper.class);
                job.setReducerClass(MyReducer.class);
    
                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(Text.class);
    
                Path inPath =new Path("/input/QuChong.txt");
                FileInputFormat.addInputPath(job, inPath);
    
                Path outpath=new Path("/output/QuChong/");
                if(outpath.getFileSystem(conf).exists(outpath)){
        
                    outpath.getFileSystem(conf).delete(outpath, true);
                }
                FileOutputFormat.setOutputPath(job, outpath);
    
                job.waitForCompletion(true);
            } catch (Exception e) {
        
                // TODO Auto-generate
                //  .d catch block
                e.printStackTrace();
            }
        }
        public static class MyMapper extends Mapper<LongWritable, Text, Text, Text>{
        
            Text keyinfo = new Text();
            Text valueinfo = new Text();
            @Override
            protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        
                keyinfo.set(value.toString());
                valueinfo.set("");
                context.write(keyinfo,valueinfo);
            }
        }
        public static class MyReducer extends Reducer<Text, Text, Text, Text> {
        
            Text valueinfo = new Text();
            @Override
            protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        
                valueinfo.set("");
                context.write(key,valueinfo);
            }
        }
    }