MapReduce编程文件合并和去重

喜欢ヅ旅行 2022-05-26 09:51 249阅读 0赞

# 、问题描述 #

> 对输入的多个文件进行合并，并剔除其中重复的内容，去重后的内容输出到一个文件中。

file1.txt中的内容:

20150101     x
    20150102     y
    20150103     x
    20150104     y

file2.txt中的内容:

20150105     z
    20150106     x
    20150101     y
    20150102     y

file3.txt中的内容:

20150103     x
    20150104     z
    20150105     y

# 二、MapReduce程序 #

编写MapReduce程序，运行环境参考我的上一篇博客[Intellij Idea配置MapReduce编程环境][Intellij Idea_MapReduce]

package com.javacore.hadoop;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import java.io.IOException;
    
    
    /**
     * Created by bee on 17/3/25.
     */
    public class FileMerge {
        
    
        public static class Map extends Mapper<Object, Text, Text, Text> {
        
            private static Text text = new Text();
    
            public void map(Object key, Text value, Context content) throws IOException, InterruptedException {
    
                text = value;
                content.write(text, new Text(""));
            }
        }
    
        public static class Reduce extends Reducer<Text, Text, Text, Text> {
        
            public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
                context.write(key, new Text(""));
            }
        }
    
    
        public static void main(String[] args) throws Exception {
    
            // delete output directory
            FileUtil.deleteDir("output");
            Configuration conf = new Configuration();
            conf.set("fs.defaultFS", "hdfs://localhost:9000");
            String[] otherArgs = new String[]{
       "input/filemerge/f*.txt",
                    "output"};
            if (otherArgs.length != 2) {
                System.err.println("Usage:Merge and duplicate removal <in> <out>");
                System.exit(2);
            }
    
            Job job = Job.getInstance();
            job.setJarByClass(FileMerge.class);
            job.setMapperClass(Map.class);
            job.setReducerClass(Reduce.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
    
            FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
            FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
            System.exit(job.waitForCompletion(true) ? 0 : 1);
    
        }
    }

# 三、输出 #

20150101     x
    20150101     y
    20150102     y
    20150103     x
    20150104     y
    20150104     z
    20150105     y
    20150105     z
    20150106     x

[Intellij Idea_MapReduce]: http://blog.csdn.net/napoay/article/details/68491469