MapReduce编程实现按词频统计的排序输出

た入场券 2023-02-20 02:34 2阅读 0赞

先计数后排序  
**`计数`**

import java.io.IOException;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    public class WordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{ 
    	
    	Text k = new Text();
    	IntWritable v = new IntWritable(1);
    	@Override
    	protected void map(LongWritable key, Text value,Context context)
    			throws IOException, InterruptedException { 
    		System.out.println(key.toString());
    		//获取一行
    		String line = value.toString();
    		//切割单词
    		String[] words = line.split(" ");
    		//循环翻译
    		for (String word: words) { 
    		k.set(word);
    		context.write(k, v);
    		}
    	}
    }

import java.io.IOException;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    public class WordcountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{ 
    	IntWritable v = new IntWritable();
    	@Override
    	protected void reduce(Text key, Iterable<IntWritable> values,
    			Context context) throws IOException, InterruptedException { 
    		int sum = 0;
    		//累计求和
    		for (IntWritable value : values) { 
    			sum += value.get();
    			}
    		v.set(sum);
    		//输出
    		context.write(key, v);
    	}
    
    }

import java.io.IOException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class WordcountDriver { 
    	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 
    		args = new String[] { "f:/input1","f:/in"};
    		Configuration conf = new Configuration();
    		//获取job
    		Job job = Job.getInstance(conf);
    		//设置jar包
    		job.setJarByClass(WordcountDriver.class);
    		//关联mapper和reducer
    		job.setMapperClass(WordcountMapper.class);
            job.setReducerClass(WordcountReducer.class);
    		//map输出的k和v
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(IntWritable.class);
    		//最终输出kv
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(IntWritable.class);
    		//输入和输出路径
            FileInputFormat.setInputPaths(job, new Path(args[0]));
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
    		//提交job
            boolean result = job.waitForCompletion(true);
            System.exit(result?0:1);
    	}
    
    }

第一个任务输出的结果  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQ2NTQ4ODU1_size_16_color_FFFFFF_t_70]

**`按词频排序`**

import org.apache.hadoop.io.WritableComparable;
    
    import java.io.DataInput;
    import java.io.DataOutput;
    import java.io.IOException;
    
    
    public class tongji implements WritableComparable <tongji>{ 
        private long number;
    
        public tongji(long number) { 
            this.number = number;
        }
    
        public tongji() { 
        }
    
        public long getNumber() { 
            return number;
        }
    
        public void setNumber(long number) { 
            this.number = number;
        }
    
        @Override
        public String toString() { 
            return String.valueOf(number);
        }
    
    
    
    
        @Override
        public int compareTo(tongji o) { 
            int result;
    
            // 按照总流量大小，倒序排列
            if (number > o.getNumber()) { 
                result = -1;
            }else if (number < o.getNumber()) { 
                result = 1;
            }else { 
                result = 0;
            }
    
            return result;
        }
    
    
    
    
        @Override
        public void write(DataOutput out) throws IOException { 
            out.writeLong(number);
    
        }
    
        @Override
        public void readFields(DataInput in) throws IOException { 
            number =in.readLong();
    
        }
    }

import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    import java.io.IOException;
    
    public class WordcountMapper extends Mapper<LongWritable, Text, tongji, Text>{ 
    
        tongji k = new tongji();
        Text v = new Text();
        @Override
        protected void map(LongWritable key, Text value,Context context)
                throws IOException, InterruptedException { 
            //获取一行
            String line = value.toString();
            //切割单词
            String[] fields = line.split("\t");
            // 3 封装对象
            String danci = fields[0];
            long number = Long.parseLong(fields[1]);
            k.setNumber(number);
            v.set(danci);
            context.write(k,v);
        }
    }

import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    
    public class WordCountReducer extends Reducer<tongji,Text,Text, tongji> { 
    
        @Override
        protected void reduce(tongji key, Iterable<Text> values, Context context)
                throws IOException, InterruptedException { 
            for (Text value : values) { 
                context.write(value,key);
            }
    
        }
    }

import java.io.IOException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class WordcountDriver { 
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 
            args = new String[] { "f:/input1","f:/in"};
            Configuration conf = new Configuration();
            //获取job
            Job job = Job.getInstance(conf);
            //设置jar包
            job.setJarByClass(WordcountDriver.class);
            //关联mapper和reducer
            job.setMapperClass(WordcountMapper.class);
            job.setReducerClass(WordCountReducer.class);
            //map输出的k和v
            job.setMapOutputKeyClass(tongji.class);
            job.setMapOutputValueClass(Text.class);
            //最终输出kv
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(tongji.class);
            //输入和输出路径
            FileInputFormat.setInputPaths(job, new Path(args[0]));
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            //提交job
            boolean result = job.waitForCompletion(true);
            System.exit(result?0:1);
        }
    
    }

最终输出结果  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQ2NTQ4ODU1_size_16_color_FFFFFF_t_70 1]  
如果你想打包在集群上运行，记得把\*\*`args = new String[] {"f:/input1","f:/in"}`\*\*删掉 然后在打包  
打包到集群上运行  
[https://blog.csdn.net/qq\_46548855/article/details/105954338][https_blog.csdn.net_qq_46548855_article_details_105954338]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQ2NTQ4ODU1_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200706132145229.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQ2NTQ4ODU1,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQ2NTQ4ODU1_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/20200706132407221.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQ2NTQ4ODU1,size_16,color_FFFFFF,t_70
[https_blog.csdn.net_qq_46548855_article_details_105954338]: https://blog.csdn.net/qq_46548855/article/details/105954338