MapReduce - 词频统计

布满荆棘的人生 2024-04-17 23:15 109阅读 0赞

> 统计一个文本的词频

package Test01;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    
    public class WordCount {
        public static void main(String[] args){
            try{
                //获取配置对象
                Configuration conf = new Configuration();
                //对conf设置
                //获取job
                Job job = Job.getInstance(conf, "WordCount02");
                //对job设置运行主类
                job.setJarByClass(WordCount.class);
                //对job的map端属性设置
                job.setMapperClass(MyMapper.class);
                job.setMapOutputKeyClass(Text.class);
                job.setMapOutputValueClass(LongWritable.class);
                //对job的reduce端属性设置
                job.setReducerClass(MyReducer.class);
                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(IntWritable.class);
                //设置job的输入路径和输出路径
                FileInputFormat.addInputPath(job, new Path(args[0]));
                FileOutputFormat.setOutputPath(job, new Path(args[1]));
                //提交作业
                int success = job.waitForCompletion(true) ? 0: 1;
                //退出
                System.exit(success);
            }
            catch (Exception e){
                e.printStackTrace();
            }
        }
        //自定义Mapper类
        public static class MyMapper extends Mapper<Object, Text, Text, LongWritable> {
            public Text k =  new Text();
            public LongWritable v = new LongWritable(1L);
            //map函数，map阶段的核心业务逻辑
            @Override
            protected void map(Object key, Text value, Context context){
                try{
                    //获取行值
                    String row = value.toString();//map中每次读取一行
                    //拆分行值
                    String[] words = row.split(" ");
                    for(String st: words){
                        String st2 = "";
                        char[] ct = st.toCharArray();
                        if ((ct[st.length()-1] >= 'a' && ct[st.length()-1] <= 'z') || (ct[st.length()-1] >= 'A' && ct[st.length()-1] <= 'Z')){
                            st2 = String.valueOf(ct);
                        }
                        else {
                            for(int i = 0; i < st.length()-1; i++){
                                st2 += String.valueOf(ct[i]);
                            }
                        }
                        k.set(st2);
                        context.write(k, v);
                    }
                }
                catch (Exception e){
                    e.printStackTrace();
                }
            }
        }
        //自定义reducer类
        public static class MyReducer extends Reducer<Text, LongWritable, Text, IntWritable> {
            public IntWritable v = new IntWritable();
            //reduce函数，reduce阶段的核心业务逻辑
            @Override
            protected void reduce(Text key, Iterable<LongWritable> value, Context context){
                try{
                    //定义计数器
                    int cnt = 0;
                    for(LongWritable i: value){
                        cnt += i.get();
                    }
                    v.set(cnt);
                    //reduce的输出
                    context.write(key, v);
                }
                catch(Exception e){
                    e.printStackTrace();
                }
            }
        }
    }

（1）点开IDEA右边的maven，依次点击 Lifecycle - > clean（右键）-> Run Maven Build  
![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JsZXNzaW5nWFJZ_size_16_color_FFFFFF_t_70][]  
（2）依次点击 install（右键）-> Run Maven Build  
![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JsZXNzaW5nWFJZ_size_16_color_FFFFFF_t_70 1][]  
（3）将target下的jar包发送到Hadoop服务器  
![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JsZXNzaW5nWFJZ_size_16_color_FFFFFF_t_70 2][]  
（4）输入命令运行

yarn jar MapReducePractice-1.0-SNAPSHOT.jar Test01.WordCount /test/input/t1.txt /test/output/01

（5）查看运行结果

> \[hadoop@hadoop105 ~\]$ hdfs dfs -cat /test/output/01/part-r-00000  
> Dream 1  
> Hello 2  
> I 1  
> Love 1  
> World 1  
> You 2  
> are 1  
> hope 1  
> my 1

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JsZXNzaW5nWFJZ_size_16_color_FFFFFF_t_70]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/41172bdf12c344ce84426d6a7be445b4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JsZXNzaW5nWFJZ_size_16_color_FFFFFF_t_70 1]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/b9723be6d99c45b2960c799ca7329d99.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JsZXNzaW5nWFJZ_size_16_color_FFFFFF_t_70 2]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/c5b7b62af4444609b887ab42b61a4812.png