Hadoop按日期统计访问次数

深碍√TFBOYSˉ_ 2022-02-21 00:05 551阅读 0赞

*  **MapReduce编程主要组件**  
    InputFormat类：分割成多个splits和每行怎么解析。  
    Mapper类：对输入的每对<key,value>生成中间结果。  
    Combiner类：在map端，对相同的key进行合并。  
    Partitioner类：在shuffle过程中，将按照key值将中间结果分为R份，每一份都由一个reduce去完成。  
    Reducer类：对所有的map中间结果，进行合并。  
    OutputFormat类：负责输出结果格式。
 *  **执行原理图**  
    ![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzMyMTY4MDg3_size_16_color_FFFFFF_t_70]
 *  **代码示例**
    
    **引入依赖包**

<dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>2.4.0</version>
         </dependency>
         <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>2.4.0</version>
         </dependency>
         <dependency>
    	    <groupId>org.apache.hadoop</groupId>
    	    <artifactId>hadoop-client</artifactId>
    	    <version>2.4.0</version>
         </dependency>

**Mapper类代码**

package cn.demo.myfriend.data;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    /**
     * 
     * @author zhongyulin
     *   LongWritable   输入的偏移量
     *   Text    输入的数据
     *   Text    输出的key
     *   IntWritable    输出的value
     */
    public class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        
    	private final static IntWritable one=new IntWritable(1);
    	
    	@Override
    	protected void map(LongWritable key, Text value, Context context)
    			throws IOException, InterruptedException {
    		  String line=value.toString();
    		  //按规则拆分成数组
    		  String[] arry=line.split(",");
    		  String keyout=arry[1];
    		  context.write(new Text(keyout),one);
    	}
    }

**Reducer代码类**

package cn.demo.myfriend.data;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    /**
     * Reducer模块
     * @author zhongyulin
     *
     */
    public class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
        
    	private IntWritable result=new IntWritable();
    	@Override
    	protected void reduce(Text key, Iterable<IntWritable> values,
    			Context context) throws IOException, InterruptedException {
    		 int sum=0;
    		 for (IntWritable val : values) {
    			sum +=val.get();
    		}
    		 result.set(sum);
    		 context.write(key, result);
    	}
    }

**程序启动入口**

package cn.demo.myfriend.data;
    
    import java.io.IOException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    
    public class JobMain {
       public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    	   Configuration conf =new Configuration();
    		//创建任务
    		Job job= Job.getInstance(conf);
    		//指定执行的map和Reduce的类
    		job.setMapperClass(MyMapper.class);
    		job.setReducerClass(MyReducer.class);
    		
    	    // 指定map和任务输出的类型
    		job.setMapOutputKeyClass(Text.class);
    		job.setMapOutputValueClass(IntWritable.class);
    		job.setOutputKeyClass(Text.class);
    		job.setOutputValueClass(IntWritable.class);
    		
    		//设置文件输出路径
    	    FileInputFormat.addInputPath(job,new Path("hdfs://172.80.2.207:9000/test/user_login.txt"));
    	    FileOutputFormat.setOutputPath(job,new Path("hdfs://172.80.2.207:9000/test/dataLogin"));
    	    
    	    boolean flag=job.waitForCompletion(true);
       }
    }

**数据示例格式**

Nehru,2016-01-01
    Dane,2016-01-01
    Walter,2016-01-01
    Gloria,2016-01-01
    Clarke,2016-01-01
    Madeline,2016-01-01
    Kevyn,2016-01-01
    Rebecca,2016-01-01
    Calista,2016-01-01
    Lana,2016-01-01
    Phoebe,2016-01-01
    Clayton,2016-01-01
    Kimberly,2016-01-01
    Drew,2016-01-01
    Giselle,2016-01-01
    Nolan,2016-01-01
    Madeson,2016-01-01
    Janna,2016-01-01
    Raja,2016-01-01
    Merrill,2016-01-01
    Aquila,2016-01-01
    Idona,2016-01-01
    Connor,2016-01-01
    acqueline,2016-01-01
    Shaeleigh,2016-01-01
    Bevis,2016-01-01
    Howard,2016-01-01
    Sylvia,2016-01-01
    Molly,2016-01-01
    Julie,2016-01-01
    Hedy,2016-01-01
    Kuame,2016-01-01
    Kerry,2016-01-01
    Burton,2016-01-01
    Abra,2016-01-01

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzMyMTY4MDg3_size_16_color_FFFFFF_t_70]: /images/20220221/aacc7ccfe43b4df2b993b528b32fd755.png