Hadoop实战演练：搜索数据分析----数据去重（1）

快来打我* 2021-09-11 03:40 520阅读 0赞

林炳文Evankaka原创作品。转载请注明出处[http://blog.csdn.net/evankaka][http_blog.csdn.net_evankaka]

摘要：本文讲了如何使用hadoop进行数据去重

工程源码下载：[https://github.com/appleappleapple/BigDataLearning/tree/master/Hadoop-Demo][https_github.com_appleappleapple_BigDataLearning_tree_master_Hadoop-Demo]

一、环境与数据

1、本地开发环境

Windows7 + Eclipse Luna

hadoop版本：2.6.2

JDK版本：1.8

2、数据来源：

搜狗实验室

http://www.sogou.com/labs/resource/q.php

3、数据格式

用户ID\\t\[查询词\]\\t该URL在返回结果中的排名\\t用户点击的顺序号\\t用户点击的URL  
其中，用户ID是根据用户使用浏览器访问搜索引擎时的Cookie信息自动赋值，即同一次使用浏览器输入的不同查询对应同一个用户ID

样例：

![20160915144919379][]

3、数据去重

每一人的同一个搜索关键词能可会有多个点击条数。这里要去除同一个人同一个关键词重复的条数

二、编程实现

package com.lin.keyword;
    
    import java.io.IOException;
    import java.util.StringTokenizer;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
    import org.apache.hadoop.util.GenericOptionsParser;
    
    /**
     * 功能概要：数据去重
     * 
     * @author linbingwen
     * @since 2016年7月31日
     */
    public class CleanSameData {
    	// map将输入中的value复制到输出数据的key上，并直接输出
    	public static class Map extends Mapper<Object, Text, Text, Text> {
    
    		// 实现map函数
            @Override
    		public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    			// 将输入的纯文本文件的数据转化成String
    			String line = value.toString();
    			// 将输入的数据首先按行进行分割
    			StringTokenizer tokenizerArticle = new StringTokenizer(line, "\n");
    			// 分别对每一行进行处理
    			while (tokenizerArticle.hasMoreElements()) {
    				// 每行按空格划分
    				StringTokenizer tokenizerLine = new StringTokenizer(tokenizerArticle.nextToken());
    				String c1 = tokenizerLine.nextToken();//
    				String c2 = tokenizerLine.nextToken();// 关键词
    				c2 = c2.substring(1, c2.length() - 1);
    				Text newline = new Text(c1 + "    " + c2);
    				context.write(newline, new Text(""));
    			}
    
    		}
    
    	}
    
    	// reduce将输入中的key复制到输出数据的key上，并直接输出
    	public static class Reduce extends Reducer<Text, Text, Text, Text> {
    		// 实现reduce函数
            @Override
    		public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
    			context.write(key, new Text(""));
    		}
    	}
    
    	public static void main(String[] args) throws Exception {
    
    		Configuration conf = new Configuration();
    		// 设置hadoop的机器、端口
    		conf.set("mapred.job.tracker", "10.75.201.125:9000");
    		// 设置输入输出文件目录
    		String[] ioArgs = new String[] { "hdfs://hmaster:9000/data_in", "hdfs://hmaster:9000/clean_same_out" };
    		String[] otherArgs = new GenericOptionsParser(conf, ioArgs).getRemainingArgs();
    		if (otherArgs.length != 2) {
    			System.err.println("Usage:  <in> <out>");
    			System.exit(2);
    		}
    		// 设置一个job
    		Job job = Job.getInstance(conf, "clean same data");
    		job.setJarByClass(CleanSameData.class);
    
    		// 设置Map、Combine和Reduce处理类
    		job.setMapperClass(Map.class);
    		job.setCombinerClass(Reduce.class);
    		job.setReducerClass(Reduce.class);
    
    		// 设置输出类型
    		job.setOutputKeyClass(Text.class);
    		job.setOutputValueClass(Text.class);
    
    		// 将输入的数据集分割成小数据块splites，提供一个RecordReder的实现
    		job.setInputFormatClass(TextInputFormat.class);
    
    		// 提供一个RecordWriter的实现，负责数据输出
    		job.setOutputFormatClass(TextOutputFormat.class);
    
    		// 设置输入和输出目录
    		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    		System.exit(job.waitForCompletion(true) ? 0 : 1);
    
    	}
    
    }

maven文件：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    	<modelVersion>4.0.0</modelVersion>
    	<groupId>com.lin</groupId>
    	<artifactId>zdy-hadoop</artifactId>
    	<version>0.0.1-SNAPSHOT</version>
    	<properties>
    		
    		<spring.version>4.3.0.RELEASE</spring.version>
    		
    		<slf4j.version>1.6.6</slf4j.version>
    		<log4j.version>1.2.12</log4j.version>
    		
    		<junit.version>4.12</junit.version>
    		<hadoop.version>2.6.0</hadoop.version>
    		
    		<mybatis.version>3.2.1</mybatis.version>
    		<mysql.connect.version>6.0.3</mysql.connect.version>
    	</properties>
    
    	<dependencies>		
    		
    		<dependency>
    			<groupId>junit</groupId>
    			<artifactId>junit</artifactId>
    			<version>${junit.version}</version>
    			<scope>test</scope>
    		</dependency>
    		
    		<dependency>
    			<groupId>log4j</groupId>
    			<artifactId>log4j</artifactId>
    			<version>${log4j.version}</version>
    		</dependency>
    		<dependency>
    			<groupId>org.slf4j</groupId>
    			<artifactId>slf4j-api</artifactId>
    			<version>${slf4j.version}</version>
    		</dependency>
    		<dependency>
    			<groupId>ch.qos.logback</groupId>
    			<artifactId>logback-classic</artifactId>
    			<version>1.1.2</version>
    		</dependency>
    		<dependency>
    			<groupId>ch.qos.logback</groupId>
    			<artifactId>logback-core</artifactId>
    			<version>1.1.2</version>
    		</dependency>
    		<dependency>
    			<groupId>org.logback-extensions</groupId>
    			<artifactId>logback-ext-spring</artifactId>
    			<version>0.1.1</version>
    		</dependency>
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-common</artifactId>
    			<version>2.6.0</version>
    		</dependency>
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-hdfs</artifactId>
    			<version>2.6.0</version>
    		</dependency>
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-client</artifactId>
    			<version>2.6.0</version>
    		</dependency>
    		<dependency>
    			<groupId>jdk.tools</groupId>
    			<artifactId>jdk.tools</artifactId>
    			<version>1.8</version>
    			<scope>system</scope>
    			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
    		</dependency>
    	     
    		<dependency>
    			<groupId>mysql</groupId>
    			<artifactId>mysql-connector-java</artifactId>
    			<version>${mysql.connect.version}</version>
    		</dependency>
    	</dependencies>
    
    	<build>
    		<finalName>zdy-hadoop</finalName>
    		<filters>
    		    
    			<filter>src/main/resources/env/${env}.properties</filter>
    		</filters>
    		
    		<resources>
    			<resource>
    				<directory>src/main/resources</directory>
    				<targetPath>${basedir}/target/classes</targetPath>
    				<includes>
    					<include>**/*.properties</include>
    					<include>**/*.xml</include>
    				</includes>
    				<filtering>true</filtering>
    			</resource>
    			<resource>
    				<directory>src/main/resources</directory>
    				<targetPath>${basedir}/target/resources</targetPath>
    				<includes>
    					<include>**/*.properties</include>
    					<include>**/*.xml</include>
    				</includes>
    				<filtering>true</filtering>
    			</resource>
    		</resources>
    
    		
    		<plugins>
    			<plugin>
    				<groupId>org.apache.maven.plugins</groupId>
    				<artifactId>maven-compiler-plugin</artifactId>
    				<version>2.0.2</version>
    				<configuration>
    					<source>1.8</source>
    					<target>1.8</target>
    					<encoding>UTF-8</encoding>
    				</configuration>
    			</plugin>
    			<plugin>
    				<groupId>org.apache.maven.plugins</groupId>
    				<artifactId>maven-jar-plugin</artifactId>
    				<executions>
    					
    					<execution>
    						<phase>prepare-package</phase>
    						<goals>
    							<goal>jar</goal>
    						</goals>
    						<configuration>
    							<includes>
    								<include>com/**</include>
    							</includes>
    						</configuration>
    					</execution>
    				</executions>
    			</plugin>
    			<plugin>
    				<artifactId>maven-assembly-plugin</artifactId>
    				<configuration>
    					
    					<appendAssemblyId>false</appendAssemblyId>
    					<finalName>${project.artifactId}</finalName>
    					<descriptors>
    						<descriptor>src/main/assemble/assembly.xml</descriptor>
    					</descriptors>
    				</configuration>
    				<executions>
    					<execution>
    						<id>make-assembly</id>
    						<phase>package</phase>
    						<goals>
    							<goal>single</goal>
    						</goals>
    					</execution>
    				</executions>
    			</plugin>
    	</plugins>
    	</build>
    	<profiles>
    		
    		<profile>
    			<id>dev</id>
    			<activation>
    				<activeByDefault>true</activeByDefault>
    			</activation>
    			<properties>
    				<env>dev</env>
    			</properties>
    		</profile>
    	</profiles>
    </project>

输出结果：

![20160915145200099][]

[http_blog.csdn.net_evankaka]: http://blog.csdn.net/evankaka
[https_github.com_appleappleapple_BigDataLearning_tree_master_Hadoop-Demo]: https://github.com/appleappleapple/BigDataLearning/tree/master/Hadoop-Demo
[20160915144919379]: /images/20210811/1e7ffb56976948fe9c62725494d44a77.png
[20160915145200099]: /images/20210811/9d7761678a114dcbb5c6d7603daf37ee.png