java hadoop wordcount
是什么？
hadoop 入门栗子，跑一个单词统计功能。
为什么？
理解一个MR，后面都是小事。万事开头难，一旦上了路接下来的借简单了。
怎么做？
一、先从网上找个Demo：https://www.programcreek.com/java-api-examples/?code=Nextzero/hadoop-2.6.0-cdh5.4.3/hadoop-2.6.0-cdh5.4.3-master/hadoop-mapreduce1-project/src/test/org/apache/hadoop/mapreduce/TestMapReduceLocal.java#
代码如下：
package com.geotmt.hadoop.hdfs;import java.io.IOException;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;/*** 单词计数** Created by chao,zhao on 2018/8/9. */public class FirstMapReduceJob {public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{private final static IntWritable one = new IntWritable(1);private Text word = new Text();public void map(Object key, Text value, Context context) throws IOException, InterruptedException {StringTokenizer itr = new StringTokenizer(value.toString());while (itr.hasMoreTokens()) {word.set(itr.nextToken());context.write(word, one);}}}public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable> {private IntWritable result = new IntWritable();public void reduce(Text key, Iterable<IntWritable> values,Context context ) throws IOException, InterruptedException {int sum = 0;for (IntWritable val : values) {sum += val.get();}result.set(sum);context.write(key, result);}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();if (otherArgs.length != 2) {System.err.println("Usage: wordcount <in> <out>");System.exit(2);}Job job = new Job(conf, "word count");job.setJarByClass(FirstMapReduceJob.class);job.setMapperClass(TokenizerMapper.class);job.setCombinerClass(IntSumReducer.class);job.setReducerClass(IntSumReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);FileInputFormat.addInputPath(job, new Path(otherArgs[0]));FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));System.exit(job.waitForCompletion(true) ? 0 : 1);}}
二、打包把依赖全部打进去，当然还有种方案是放在一个目录然后配置环境变量，个人觉得不可取，问题不断。
<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>myHadoop</groupId><artifactId>myHadoop</artifactId><version>1.0-SNAPSHOT</version><properties><hadoop.hdfs.version>2.4.0</hadoop.hdfs.version><hadoop.mapreduce.client.core.version>2.4.0</hadoop.mapreduce.client.core.version><hadoop.common.version>2.4.0</hadoop.common.version><hadoop.mapreduce.client.common.version>2.4.0</hadoop.mapreduce.client.common.version><hadoop.mapreduce.client.jobclient.version>2.4.0</hadoop.mapreduce.client.jobclient.version><commons.lang.version>3.3.2</commons.lang.version><commons.io.version>2.4</commons.io.version></properties><dependencies><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>${hadoop.hdfs.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-core</artifactId><version>${hadoop.mapreduce.client.core.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>${hadoop.common.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-common</artifactId><version>${hadoop.mapreduce.client.common.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-jobclient</artifactId><version>${hadoop.mapreduce.client.jobclient.version}</version></dependency><dependency><groupId>org.apache.commons</groupId><artifactId>commons-lang3</artifactId><version>${commons.lang.version}</version></dependency><dependency><groupId>commons-io</groupId><artifactId>commons-io</artifactId><version>${commons.io.version}</version></dependency></dependencies><build><plugins><plugin><artifactId>maven-assembly-plugin</artifactId><configuration><appendAssemblyId>false</appendAssemblyId><descriptorRefs><descriptorRef>jar-with-dependencies</descriptorRef></descriptorRefs><archive><manifest><!-- 此处指定main方法入口的class --><mainClass>com.geotmt.hadoop.hdfs.FirstMapReduceJob</mainClass></manifest></archive></configuration><executions><execution><id>make-assembly</id><phase>package</phase><goals><goal>assembly</goal></goals></execution></executions></plugin><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><configuration><source>1.7</source><target>1.7</target></configuration></plugin></plugins></build></project>
三、上传集群，记住用户和权限一致，执行命令：
hadoop jar myHadoop-1.0-SNAPSHOT.jar /user/zhaochao/20180727/sms.1532681090912.log.tmp /user/zhaochao/20180727/mr_result.txt
四、查看结果：
hadoop fs -cat /user/zhaochao/20180727/mr_result/part-r-00000
淡淡的烟草味﹌
我就是我
我会带着你远行
系统管理员![[C#版剑指offer]旋转数组的最小数字查询](https://image.dandelioncloud.cn/dist/img/NoSlightly.png "[C#版剑指offer]旋转数组的最小数字查询")
还没有评论，来说两句吧...