WordCount运行测试

浅浅的花香味﹌ 2022-07-24 10:12 172阅读 0赞

在hadoop目录下创建wordcount文件夹

>     mkdir wordcount
> 
>     cd wordcount

创建输入文件夹input和两个输入文件file0、file1，在这里将wordcount源码作为file0内容。

![20160519210718245][]

创建WordCount.java文件，源码附在下方

>     vim WordCount.java

> 编译java文件

>     javac WordCount.java

![20160519210651933][]

> 将编译后的.class文件打成jar包

>     jar -cvf WordCount.jar *.class

![20160519210708525][]

上传测试文件到hdfs

>     hadoop fs -put input/* input

![20160519210740246][]

> 查看上传

>     hadoop fs -ls input/

> 运行Wordcount

>     hadoop jar WordCount.jar WordCount input output

> 如果hdfs中已有output存在则删除后再运行

>     hadoop fs -rmr output

> WordCount.java
> 
>     import java.io.IOException;
>     import java.util.*;
>             
>     import org.apache.hadoop.fs.Path;
>     import org.apache.hadoop.conf.*;
>     import org.apache.hadoop.io.*;
>     import org.apache.hadoop.mapreduce.*;
>     import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
>     import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
>     import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>     import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>             
>     public class WordCount {
>             
>      public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {
>         private final static IntWritable one = new IntWritable(1);
>         private Text word = new Text();
>             
>         public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
>             String line = value.toString();
>             StringTokenizer tokenizer = new StringTokenizer(line,"\t\n\r\f\b, . :;?![]()<>{}\"=+-'");
>             while (tokenizer.hasMoreTokens()) {
>                 word.set(tokenizer.nextToken());
>                 context.write(word, one);
>             }
>         }
>      } 
>             
>      public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> {
>     
>         public void reduce(Text key, Iterable<IntWritable> values, Context context) 
>           throws IOException, InterruptedException {
>             int sum = 0;
>             for (IntWritable val : values) {
>                 sum += val.get();
>             }
>             context.write(key, new IntWritable(sum));
>         }
>      }
>             
>      public static void main(String[] args) throws Exception {
>         Configuration conf = new Configuration();
>             
>         Job job = new Job(conf, "wordcount");
>         
>         job.setOutputKeyClass(Text.class);
>         job.setOutputValueClass(IntWritable.class);
>             
>         job.setMapperClass(Map.class);
>         job.setReducerClass(Reduce.class);
>             
>         job.setInputFormatClass(TextInputFormat.class);
>         job.setOutputFormatClass(TextOutputFormat.class);
>             
>         FileInputFormat.addInputPath(job, new Path(args[0]));
>         FileOutputFormat.setOutputPath(job, new Path(args[1]));
>             
>         job.waitForCompletion(true);
>      }
>             
>     }
> 
>   
>   
>

[20160519210718245]: /images/20220722/9ec209a7b6004dd0a16a37f405e77360.png
[20160519210651933]: /images/20220722/2e9c496abbb841e39a047b36725917ca.png
[20160519210708525]: /images/20220722/8c822c832cac4fc982f55f43e18d06a3.png
[20160519210740246]: /images/20220722/aecb8f6e10594f6bb0a2845ed1ca6ba8.png