mapreduce读取文件内容输出到hbase

柔光的暖阳◎ 2022-06-13 06:57 250阅读 0赞

package hadoop;
    
    import java.io.IOException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.hbase.HBaseConfiguration;
    import org.apache.hadoop.hbase.client.Put;
    import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
    import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
    import org.apache.hadoop.hbase.mapreduce.TableReducer;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    
    public class FileToHbase {
    
    	static class FileMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
    
    		@Override
    		protected void map(LongWritable key, Text value,Context context)
    				throws IOException, InterruptedException {
    			String line = value.toString();
    			String[] lines = line.split(" ");
    			for(String string:lines){
    				context.write(new Text(string), new IntWritable(1));
    			}
    		}
    		
    		
    	}
    	
    	static class FileReduce extends TableReducer<Text, IntWritable, ImmutableBytesWritable>{
    
    		@Override
    		protected void reduce(Text key, Iterable<IntWritable> values,Context context)
    				throws IOException, InterruptedException {
    			int sum=0;
    			for(IntWritable ints:values){
    				sum += ints.get();
    			}
    			Put put = new Put(("row"+key).getBytes());
    			put.add("f".getBytes(), key.getBytes(), (sum+"").getBytes());
    			context.write(new ImmutableBytesWritable("result".getBytes()), put);
    		}
    		
    	}
    	
    	public static void main(String[] args) throws Exception {
    		Configuration conf = HBaseConfiguration.create();
    		conf.set("hbase.zookeeper.quorum", "192.168.10.250");
    		conf.set("hbase.zookeeper.property.clientPort", "2181");
    		Job job = Job.getInstance(conf,"hbasedemo");
    		job.setJarByClass(FileToHbase.class);
    		
    		FileInputFormat.addInputPath(job, new Path("/test/test.txt"));
    		job.setMapperClass(FileMapper.class);
    		job.setMapOutputKeyClass(Text.class);
    		job.setMapOutputValueClass(IntWritable.class);
    		
    		TableMapReduceUtil.initTableReducerJob("result", FileReduce.class, job);
    		boolean b = job.waitForCompletion(true);
    		if(!b){
    			System.exit(1);
    		}
    	}
    }