HDFS的API学习

待我称王封你为后i 2022-04-16 04:06 216阅读 0赞

package com.jt;

import java.io.ByteArrayInputStream;  
import java.io.File;  
import java.io.FileInputStream;  
import java.io.FileOutputStream;  
import java.io.IOException;  
import java.io.InputStream;  
import java.io.OutputStream;  
import java.net.URI;  
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.BlockLocation;  
import org.apache.hadoop.fs.FileStatus;  
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.LocatedFileStatus;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.fs.RemoteIterator;  
import org.apache.hadoop.io.IOUtils;  
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication  
public class Starter \{

public static void main(String[] args) throws Exception {
    	testConnectNamenode();

// testPut();

// testDelete();  
// testLs1();  
// testGetBlock();  
\}

//连接name节点并读取数据
    public static void testConnectNamenode() throws Exception {
    	//获取hadoop的环境变量信息，可以通过此对象设置环境，通过代码设定的优先级要高于文件设置的优先级
    	//代码设定的生效范围是局部的
    	Configuration conf=new Configuration();

// conf.set(“hadoop.home.dir”, “/Users/joe/Documents/software/hadoop-2.7.1”);  
System.setProperty(“hadoop.home.dir”, “/Users/joe/Documents/software/hadoop-2.7.1”);  
//得到文件输入流

FileSystem  fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"), conf);
    	
    	InputStream in=fs.open(new Path("/ex/1.txt"));
    	
    	OutputStream out=new FileOutputStream("1.txt");
    	
    	IOUtils.copyBytes(in, out, conf);
    	
    	out.close();
    	in.close();
    	fs.close();
    	
    	
    }
    
    //上传文件到hdfs上
    public static void  testPut() throws Exception{
    	Configuration conf=new Configuration();
    	
    	conf.set("dfs.replication","1");
    	
    	System.setProperty("hadoop.home.dir", "/Users/joe/Documents/software/hadoop-2.7.1");
    	
    	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf,"root");

// ByteArrayInputStream in=new ByteArrayInputStream(“hello hdfs”.getBytes());  
InputStream in=new FileInputStream(new File("/Users/joe/Downloads/jtday15pm.mp4"));

OutputStream  out=fs.create(new Path("/park02/jtday15pm.mp4"));
    	IOUtils.copyBytes(in, out, conf);
    	
    	in.close();
    	out.close();
    	fs.close();
    	
    	
    	
    }
    
    //从hdfs上删除文件
    public static void testDelete() throws Exception{
    	Configuration conf=new Configuration();
    	System.setProperty("hadoop.home.dir", "/Users/joe/Documents/software/hadoop-2.7.1");
    	
    	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"), conf);
    	
       fs.deleteOnExit(new Path("/park02/jtday15pm.mp4"));
       
       fs.close();
        
    			
    	
    }
    
    //查询指定目录下文件
    
    public static void testLs() throws Exception{
    	Configuration conf=new Configuration();
    	System.setProperty("hadoop.home.dir","/Users/joe/Documents/software/hadoop-2.7.1");
    	
    	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf);
    	
    	FileStatus[] listStatus = fs.listStatus(new Path("/"));
    	
    	for(FileStatus status:listStatus){
    		System.out.println(status);
    	}
    	fs.close();
    }
    
    //递归查询
    
    public static void testLs1() throws Exception{
    	Configuration conf=new Configuration();
    	System.setProperty("hadoop.home.dir","/Users/joe/Documents/software/hadoop-2.7.1");
    	
    	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf);
    	
        RemoteIterator<LocatedFileStatus> lf = fs.listFiles(new Path("/"),true);
    	
    	while (lf.hasNext()){
    		System.err.println(lf.next());
    	}
    	
    	fs.close();
    	
    }
    
    //得到文件块信息
    
    public static void testGetBlock() throws Exception{
    	Configuration conf=new Configuration();
    	System.setProperty("hadoop.home.dir","/Users/joe/Documents/software/hadoop-2.7.1");
    	
    	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf);
    	
       BlockLocation[] fbl = fs.getFileBlockLocations(new Path("/park02/jtday15pm.mp4"), 0, Integer.MAX_VALUE);
    	for(BlockLocation bl:fbl){
    		System.err.println(bl);
    	}
       
       fs.close();
    	
    }