Java操作HDFS(常用API)

落日映苍穹つ 2021-09-27 14:34 548阅读 0赞

### Java操作HDFS ###

*  先前准备
 *  API操作
 *   *  查看文件
     *  创建新文件夹
     *  上传文件
     *  下载文件
     *  删除文件
     *  内部复制和内部移动(剪切)
     *  重命名
     *  创建新的文件
     *  写文件
     *  读文件
     *  追加写
     *  获取数据位置

# 先前准备 #

*  确保HDFS集群已经搭建 如有不会请查看[分布式HDFS集群搭建][HDFS]
 *  在本机配好hadoop环境变量，必要时在hosts文件下加入node和ip
 *  在eclipse中配置Hadoop插件
    
     *  在eclipse安装目录下打开dropins文件夹新建plugins放入[hadoop-eclipse-plugin-2.6.0.jar][]  
        ![eclipse配置信息][eclipse]
 *  导入依赖的jar包(在hadoop安装包下)

# API操作 #

*  首先先创建配置对象

// 操作HDFS之前得先创建配置对象
    Configuration conf = new Configuration(true);
    // 创建操作HDFS的对象
    FileSystem fs=fileSystem.get(conf);

## 查看文件 ##

// 调用listStatus(Path path)方法
    private static List listFileSystem(FileSystem fs, String path) throws FileNotFoundException, IOException { 
    		Path ppath = new Path(path);// 获取路径
    		FileStatus[] listStatus = fs.listStatus(ppath);
    		
    		for (FileStatus fileStatus : listStatus) { 
    			System.out.println(fileStatus.getPath());
    		}
    		return null;
    	}

## 创建新文件夹 ##

private static void createDir(FileSystem fs, String string) 				throws IllegalArgumentException, IOException { 
    	Path path = new Path(string);
    	if(fs.exists(path)){ // 判断文件是否存在
    		fs.delete(path, true);
    	}	
    	fs.mkdirs(path);
    }

## 上传文件 ##

// copyFromLocalFile()
    private static void uploadFileToHDFS(FileSystem fs, String src, String dest) throws IOException { 
    	Path srcPath = new Path(src);
    	Path destPath = new Path(dest);
    	//copyFromLocal 复制
    	fs.copyFromLocalFile(srcPath, destPath);
    	//moveFromLocal 剪切
    	fs.copyFromLocalFile(true,srcPath, destPath);
    }

## 下载文件 ##

private static void downLoadFileFromHDFS(FileSystem fs, String src, String dest) throws IOException { 
    	Path srcPath = new Path(src);
    	Path destPath = new Path(dest);
    	//copyToLocal 复制到本地
    // fs.copyToLocalFile(srcPath, destPath);
    	//moveToLocal 剪切到本地
    	fs.copyToLocalFile(true,srcPath, destPath);
    }

## 删除文件 ##

fs.delete(path,true);

## 内部复制和内部移动(剪切) ##

private static void innerCopyAndMoveFile(FileSystem fs, Configuration conf,String src, String dest) throws IOException { 
    	Path srcPath = new Path(src);
    	Path destPath = new Path(dest);	
    	//内部复制
    	FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,false, conf);
    	//内部移动
    	FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,true, conf);
    }

## 重命名 ##

private static void renameFile(FileSystem fs, String src, String dest) throws IOException { 
    	Path srcPath = new Path(src);
    	Path destPath = new Path(dest);	
    	fs.rename(srcPath, destPath);
    	}

## 创建新的文件 ##

private static void createNewFile(FileSystem fs, String string) throws IllegalArgumentException, IOException { 
    	fs.createNewFile(new Path(string));
    }

## 写文件 ##

private static void writeToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException { 
    	FSDataOutputStream outputStream = fs.create(new Path(filePath));
    	outputStream.write(content.getBytes("UTF-8"));
    	outputStream.flush();
    	outputStream.close();
    }

## 读文件 ##

private static void readFromHDFSFile(FileSystem fs, String string) throws IllegalArgumentException, IOException { 
    	FSDataInputStream inputStream = fs.open(new Path(string));	
    	FileStatus fileStatus = fs.getFileStatus(new Path(string));
    	long len = fileStatus.getLen();	
    		byte[] b = new byte[(int)len];
    		int read = inputStream.read(b);
    		while(read != -1){ 
    			System.out.println(new String(b));
    			read = inputStream.read(b);
    		}
    }

## 追加写 ##

private static void appendToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException { 
    	FSDataOutputStream append = fs.append(new Path(filePath));
    	append.write(content.getBytes("UTF-8"));
    	append.flush();
    	append.close();
    }

## 获取数据位置 ##

private static void getFileLocation(FileSystem fs, String string) throws IOException { 
    	FileStatus fileStatus = fs.getFileStatus(new Path(string));
    	long len = fileStatus.getLen();
    	BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
    	String[] hosts = fileBlockLocations[0].getHosts();
    	for (String string2 : hosts) { 
    		System.out.println(string2);
    	}	
    	HdfsBlockLocation blockLocation = (HdfsBlockLocation)fileBlockLocations[0];
    	long blockId = blockLocation.getLocatedBlock().getBlock().getBlockId();
    	System.out.println(blockId);
    }

[HDFS]: https://blog.csdn.net/qq_37162090/article/details/83034095
[hadoop-eclipse-plugin-2.6.0.jar]: https://pan.baidu.com/s/1x_bE9hYdUCOs4pvFy3XnVg
[eclipse]: /images/20210923/67348070445b4c2d8c58022cd4c1bbcb.png