利用 Hadoop FileSystem listStatus 遍历文件目录实现HDFS操作

深碍√TFBOYSˉ_ 2022-06-12 07:56 529阅读 0赞

[Hadoop][] 版本 2.7.0

利用 [hadoop][Hadoop] FileSystem listStatus 遍历文件目录 实现HDFS操作

package com.feng.test.hdfs.filesystem;
    
    import java.net.URI;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    
    /**
     * 遍历文件目录
     * 远程调用机器 需要 liunx 修改 /etc/hosts 添加 10.11.12.4 master
     * @author feng
     *
     */
    public class FileList {
          
    
        public static void main(String[] args) {
            FileSystem hdfs = null;
            try{
                Configuration config = new Configuration();
                // 程序配置
                config.set("fs.default.name", "hdfs://master:9000");
                //config.set("hadoop.job.ugi", "feng,111111");
                //config.set("hadoop.tmp.dir", "/tmp/hadoop-fengClient");
                //config.set("dfs.replication", "1");
                //config.set("mapred.job.tracker", "master:9001");
    
                hdfs = FileSystem.get(new URI("hdfs://master:9000"),
                        config, "feng");
                Path path = new Path("/");
    
                iteratorShowFiles(hdfs, path);
    
            }catch(Exception e){
                e.printStackTrace();
            }finally{
                if(hdfs != null){
                    try {
                        hdfs.closeAll();
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    
        /**
         * 
         * @param hdfs FileSystem 对象
         * @param path 文件路径
         */
        public static void iteratorShowFiles(FileSystem hdfs, Path path){
            try{
                if(hdfs == null || path == null){
                    return;
                }
                //获取文件列表
                FileStatus[] files = hdfs.listStatus(path);
    
                //展示文件信息
                for (int i = 0; i < files.length; i++) {
                    try{
                        if(files[i].isDirectory()){
                            System.out.println(">>>" + files[i].getPath()
                                    + ", dir owner:" + files[i].getOwner());
                            //递归调用
                            iteratorShowFiles(hdfs, files[i].getPath());
                        }else if(files[i].isFile()){
                            System.out.println("   " + files[i].getPath()
                                    + ", length:" + files[i].getLen()
                                    + ", owner:" + files[i].getOwner());
                        }
                    }catch(Exception e){
                        e.printStackTrace();
                    }
                }
            }catch(Exception e){
                e.printStackTrace();
            }
        }
    
    }
       
        
         1 
         2 
         3 
         4 
         5 
         6 
         7 
         8 
         9 
         10 
         11 
         12 
         13 
         14 
         15 
         16 
         17 
         18 
         19 
         20 
         21 
         22 
         23 
         24 
         25 
         26 
         27 
         28 
         29 
         30 
         31 
         32 
         33 
         34 
         35 
         36 
         37 
         38 
         39 
         40 
         41 
         42 
         43 
         44 
         45 
         46 
         47 
         48 
         49 
         50 
         51 
         52 
         53 
         54 
         55 
         56 
         57 
         58 
         59 
         60 
         61 
         62 
         63 
         64 
         65 
         66 
         67 
         68 
         69 
         70 
         71 
         72 
         73 
         74 
         75 
         76 
         77 
         78 
         79 
         80 
         81 
         82 
         83 
        
       
        
         1 
         2 
         3 
         4 
         5 
         6 
         7 
         8 
         9 
         10 
         11 
         12 
         13 
         14 
         15 
         16 
         17 
         18 
         19 
         20 
         21 
         22 
         23 
         24 
         25 
         26 
         27 
         28 
         29 
         30 
         31 
         32 
         33 
         34 
         35 
         36 
         37 
         38 
         39 
         40 
         41 
         42 
         43 
         44 
         45 
         46 
         47 
         48 
         49 
         50 
         51 
         52 
         53 
         54 
         55 
         56 
         57 
         58 
         59 
         60 
         61 
         62 
         63 
         64 
         65 
         66 
         67 
         68 
         69 
         70 
         71 
         72 
         73 
         74 
         75 
         76 
         77 
         78 
         79 
         80 
         81 
         82 
         83

[Hadoop]: http://lib.csdn.net/base/hadoop