【Java-HDFS】使用Java操作HDFS获取HDFS指定目录下的数据量大小

拼搏现实的明天。 2023-10-15 16:58 10阅读 0赞

*  Maven依赖

<dependencies>
            <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>RELEASE</version>
            </dependency>
            <dependency>
                <groupId>org.apache.logging.log4j</groupId>
                <artifactId>log4j-core</artifactId>
                <version>2.8.2</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>3.1.1</version>
            </dependency>
            
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>3.1.1</version>
            </dependency>
            
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>3.1.1</version>
            </dependency>
    <dependencies>

*  HDFSUtils工具类

package gaei.cn.x5l.x5lhive2cos.utils;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.security.UserGroupInformation;
    
    import java.io.FileNotFoundException;
    import java.io.IOException;
    
    public class HdfsUtils {
        
        private static final Configuration conf = new Configuration();
        private static FileSystem fs;
    
        public static void init() {
        
            //本地认证kerberos认证
    //        System.setProperty("java.security.krb5.conf", "D:/cyp/Chinasoft/keytab/prd_keytab/krb5.conf");
    //        conf.set("hadoop.security.authentication", "kerberos");
    //        UserGroupInformation.setConfiguration(conf);
    //        try {
        
    //            UserGroupInformation.loginUserFromKeytab("x9e@PRD.X8V.COM", "D:/cyp/Chinasoft/keytab/prd_keytab/x9e.keytab");
    //        } catch (IOException e) {
        
    //            e.printStackTrace();
    //        }
    //        System.out.println(Thread.currentThread().getContextClassLoader());
            try {
        
    			//将core-site.xml和hdfs-site.xml放在resource目录下即可，必须指定，不然获取不到HDFS的nameserver等信息
                conf.addResource(Thread.currentThread().getContextClassLoader().getResource("./core-site.xml"));
                conf.addResource(Thread.currentThread().getContextClassLoader().getResource("./hdfs-site.xml"));
                fs = FileSystem.get(conf);
            } catch (FileNotFoundException fnfe) {
        
                fnfe.printStackTrace();
            } catch (IOException ioe) {
        
                ioe.printStackTrace();
            }
        }
    
        public static long getHdfsSampleDateSize(String hdfsPath) {
        
            long length = 0L;
            try {
        
                length = fs.getContentSummary(new Path(hdfsPath)).getLength();
            } catch (IOException e) {
        
                e.printStackTrace();
            }
            return length;
        }
    
        public static void close() {
        
            try {
        
                fs.close();
            } catch (IOException ex) {
        
                ex.printStackTrace();
            }
        }
    }