Hadoop集群的hdfs的java 的api的编写过程-蒲公英云

Hadoop集群的hdfs的java 的api的编写过程

矫情吗；* 2022-12-09 04:52 137阅读 0赞

开发环境：windows10、idea2020.2.1

需要配置的环境：

hadoop3.3.0，以及winutils.exe等文件的配置：

链接：https://pan.baidu.com/s/1dJDuAejepxsZQcHE7HCPQA 提取码：3utf

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d0bDE5OTI_size_16_color_FFFFFF_t_70

将上述的文件拷贝到windows里的hadoop的bin里。

然后配置HADOOP_HOME的环境变量，并添加到Path环境变量中。

这时候可能你打开的idea依然报错：

java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset.

然后重启idea，再次运行就可以了！！！

maven依赖：

<!--hadoop-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.3.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.3.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.3.0</version>
        </dependency>
package application.service.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
/**
 * @author: wtl
 * @License: (C) Copyright 2020, wtl Corporation Limited.
 * @Contact: 1050100468@qq.com
 * @Date: 2020-09-21 20:15
 * @Version: 1.0
 * @Description:
 */
public class HdfsOperateTest {
    private FileSystem fileSystem = null;
    @Before
    public void init() throws Exception {
        Configuration configuration = new Configuration();
        //设置要连接的hdfs集群
        configuration.set("fs.defaultFS","hdfs://hadoop01:9000");
        fileSystem = FileSystem.get(configuration);
    }
    @Test
    public void copyFromLocalFile() throws Exception {
        fileSystem.copyFromLocalFile(new Path("D:/datas/wordcount.txt"),new Path("/"));
    }
    @Test
    public void copyToLocalFile() throws Exception {
        fileSystem.copyToLocalFile(new Path("/wordcount.txt"),new Path("D:/datas/"));
    }
    @Test
    public void delete() throws Exception {
        boolean delete = fileSystem.delete(new Path("/ideaIU-2019.2.4.tar.gz"),true);
        System.out.println(delete);
    }
    @Test
    public void listFiles() throws Exception {
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus locatedFileStatus = locatedFileStatusRemoteIterator.next();
            System.out.println(locatedFileStatus.getPath());
            System.out.println(locatedFileStatus.getLen());
            System.out.println(locatedFileStatus.getReplication());
        }
    }
}