环境配置

谁借莪１个温暖的怀抱￠ 2021-12-24 15:55 357阅读 0赞

一。生成数据库表

1. 创建数据库和表

/*
    SQLyog Ultimate v12.08 (64 bit)
    MySQL - 5.6.42 : Database - zghPortrait
    *********************************************************************
    */
    
    
    /*!40101 SET NAMES utf8 */;
    
    /*!40101 SET SQL_MODE=''*/;
    
    /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
    /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
    /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
    /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
    CREATE DATABASE /*!32312 IF NOT EXISTS*/`zghPortrait` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */;
    
    USE `zghPortrait`;
    
    /*Table structure for table `userInfo` */
    
    DROP TABLE IF EXISTS `userInfo`;
    
    CREATE TABLE `userInfo` (
      `userid` int(20) DEFAULT NULL COMMENT '用户id',
      `username` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '名称',
      `password` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '密码',
      `sex` int(1) DEFAULT NULL COMMENT '性别',
      `telphone` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '手机号',
      `email` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱',
      `age` int(20) DEFAULT NULL COMMENT '年龄',
      `idCard` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '身份证',
      `registerTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '注册时间',
      `usetype` int(1) DEFAULT NULL COMMENT '使用终端：0 pc端：1 移动端：2 小程序端'
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
    
    /*Data for the table `userInfo` */
    
    /*Table structure for table `userdetail` */
    
    DROP TABLE IF EXISTS `userdetail`;
    
    CREATE TABLE `userdetail` (
      `userdetailid` int(20) NOT NULL COMMENT '详情id',
      `userid` int(20) DEFAULT NULL COMMENT '用户id',
      `edu` int(1) DEFAULT NULL COMMENT '学历',
      `profession` varchar(20) COLLATE utf8_bin DEFAULT NULL COMMENT '职业',
      `marriage` int(1) DEFAULT NULL COMMENT '婚姻状态:1 未婚：2 已婚：3 离异',
      `haschild` int(1) DEFAULT NULL COMMENT '1.没有小孩：2.有小孩 3.未知 4. 未知',
      `hascar` int(1) DEFAULT NULL COMMENT '1.有车；2.没车；3. 未知',
      `hashourse` int(1) DEFAULT NULL COMMENT '1.有房；2.没房；3. 未知',
      `telphonebrand` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '手机品牌',
      PRIMARY KEY (`userdetailid`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
    
    /*Data for the table `userdetail` */
    
    /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
    /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
    /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
    /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

二。创建maven工程

1. 生成父工程：zghPortraitMain ，并在此工程下生成工程：analySrv ，工程 analySrv 继承且聚合于工程 zghPortraitMain

2. 在工程 analySrv 下添加依赖，如果依赖下载无进度，停止任务并对应删除本地仓库的依赖，重新载入或者刷新下载

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    
        <modelVersion>4.0.0</modelVersion>
        <groupId>com.zgh.portrait</groupId>
        
        
        <artifactId>analySrv</artifactId>
    
        <parent>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-examples</artifactId>
            <version>1.7.0</version>
        </parent>
    
        <properties>
            <scala.version>2.11.12</scala.version>
            <scala.binary.version>2.11</scala.binary.version>
        </properties>
    
        <dependencies>
            <dependency>
                <groupId>org.apache.flink</groupId>
                <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
                <version>${project.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.flink</groupId>
                <artifactId>flink-scala_2.11</artifactId>
                <version>${project.version}</version>
            </dependency>
        </dependencies>
    
    </project>

3. 在 analySrv 下新建包：com.zgh.task ，并创建 Test 类

![1271822-20190228212652012-957909689.png][]

package com.zgh.task;
    
    import org.apache.flink.api.java.utils.ParameterTool;
    import org.apache.flink.api.java.DataSet;
    import org.apache.flink.api.java.ExecutionEnvironment;
    
    public class Test {
        public static void main(String[] args){
            final ParameterTool params = ParameterTool.fromArgs(args);
    
            // set up the execution environment
            final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
            // make parameters available in the web interface
            env.getConfig().setGlobalJobParameters(params);
    
            // get input data     hdfs
            DataSet<String> text = env.readTextFile(params.get("input"));
            DataSet map = text.flatMap(null);
            DataSet reduce = map.groupBy("groupbyfield").reduce(null);
            try {
                env.execute("test");
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

三。搭建hadoop

1. 修改主机名为 master ，并对应 ip（dns），然后重新登录

[root@localhost ~]# vim /etc/hostname 
    [root@localhost ~]# cat /etc/hostname 
    master
    [root@localhost ~]# cat /etc/hosts
    127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
    ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
    
    10.20.0.129    master
    [root@localhost ~]# 
    Last login: Fri Mar  1 10:16:35 2019
    [root@master ~]#

2.  配置秘钥，回车默认，再配置秘钥登录

[root@master ~]# ssh-keygen -t rsa
    Generating public/private rsa key pair.
    Enter file in which to save the key (/root/.ssh/id_rsa): 
    [root@master ~]# ssh-copy-id master
    /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub"
    [root@master ~]# ls ~/.ssh/
    authorized_keys  id_rsa  id_rsa.pub  known_hosts
    [root@master ~]# cat ~/.ssh/authorized_keys 
    ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDmisZgUIF36e10nYnff1YqoqXFabUW7FTEZ1uO2w6W2Xq9R95ywaRqElMUccNxaruI8T5NFJ0uGpD3Bbv7TOHr42DTalA7MYU3gLUCs5rFXQATKbRA3veQa9IN2KYSxwrvqfmJCVbJbLMEsTPvobWDaOvDAacWdwh5ppliY3ngI8tgevttXblFWPrfbThXdIws3/vu04plRSEYxUn0Dvp9y9s4fOBClSCCvAmo2y2D3wK3ut2qnwyM8jVL1Fy58tke/w4ZMtRb83Tt7lRFpY5InxGmxNH68sB1ql2YXzKrIPP3ShuLFldQkp8qK/uvLC8GwcHT7trL+X3wyDBrkX9 root@master
    [root@master ~]#

[root@master ~]# ssh master 
    Last login: Fri Mar  1 10:17:55 2019 from 10.20.0.1
    [root@master ~]#

3.  下载jdk，并解压

[root@master hadoop]# ls
    jdk-8u201-linux-x64.tar.gz
    [root@master hadoop]# tar -zxvf jdk-8u201-linux-x64.tar.gz

[root@master hadoop]# ls
    jdk1.8.0_201  jdk-8u201-linux-x64.tar.gz
    [root@master hadoop]# mv jdk1.8.0_201/ /usr/local/jdk
    [root@master hadoop]#

4. 下载 hadoop ：http://archive.apache.org/dist/hadoop/core/

[root@master hadoop]# tar -zxvf hadoop-2.6.0.tar.gz 
    [root@master hadoop]# mv hadoop-2.6.0/ /usr/local/hadoop
    [root@master hadoop]#

5. 配置 hadoop jdk环境

[root@master ~]# cd /usr/local/hadoop/etc/hadoop/
    [root@master hadoop]# vim hadoop-env.sh

export JAVA_HOME=/usr/local/jdk

6. 配置 core-site.xml

[root@master hadoop]# vim core-site.xml 
    [root@master hadoop]#

<configuration>
            <property>
                    <name>fs.default.name</name>
                    <value>hdfs://master:9000</value>
            </property>
            <property>
                    <name>hadoop.tmp.dir</name>
                    <value>/usr/local/hadoop/tmp</value>
            </property>
    </configuration>

7. 配置 hdfs-site.xml

[root@master hadoop]# vim hdfs-site.xml 
    [root@master hadoop]#

<configuration>
            <property>
                    <name>dfs.replication</name>
                    <value>1</value>
                    </property>
            <property>
                    <name>dfs.permissions</name>
                    <value>true</value>
            </property>
    </configuration>

8. 配置 mapred-site.xml

[root@master hadoop]# cp mapred-site.xml.template mapred-site.xml
    [root@master hadoop]# vim mapred-site.xml
    [root@master hadoop]#

<configuration>
            <property>
                    <name>mapreduce.framework.name</name>
                    <value>yarn</value>
            </property>
            <property>
                    <name>mapreduce.jobhistory.address</name>
                    <value>master:10020</value>
            </property>
    </configuration>

9. 配置 yarn-site.xml

[root@master hadoop]# vim yarn-site.xml 
    [root@master hadoop]#

<configuration>
    
    
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
      </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
      </property>
    <property>
        <name>mapreduce.job.ubertask.enable</name>
        <value>true</value>
    </property>
    </configuration>

10. 配置环境变量

[root@master hadoop]# vim /etc/profile
    [root@master hadoop]#

在末尾加上

export JAVA_HOME=/usr/local/jdk
    export HADOOP_HOME=/usr/local/hadoop
    export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

让环境生效

[root@master hadoop]# source /etc/profile
    [root@master hadoop]# jps
    31134 Jps
    [root@master hadoop]#

11. 启动 hadoop , 一键启动：start-all.sh

[root@master hadoop]# hadoop namenode -format
    [root@master hadoop]# start-dfs.sh 
    [root@master hadoop]# start-yarn.sh 
    [root@master ~]# jps
    2518 DataNode
    2886 NodeManager
    2791 ResourceManager
    2648 SecondaryNameNode
    3180 Jps
    2429 NameNode
    [root@master hadoop]# systemctl stop firewalld.service
    [root@master hadoop]#

12. 浏览器访问：http://10.20.0.129:50070/

![1271822-20190301132659884-751660340.png][]

四。安装hbase

1. 下载 zookeeper：http://mirror.bit.edu.cn/apache/zookeeper/

[root@master hadoop]# tar -zxvf zookeeper-3.4.10.tar.gz
    [root@master hadoop]# mv zookeeper-3.4.10/ /usr/local/zk

2. 配置 zookeeper 数据路径

[root@master hadoop]# cd /usr/local/zk/conf/
    [root@master conf]# cp zoo_sample.cfg zoo.cfg 
    [root@master conf]# vim zoo.cfg

dataDir=/usr/local/zk/data

3. 下载 hbase： http://archive.cloudera.com/cdh5/cdh/5/

[root@master hadoop]# tar -zxvf hbase-1.0.0-cdh5.5.1.tar.gz 
    [root@master hadoop]# mv hbase-1.0.0-cdh5.5.1/ /usr/local/hbase

4. 配置 hbase jdk路径

[root@master hadoop]# cd /usr/local/hbase/conf/
    [root@master conf]# vim hbase-env.sh

export JAVA_HOME=/usr/local/jdk

5. 配置 hbase-site.xml

[root@master conf]# vim hbase-site.xml

<configuration>
    <property>
      <name>hbase.rootdir</name>
      <value>hdfs://master:9000/hbase</value>
    </property>
    <property>
      <name>hbase.cluster.distributed</name>
      <value>true</value>
    </property>
    <property>
      <name>hbase.zookeeper.quorum</name>
      <value>master</value>
    </property>
    <property>
      <name>dfs.replication</name>
      <value>1</value>
    </property>
    </configuration>

6. 配置 regionservers 为 master

[root@master conf]# vim regionservers

![1271822-20190301151957409-662134544.png][]

7. 配置环境

[root@master conf]# vim /etc/profile

export JAVA_HOME=/usr/local/jdk
    export HADOOP_HOME=/usr/local/hadoop
    export ZK_HOME=/usr/local/zk
    export HBASE_HOME=/usr/local/hbase
    export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$PATH

[root@master conf]# source /etc/profile

8. 启动 hbase

[root@master conf]# zkServer.sh start
    ZooKeeper JMX enabled by default
    Using config: /usr/local/zk/bin/../conf/zoo.cfg
    Starting zookeeper ... STARTED
    [root@master conf]# zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /usr/local/zk/bin/../conf/zoo.cfg
    Mode: standalone
    [root@master conf]# start-hbase.sh 
    master: starting zookeeper, logging to /usr/local/hbase/bin/../logs/hbase-root-zookeeper-master.out
    starting master, logging to /usr/local/hbase/logs/hbase-root-master-master.out
    master: starting regionserver, logging to /usr/local/hbase/bin/../logs/hbase-root-regionserver-master.out
    [root@master conf]#

[root@master conf]# jps
    4545 HMaster
    5010 Jps
    4197 QuorumPeerMain
    2518 DataNode
    2886 NodeManager
    4678 HRegionServer
    2791 ResourceManager
    2648 SecondaryNameNode
    4472 HQuorumPeer
    2429 NameNode
    [root@master conf]#

9 . 浏览器访问hbase控制台：http://10.20.0.129:60010/

![1271822-20190301153028162-1675931441.png][]

10. 浏览器访问 hadoop 控制台：http://10.20.0.129:50070

![1271822-20190301153314456-1527345158.png][]

五。安装mongodb

1. 下载 mongodb ：https://www.mongodb.com/download-center/community

[root@master hadoop]# tar -zxvf mongodb-linux-x86_64-3.2.22.tgz 
    [root@master hadoop]# mv mongodb-linux-x86_64-3.2.22/ /usr/local/mongodb
    [root@master hadoop]#

2. 配置 mongodb

[root@master hadoop]# cd /usr/local/mongodb/
    [root@master mongodb]# mkdir data
    [root@master mongodb]# mkdir conf
    [root@master mongodb]# mkdir log
    [root@master mongodb]# vim conf/mongod.conf
    [root@master mongodb]# cat conf/mongod.conf 
    #port = 12345
    dbpath = /usr/local/mongodb/data
    logpath = /usr/local/mongodb/log/mongod.log
    fork = true
    logappend = true
    #bind_ip = 127.0.0.1
    [root@master mongodb]#

3. 配置环境

[root@master mongodb]# vim /etc/profile
    [root@master mongodb]#

export JAVA_HOME=/usr/local/jdk
    export HADOOP_HOME=/usr/local/hadoop
    export ZK_HOME=/usr/local/zk
    export HBASE_HOME=/usr/local/hbase
    export MONGO_HOME=/usr/local/mongodb
    export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$MONGO_HOME/bin:$PATH

[root@master mongodb]# source /etc/profile
    [root@master mongodb]#

3. 启动 mongodb

[root@master mongodb]# mongod -f conf/mongod.conf
    about to fork child process, waiting until server is ready for connections.
    forked process: 5662
    child process started successfully, parent exiting
    [root@master mongodb]#

4. 登录

> show dbs
    local  0.000GB
    > exit
    bye
    [root@master mongodb]#

转载于:https://www.cnblogs.com/GH-123/p/10453776.html

[1271822-20190228212652012-957909689.png]: /images/20211223/cc7b49244aa14ea6a53b679b301dc8f7.png
[1271822-20190301132659884-751660340.png]: /images/20211223/e2425154cf7e48eab56ead3d2e614076.png
[1271822-20190301151957409-662134544.png]: /images/20211223/35a0c399995e4e9ab5434bb183c320a9.png
[1271822-20190301153028162-1675931441.png]: /images/20211223/e80795fda5a7472e90ba8f554e2e79f6.png
[1271822-20190301153314456-1527345158.png]: /images/20211223/2650207f373b4f108a5318f919168115.png