【部署hadoop高可用集群】所有步骤

- 日理万妓 2024-02-19 13:49 24阅读 0赞

过程都很艰难，都在咬牙坚持，如果你不去努力，你都不敢相信自己竟然这么优秀，每一次的突破，都是淡然的欣慰，也许多年之后，你就可以跟你的孩子有了吹牛的资本，当年你爹如何如何，，回首时，你会为现在的你感动，，  
————————————-前言：送给挣扎中的你  
今天的学习  
部署:  
1.阿里云机器准备好  
2.CRT配置三台  
3.hadoop用户  
4.上传文件  
5.hosts文件  
6.ssh互相信任  
http://blog.itpub.net/30089851/viewspace-1992210/  
7.jdk部署  
8.解压hadoop zk

$HADOOP_HOME /home/hadoop/app/hadoop-2.6.0-cdh5.7.0
    $ZOOKEEPER_HOME /home/hadoop/app/zookeeper-3.4.6

9.zk hadoop启动和web

[root@hadoop002 ~]# useradd hadoop
    [root@hadoop002 ~]# id hadoop
    uid=500(hadoop) gid=500(hadoop) groups=500(hadoop)
    [root@hadoop002 ~]# su - hadoop
    [hadoop@hadoop002 ~]$ pwd
    /home/hadoop
    [hadoop@hadoop002 ~]$ mkdir software app data lib source
    [hadoop@hadoop002 ~]$ ll
    total 20
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 app
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 data
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 lib
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 software
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 source

第一台机器

[root@hadoop002 ~]# yum install -y lrzsz  （root用户下面）
    [root@hadoop002 ~]# rz
    [root@hadoop002 ~]# ll
    total 17288
    -rw-r--r-- 1 root root 17699306 Nov 27 09:08 zookeeper-3.4.6.tar.gz
    [root@hadoop002 ~]# rz
    
    [root@hadoop002 ~]# ll
    total 18140
    -rw-r--r-- 1 root root   872300 Nov 19 20:06 mysql-connector-java-5.1.27.jar
    -rw-r--r-- 1 root root 17699306 Nov 27 09:08 zookeeper-3.4.6.tar.gz
    [root@hadoop002 ~]# rz
    
    [root@hadoop002 ~]# ll
    total 322428
    -rw-r--r-- 1 root root 311585484 Nov 27 09:10 hadoop-2.6.0-cdh5.7.0.tar.gz
    -rw-r--r-- 1 root root    872300 Nov 19 20:06 mysql-connector-java-5.1.27.jar
    -rw-r--r-- 1 root root  17699306 Nov 27 09:08 zookeeper-3.4.6.tar.gz
    [root@hadoop002 ~]# mv * /home/hadoop/software
    [root@hadoop002 ~]# ll
    total 0
    [root@hadoop002 ~]# su - hadoop 
    [hadoop@hadoop002 ~]$ ll
    total 20
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 app
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 data
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 lib
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 09:31 software
    drwxrwxr-x 2 hadoop hadoop 4096 Nov 27 08:58 source
    [hadoop@hadoop002 ~]$ cd software
    [hadoop@hadoop002 software]$ ll
    total 322428
    -rw-r--r-- 1 root root 311585484 Nov 27 09:10 hadoop-2.6.0-cdh5.7.0.tar.gz
    -rw-r--r-- 1 root root    872300 Nov 19 20:06 mysql-connector-java-5.1.27.jar
    -rw-r--r-- 1 root root  17699306 Nov 27 09:08 zookeeper-3.4.6.tar.gz
    [hadoop@hadoop002 ~]$ cd software
    [hadoop@hadoop002 software]$  scp * root@172.26.227.26:/home/hadoop/software/
    The authenticity of host '172.26.227.26 (172.26.227.26)' can't be established.
    RSA key fingerprint is 26:ea:51:d0:ff:e0:4c:f6:e4:7e:b7:25:67:7f:4b:c3.
    Are you sure you want to continue connecting (yes/no)? yes
    Warning: Permanently added '172.26.227.26' (RSA) to the list of known hosts.
    root@172.26.227.26's password: 
    hadoop-2.6.0-cdh5.7.0.tar.gz                                                                      100%  297MB 297.2MB/s   00:01    
    mysql-connector-java-5.1.27.jar                                                                   100%  852KB 851.9KB/s   00:00    
    zookeeper-3.4.6.tar.gz                                                                            100%   17MB  16.9MB/s   00:01    
    [hadoop@hadoop002 software]$ ll
    total 322428
    -rw-r--r-- 1 root root 311585484 Nov 27 09:10 hadoop-2.6.0-cdh5.7.0.tar.gz
    -rw-r--r-- 1 root root    872300 Nov 19 20:06 mysql-connector-java-5.1.27.jar
    -rw-r--r-- 1 root root  17699306 Nov 27 09:08 zookeeper-3.4.6.tar.gz
    [hadoop@hadoop002 software]$  scp * root@172.26.227.27:/home/hadoop/software/
    The authenticity of host '172.26.227.27 (172.26.227.27)' can't be established.
    RSA key fingerprint is 14:51:f1:3d:4c:26:ef:19:e4:20:7c:08:63:0b:55:27.
    Are you sure you want to continue connecting (yes/no)? yes
    Warning: Permanently added '172.26.227.27' (RSA) to the list of known hosts.
    root@172.26.227.27's password: 
    hadoop-2.6.0-cdh5.7.0.tar.gz                                                                      100%  297MB 148.6MB/s   00:02    
    mysql-connector-java-5.1.27.jar                                                                   100%  852KB 851.9KB/s   00:00    
    zookeeper-3.4.6.tar.gz                                                                            100%   17MB  16.9MB/
    都退出，到root用户解压
    [hadoop@hadoop002 software]$ exit
    logout
    [root@hadoop002 ~]# 
    [root@hadoop002 ~]# chown -R hadoop:hadoop /home/hadoop/software/*
    [root@hadoop002 ~]# ll /home/hadoop/software/*
    -rw-r--r-- 1 hadoop hadoop 311585484 Nov 27 09:10 /home/hadoop/software/hadoop-2.6.0-cdh5.7.0.tar.gz
    -rw-r--r-- 1 hadoop hadoop    872300 Nov 19 20:06 /home/hadoop/software/mysql-connector-java-5.1.27.jar
    -rw-r--r-- 1 hadoop hadoop  17699306 Nov 27 09:08 /home/hadoop/software/zookeeper-3.4.6.tar.gz
    [root@hadoop002 ~]# su - hadoop
    [root@hadoop002 ~]# vi /etc/hosts
    [root@hadoop002 ~]# scp /etc/hosts 172.26.227.26:/etc/hosts

多台机器相互无密码访问

[root@hadoop004 ~]# su - hadoop
    [hadoop@hadoop004 ~]$ ll -a
    total 44
    drwx------  7 hadoop hadoop 4096 Nov 27 09:45 .
    drwxr-xr-x. 3 root   root   4096 Nov 27 08:54 ..
    drwxrwxr-x  2 hadoop hadoop 4096 Nov 27 08:58 app
    -rw-------  1 hadoop hadoop   82 Nov 27 10:04 .bash_history
    -rw-r--r--  1 hadoop hadoop   18 Mar 23  2017 .bash_logout
    -rw-r--r--  1 hadoop hadoop  176 Mar 23  2017 .bash_profile
    -rw-r--r--  1 hadoop hadoop  124 Mar 23  2017 .bashrc
    drwxrwxr-x  2 hadoop hadoop 4096 Nov 27 08:58 data
    drwxrwxr-x  2 hadoop hadoop 4096 Nov 27 08:58 lib
    drwxrwxr-x  2 hadoop hadoop 4096 Nov 27 09:42 software
    drwxrwxr-x  2 hadoop hadoop 4096 Nov 27 08:58 source
    [hadoop@hadoop004 ~]$ rm -rf .ssh
    [hadoop@hadoop004 ~]$ ssh-keygen
    Generating public/private rsa key pair.
    Enter file in which to save the key (/home/hadoop/.ssh/id_rsa): 
    Created directory '/home/hadoop/.ssh'.
    Enter passphrase (empty for no passphrase): 
    Enter same passphrase again: 
    Your identification has been saved in /home/hadoop/.ssh/id_rsa.
    Your public key has been saved in /home/hadoop/.ssh/id_rsa.pub.
    The key fingerprint is:
    f8:62:df:99:83:c3:77:1e:c0:95:5f:9b:ac:53:08:e5 hadoop@hadoop004
    The key's randomart image is:
    +--[ RSA 2048]----+
    |             .   |
    |            o.   |
    |           .oE  .|
    |       . . ...o.o|
    |      . S o  ..= |
    |       .   .  o  |
    |      o...  .o   |
    |     . o+.oo...  |
    |        .o++.    |
    +-----------------+
    [hadoop@hadoop002 ~]$ cd .ssh
    [hadoop@hadoop002 .ssh]$ ll
    total 16
    -rw------- 1 hadoop hadoop 1675 Nov 27 10:32 id_rsa
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:32 id_rsa.pub
    -rw-r--r-- 1 root   root    398 Nov 27 10:44 id_rsa.pub2
    -rw-r--r-- 1 root   root    398 Nov 27 10:44 id_rsa.pub3
    [hadoop@hadoop002 .ssh]$ cat id_rsa.pub >> authorized_keys
    [hadoop@hadoop002 .ssh]$ cat id_rsa.pub2 >> authorized_keys
    [hadoop@hadoop002 .ssh]$ cat id_rsa.pub3 >> authorized_keys
    [hadoop@hadoop002 .ssh]$ cat authorized_keys
    ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0p+SFoJxarvAvPenEEbP8EsAponVoiM9mjIta66pHCykj+HpCz1JWz7bXovM+lPZ3e20tSHKUxGuaEP6i1j7xKdOPIwfA2pciCXZf7+F3mziTQF5pUFQQQtEhpalzyey2l8Zzn8AKiMdKsfwzEWdh9xSWeviy2ET9MSyWM0PjYYlI6P/1+uKWz+MDQHvW0/Qj3mACSGV7Jv8om6rbX43fAExoOP4qkwyxOeKZfH+fxQUuTJWQNUTnndmJTjYul0NvWXWv0E1M6Shv9kitbzhPM57MqgrhNzBvQ3e7CjFlmLQRsbbuEz/HYbXWk1M9rmJy6EqXIv+4ZStXLM9tJkYdQ== hadoop@hadoop002
    ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA02+w+EIPu7CfFfg/pc2D6o7u8wdPF2y01t4SPXinoCJ0XZX/fkRop6NV9dF4U89cHblIRKmI5YTzBjKlEDiFiAlDm4lXvpHv8uzLBi11195ZcSixtXEHvazvuw9J2BwfwI5hWGdg2WKffILPzsMu4sKYJTQxlgNGR1PIx9wtf1dJvFZvob5DS1rpZLxR75yqhI5YzwgWLGVlYVLcDiqA+AWhunFcIduN8Aa2wCpfE/TzZF9g8tiUAnKRIYRKXnlknQZmcf7eU1hJpjqO/+0hW3slUFZKODO9XzBVt+euIngeVFrfCWMfNnvJv5kq4kQvIJx16DmCcOkgFnUxtV1frQ== hadoop@hadoop003
    ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyyk79i49YMPoczxt/w/6+2SuWXqjvrs8UHML+gdOk5GGzpp+dAa+KAzGbAFs2bwr35T2YUJDbDFSTlCltgxQ5UXcKHNkh9/dsS/SdverO2RKg5CALQI4R83+7ENCIyfj6WtIeF1gw1xJA/qiACMdg+ovkYvqQCbuq6kDUtgJrEZV3sNJ/9x/1HjchOc45ZUPJdYRbCtR4VkiiN6/R38T+hWioKoNVxyao43wR85tXTOv8JOBFrpWqpA4UkkKEOHDqYI/+foXNs7LSICJoev2YAQgPMdrpyVXzojV9CiEWOzt0r9Jm3o6JkfQAm980T3KO/MtCM/aKirEE6kQx6MMcQ== hadoop@hadoop004
    [hadoop@hadoop002 .ssh]$ scp authorized_keys root@hadoop003:/home/hadoop/.ssh/
    The authenticity of host 'hadoop003 (172.26.227.26)' can't be established.
    RSA key fingerprint is 26:ea:51:d0:ff:e0:4c:f6:e4:7e:b7:25:67:7f:4b:c3.
    Are you sure you want to continue connecting (yes/no)? yes
    Warning: Permanently added 'hadoop003,172.26.227.26' (RSA) to the list of known hosts.
    root@hadoop003's password: 
    [hadoop@hadoop002 .ssh]$ scp authorized_keys root@hadoop004:/home/hadoop/.ssh/
    The authenticity of host 'hadoop004 (172.26.227.27)' can't be established.
    RSA key fingerprint is 14:51:f1:3d:4c:26:ef:19:e4:20:7c:08:63:0b:55:27.
    Are you sure you want to continue connecting (yes/no)? yes
    Warning: Permanently added 'hadoop004,172.26.227.27' (RSA) to the list of known hosts.
    root@hadoop004's password: 
    authorized_keys        
    [hadoop@hadoop002 .ssh]$ exit
    logout
    [root@hadoop002 ~]# chown -R hadoop:hadoop /home/hadoop/.ssh
    [root@hadoop002 ~]# chown -R hadoop:hadoop /home/hadoop/.ssh/*
    [root@hadoop002 ~]# su - hadoop
    [hadoop@hadoop002 ~]$ cd .ssh
    [hadoop@hadoop002 .ssh]$ ll
    total 24
    -rw-rw-r-- 1 hadoop hadoop 1194 Nov 27 11:26 authorized_keys
    -rw------- 1 hadoop hadoop 1675 Nov 27 10:32 id_rsa
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:32 id_rsa.pub
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:44 id_rsa.pub2
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:44 id_rsa.pub3
    -rw-r--r-- 1 hadoop hadoop  810 Nov 27 11:34 known_hosts
    [hadoop@hadoop002 .ssh]$ chmod 600 authorized_keys
    [hadoop@hadoop002 .ssh]$ ll
    total 24
    -rw------- 1 hadoop hadoop 1194 Nov 27 11:26 authorized_keys
    -rw------- 1 hadoop hadoop 1675 Nov 27 10:32 id_rsa
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:32 id_rsa.pub
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:44 id_rsa.pub2
    -rw-r--r-- 1 hadoop hadoop  398 Nov 27 10:44 id_rsa.pub3
    -rw-r--r-- 1 hadoop hadoop  810 Nov 27 11:34 known_hosts
    [hadoop@hadoop004 .ssh]$ 
    [hadoop@hadoop004 .ssh]$ ssh hadoop002 date
    Tue Nov 27 11:54:11 CST 2018
    [hadoop@hadoop004 .ssh]$ ssh hadoop003 date
    Tue Nov 27 11:54:23 CST 2018
    [hadoop@hadoop004 .ssh]$ ssh hadoop004 date
    Tue Nov 27 11:54:31 CST 2018
    [hadoop@hadoop004 .ssh]$ ssh hadoop002 date && ssh hadoop003 date && ssh hadoop004 date
    Tue Nov 27 11:58:02 CST 2018
    Tue Nov 27 11:58:02 CST 2018
    Tue Nov 27 11:58:03 CST 2018
    [hadoop@hadoop004 .ssh]$

Jdk部署

drwxr-xr-x 8 uucp 143 4096 Apr 11  2015 jdk1.8.0_45
    [root@hadoop002 ~]# mv jdk1.8.0_45 /usr/java
    [root@hadoop002 ~]# ll
    total 0
    [root@hadoop002 ~]# cd java
    -bash: cd: java: No such file or directory
    [root@hadoop002 ~]# cd /usr/java
    [root@hadoop002 java]# ll
    total 4
    drwxr-xr-x 8 uucp 143 4096 Apr 11  2015 jdk1.8.0_45
    [root@hadoop002 java]# chown -R root:root /usr/java/jdk1.8.0_45
    [root@hadoop002 java]# ll
    total 4
    drwxr-xr-x 8 root root 4096 Apr 11  2015 jdk1.8.0_45
    [root@hadoop002 java]# 
    [root@hadoop004 software]# cd
    [root@hadoop004 ~]# cd /usr/java
    [root@hadoop004 java]# ll
    total 4
    drwxr-xr-x 8 hadoop hadoop 4096 Apr 11  2015 jdk1.8.0_45
    [root@hadoop004 java]# vi /etc/profile
    [root@hadoop004 java]#  source /etc/profile
    [root@hadoop004 java]#  java -version
    java version "1.8.0_45"
    Java(TM) SE Runtime Environment (build 1.8.0_45-b14)
    Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)
    [root@hadoop004 java]# 
    [hadoop@hadoop002 software]$ cd ../app
    [hadoop@hadoop002 app]$ ll
    total 8
    drwxr-xr-x 14 hadoop hadoop 4096 Mar 24  2016 hadoop-2.6.0-cdh5.7.0
    drwxr-xr-x 10 hadoop hadoop 4096 Feb 20  2014 zookeeper-3.4.6
    [hadoop@hadoop002 app]$ 
    [hadoop@hadoop002 app]$ 
    [hadoop@hadoop002 app]$ 
    [hadoop@hadoop002 app]$ cd
    [hadoop@hadoop002 ~]$ vi .bash_profile
    [hadoop@hadoop002 ~]$ . .bash_profile
    [hadoop@hadoop002 ~]$ cd $HADOOP_HOME
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ mkdir $HADOOP_HOME/data && mkdir $HADOOP_HOME/logs && mkdir $HADOOP_HOME/tmp
    -bash: mkdir: command not found
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ 
    [root@hadoop002 ~]# su - hadoop
    [hadoop@hadoop002 ~]$ cd $HADOOP_HOME
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ mkdir $HADOOP_HOME/data &&  $HADOOP_HOME/logs &&$HADOOP_HOME/tmp
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$  chmod -R 777 $HADOOP_HOME/tmp
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ ll
    total 88
    drwxr-xr-x  2 hadoop hadoop  4096 Mar 24  2016 bin
    drwxr-xr-x  2 hadoop hadoop  4096 Mar 24  2016 bin-mapreduce1
    drwxr-xr-x  3 hadoop hadoop  4096 Mar 24  2016 cloudera
    drwxrwxr-x  2 hadoop hadoop  4096 Nov 27 20:30 data
    drwxr-xr-x  6 hadoop hadoop  4096 Mar 24  2016 etc
    drwxr-xr-x  5 hadoop hadoop  4096 Mar 24  2016 examples
    drwxr-xr-x  3 hadoop hadoop  4096 Mar 24  2016 examples-mapreduce1
    drwxr-xr-x  2 hadoop hadoop  4096 Mar 24  2016 include
    drwxr-xr-x  3 hadoop hadoop  4096 Mar 24  2016 lib
    drwxr-xr-x  2 hadoop hadoop  4096 Mar 24  2016 libexec
    -rw-r--r--  1 hadoop hadoop 17087 Mar 24  2016 LICENSE.txt
    drwxrwxr-x  2 hadoop hadoop  4096 Nov 27 20:36 logs
    -rw-r--r--  1 hadoop hadoop   101 Mar 24  2016 NOTICE.txt
    -rw-r--r--  1 hadoop hadoop  1366 Mar 24  2016 README.txt
    drwxr-xr-x  3 hadoop hadoop  4096 Mar 24  2016 sbin
    drwxr-xr-x  4 hadoop hadoop  4096 Mar 24  2016 share
    drwxr-xr-x 17 hadoop hadoop  4096 Mar 24  2016 src
    drwxrwxrwx  2 hadoop hadoop  4096 Nov 27 20:36 tmp
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ cd ../
    [hadoop@hadoop002 app]$ ll
    total 8
    drwxr-xr-x 17 hadoop hadoop 4096 Nov 27 20:36 hadoop-2.6.0-cdh5.7.0
    drwxr-xr-x 10 hadoop hadoop 4096 Feb 20  2014 zookeeper-3.4.6
    [hadoop@hadoop002 app]$ cd zookeeper-3.4.6
    [hadoop@hadoop002 zookeeper-3.4.6]$ ll
    total 1552
    drwxr-xr-x  2 hadoop hadoop    4096 Feb 20  2014 bin
    -rw-rw-r--  1 hadoop hadoop   82446 Feb 20  2014 build.xml
    -rw-rw-r--  1 hadoop hadoop   80776 Feb 20  2014 CHANGES.txt
    drwxr-xr-x  2 hadoop hadoop    4096 Feb 20  2014 conf
    drwxr-xr-x 10 hadoop hadoop    4096 Feb 20  2014 contrib
    drwxr-xr-x  2 hadoop hadoop    4096 Feb 20  2014 dist-maven
    drwxr-xr-x  6 hadoop hadoop    4096 Feb 20  2014 docs
    -rw-rw-r--  1 hadoop hadoop    1953 Feb 20  2014 ivysettings.xml
    -rw-rw-r--  1 hadoop hadoop    3375 Feb 20  2014 ivy.xml
    drwxr-xr-x  4 hadoop hadoop    4096 Feb 20  2014 lib
    -rw-rw-r--  1 hadoop hadoop   11358 Feb 20  2014 LICENSE.txt
    -rw-rw-r--  1 hadoop hadoop     170 Feb 20  2014 NOTICE.txt
    -rw-rw-r--  1 hadoop hadoop    1770 Feb 20  2014 README_packaging.txt
    -rw-rw-r--  1 hadoop hadoop    1585 Feb 20  2014 README.txt
    drwxr-xr-x  5 hadoop hadoop    4096 Feb 20  2014 recipes
    drwxr-xr-x  8 hadoop hadoop    4096 Feb 20  2014 src
    -rw-rw-r--  1 hadoop hadoop 1340305 Feb 20  2014 zookeeper-3.4.6.jar
    -rw-rw-r--  1 hadoop hadoop     836 Feb 20  2014 zookeeper-3.4.6.jar.asc
    -rw-rw-r--  1 hadoop hadoop      33 Feb 20  2014 zookeeper-3.4.6.jar.md5
    -rw-rw-r--  1 hadoop hadoop      41 Feb 20  2014 zookeeper-3.4.6.jar.sha1
    [hadoop@hadoop002 zookeeper-3.4.6]$ cd conf
    [hadoop@hadoop002 conf]$ ll
    total 12
    -rw-rw-r-- 1 hadoop hadoop  535 Feb 20  2014 configuration.xsl
    -rw-rw-r-- 1 hadoop hadoop 2161 Feb 20  2014 log4j.properties
    -rw-rw-r-- 1 hadoop hadoop  922 Feb 20  2014 zoo_sample.cfg
    [hadoop@hadoop002 conf]$ cp zoo_sample.cfg zoo.cfg
    [hadoop@hadoop002 conf]$ ll
    total 16
    -rw-rw-r-- 1 hadoop hadoop  535 Feb 20  2014 configuration.xsl
    -rw-rw-r-- 1 hadoop hadoop 2161 Feb 20  2014 log4j.properties
    -rw-rw-r-- 1 hadoop hadoop  922 Nov 27 20:45 zoo.cfg
    -rw-rw-r-- 1 hadoop hadoop  922 Feb 20  2014 zoo_sample.cfg
    [hadoop@hadoop002 zookeeper-3.4.6]$ cd conf
    [hadoop@hadoop002 conf]$ ll
    total 12
    -rw-rw-r-- 1 hadoop hadoop  535 Feb 20  2014 configuration.xsl
    -rw-rw-r-- 1 hadoop hadoop 2161 Feb 20  2014 log4j.properties
    -rw-rw-r-- 1 hadoop hadoop  922 Feb 20  2014 zoo_sample.cfg
    [hadoop@hadoop002 conf]$ cp zoo_sample.cfg zoo.cfg
    [hadoop@hadoop002 conf]$ ll
    total 16
    -rw-rw-r-- 1 hadoop hadoop  535 Feb 20  2014 configuration.xsl
    -rw-rw-r-- 1 hadoop hadoop 2161 Feb 20  2014 log4j.properties
    -rw-rw-r-- 1 hadoop hadoop  922 Nov 27 20:45 zoo.cfg
    -rw-rw-r-- 1 hadoop hadoop  922 Feb 20  2014 zoo_sample.cfg
    [hadoop@hadoop002 conf]$ 
    [hadoop@hadoop002 conf]$ 
    [hadoop@hadoop002 conf]$ vi zoo.cfg
    [hadoop@hadoop002 conf]$ scp zoo.cfg hadoop003:/home/hadoop/app/zookeeper-3.4.6/conf
    zoo.cfg                                                                                 100% 1040     1.0KB/s   00:00    
    [hadoop@hadoop002 conf]$ scp zoo.cfg hadoop004:/home/hadoop/app/zookeeper-3.4.6/conf
    zoo.cfg            
                     [hadoop@hadoop002 conf]$ cd ../
    [hadoop@hadoop002 zookeeper-3.4.6]$ ll
    total 1552
    drwxr-xr-x  2 hadoop hadoop    4096 Feb 20  2014 bin
    -rw-rw-r--  1 hadoop hadoop   82446 Feb 20  2014 build.xml
    -rw-rw-r--  1 hadoop hadoop   80776 Feb 20  2014 CHANGES.txt
    drwxr-xr-x  2 hadoop hadoop    4096 Nov 27 20:53 conf
    drwxr-xr-x 10 hadoop hadoop    4096 Feb 20  2014 contrib
    drwxr-xr-x  2 hadoop hadoop    4096 Feb 20  2014 dist-maven
    drwxr-xr-x  6 hadoop hadoop    4096 Feb 20  2014 docs
    -rw-rw-r--  1 hadoop hadoop    1953 Feb 20  2014 ivysettings.xml
    -rw-rw-r--  1 hadoop hadoop    3375 Feb 20  2014 ivy.xml
    drwxr-xr-x  4 hadoop hadoop    4096 Feb 20  2014 lib
    -rw-rw-r--  1 hadoop hadoop   11358 Feb 20  2014 LICENSE.txt
    -rw-rw-r--  1 hadoop hadoop     170 Feb 20  2014 NOTICE.txt
    -rw-rw-r--  1 hadoop hadoop    1770 Feb 20  2014 README_packaging.txt
    -rw-rw-r--  1 hadoop hadoop    1585 Feb 20  2014 README.txt
    drwxr-xr-x  5 hadoop hadoop    4096 Feb 20  2014 recipes
    drwxr-xr-x  8 hadoop hadoop    4096 Feb 20  2014 src
    -rw-rw-r--  1 hadoop hadoop 1340305 Feb 20  2014 zookeeper-3.4.6.jar
    -rw-rw-r--  1 hadoop hadoop     836 Feb 20  2014 zookeeper-3.4.6.jar.asc
    -rw-rw-r--  1 hadoop hadoop      33 Feb 20  2014 zookeeper-3.4.6.jar.md5
    -rw-rw-r--  1 hadoop hadoop      41 Feb 20  2014 zookeeper-3.4.6.jar.sha1
    [hadoop@hadoop002 zookeeper-3.4.6]$ mkdir data
    [hadoop@hadoop002 zookeeper-3.4.6]$ mkdir data/myid
    [hadoop@hadoop002 zookeeper-3.4.6]$ rm -rf data/myid
    [hadoop@hadoop002 zookeeper-3.4.6]$ touch data/myid
    [hadoop@hadoop002 zookeeper-3.4.6]$ echo 1 > data/myid
    [hadoop@hadoop002 zookeeper-3.4.6]$ cat data/myid
    1
    [hadoop@hadoop003 zookeeper-3.4.6]$ touch data/myid
    [hadoop@hadoop003 zookeeper-3.4.6]$ echo 2 > data/myid
    [hadoop@hadoop003 zookeeper-3.4.6]$ cat data/myid
    2
    [hadoop@hadoop004 zookeeper-3.4.6]$  touch data/myid
    [hadoop@hadoop004 zookeeper-3.4.6]$ echo 3 > data/myid
    [hadoop@hadoop004 zookeeper-3.4.6]$ cat data/myid
    3
    [hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ cd  ../
    [hadoop@hadoop002 app]$ cd hadoop-2.6.0-cdh5.7.0/etc/hadoop
    [hadoop@hadoop002 hadoop]$ ll
    total 152
    -rw-r--r-- 1 hadoop hadoop  4436 Mar 24  2016 capacity-scheduler.xml
    -rw-r--r-- 1 hadoop hadoop  1335 Mar 24  2016 configuration.xsl
    -rw-r--r-- 1 hadoop hadoop   318 Mar 24  2016 container-executor.cfg
    -rw-r--r-- 1 hadoop hadoop   774 Mar 24  2016 core-site.xml
    -rw-r--r-- 1 hadoop hadoop  3670 Mar 24  2016 hadoop-env.cmd
    -rw-r--r-- 1 hadoop hadoop  4224 Mar 24  2016 hadoop-env.sh
    -rw-r--r-- 1 hadoop hadoop  2598 Mar 24  2016 hadoop-metrics2.properties
    -rw-r--r-- 1 hadoop hadoop  2490 Mar 24  2016 hadoop-metrics.properties
    -rw-r--r-- 1 hadoop hadoop  9683 Mar 24  2016 hadoop-policy.xml
    -rw-r--r-- 1 hadoop hadoop   775 Mar 24  2016 hdfs-site.xml
    -rw-r--r-- 1 hadoop hadoop  1449 Mar 24  2016 httpfs-env.sh
    -rw-r--r-- 1 hadoop hadoop  1657 Mar 24  2016 httpfs-log4j.properties
    -rw-r--r-- 1 hadoop hadoop    21 Mar 24  2016 httpfs-signature.secret
    -rw-r--r-- 1 hadoop hadoop   620 Mar 24  2016 httpfs-site.xml
    -rw-r--r-- 1 hadoop hadoop  3523 Mar 24  2016 kms-acls.xml
    -rw-r--r-- 1 hadoop hadoop  1611 Mar 24  2016 kms-env.sh
    -rw-r--r-- 1 hadoop hadoop  1631 Mar 24  2016 kms-log4j.properties
    -rw-r--r-- 1 hadoop hadoop  5511 Mar 24  2016 kms-site.xml
    -rw-r--r-- 1 hadoop hadoop 11291 Mar 24  2016 log4j.properties
    -rw-r--r-- 1 hadoop hadoop   938 Mar 24  2016 mapred-env.cmd
    -rw-r--r-- 1 hadoop hadoop  1383 Mar 24  2016 mapred-env.sh
    -rw-r--r-- 1 hadoop hadoop  4113 Mar 24  2016 mapred-queues.xml.template
    -rw-r--r-- 1 hadoop hadoop   758 Mar 24  2016 mapred-site.xml.template
    -rw-r--r-- 1 hadoop hadoop    10 Mar 24  2016 slaves
    -rw-r--r-- 1 hadoop hadoop  2316 Mar 24  2016 ssl-client.xml.example
    -rw-r--r-- 1 hadoop hadoop  2268 Mar 24  2016 ssl-server.xml.example
    -rw-r--r-- 1 hadoop hadoop  2237 Mar 24  2016 yarn-env.cmd
    -rw-r--r-- 1 hadoop hadoop  4567 Mar 24  2016 yarn-env.sh
    -rw-r--r-- 1 hadoop hadoop   690 Mar 24  2016 yarn-site.xml
    [hadoop@hadoop002 hadoop]$ vi hadoop-env.sh
    [hadoop@hadoop002 hadoop]$ scp hadoop-env.sh hadoop003:/home/hadoop/app/hadoop-2.6.0-cdh5.7.0/etc/hadoop
    hadoop-env.sh                                                                           100% 4233     4.1KB/s   00:00    
    [hadoop@hadoop002 hadoop]$ scp hadoop-env.sh hadoop004:/home/hadoop/app/hadoop-2.6.0-cdh5.7.0/etc/hadoop
    hadoop-env.sh                                                                           100% 4233     4.1KB/s   00:00    
    [hadoop@hadoop002 hadoop]$ 
    [hadoop@hadoop002 hadoop]$  scp core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml hadoop-env.sh slaves hadoop003:/home/hadoop/app/hadoop-2.6.0-cdh5.7.0/etc/hadoop
    core-site.xml                                                                           100% 2373     2.3KB/s   00:00    
    hdfs-site.xml                                                                           100% 4619     4.5KB/s   00:00    
    mapred-site.xml                                                                         100% 1060     1.0KB/s   00:00    
    yarn-site.xml                                                                           100% 5276     5.2KB/s   00:00    
    hadoop-env.sh                                                                           100% 4233     4.1KB/s   00:00    
    slaves: No such file or directory
    [hadoop@hadoop002 hadoop]$  scp core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml hadoop-env.sh slaves hadoop004:/home/hadoop/app/hadoop-2.6.0-cdh5.7.0/etc/hadoop
    core-site.xml                                                                           100% 2373     2.3KB/s   00:00    
    hdfs-site.xml                                                                           100% 4619     4.5KB/s   00:00    
    mapred-site.xml                                                                         100% 1060     1.0KB/s   00:00    
    yarn-site.xml                                                                           100% 5276     5.2KB/s   00:00    
    hadoop-env.sh                                                                           100% 4233     4.1KB/s   00:00    
    slaves: No such file or directory
    [hadoop@hadoop002 hadoop]$

启动

[hadoop@hadoop002 hadoop-2.6.0-cdh5.7.0]$ cd
    [hadoop@hadoop002 ~]$ $ZOOKEEPER_HOME/bin/zkServer.sh start
    JMX enabled by default
    Using config: /home/hadoop/app/zookeeper-3.4.6/bin/../conf/zoo.cfg
    Starting zookeeper ... STARTED
    [hadoop@hadoop002 ~]$  $ZOOKEEPER_HOME/bin/zkServer.sh status
    JMX enabled by default
    Using config: /home/hadoop/app/zookeeper-3.4.6/bin/../conf/zoo.cfg
    Mode: follower
    [hadoop@hadoop002 ~]$ jps
    1834 Jps
    1771 QuorumPeerMain
    zookper

启动zookeeper服务（3台）

[hadoop@hadoop001 ~]$ $ZOOKEEPER_HOME/bin/zkServer.sh start
    JMX enabled by default
    Using config: /opt/app/zookeeper-3.4.5-cdh5.15.1/bin/../conf/zoo.cfg
    Starting zookeeper ... STARTED
    [hadoop@hadoop001 ~]$
    查看zookeeper状态（3台）
    [hadoop@hadoop001 ~]$ $ZOOKEEPER_HOME/bin/zkServer.sh status
    JMX enabled by default
    Using config: /opt/app/zookeeper-3.4.5-cdh5.15.1/bin/../conf/zoo.cfg
    Mode: follower
    [hadoop@hadoop001 ~]$ 
    jps查看zookeeper进程（3台）
    [hadoop@hadoop001 ~]$ jps
    20483 QuorumPeerMain
    20516 Jps
    [hadoop@hadoop001 ~]$

验证zookeeper服务

[hadoop@hadoop001 ~]$ $ZOOKEEPER_HOME/bin/zkCli.sh
    [zk: localhost:2181(CONNECTED) 0] ls /
    [zookeeper]
    [zk: localhost:2181(CONNECTED) 1] quit
    Quitting...
    [hadoop@hadoop001 ~]$

HDFS  
格式化hdfs的zookeeper存储目录

[hadoop@hadoop001 ~]$ hdfs zkfc -formatZK

查看zookeeper信息

[hadoop@hadoop001 ~]$ $ZOOKEEPER_HOME/bin/zkCli.sh
    [zk: localhost:2181(CONNECTED) 0] ls /
    [zookeeper, hadoop-ha]
    [zk: localhost:2181(CONNECTED) 1] quit
    Quitting...

[hadoop@hadoop001 ~]$

启动JournalNode服务（3台）

[hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode
    starting journalnode, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/hadoop-hadoop-journalnode-hadoop001.out
    [hadoop@hadoop001 ~]$

jps查看JournalNode进程（3台）

[hadoop@hadoop001 ~]$ jps
    21107 JournalNode
    21156 Jps
    20893 QuorumPeerMain
    [hadoop@hadoop001 ~]$

格式化并启动第一个NameNode(hadoop001)  
\[hadoop@hadoop001 ~\]$ hdfs namenode -format \#\#格式化当前节点的namenode 数据  
格式化journalnode 的数据，这个是ha需要做的

[hadoop@hadoop001 ~]$ hdfs namenode -initializeSharedEdits
    18/11/27 01:51:28 INFO namenode.NameNode: STARTUP_MSG: 
    /************************************************************
    STARTUP_MSG: Starting NameNode
    
    Re-format filesystem in QJM to [172.26.239.216:8485, 172.26.239.214:8485, 172.26.239.215:8485] ? (Y or N) Y
    18/11/27 01:51:39 INFO namenode.FileJournalManager: Recovering unfinalized segments in /opt/app/hadoop-2.6.0-cdh5.15.1/data/dfs/name/current
    18/11/27 01:51:39 INFO client.QuorumJournalManager: Starting recovery process for unclosed journal segments...
    18/11/27 01:51:39 INFO client.QuorumJournalManager: Successfully started new epoch 1
    18/11/27 01:51:39 INFO util.ExitUtil: Exiting with status 0
    18/11/27 01:51:39 INFO namenode.NameNode: SHUTDOWN_MSG: 
    /************************************************************
    SHUTDOWN_MSG: Shutting down NameNode at hadoop001/172.26.239.216
    ************************************************************/
    [hadoop@hadoop001 ~]$ 
    启动当前节点的namenode 服务
    [hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
    starting namenode, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/hadoop-hadoop-namenode-hadoop001.out
    [hadoop@hadoop001 ~]$
    jps查看namenode进程
    [hadoop@hadoop001 ~]$ jps
    21107 JournalNode
    21350 Jps
    21276 NameNode
    20893 QuorumPeerMain
    [hadoop@hadoop001 ~]
    格式化并启动第二个NameNode(hadoop002)
    [hadoop@hadoop002 ~]$ hdfs namenode -bootstrapStandby  #hadoop001已经格式化过，同步至hadoop002
    启动当前节点的namenode 服务(hadoop002)
    [hadoop@hadoop002 ~]$ $HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
    starting namenode, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/hadoop-hadoop-namenode-hadoop002.out
    [hadoop@hadoop002 ~]$ 
    jps查看namenode进程(hadoop002)
    [hadoop@hadoop002 ~]$ jps
    20690 QuorumPeerMain
    20788 JournalNode
    21017 Jps
    20923 NameNode
    [hadoop@hadoop002 ~]$
    启动datanode服务（hadoop001）
    [hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/hadoop-daemons.sh start namenode
    jps查看datanode进程（3台）
    [hadoop@hadoop001 ~]$ jps
    21857 Jps
    21107 JournalNode
    21766 DataNode
    21276 NameNode
    20893 QuorumPeerMain
    [hadoop@hadoop001 ~]$
    启动ZooKeeperFailoverController（hadoop001,hadoop002）
    [hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc  #所有namenode节点分别执行
    starting zkfc, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/hadoop-hadoop-zkfc-hadoop001.out
    [hadoop@hadoop001 ~]$
    [hadoop@hadoop002 ~]$ $HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc  #所有namenode节点分别执行
    starting zkfc, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/hadoop-hadoop-zkfc-hadoop002.out
    [hadoop@hadoop002 ~]$
    查看DFSZKFailoverController
    [hadoop@hadoop001 ~]$ jps
    21107 JournalNode
    21766 DataNode
    21276 NameNode
    20893 QuorumPeerMain
    21950 DFSZKFailoverController
    22030 Jps
    [hadoop@hadoop001 ~]$
    登陆http://39.98.44.126:50070（hadoop001）
    其中一个为active,另一个为standby状态
    
    登陆http://39.98.37.133:50070（hadoop002）

YARN  
hadoop001上启动yarn

[hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/start-yarn.sh
    starting yarn daemons
    starting resourcemanager, logging to /opt/app/hadoop-2.6.0-cdh5.15.1logs/yarn-hadoop-resourcemanager-hadoop001.out
    hadoop002: starting nodemanager, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/yarn-hadoop-nodemanager-hadoop002.out
    hadoop003: starting nodemanager, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/yarn-hadoop-nodemanager-hadoop003.out
    hadoop001: starting nodemanager, logging to /opt/app/hadoop-2.6.0-cdh5.15.1logs/yarn-hadoop-nodemanager-hadoop001.out
    [hadoop@hadoop001 ~]$ 
    jps查看yarn进程(3台)
    [hadoop@hadoop001 ~]$ jps
    22624 Jps
    21107 JournalNode
    22212 ResourceManager
    21766 DataNode
    22310 NodeManager
    21276 NameNode
    20893 QuorumPeerMain
    21950 DFSZKFailoverController
    [hadoop@hadoop001 ~]
    hadoop002上启动resourcemanager
    [hadoop@hadoop002 ~]$ $HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager
    starting resourcemanager, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/yarn-hadoop-resourcemanager-hadoop002.out
    [hadoop@hadoop002 ~]$ 
    jps查看resourcemanager进程
    [hadoop@hadoop002 ~]$ jps
    20690 QuorumPeerMain
    20788 JournalNode
    21908 Jps
    21399 DFSZKFailoverController
    20923 NameNode
    21675 NodeManager
    21117 DataNode
    21853 ResourceManager
    [hadoop@hadoop002 ~]$ 
    登陆http://39.98.44.126:8088（hadoop001）
    其中一个为active,另一个为standby状态
    
    登陆http://39.98.37.133:8088/cluster/cluster（hadoop002）

启动jobhistory  
在hadoop001上启动jobhistory服务

[hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver
    starting historyserver, logging to /opt/app/hadoop-2.6.0-cdh5.15.1/logs/mapred-hadoop-historyserver-hadoop001.out
    [hadoop@hadoop001 ~]$
    jps查看JobHistoryServer进程
    [hadoop@hadoop001 ~]$ jps
    22785 Jps
    21107 JournalNode
    22212 ResourceManager
    21766 DataNode
    22310 NodeManager
    22680 JobHistoryServer
    21276 NameNode
    20893 QuorumPeerMain
    21950 DFSZKFailoverController
    [hadoop@hadoop001 ~]$
    登陆jobhistory服务器web端查看job状态
    http://39.98.44.126:19888（hadoop001）

集群关闭

停止YARN相关服务
    停止hadoop001上historyserver
    [hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh stop historyserver
    stopping historyserver
    [hadoop@hadoop001 ~]$
    停止hadoop001上yarn任务
    [hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/stop-yarn.sh  #停止hadoop001上resourcemanager及所有的nodemanager
    stopping yarn daemons
    stopping resourcemanager
    hadoop003: stopping nodemanager
    hadoop001: stopping nodemanager
    hadoop002: stopping nodemanager
    no proxyserver to stop
    [hadoop@hadoop001 ~]$ 
    停止hadoop002上resourcemanager
    [hadoop@hadoop002 ~]$ $HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager
    stopping resourcemanager
    [hadoop@hadoop002 ~]$ 
    停止HDFS相关服务
    停止namenode，datanode，journalnode，zkfc服务
    [hadoop@hadoop001 ~]$ $HADOOP_HOME/sbin/stop-dfs.sh
    18/11/27 03:32:43 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
    Stopping namenodes on [hadoop001 hadoop002]
    hadoop001: stopping namenode
    hadoop002: stopping namenode
    hadoop003: stopping datanode
    hadoop001: stopping datanode
    hadoop002: stopping datanode
    Stopping journal nodes [hadoop001 hadoop002 hadoop003]
    hadoop002: stopping journalnode
    hadoop003: stopping journalnode
    hadoop001: stopping journalnode
    18/11/27 03:33:01 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
    Stopping ZK Failover Controllers on NN hosts [hadoop001 hadoop002]
    hadoop001: stopping zkfc
    hadoop002: stopping zkfc
    [hadoop@hadoop001 ~]$ 
     停止Zookeeper相关服务
    Zookeeper(3台)
    [hadoop@hadoop001 ~]$ $ZOOKEEPER_HOME/bin/zkServer.sh stop
    JMX enabled by default
    Using config: /opt/app/zookeeper-3.4.5-cdh5.15.1/bin/../conf/zoo.cfg
    Stopping zookeeper ... STOPPED
    [hadoop@hadoop001 ~]$
    jps查看进程（3台）
    [hadoop@hadoop001 ~]$ jps
    23881 Jps
    [hadoop@hadoop001 ~]$

如果你的人生没有一件为之而努力奋斗的事情，当你回首往事的时候，你将为自己一生碌碌无为而羞愧，你永远不知道的潜力有多大，连你自己你都不了解，岂不少了一些什么，，，奋斗是为了更好的生活，生活因奋斗而精彩。  
——————————————结束语：送给深夜还在努力的你