hadoop2安装脚本

待我称王封你为后i 2022-10-02 02:58 211阅读 0赞

\#!/bin/bash  
\#  
\# Install Hadoop 2 using pdsh/pdcp where possible.  
\#  
\# Command can be interactive or file-based. This script sets up  
\# a Hadoop 2 cluster with basic configuration. Modify data, log, and pid  
\# directories as desired. Further configure your cluster with ./conf-hadoop2.sh  
\#  
\#  
  
\# Basic environment variables. Edit as necessary  
HADOOP\_VERSION=2.0.5-alpha  
HADOOP\_HOME="/opt/hadoop-$\{HADOOP\_VERSION\}"  
NN\_DATA\_DIR=/var/data/hadoop/hdfs/nn  
SNN\_DATA\_DIR=/var/data/hadoop/hdfs/snn  
DN\_DATA\_DIR=/var/data/hadoop/hdfs/dn  
YARN\_LOG\_DIR=/var/log/hadoop/yarn  
HADOOP\_LOG\_DIR=/var/log/hadoop/hdfs  
HADOOP\_MAPRED\_LOG\_DIR=/var/log/hadoop/mapred  
YARN\_PID\_DIR=/var/run/hadoop/yarn  
HADOOP\_PID\_DIR=/var/run/hadoop/hdfs  
HADOOP\_MAPRED\_PID\_DIR=/var/run/hadoop/mapred  
HTTP\_STATIC\_USER=hdfs  
YARN\_PROXY\_PORT=8081  
  
source hadoop-xml-conf.sh  
CMD\_OPTIONS=$(getopt -n "$0" -o hif --long "help,interactive,file" -- "$@")  
  
\# Take care of bad options in the command  
if \[ $? -ne 0 \];  
then  
exit 1  
fi  
eval set -- "$CMD\_OPTIONS"  
  
all\_hosts="all\_hosts"  
nn\_host="nn\_host"  
snn\_host="snn\_host"  
dn\_hosts="dn\_hosts"  
rm\_host="rm\_host"  
nm\_hosts="nm\_hosts"  
mr\_history\_host="mr\_history\_host"  
yarn\_proxy\_host="yarn\_proxy\_host"  
  
install()  
\{  
echo "Copying Hadoop $HADOOP\_VERSION to all hosts..."  
pdcp -w ^all\_hosts hadoop-"$HADOOP\_VERSION".tar.gz /opt  
  
echo "Copying JDK 1.6.0\_31 to all hosts..."  
pdcp -w ^all\_hosts jdk-6u31-linux-x64-rpm.bin /opt  
  
echo "Installing JDK 1.6.0\_31 on all hosts..."  
pdsh -w ^all\_hosts chmod a+x /opt/jdk-6u31-linux-x64-rpm.bin  
pdsh -w ^all\_hosts /opt/jdk-6u31-linux-x64-rpm.bin -noregister 1>&- 2>&-  
  
echo "Setting JAVA\_HOME and HADOOP\_HOME environment variables on all hosts..."  
pdsh -w ^all\_hosts 'echo export JAVA\_HOME=/usr/java/jdk1.6.0\_31 > /etc/profile.d/java.sh'  
pdsh -w ^all\_hosts "source /etc/profile.d/java.sh"  
pdsh -w ^all\_hosts "echo export HADOOP\_HOME=$HADOOP\_HOME > /etc/profile.d/hadoop.sh"  
pdsh -w ^all\_hosts 'echo export HADOOP\_PREFIX=$HADOOP\_HOME >> /etc/profile.d/hadoop.sh'  
pdsh -w ^all\_hosts "source /etc/profile.d/hadoop.sh"  
  
echo "Extracting Hadoop $HADOOP\_VERSION distribution on all hosts..."  
pdsh -w ^all\_hosts tar -zxf /opt/hadoop-"$HADOOP\_VERSION".tar.gz -C /opt  
  
echo "Creating system accounts and groups on all hosts..."  
pdsh -w ^all\_hosts groupadd hadoop  
pdsh -w ^all\_hosts useradd -g hadoop yarn  
pdsh -w ^all\_hosts useradd -g hadoop hdfs  
pdsh -w ^all\_hosts useradd -g hadoop mapred  
  
echo "Creating HDFS data directories on NameNode host, Secondary NameNode host, and DataNode hosts..."  
pdsh -w ^nn\_host "mkdir -p $NN\_DATA\_DIR && chown hdfs:hadoop $NN\_DATA\_DIR"  
pdsh -w ^snn\_host "mkdir -p $SNN\_DATA\_DIR && chown hdfs:hadoop $SNN\_DATA\_DIR"  
pdsh -w ^dn\_hosts "mkdir -p $DN\_DATA\_DIR && chown hdfs:hadoop $DN\_DATA\_DIR"  
  
echo "Creating log directories on all hosts..."  
pdsh -w ^all\_hosts "mkdir -p $YARN\_LOG\_DIR && chown yarn:hadoop $YARN\_LOG\_DIR"  
pdsh -w ^all\_hosts "mkdir -p $HADOOP\_LOG\_DIR && chown hdfs:hadoop $HADOOP\_LOG\_DIR"  
pdsh -w ^all\_hosts "mkdir -p $HADOOP\_MAPRED\_LOG\_DIR && chown mapred:hadoop $HADOOP\_MAPRED\_LOG\_DIR"  
  
echo "Creating pid directories on all hosts..."  
pdsh -w ^all\_hosts "mkdir -p $YARN\_PID\_DIR && chown yarn:hadoop $YARN\_PID\_DIR"  
pdsh -w ^all\_hosts "mkdir -p $HADOOP\_PID\_DIR && chown hdfs:hadoop $HADOOP\_PID\_DIR"  
pdsh -w ^all\_hosts "mkdir -p $HADOOP\_MAPRED\_PID\_DIR && chown mapred:hadoop $HADOOP\_MAPRED\_PID\_DIR"  
  
echo "Editing Hadoop environment scripts for log directories on all hosts..."  
pdsh -w ^all\_hosts echo "export HADOOP\_LOG\_DIR=$HADOOP\_LOG\_DIR >> $HADOOP\_HOME/etc/hadoop/hadoop-env.sh"  
pdsh -w ^all\_hosts echo "export YARN\_LOG\_DIR=$YARN\_LOG\_DIR >> $HADOOP\_HOME/etc/hadoop/yarn-env.sh"  
pdsh -w ^all\_hosts echo "export HADOOP\_MAPRED\_LOG\_DIR=$HADOOP\_MAPRED\_LOG\_DIR >> $HADOOP\_HOME/etc/hadoop/mapred-env.sh"  
  
echo "Editing Hadoop environment scripts for pid directories on all hosts..."  
pdsh -w ^all\_hosts echo "export HADOOP\_PID\_DIR=$HADOOP\_PID\_DIR >> $HADOOP\_HOME/etc/hadoop/hadoop-env.sh"  
pdsh -w ^all\_hosts echo "export YARN\_PID\_DIR=$YARN\_PID\_DIR >> $HADOOP\_HOME/etc/hadoop/yarn-env.sh"  
pdsh -w ^all\_hosts echo "export HADOOP\_MAPRED\_PID\_DIR=$HADOOP\_MAPRED\_PID\_DIR >> $HADOOP\_HOME/etc/hadoop/mapred-env.sh"  
  
echo "Creating base Hadoop XML config files..."  
create\_config --file core-site.xml  
put\_config --file core-site.xml --property fs.default.name --value "hdfs://$nn:9000"  
put\_config --file core-site.xml --property hadoop.http.staticuser.user --value "$HTTP\_STATIC\_USER"  
  
create\_config --file hdfs-site.xml  
put\_config --file hdfs-site.xml --property dfs.namenode.name.dir --value "$NN\_DATA\_DIR"  
put\_config --file hdfs-site.xml --property fs.checkpoint.dir --value "$SNN\_DATA\_DIR"  
put\_config --file hdfs-site.xml --property fs.checkpoint.edits.dir --value "$SNN\_DATA\_DIR"  
put\_config --file hdfs-site.xml --property dfs.datanode.data.dir --value "$DN\_DATA\_DIR"  
put\_config --file hdfs-site.xml --property dfs.namenode.http-address --value "$nn:50070"  
put\_config --file hdfs-site.xml --property dfs.namenode.secondary.http-address --value "$snn:50090"  
  
create\_config --file mapred-site.xml  
put\_config --file mapred-site.xml --property mapreduce.framework.name --value yarn  
put\_config --file mapred-site.xml --property mapreduce.jobhistory.address --value "$mr\_hist:10020"  
put\_config --file mapred-site.xml --property mapreduce.jobhistory.webapp.address --value "$mr\_hist:19888"  
put\_config --file mapred-site.xml --property yarn.app.mapreduce.am.staging-dir --value /mapred  
  
create\_config --file yarn-site.xml  
put\_config --file yarn-site.xml --property yarn.nodemanager.aux-services --value mapreduce.shuffle  
put\_config --file yarn-site.xml --property yarn.nodemanager.aux-services.mapreduce.shuffle.class --value org.apache.hadoop.mapred.ShuffleHandler  
put\_config --file yarn-site.xml --property yarn.web-proxy.address --value "$yarn\_proxy:$YARN\_PROXY\_PORT"  
put\_config --file yarn-site.xml --property yarn.resourcemanager.scheduler.address --value "$rmgr:8030"  
put\_config --file yarn-site.xml --property yarn.resourcemanager.resource-tracker.address --value "$rmgr:8031"  
put\_config --file yarn-site.xml --property yarn.resourcemanager.address --value "$rmgr:8032"  
put\_config --file yarn-site.xml --property yarn.resourcemanager.admin.address --value "$rmgr:8033"  
put\_config --file yarn-site.xml --property yarn.resourcemanager.webapp.address --value "$rmgr:8088"  
  
echo "Copying base Hadoop XML config files to all hosts..."  
pdcp -w ^all\_hosts core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml $HADOOP\_HOME/etc/hadoop/  
  
echo "Creating configuration, command, and script links on all hosts..."  
pdsh -w ^all\_hosts "ln -s $HADOOP\_HOME/etc/hadoop /etc/hadoop"  
pdsh -w ^all\_hosts "ln -s $HADOOP\_HOME/bin/\* /usr/bin"  
pdsh -w ^all\_hosts "ln -s $HADOOP\_HOME/libexec/\* /usr/libexec"  
  
echo "Formatting the NameNode..."  
pdsh -w ^nn\_host "su - hdfs -c '$HADOOP\_HOME/bin/hdfs namenode -format'"  
  
echo "Copying startup scripts to all hosts..."  
pdcp -w ^nn\_host hadoop-namenode /etc/init.d/  
pdcp -w ^snn\_host hadoop-secondarynamenode /etc/init.d/  
pdcp -w ^dn\_hosts hadoop-datanode /etc/init.d/  
pdcp -w ^rm\_host hadoop-resourcemanager /etc/init.d/  
pdcp -w ^nm\_hosts hadoop-nodemanager /etc/init.d/  
pdcp -w ^mr\_history\_host hadoop-historyserver /etc/init.d/  
pdcp -w ^yarn\_proxy\_host hadoop-proxyserver /etc/init.d/  
  
echo "Starting Hadoop $HADOOP\_VERSION services on all hosts..."  
pdsh -w ^nn\_host "chmod 755 /etc/init.d/hadoop-namenode && chkconfig hadoop-namenode on && service hadoop-namenode start"  
pdsh -w ^snn\_host "chmod 755 /etc/init.d/hadoop-secondarynamenode && chkconfig hadoop-secondarynamenode on && service hadoop-secondarynamenode start"  
pdsh -w ^dn\_hosts "chmod 755 /etc/init.d/hadoop-datanode && chkconfig hadoop-datanode on && service hadoop-datanode start"  
pdsh -w ^rm\_host "chmod 755 /etc/init.d/hadoop-resourcemanager && chkconfig hadoop-resourcemanager on && service hadoop-resourcemanager start"  
pdsh -w ^nm\_hosts "chmod 755 /etc/init.d/hadoop-nodemanager && chkconfig hadoop-nodemanager on && service hadoop-nodemanager start"  
  
pdsh -w ^yarn\_proxy\_host "chmod 755 /etc/init.d/hadoop-proxyserver && chkconfig hadoop-proxyserver on && service hadoop-proxyserver start"  
  
echo "Creating MapReduce Job History directories..."  
su - hdfs -c "hadoop fs -mkdir -p /mapred/history/done\_intermediate"  
su - hdfs -c "hadoop fs -chown -R mapred:hadoop /mapred"  
su - hdfs -c "hadoop fs -chmod -R g+rwx /mapred"  
  
pdsh -w ^mr\_history\_host "chmod 755 /etc/init.d/hadoop-historyserver && chkconfig hadoop-historyserver on && service hadoop-historyserver start"  
  
echo "Running YARN smoke test..."  
pdsh -w ^all\_hosts "usermod -a -G hadoop $(whoami)"  
su - hdfs -c "hadoop fs -mkdir -p /user/$(whoami)"  
su - hdfs -c "hadoop fs -chown $(whoami):$(whoami) /user/$(whoami)"  
source /etc/profile.d/java.sh  
source /etc/profile.d/hadoop.sh  
source /etc/hadoop/hadoop-env.sh  
source /etc/hadoop/yarn-env.sh  
hadoop jar $HADOOP\_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-$HADOOP\_VERSION.jar pi -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -libjars $HADOOP\_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-$HADOOP\_VERSION.jar 16 10000  
\}  
  
interactive()  
\{  
echo -n "Enter NameNode hostname: "  
read nn  
echo -n "Enter Secondary NameNode hostname: "  
read snn  
echo -n "Enter ResourceManager hostname: "  
read rmgr  
echo -n "Enter Job History Server hostname: "  
read mr\_hist  
echo -n "Enter YARN Proxy hostname: "  
read yarn\_proxy  
echo -n "Enter DataNode hostnames (comma separated or hostlist syntax): "  
read dns  
echo -n "Enter NodeManager hostnames (comma separated or hostlist syntax): "  
read nms  
  
echo "$nn" > "$nn\_host"  
echo "$snn" > "$snn\_host"  
echo "$rmgr" > "$rm\_host"  
echo "$mr\_hist" > "$mr\_history\_host"  
echo "$yarn\_proxy" > "$yarn\_proxy\_host"  
dn\_hosts\_var=$(sed 's/\\,/\\n/g' <<< $dns)  
nm\_hosts\_var=$(sed 's/\\,/\\n/g' <<< $nms)  
echo "$dn\_hosts\_var" > "$dn\_hosts"  
echo "$nm\_hosts\_var" > "$nm\_hosts"  
echo "$(echo "$nn $snn $rmgr $mr\_hist $yarn\_proxy $dn\_hosts\_var $nm\_hosts\_var" | tr ' ' '\\n' | sort -u)" > "$all\_hosts"  
\}  
  
file()  
\{  
nn=$(cat nn\_host)  
snn=$(cat snn\_host)  
rmgr=$(cat rm\_host)  
mr\_hist=$(cat mr\_history\_host)  
yarn\_proxy=$(cat yarn\_proxy\_host)  
dns=$(cat dn\_hosts)  
nms=$(cat nm\_hosts)  
  
echo "$(echo "$nn $snn $rmgr $mr\_hist $dns $nms" | tr ' ' '\\n' | sort -u)" > "$all\_hosts"  
\}  
  
help()  
\{  
cat << EOF  
install-hadoop2.sh  
  
This script installs Hadoop 2 with basic data, log, and pid directories.  
  
USAGE: install-hadoop2.sh \[options\]  
  
OPTIONS:  
\-i, --interactive Prompt for fully qualified domain names (FQDN) of the NameNode,  
Secondary NameNode, DataNodes, ResourceManager, NodeManagers,  
MapReduce Job History Server, and YARN Proxy server. Values  
entered are stored in files in the same directory as this command.  
  
\-f, --file Use files with fully qualified domain names (FQDN), new-line  
separated. Place files in the same directory as this script.  
Services and file name are as follows:  
NameNode = nn\_host  
Secondary NameNode = snn\_host  
DataNodes = dn\_hosts  
ResourceManager = rm\_host  
NodeManagers = nm\_hosts  
MapReduce Job History Server = mr\_history\_host  
YARN Proxy Server = yarn\_proxy\_host  
  
\-h, --help Show this message.  
  
EXAMPLES:  
Prompt for host names:  
install-hadoop2.sh -i  
install-hadoop2.sh --interactive  
  
Use values from files in the same directory:  
install-hadoop2.sh -f  
install-hadoop2.sh --file  
  
EOF  
\}  
  
while true;  
do  
case "$1" in  
  
\-h|--help)  
help  
exit 0  
;;  
\-i|--interactive)  
interactive  
install  
shift  
;;  
\-f|--file)  
file  
install  
shift  
;;  
\--)  
shift  
break  
;;  
esac  
done

转载于:https://blog.51cto.com/huanghonglin/1754130