695
阿裏雲一鍵部署 Hadoop 分布式集群
Hadoop是一個能夠讓用戶輕鬆架構和使用的開源分布式計算框架，一種可靠、高效、可伸縮的方式進行數據處理。本文主要目的是為大家提供一種非常簡單的方法，在阿裏雲上部署Hadoop集群。
通過<阿裏雲ROS資源編排服務>，將VPC、NAT Gateway、ECS創建和Hadoop部署過程自動化，使大家能夠非常方便地部署一個Hadoop集群。本文創建的Hadoop集群包含三個節點：master.hadoop，slave1.hadoop，slave2.hadoop。
正常安裝Hadoop分為三步：配置ssh無密碼登錄，安裝Java JDK，安裝配置Hadoop tar包，和啟動測試集群。
集群中的所有主機應該可以無密碼ssh登錄，因此3台ECS都應該執行如下命令，保證所有的主機中的密鑰和公鑰均相同，這樣3台ECS就可以無密碼登錄了。
"ssh-keygen -t rsa -P '' -f '/root/.ssh/id_rsa' \n",
"cd /root/.ssh \n",
"mv id_rsa bak.id_rsa \n",
"mv id_rsa.pub bak.id_rsa.pub \n",
"echo '-----BEGIN RSA PRIVATE KEY-----' > id_rsa \n",
"echo 'MIIEowIBAAKCAQEAzfQ/QHwWB1njU9+Wu3RYi9g+g5rydSpAE0klefTJuZjtcaic' >> id_rsa \n",
"echo 'SCeBN5avih8UToZ148+Ef2YzOtoosqluZpoYLCPSpAqr8pmviBJIU3vfu9mnDG9L' >> id_rsa \n",
"echo 'oevT6K8w3wCBRCmqu+vc0Tpju/EeCuYK3+8w7e2I6F3+zIYzhhX3qmkocje7ACnV' >> id_rsa \n",
"echo 'yh2DB/2m3sogTMc0RT+5y3kJAnC6TOIlGsYjOOkPbEF3Ifn1o4ZjOFOmQlcRJer4' >> id_rsa \n",
"echo '1E6rTdXAxS2uDNMFBCf9Xyx7O9J+ELTCAXc4h7AE6WLdQb5Apzv4t1KswCAtRenP' >> id_rsa \n",
"echo '1xGcUYY8I/JUT2VvBWtQJennrk9jrPZUFDcmcwIDAQABAoIBAAwzDZQaRYvF7UtI' >> id_rsa \n",
"echo 'kTslVyFhe8J76SS7jfQWfxvMPi66OkZjQG6duG+8g0VhNei42j7WSfjp6trvlT2P' >> id_rsa \n",
"echo '/7QgKJJkxNNmtmy2Ycljm9kmG0ibSePYq9g5ieHcjr6G3yFUfoKHJBtYpBO74pWu' >> id_rsa \n",
"echo 'rrI5DuLpERUCjFc9E8w7fOIhPH4XZ6wk/EmPxHTgxZk+aMvqptyPSbUyiUOvCiZf' >> id_rsa \n",
"echo 'MD0ircs9vgtslMVDlz9m6CoiNz6B3Yf6eLRoGGMiGnsQzZHIfnHCMX8i65Jc+TvQ' >> id_rsa \n",
"echo 'fLopIHzwBwI55xpcOIBRgYiEAQJhLsSNSFugoxMcwe9RalTGGS21HOQu4b3ZylKM' >> id_rsa \n",
"echo '8ofEVKECgYEA/Y4MzN04wAsM1yNuHN+9sdiVLG28LWH3dgpcXqa9gyNsWs9Gf8uf' >> id_rsa \n",
"echo 'qbuvQGeKDRXCW93wO1jO+pCYVrkyY3l+KhBKIqmkJT5gFsa8dBUvEBLALiHNg3+o' >> id_rsa \n",
"echo 'jR2Vqsemk8kMZA8zfJ3FKcMb1pt4S2GqepqsdC3DgzIIsxufCh7jSzsCgYEAz/Cv' >> id_rsa \n",
"echo 'Z7gAdSFC8q6QxFxSyhfZwGA6QW6ZU7rZv9ZdGySvZg+vHbpNVmQ0BAQzJui3Qs9u' >> id_rsa \n",
"echo 'XQMpYafXWzKsPzG6ZWvYXTF7fuxlovvG8Qd2A2QnGtGMB9YtQMHVqbsUDwxMjiTn' >> id_rsa \n",
"echo 'VBZILkDf+WCwQ98P5UMoumI0goIcNZ+AXhcqrikCgYBkSwLvKfYfqH9Mvfv5Odsr' >> id_rsa \n",
"echo '9NKUv1c20FB1BYYh/mxp6eIbTW/CbwXZup6IqCvoHxpBAlna77b3T6iibSDsTgtE' >> id_rsa \n",
"echo 'kirw6Q8/mBukBrpWZGa4QeJ4nPBQuncuUmx4H/7Y6CaZkZW5DiMF8OIbEmYT0y7+' >> id_rsa \n",
"echo 'zh222r9CLtFYH23aL/uSLwKBgQCS1xyG2eE41aw5RBznDWtJW15iA5If8sJD5ocu' >> id_rsa \n",
"echo 'eWp2aImUQS8ghxdmEozI6U5WA7CmdWUyObFXTPc/Z6FLXwqJ5IZ+CRt0neuIFNSA' >> id_rsa \n",
"echo 'EQy9iFQ1FBUW06BRQpBns7yOg9jr6BOTxchjIV0I9caDp1nKRIrWU9NQ9iCFnYVA' >> id_rsa \n",
"echo '7IsvQQKBgFxgF7UOhwaiMb/ATSuhm2v9kVvRPEO9umdo7YJ9I4L09lYbAtpcnusQ' >> id_rsa \n",
"echo 'fIROYL25VeEMgYcQyInc3Fm/sgJdbXQnUy+3QbbCcBCmcLj27LPQyxuu7p9hbPPT' >> id_rsa \n",
"echo 'Mxx37OmYvOkSVTQz0T9HfDGvOJgt4t4cXD4T/7ewk62p6jdpSQrt' >> id_rsa \n",
"echo '-----END RSA PRIVATE KEY-----' >> id_rsa \n",
"echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN9D9AfBYHWeNT35a7dFiL2D6DmvJ1KkATSSV59Mm5mO1xqJxIJ4E3lq+KHxROhnXjz4R/ZjM62iiyqW5mmhgsI9KkCqvyma+IEkhTe9+72acMb0uh69PorzDfAIFEKaq769zROmO78R4K5grf7zDt7YjoXf7MhjOGFfeqaShyN7sAKdXKHYMH/abeyiBMxzRFP7nLeQkCcLpM4iUaxiM46Q9sQXch+fWjhmM4U6ZCVxEl6vjUTqtN1cDFLa4M0wUEJ/1fLHs70n4QtMIBdziHsATpYt1BvkCnO/i3UqzAIC1F6c/XEZxRhjwj8lRPZW8Fa1Al6eeuT2Os9lQUNyZz root@iZ2zee53wf4ndvajz30cdvZ' > id_rsa.pub \n",
"cp id_rsa.pub authorized_keys \n",
"chmod 600 authorized_keys \n",
"chmod 600 id_rsa \n",
"sed -i 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/' /etc/ssh/ssh_config \n",
"sed -i 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config \n",
"service sshd restart \n",
為了保證安全，防止對外泄露密鑰和公鑰。我們要在master上執行：
"scp -r /root/.ssh/bak.* root@$ipaddr_slave1:/root/.ssh/ \n",
"scp -r /root/.ssh/bak.* root@$ipaddr_slave2:/root/.ssh/ \n",
"ssh root@$ipaddr_slave1 \"cd /root/.ssh; rm -f id_rsa; mv bak.id_rsa id_rsa; rm -f id_rsa.pub; mv bak.id_rsa.pub id_rsa.pub; rm -f authorized_keys; cp id_rsa.pub authorized_keys; chmod 600 authorized_keys; chmod 600 id_rsa; exit\" \n",
"ssh root@$ipaddr_slave2 \"cd /root/.ssh; rm id_rsa; mv bak.id_rsa id_rsa; rm id_rsa.pub; mv bak.id_rsa.pub id_rsa.pub; rm authorized_keys; cp id_rsa.pub authorized_keys; chmod 600 authorized_keys; chmod 600 id_rsa; exit\" \n",
"cd /root/.ssh \n rm id_rsa \n mv bak.id_rsa id_rsa \n rm id_rsa.pub \n mv bak.id_rsa.pub id_rsa.pub \n rm authorized_keys \n cp id_rsa.pub authorized_keys \n chmod 600 authorized_keys \n chmod 600 id_rsa \n cd \n",
master 上安裝JDK，並遠程控製在slaves上安裝JDK。
"yum -y install aria2 \n",
"aria2c --header='Cookie: oraclelicense=accept-securebackup-cookie' $JdkUrl \n",
"mkdir -p $JAVA_HOME \ntar zxvf jdk*-x64.tar.gz -C $JAVA_HOME \ncd $JAVA_HOME \nmv jdk*.*/* ./ \nrmdir jdk*.* \n",
"echo >> /etc/profile \n",
"echo export JAVA_HOME=$JAVA_HOME >> /etc/profile \n",
"echo export JRE_HOME=$JAVA_HOME/jre >> /etc/profile \n",
"echo export CLASSPATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$CLASSPATH >> /etc/profile \n",
"echo export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH >> /etc/profile \n",
"source /etc/profile \n",
"ssh root@$ipaddr_slave1 \"mkdir -p $JAVA_HOME; exit\" \n",
"ssh root@$ipaddr_slave2 \"mkdir -p $JAVA_HOME; exit\" \n",
"scp -r $JAVA_HOME/* root@$ipaddr_slave1:$JAVA_HOME \n",
"scp -r $JAVA_HOME/* root@$ipaddr_slave2:$JAVA_HOME \n",
"ssh root@$ipaddr_slave1 \"echo >> /etc/profile; echo export JAVA_HOME=$JAVA_HOME >> /etc/profile; echo export JRE_HOME=$JAVA_HOME/jre >> /etc/profile; echo export CLASSPATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$CLASSPATH >> /etc/profile; echo export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH >> /etc/profile; source /etc/profile; exit\" \n",
"ssh root@$ipaddr_slave2 \"echo >> /etc/profile; echo export JAVA_HOME=$JAVA_HOME >> /etc/profile; echo export JRE_HOME=$JAVA_HOME/jre >> /etc/profile; echo export CLASSPATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$CLASSPATH >> /etc/profile; echo export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH >> /etc/profile; source /etc/profile; exit\" \n",
" \n",
下載安裝Hadoop：
"aria2c $HadoopUrl \n", "mkdir -p $HADOOP_HOME \ntar zxvf hadoop-*.tar.gz -C $HADOOP_HOME \ncd $HADOOP_HOME \nmv hadoop-*.*/* ./ \nrmdir hadoop-*.* \n", "echo >> /etc/profile \n", "echo export HADOOP_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_MAPRED_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_COMMON_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native >> /etc/profile \n", "echo export YARN_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_HDFS_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop >> /etc/profile \n", "echo export HADOOP_OPTS=-Djava.library.path=$HADOOP_HOME/lib:$HADOOP_HOME/lib/native >> /etc/profile \n", "echo export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH >> /etc/profile \n", "echo export CLASSPATH=.:$CLASSPATH:$HADOOP__HOME/lib >> /etc/profile \n", "source /etc/profile \n", " \n", "mkdir -p $HADOOP_HOME/tmp \n", "mkdir -p $HADOOP_HOME/dfs/name \n", "mkdir -p $HADOOP_HOME/dfs/data \n", ###此處要對這4個文件進行正確配置，具體內容請參考模板Hadoop_Distributed_Env_3_ecs.json. core-site.xml，hdfs-site.xml，mapred-site.xml，yarn-site.xml### "ssh root@$ipaddr_slave1 \"mkdir -p $HADOOP_HOME; exit\" \n", "ssh root@$ipaddr_slave2 \"mkdir -p $HADOOP_HOME; exit\" \n", "scp -r $HADOOP_HOME/* root@$ipaddr_slave1:$HADOOP_HOME \n", "scp -r $HADOOP_HOME/* root@$ipaddr_slave2:$HADOOP_HOME \n", " \n", "cd $HADOOP_HOME/etc/hadoop \n", "echo $ipaddr_slave1 > slaves \n", "echo $ipaddr_slave2 >> slaves \n",
最後格式化HDFS，關閉防火牆，啟動集群。
"hadoop namenode -format \n",
"systemctl stop firewalld \n",
"start-dfs.sh \n",
"start-yarn.sh \n",
注意：
- 必須確保可以正確下載Jdk和Hadoop tar 包，我們可以選擇類似如下的URL：
- 利用該模板創建時，隻能選擇CentOS係統；
- 為了防止Timeout 失敗，可以設置為120分鍾；
- 我們選擇的數據中心在上海。
創建完成後，查看資源棧概況：
瀏覽器中輸入圖中的的WebsiteUrl，得到如下結果，則部署成功：
Hadoop_Distributed_Env_3_ecs.json
最後更新：2017-04-17 12:30:28