{ "Description": "This template shows how to deploy a Hadoop-Spark environment. One ECS instance plays role of master, and one instance group plays role of worker. Step 1 configs the ssh login without password. Step 2 installs and configs Java env. Step 3 installs and configs hadoop env. Step 4 installs and configs Spark env. *** WARNING *** Only test in CentOS-7. Maybe stopping firewall is needed. The deploying time mainly depends on the speed of downloading 4 packages.", "Parameters": { "VpcName": { "Type": "String", "Description": { "zh-cn": "VPC 名称，[2，128]英文或中文字符，必须以字母或中文开头，可以包含数字、下划线或“.”、“-”。", "en": "Name of VPC, [2, 128] English or Chinese characters, must start with a letter or Chinese in size, can contain numbers, '_' or '.', '-'." }, "Label": "VPC Name", "ConstraintDescription": "[2, 128] English or Chinese characters", "MinLength": 2, "MaxLength": 128, "Default": "MyVPC" }, "VpcCidrBlock": { "Type": "String", "Description": { "zh-cn": "VPC的IP地址范围；
您可以使用以下IP地址范围及其子网：
[10.0.0.0/8]
[172.16.0.0/12]
[192.168.0/16]", "en": "The IP address range of the VPC in the CIDR Block form;
you can use the following IP address ranges and their subnets:
[10.0.0.0/8]
[172.16.0.0/12]
[192.168.0.0/16]" }, "AllowedValues": [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" ], "Label": "VPC CIDR Block", "Default": "192.168.0.0/16" }, "VSwitchCidrBlock": { "Type": "String", "Description": { "zh-cn": "创建的虚拟交换机的子网，它必须属于自己的VPC网段内", "en": "CIDR Block of created VSwitch,It must belong to itself VPC CIDR block." }, "Label": "VSwitch CIDR Block", "Default": "192.168.1.0/24" }, "ZoneId": { "AssociationProperty": "ALIYUN::ECS::Instance:ZoneId", "Type": "String", "Description": { "zh-cn": "ECS 可用区ID, 查看可用区信息.", "en": "ECS Available Zone ID, View region and zone info." }, "Label": "Zone ID" }, "InstanceName": { "Type": "String", "Description": { "zh-cn": "ECS实例名称，[2，128]英文或中文字符，必须以字母或中文开头，可以包含数字、下划线或“.”、“-”。", "en": "The name of ECS instance,[2, 128] English or Chinese characters, must start with a letter or Chinese in size, can contain numbers, '_' or '.', '-'." }, "Label": "Instance Name", "Default": "ecsinstance" }, "InstanceType": { "Type": "String", "Description": { "zh-cn": "ECS实例类型，进入产品控制台确保当前实例可用, 查看实例类型", "en": "The ECS instance type,go to the product console to ensure the current instance is available, View instance types." }, "Label": "Instance Type", "Default": "ecs.c5.large" }, "SystemDiskCategory": { "Type": "String", "AllowedValues": [ "cloud", "cloud_efficiency", "cloud_ssd" ], "Description": { "zh-cn": "系统磁盘类别：普通云盘（cloud）、高效云盘（cloud_efficiency）或固态盘（cloud_ssd）", "en": "System disk category: average cloud disk(cloud), efficient cloud disk(cloud_efficiency) or SSD cloud disk(cloud_ssd)" }, "Label": "Instance System Disk Category", "Default": "cloud_ssd" }, "ImageId": { "Type": "String", "Description": { "zh-cn": "镜像ID，ECS实例的镜像资源, 查看镜像资源", "en": "Image ID, represents the image resource to startup one ECS instance, View image resources" }, "Label": "Image ID", "Default": "centos_7" }, "InstancePassword": { "NoEcho": true, "Type": "String", "Description": { "zh-cn": "长度8-30，必须包含大写字母、小写字母、数字、特殊符号三种；
特殊字符包括：（）'~！@#$%^&*_-+=|{}[]:;'<>,.?/", "en": "The Instance Name，A maximum of 16 characters, consisting of lowercase letters, Numbers, underscores, letters beginning, letters or Numbers ending." }, "AllowedPattern": "[a-zA-Z0-9-\\(\\)\\`\\~\\!@\\#\\$%\\^&\\*-+=\\|\\{\\}\\[\\]\\:\\;\\‘\\,\\.\\?\\/]*", "Label": "Instance Password", "ConstraintDescription": "The password is a string of 8 to 30 characters and must contain uppercase/lowercase letters, numbers.", "MinLength": "8", "MaxLength": "30" }, "MasterName": { "Type": "String", "Description": { "zh-cn": "主机名，[2，128]英文或中文字符，必须以字母或中文开头，可以包含数字、下划线或“.”、“-”。", "en": "Master hostname, [2, 128] English or Chinese characters, must start with a letter or Chinese in size, can contain numbers, '_' or '.', '-'." }, "Label": "Master Name", "Default": "master.hadoop" }, "HadoopHome": { "Type": "String", "Description": { "zh-cn": "Hadoop 主目录必须是新的", "en": "HADOOP_HOME directory should be new." }, "Label": "Hadoop Home", "Default": "/usr/local/hadoop/hadoop-2.7.7" }, "HadoopUrl": { "Type": "String", "Description": { "zh-cn": ".tar.gz的官方下载路径", "en": "The official download path of .tar.gz" }, "Label": "Hadoop Url", "Default": "http://apache.claz.org/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz" }, "SlaveAmount": { "Type": "Number", "Description": { "zh-cn": "ECS实例的最小值必须小于或等于最大值。", "en": "The minimum of ECS instances, must be less than or equal to the maximum." }, "Label": "Slave Amount", "MinValue": 1, "ConstraintDescription": "An integer within [1, 100]", "MaxValue": 100, "Default": 1 }, "ScalaHome": { "Type": "String", "Description": { "zh-cn": "SCALA_HOME 主目录必须是新的", "en": "SCALA_HOME directory should be new." }, "Label": "Scala Home", "Default": "/usr/local/scala/scala-2.12.1" }, "ScalaUrl": { "Type": "String", "Description": { "zh-cn": ".tar.gz的官方下载路径", "en": "The official download path of .tar.gz" }, "Label": "Scala Url", "Default": "https://downloads.lightbend.com/scala/2.12.1/scala-2.12.1.tgz" }, "SparkHome": { "Type": "String", "Description": { "zh-cn": "SPARK_HOME 主目录必须是新的", "en": "SPARK_HOME directory should be new." }, "Label": "Spark Home", "Default": "/usr/local/hadoop/spark-2.1.0-bin-hadoop2.7" }, "SparkUrl": { "Type": "String", "Description": { "zh-cn": "-bin-hadoopx.x.tgz的官方下载路径", "en": "The official download path of -bin-hadoopx.x.tgz" }, "Label": "Spark Url", "Default": "http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz" } }, "Metadata": { "ALIYUN::ROS::Interface": { "ParameterGroups": [ { "Parameters": [ "VpcName", "VpcCidrBlock", "VSwitchCidrBlock" ], "Label": { "default": "VPC" } }, { "Parameters": [ "ZoneId", "InstanceName", "InstanceType", "SystemDiskCategory", "ImageId", "SlaveAmount", "InstancePassword" ], "Label": { "default": "ECS" } }, { "Parameters": [ "MasterName", "HadoopHome", "HadoopUrl" ], "Label": { "default": "Hadoop" } }, { "Parameters": [ "ScalaHome", "ScalaUrl" ], "Label": { "default": "Scala" } }, { "Parameters": [ "SparkHome", "SparkUrl" ], "Label": { "default": "Spark" } } ], "TemplateTags": [ "Deploy a Hadoop-Spark environment" ] } }, "ROSTemplateFormatVersion": "2015-09-01", "Outputs": { "SparkWebsiteURL": { "Description": "URL for newly created Spark environment.", "Value": { "Fn::Join": [ "", [ "http://", { "Fn::GetAtt": [ "EipForward", "EipAddress" ] }, ":8080" ] ] } }, "HadoopWebsiteURL": { "Description": "URL for newly created Hadoop environment.", "Value": { "Fn::Join": [ "", [ "http://", { "Fn::GetAtt": [ "EipForward", "EipAddress" ] }, ":50070" ] ] } } }, "Resources": { "EipSNat": { "Type": "ALIYUN::VPC::EIP", "Properties": { "InternetChargeType": "PayByTraffic", "Bandwidth": 5 } }, "EIpSnatAssociation": { "DependsOn": "NatGateway", "Type": "ALIYUN::VPC::EIPAssociation", "Properties": { "InstanceId": { "Ref": "NatGateway" }, "AllocationId": { "Ref": "EipSNat" } } }, "EipForward": { "Type": "ALIYUN::VPC::EIP", "Properties": { "InternetChargeType": "PayByTraffic", "Bandwidth": 5 } }, "EIpForwardAssociation": { "DependsOn": "EIpSnatAssociation", "Type": "ALIYUN::VPC::EIPAssociation", "Properties": { "InstanceId": { "Ref": "NatGateway" }, "AllocationId": { "Ref": "EipForward" } } }, "ForwardEntry": { "Type": "ALIYUN::ECS::ForwardEntry", "DependsOn": [ "Master", "EIpForwardAssociation" ], "Properties": { "ExternalPort": "Any", "ExternalIp": { "Fn::GetAtt": [ "EipForward", "EipAddress" ] }, "IpProtocol": "Any", "InternalPort": "Any", "InternalIp": { "Fn::GetAtt": [ "Master", "PrivateIp" ] }, "ForwardTableId": { "Fn::GetAtt": [ "NatGateway", "ForwardTableId" ] } } }, "Master": { "Type": "ALIYUN::ECS::Instance", "DependsOn": [ "SlaveGroupWaitCondition", "SNatEntry" ], "Properties": { "InstanceName": { "Ref": "InstanceName" }, "IoOptimized": "optimized", "VpcId": { "Fn::GetAtt": [ "Vpc", "VpcId" ] }, "UserData": { "Fn::Replace": [ { "ros-notify": { "Fn::GetAtt": [ "MasterConditionHandle", "CurlCli" ] } }, { "Fn::Join": [ "", [ "#!/bin/bash \n", "# set -e \n", "ips_slave=", { "Fn::Join": [ ",", { "Fn::GetAtt": [ "SlaveGroup", "PrivateIps" ] } ] }, "\n", "hostnames_slave=", { "Fn::Join": [ ",", { "Fn::GetAtt": [ "SlaveGroup", "HostNames" ] } ] }, "\n", "HADOOP_HOME=", { "Ref": "HadoopHome" }, "\n", "SCALA_HOME=", { "Ref": "ScalaHome" }, "\n", "SPARK_HOME=", { "Ref": "SparkHome" }, "\n", "HadoopUrl=", { "Ref": "HadoopUrl" }, "\n", "ScalaUrl=", { "Ref": "ScalaUrl" }, "\n", "SparkUrl=", { "Ref": "SparkUrl" }, "\n", "MasterName=", { "Ref": "MasterName" }, "\n", "OLD_IFS=$IFS \nIFS=, \nips=($ips_slave) \nhostnames=($hostnames_slave) \nIFS=$OLD_IFS \n", "hostnamectl --static set-hostname \"$MasterName\" \n", "export HOME=/root \nexport HOSTNAME=`hostname` \n", "ip_addr=`ifconfig eth0 | awk '/inet /{print $2}'` \n", "cd /root \n", "rm -rf .ssh \n", "ssh-keygen -t rsa -P '' -f '/root/.ssh/id_rsa' \n", "cd /root/.ssh \n", "mv id_rsa bak.id_rsa \nmv id_rsa.pub bak.id_rsa.pub \n", "echo '-----BEGIN RSA PRIVATE KEY-----' > id_rsa \n", "echo 'MIIEowIBAAKCAQEAzfQ/QHwWB1njU9+Wu3RYi9g+g5rydSpAE0klefTJuZjtcaic' >> id_rsa \n", "echo 'SCeBN5avih8UToZ148+Ef2YzOtoosqluZpoYLCPSpAqr8pmviBJIU3vfu9mnDG9L' >> id_rsa \n", "echo 'oevT6K8w3wCBRCmqu+vc0Tpju/EeCuYK3+8w7e2I6F3+zIYzhhX3qmkocje7ACnV' >> id_rsa \n", "echo 'yh2DB/2m3sogTMc0RT+5y3kJAnC6TOIlGsYjOOkPbEF3Ifn1o4ZjOFOmQlcRJer4' >> id_rsa \n", "echo '1E6rTdXAxS2uDNMFBCf9Xyx7O9J+ELTCAXc4h7AE6WLdQb5Apzv4t1KswCAtRenP' >> id_rsa \n", "echo '1xGcUYY8I/JUT2VvBWtQJennrk9jrPZUFDcmcwIDAQABAoIBAAwzDZQaRYvF7UtI' >> id_rsa \n", "echo 'kTslVyFhe8J76SS7jfQWfxvMPi66OkZjQG6duG+8g0VhNei42j7WSfjp6trvlT2P' >> id_rsa \n", "echo '/7QgKJJkxNNmtmy2Ycljm9kmG0ibSePYq9g5ieHcjr6G3yFUfoKHJBtYpBO74pWu' >> id_rsa \n", "echo 'rrI5DuLpERUCjFc9E8w7fOIhPH4XZ6wk/EmPxHTgxZk+aMvqptyPSbUyiUOvCiZf' >> id_rsa \n", "echo 'MD0ircs9vgtslMVDlz9m6CoiNz6B3Yf6eLRoGGMiGnsQzZHIfnHCMX8i65Jc+TvQ' >> id_rsa \n", "echo 'fLopIHzwBwI55xpcOIBRgYiEAQJhLsSNSFugoxMcwe9RalTGGS21HOQu4b3ZylKM' >> id_rsa \n", "echo '8ofEVKECgYEA/Y4MzN04wAsM1yNuHN+9sdiVLG28LWH3dgpcXqa9gyNsWs9Gf8uf' >> id_rsa \n", "echo 'qbuvQGeKDRXCW93wO1jO+pCYVrkyY3l+KhBKIqmkJT5gFsa8dBUvEBLALiHNg3+o' >> id_rsa \n", "echo 'jR2Vqsemk8kMZA8zfJ3FKcMb1pt4S2GqepqsdC3DgzIIsxufCh7jSzsCgYEAz/Cv' >> id_rsa \n", "echo 'Z7gAdSFC8q6QxFxSyhfZwGA6QW6ZU7rZv9ZdGySvZg+vHbpNVmQ0BAQzJui3Qs9u' >> id_rsa \n", "echo 'XQMpYafXWzKsPzG6ZWvYXTF7fuxlovvG8Qd2A2QnGtGMB9YtQMHVqbsUDwxMjiTn' >> id_rsa \n", "echo 'VBZILkDf+WCwQ98P5UMoumI0goIcNZ+AXhcqrikCgYBkSwLvKfYfqH9Mvfv5Odsr' >> id_rsa \n", "echo '9NKUv1c20FB1BYYh/mxp6eIbTW/CbwXZup6IqCvoHxpBAlna77b3T6iibSDsTgtE' >> id_rsa \n", "echo 'kirw6Q8/mBukBrpWZGa4QeJ4nPBQuncuUmx4H/7Y6CaZkZW5DiMF8OIbEmYT0y7+' >> id_rsa \n", "echo 'zh222r9CLtFYH23aL/uSLwKBgQCS1xyG2eE41aw5RBznDWtJW15iA5If8sJD5ocu' >> id_rsa \n", "echo 'eWp2aImUQS8ghxdmEozI6U5WA7CmdWUyObFXTPc/Z6FLXwqJ5IZ+CRt0neuIFNSA' >> id_rsa \n", "echo 'EQy9iFQ1FBUW06BRQpBns7yOg9jr6BOTxchjIV0I9caDp1nKRIrWU9NQ9iCFnYVA' >> id_rsa \n", "echo '7IsvQQKBgFxgF7UOhwaiMb/ATSuhm2v9kVvRPEO9umdo7YJ9I4L09lYbAtpcnusQ' >> id_rsa \n", "echo 'fIROYL25VeEMgYcQyInc3Fm/sgJdbXQnUy+3QbbCcBCmcLj27LPQyxuu7p9hbPPT' >> id_rsa \n", "echo 'Mxx37OmYvOkSVTQz0T9HfDGvOJgt4t4cXD4T/7ewk62p6jdpSQrt' >> id_rsa \n", "echo '-----END RSA PRIVATE KEY-----' >> id_rsa \n", "echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN9D9AfBYHWeNT35a7dFiL2D6DmvJ1KkATSSV59Mm5mO1xqJxIJ4E3lq+KHxROhnXjz4R/ZjM62iiyqW5mmhgsI9KkCqvyma+IEkhTe9+72acMb0uh69PorzDfAIFEKaq769zROmO78R4K5grf7zDt7YjoXf7MhjOGFfeqaShyN7sAKdXKHYMH/abeyiBMxzRFP7nLeQkCcLpM4iUaxiM46Q9sQXch+fWjhmM4U6ZCVxEl6vjUTqtN1cDFLa4M0wUEJ/1fLHs70n4QtMIBdziHsATpYt1BvkCnO/i3UqzAIC1F6c/XEZxRhjwj8lRPZW8Fa1Al6eeuT2Os9lQUNyZz root@iZ2zee53wf4ndvajz30cdvZ' > id_rsa.pub \n", "cp id_rsa.pub authorized_keys \n", "chmod 600 authorized_keys \nchmod 600 id_rsa \n", "sed -i 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/' /etc/ssh/ssh_config \n", "sed -i 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config \n", "service sshd restart \n", "for ip in ${ips[*]} \ndo \nscp -r /root/.ssh/bak.* root@$ip:/root/.ssh/ \ndone \n", "for ip in ${ips[*]} \ndo \nssh root@$ip \"cd /root/.ssh; rm -f id_rsa; mv bak.id_rsa id_rsa; rm -f id_rsa.pub; mv bak.id_rsa.pub id_rsa.pub; rm -f authorized_keys; cp id_rsa.pub authorized_keys; chmod 600 authorized_keys; chmod 600 id_rsa; exit\" \ndone \n", "cd /root/.ssh \n rm id_rsa \n mv bak.id_rsa id_rsa \n rm id_rsa.pub \n mv bak.id_rsa.pub id_rsa.pub \n rm authorized_keys \n cp id_rsa.pub authorized_keys \n chmod 600 authorized_keys \n chmod 600 id_rsa \n cd \n", "echo $ip_addr $MasterName >> /etc/hosts \n", "i=0 \n", "for ip in ${ips[*]} \ndo \necho $ip ${hostnames[$i]} >> /etc/hosts \ni=$[i+1] \ndone \n", "for ip in ${ips[*]} \ndo \ni=0 \nssh root@$ip \"echo $ip_addr $MasterName >> /etc/hosts; exit\" \n", "for ip1 in ${ips[*]} \ndo \nssh root@$ip \"echo $ip1 ${hostnames[$i]} >> /etc/hosts; exit\" \ni=$[i+1] \ndone \n", "done \n", "cd /root \n", "yum -y install aria2 \n", "yum -y install java \n", "for ip in ${ips[*]} \ndo \nssh root@$ip \"yum -y install java; exit\" \ndone \n", "cd /root \n", "aria2c $HadoopUrl \n", "mkdir -p $HADOOP_HOME \ntar zxvf hadoop-*.tar.gz -C $HADOOP_HOME \ncd $HADOOP_HOME \nmv hadoop-*.*/* ./ \nrmdir hadoop-*.* \n", "echo >> /etc/profile \n", "echo export HADOOP_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_MAPRED_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_COMMON_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native >> /etc/profile \n", "echo export YARN_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_HDFS_HOME=$HADOOP_HOME >> /etc/profile \n", "echo export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop >> /etc/profile \n", "echo export HADOOP_OPTS=-Djava.library.path=$HADOOP_HOME/lib:$HADOOP_HOME/lib/native >> /etc/profile \n", "echo export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH >> /etc/profile \n", "echo export CLASSPATH=.:$CLASSPATH:$HADOOP__HOME/lib >> /etc/profile \n", "source /etc/profile \n", " \n", "mkdir -p $HADOOP_HOME/tmp \nmkdir -p $HADOOP_HOME/dfs/name \nmkdir -p $HADOOP_HOME/dfs/data \n", "echo >> /etc/profile \n", "JAVA_HOME=`find /usr/lib/jvm -name 'java-1.8.0-openjdk-1.8.0*'` \n", "echo export JAVA_HOME=$JAVA_HOME/jre >> /etc/profile \n", "echo export JRE_HOME=$JAVA_HOME/jre >> /etc/profile \n", " \n", "cd $HADOOP_HOME/etc/hadoop \n", "cp hadoop-env.sh bak.hadoop-env.sh \n", "sed -i '/export JAVA_HOME=/{s/^/# /}' hadoop-env.sh \n", "sed -i \"/# export JAVA_HOME=/a export JAVA_HOME=$JAVA_HOME\" hadoop-env.sh \n", " \n", "cd $HADOOP_HOME/etc/hadoop \n", "cp core-site.xml bak.core-site.xml \n", "echo '' > core-site.xml \n", "echo ' ' >> core-site.xml \n", "echo ' hadoop.tmp.dir' >> core-site.xml \n", "echo \" $HADOOP_HOME/tmp\" >> core-site.xml \n", "echo ' ' >> core-site.xml \n", "echo '' >> core-site.xml \n", "echo ' ' >> core-site.xml \n", "echo ' fs.default.name' >> core-site.xml \n", "echo \" hdfs://$ip_addr:9000\" >> core-site.xml \n", "echo ' ' >> core-site.xml \n", "echo '' >> core-site.xml \n", " \n", "cd $HADOOP_HOME/etc/hadoop \n", "cp hdfs-site.xml bak.hdfs-site.xml \n", "echo '' > hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo ' dfs.namenode.name.dir' >> hdfs-site.xml \n", "echo \" file://$HADOOP_HOME/dfs/name\" >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo '' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo ' dfs.datanode.data.dir' >> hdfs-site.xml \n", "echo \" file://$HADOOP_HOME/dfs/data\" >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo '' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo ' dfs.replication' >> hdfs-site.xml \n", "echo ' 1' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo '' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo ' dfs.nameservices' >> hdfs-site.xml \n", "echo ' hadoop-cluster1' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo '' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo ' dfs.namenode.secondary.http-address' >> hdfs-site.xml \n", "echo \" $ip_addr:50090\" >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo '' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo ' dfs.webhdfs.enabled' >> hdfs-site.xml \n", "echo ' true' >> hdfs-site.xml \n", "echo ' ' >> hdfs-site.xml \n", "echo '' >> hdfs-site.xml \n", " \n", "cd $HADOOP_HOME/etc/hadoop \n", "cp mapred-site.xml.template mapred-site.xml \n", "echo '' > mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo ' mapreduce.framework.name' >> mapred-site.xml \n", "echo ' yarn' >> mapred-site.xml \n", "echo ' true' >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo '' >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo ' mapreduce.jobtracker.http.address' >> mapred-site.xml \n", "echo \" $ip_addr:50030\" >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo '' >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo ' mapreduce.jobhistory.address' >> mapred-site.xml \n", "echo \" $ip_addr:10020\" >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo '' >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo ' mapreduce.jobhistory.webapp.address' >> mapred-site.xml \n", "echo \" $ip_addr:19888\" >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo '' >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo ' mapred.job.tracker' >> mapred-site.xml \n", "echo \" http://$ip_addr:9001\" >> mapred-site.xml \n", "echo ' ' >> mapred-site.xml \n", "echo '' >> mapred-site.xml \n", " \n", "cd $HADOOP_HOME/etc/hadoop \n", "cp yarn-site.xml bak.yarn-site.xml \n", "echo '' > yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.resourcemanager.hostname' >> yarn-site.xml \n", "echo \" $ip_addr\" >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.nodemanager.aux-services' >> yarn-site.xml \n", "echo ' mapreduce_shuffle' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.resourcemanager.address' >> yarn-site.xml \n", "echo \" $ip_addr:8032\" >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.resourcemanager.scheduler.address' >> yarn-site.xml \n", "echo \" $ip_addr:8030\" >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.resourcemanager.resource-tracker.address' >> yarn-site.xml \n", "echo \" $ip_addr:8031\" >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.resourcemanager.admin.address' >> yarn-site.xml \n", "echo \" $ip_addr:8033\" >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo ' yarn.resourcemanager.webapp.address' >> yarn-site.xml \n", "echo \" $ip_addr:8088\" >> yarn-site.xml \n", "echo ' ' >> yarn-site.xml \n", "echo '' >> yarn-site.xml \n", " \n", "sed -i 304s/'$JAVA'/'java'/ /usr/local/hadoop/hadoop-2.7.7/bin/hdfs \n", "for ip in ${ips[*]} \ndo \nssh root@$ip \"mkdir -p $HADOOP_HOME; exit\" \ndone \n", "for ip in ${ips[*]} \ndo \nscp -r $HADOOP_HOME/* root@$ip:$HADOOP_HOME \ndone \n", " \n", "cd $HADOOP_HOME/etc/hadoop \nrm slaves \n", "for ip in ${ips[*]} \ndo \necho $ip >> slaves \ndone \n", "cd /root \n", "aria2c $ScalaUrl \n", "mkdir -p $SCALA_HOME \ntar zxvf scala-*.tgz -C $SCALA_HOME \ncd $SCALA_HOME \nmv scala-*.*/* ./ \nrmdir scala-*.* \n", "echo >> /etc/profile \n", "echo export SCALA_HOME=$SCALA_HOME >> /etc/profile \n", "echo export PATH=$PATH:$SCALA_HOME/bin >> /etc/profile \n", "source /etc/profile \n", " \n", "cd /root \n", "aria2c $SparkUrl \n", "mkdir -p $SPARK_HOME \ntar zxvf spark-*hadoop*.tgz -C $SPARK_HOME \ncd $SPARK_HOME \nmv spark-*hadoop*/* ./ \nrmdir spark-*hadoop* \n", "echo >> /etc/profile \n", "echo export SPARK_HOME=$SPARK_HOME >> /etc/profile \n", "echo export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin >> /etc/profile \n", "source /etc/profile \n", "for ip in ${ips[*]} \ndo \necho $ip >> $SPARK_HOME/conf/slaves \ndone \n", "cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh \n", "echo export SCALA_HOME=$SCALA_HOME > $SPARK_HOME/conf/spark-env.sh \n", "echo export JAVA_HOME=$JAVA_HOME >> $SPARK_HOME/conf/spark-env.sh \n", "for ip in ${ips[*]} \ndo \nssh root@$ip \"mkdir -p $SCALA_HOME; mkdir -p $SPARK_HOME; exit\" \ndone \n", "for ip in ${ips[*]} \ndo \nscp -r $SCALA_HOME/* root@$ip:$SCALA_HOME \ndone \n", "for ip in ${ips[*]} \ndo \nscp -r $SPARK_HOME/* root@$ip:$SPARK_HOME \ndone \n", "for ip in ${ips[*]} \ndo \nssh root@$ip \"echo >> /etc/profile;echo export SCALA_HOME=$SCALA_HOME >> /etc/profile;echo export PATH=$PATH:$SCALA_HOME/bin >> /etc/profile;echo export SPARK_HOME=$SPARK_HOME >> /etc/profile;echo export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin >> /etc/profile;source /etc/profile;exit\" \ndone \n", "$HADOOP_HOME/bin/hadoop namenode -format \n", "systemctl stop firewalld \n", "$HADOOP_HOME/sbin/start-dfs.sh \n", "$HADOOP_HOME/sbin/start-yarn.sh \n", "$SPARK_HOME/sbin/start-all.sh \n", "ros-notify -d '{\"data\" : \"Install Spark.\"}'\n" ] ] } ] }, "SecurityGroupId": { "Ref": "SecurityGroup" }, "VSwitchId": { "Ref": "VSwitch" }, "ImageId": { "Ref": "ImageId" }, "AllocatePublicIP": false, "InstanceType": { "Ref": "InstanceType" }, "Password": { "Ref": "InstancePassword" }, "SystemDiskCategory": { "Ref": "SystemDiskCategory" } } }, "VSwitch": { "Type": "ALIYUN::ECS::VSwitch", "DependsOn": "Vpc", "Properties": { "ZoneId": { "Ref": "ZoneId" }, "VpcId": { "Fn::GetAtt": [ "Vpc", "VpcId" ] }, "CidrBlock": { "Ref": "VSwitchCidrBlock" } } }, "MasterConditionHandle": { "Type": "ALIYUN::ROS::WaitConditionHandle" }, "Vpc": { "Type": "ALIYUN::ECS::VPC", "Properties": { "CidrBlock": { "Ref": "VpcCidrBlock" }, "VpcName": { "Ref": "VpcName" } } }, "SlaveGroupWaitCondition": { "Type": "ALIYUN::ROS::WaitCondition", "DependsOn": "SlaveGroup", "Properties": { "Timeout": 1800, "Count": { "Ref": "SlaveAmount" }, "Handle": { "Ref": "SlaveGroupConditionHandle" } } }, "NatGateway": { "Type": "ALIYUN::ECS::NatGateway", "DependsOn": "VSwitch", "Properties": { "Description": "My NAT Gateway", "VpcId": { "Fn::GetAtt": [ "Vpc", "VpcId" ] }, "NatGatewayName": "nat_gateway_1", "VSwitchId": { "Fn::GetAtt": [ "VSwitch", "VSwitchId" ] }, "Spec": "Small" } }, "SecurityGroup": { "Type": "ALIYUN::ECS::SecurityGroup", "Properties": { "VpcId": { "Ref": "Vpc" }, "SecurityGroupIngress": [ { "PortRange": "-1/-1", "Priority": 1, "SourceCidrIp": "0.0.0.0/0", "IpProtocol": "all", "NicType": "internet" } ], "SecurityGroupEgress": [ { "PortRange": "-1/-1", "Priority": 1, "IpProtocol": "all", "DestCidrIp": "0.0.0.0/0", "NicType": "internet" } ] } }, "MasterWaitCondition": { "Type": "ALIYUN::ROS::WaitCondition", "DependsOn": "Master", "Properties": { "Timeout": 10800, "Count": 1, "Handle": { "Ref": "MasterConditionHandle" } } }, "SNatEntry": { "DependsOn": "EIpSnatAssociation", "Type": "ALIYUN::ECS::SNatEntry", "Properties": { "SourceVSwitchId": { "Fn::GetAtt": [ "VSwitch", "VSwitchId" ] }, "SNatIp": { "Fn::GetAtt": [ "EipSNat", "EipAddress" ] }, "SNatTableId": { "Fn::GetAtt": [ "NatGateway", "SNatTableId" ] } } }, "SlaveGroupConditionHandle": { "Type": "ALIYUN::ROS::WaitConditionHandle" }, "SlaveGroup": { "Type": "ALIYUN::ECS::InstanceGroup", "Properties": { "InstanceName": { "Ref": "InstanceName" }, "IoOptimized": "optimized", "VpcId": { "Fn::GetAtt": [ "Vpc", "VpcId" ] }, "UserData": { "Fn::Replace": [ { "ros-notify": { "Fn::GetAtt": [ "SlaveGroupConditionHandle", "CurlCli" ] } }, { "Fn::Join": [ "", [ "#!/bin/bash \n", "export HOME=/root \nexport HOSTNAME=`hostname` \n", "cd /root \n", "rm -rf .ssh \n", "ssh-keygen -t rsa -P '' -f '/root/.ssh/id_rsa' \n", "cd /root/.ssh \n", "rm -f * \n", "echo '-----BEGIN RSA PRIVATE KEY-----' > id_rsa \n", "echo 'MIIEowIBAAKCAQEAzfQ/QHwWB1njU9+Wu3RYi9g+g5rydSpAE0klefTJuZjtcaic' >> id_rsa \n", "echo 'SCeBN5avih8UToZ148+Ef2YzOtoosqluZpoYLCPSpAqr8pmviBJIU3vfu9mnDG9L' >> id_rsa \n", "echo 'oevT6K8w3wCBRCmqu+vc0Tpju/EeCuYK3+8w7e2I6F3+zIYzhhX3qmkocje7ACnV' >> id_rsa \n", "echo 'yh2DB/2m3sogTMc0RT+5y3kJAnC6TOIlGsYjOOkPbEF3Ifn1o4ZjOFOmQlcRJer4' >> id_rsa \n", "echo '1E6rTdXAxS2uDNMFBCf9Xyx7O9J+ELTCAXc4h7AE6WLdQb5Apzv4t1KswCAtRenP' >> id_rsa \n", "echo '1xGcUYY8I/JUT2VvBWtQJennrk9jrPZUFDcmcwIDAQABAoIBAAwzDZQaRYvF7UtI' >> id_rsa \n", "echo 'kTslVyFhe8J76SS7jfQWfxvMPi66OkZjQG6duG+8g0VhNei42j7WSfjp6trvlT2P' >> id_rsa \n", "echo '/7QgKJJkxNNmtmy2Ycljm9kmG0ibSePYq9g5ieHcjr6G3yFUfoKHJBtYpBO74pWu' >> id_rsa \n", "echo 'rrI5DuLpERUCjFc9E8w7fOIhPH4XZ6wk/EmPxHTgxZk+aMvqptyPSbUyiUOvCiZf' >> id_rsa \n", "echo 'MD0ircs9vgtslMVDlz9m6CoiNz6B3Yf6eLRoGGMiGnsQzZHIfnHCMX8i65Jc+TvQ' >> id_rsa \n", "echo 'fLopIHzwBwI55xpcOIBRgYiEAQJhLsSNSFugoxMcwe9RalTGGS21HOQu4b3ZylKM' >> id_rsa \n", "echo '8ofEVKECgYEA/Y4MzN04wAsM1yNuHN+9sdiVLG28LWH3dgpcXqa9gyNsWs9Gf8uf' >> id_rsa \n", "echo 'qbuvQGeKDRXCW93wO1jO+pCYVrkyY3l+KhBKIqmkJT5gFsa8dBUvEBLALiHNg3+o' >> id_rsa \n", "echo 'jR2Vqsemk8kMZA8zfJ3FKcMb1pt4S2GqepqsdC3DgzIIsxufCh7jSzsCgYEAz/Cv' >> id_rsa \n", "echo 'Z7gAdSFC8q6QxFxSyhfZwGA6QW6ZU7rZv9ZdGySvZg+vHbpNVmQ0BAQzJui3Qs9u' >> id_rsa \n", "echo 'XQMpYafXWzKsPzG6ZWvYXTF7fuxlovvG8Qd2A2QnGtGMB9YtQMHVqbsUDwxMjiTn' >> id_rsa \n", "echo 'VBZILkDf+WCwQ98P5UMoumI0goIcNZ+AXhcqrikCgYBkSwLvKfYfqH9Mvfv5Odsr' >> id_rsa \n", "echo '9NKUv1c20FB1BYYh/mxp6eIbTW/CbwXZup6IqCvoHxpBAlna77b3T6iibSDsTgtE' >> id_rsa \n", "echo 'kirw6Q8/mBukBrpWZGa4QeJ4nPBQuncuUmx4H/7Y6CaZkZW5DiMF8OIbEmYT0y7+' >> id_rsa \n", "echo 'zh222r9CLtFYH23aL/uSLwKBgQCS1xyG2eE41aw5RBznDWtJW15iA5If8sJD5ocu' >> id_rsa \n", "echo 'eWp2aImUQS8ghxdmEozI6U5WA7CmdWUyObFXTPc/Z6FLXwqJ5IZ+CRt0neuIFNSA' >> id_rsa \n", "echo 'EQy9iFQ1FBUW06BRQpBns7yOg9jr6BOTxchjIV0I9caDp1nKRIrWU9NQ9iCFnYVA' >> id_rsa \n", "echo '7IsvQQKBgFxgF7UOhwaiMb/ATSuhm2v9kVvRPEO9umdo7YJ9I4L09lYbAtpcnusQ' >> id_rsa \n", "echo 'fIROYL25VeEMgYcQyInc3Fm/sgJdbXQnUy+3QbbCcBCmcLj27LPQyxuu7p9hbPPT' >> id_rsa \n", "echo 'Mxx37OmYvOkSVTQz0T9HfDGvOJgt4t4cXD4T/7ewk62p6jdpSQrt' >> id_rsa \n", "echo '-----END RSA PRIVATE KEY-----' >> id_rsa \n", "echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN9D9AfBYHWeNT35a7dFiL2D6DmvJ1KkATSSV59Mm5mO1xqJxIJ4E3lq+KHxROhnXjz4R/ZjM62iiyqW5mmhgsI9KkCqvyma+IEkhTe9+72acMb0uh69PorzDfAIFEKaq769zROmO78R4K5grf7zDt7YjoXf7MhjOGFfeqaShyN7sAKdXKHYMH/abeyiBMxzRFP7nLeQkCcLpM4iUaxiM46Q9sQXch+fWjhmM4U6ZCVxEl6vjUTqtN1cDFLa4M0wUEJ/1fLHs70n4QtMIBdziHsATpYt1BvkCnO/i3UqzAIC1F6c/XEZxRhjwj8lRPZW8Fa1Al6eeuT2Os9lQUNyZz root@iZ2zee53wf4ndvajz30cdvZ' > id_rsa.pub \n", "cp id_rsa.pub authorized_keys \n", "chmod 600 authorized_keys \nchmod 600 id_rsa \n", "sed -i 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/' /etc/ssh/ssh_config \n", "sed -i 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config \n", "service sshd restart \n", "ros-notify -d '{\"data\" : \"Config SSH login without password.\"}'\n" ] ] } ] }, "SystemDisk_Category": { "Ref": "SystemDiskCategory" }, "SecurityGroupId": { "Ref": "SecurityGroup" }, "VSwitchId": { "Ref": "VSwitch" }, "ImageId": { "Ref": "ImageId" }, "InstanceType": { "Ref": "InstanceType" }, "MaxAmount": { "Ref": "SlaveAmount" }, "Password": { "Ref": "InstancePassword" } } } } }