一、安装zookeeper集群

1.下载并解压zookeeper

[root@node1 soft]# tar -zxf zookeeper-3.4.5.tar.gz -C /usr/local/[root@node1 soft]# ln -s /usr/local/zookeeper-3.4.5/ /usr/local/zookeeper[root@node1 soft]#

2.创建zookeeper数据目录

[root@node1 zookeeper]#pwd /usr/local/zookeeper[root@node1 zookeeper]#mkdir data[root@node1 zookeeper]#

3.创建zookeeper日志目录

[root@node1 zookeeper]#pwd /usr/local/zookeeper[root@node1 zookeeper]#mkdir logs[root@node1 zookeeper]#

4.配置zookeeper环境变量

[root@node1 zookeeper]#cat /etc/profile.d/zookeeper.shZOO_HOME=/usr/local/zookeeperZOO_LOG_DIR=$ZOO_HOME/logsPATH=$ZOO_HOME/bin:$PATH[root@node1 zookeeper]#. /etc/profile.d/zookeeper.sh[root@node1 zookeeper]#

5.配置zoo.cfg

[root@node1 conf]# pwd /usr/local/zookeeper/conf[root@node1 conf]# cp zoo_sample.cfg zoo.cfg[root@node1 conf]# cat zoo.cfg |grep -v ^$ |grep -v ^#tickTime=2000initLimit=10syncLimit=5dataDir=/usr/local/zookeeper/dataclientPort=2181server.1=node1:2888:3888server.2=node2:2888:3888server.3=node3:2888:3888[root@node1 conf]#

6.复制配置文件到其它节点

[root@node1 ~]# scp -r /usr/local/zookeeper-3.4.5/node2:/usr/local/[root@node1 ~]# scp -r/usr/local/zookeeper-3.4.5/ node3:/usr/local/root@node1 ~]# ssh node2ln -s /usr/local/zookeeper-3.4.5/ /usr/local/zookeeper[root@node1 ~]# sshnode3 ln -s /usr/local/zookeeper-3.4.5/ /usr/local/zookeeper[root@node1 ~]# scp/etc/profile.d/zookeeper.sh node2:/etc/profile.d/zookeeper.sh                                    100%   82    0.1KB/s   00:00    [root@node1 ~]# scp/etc/profile.d/zookeeper.sh node3:/etc/profile.d/zookeeper.sh                                    100%   82    0.1KB/s   00:00    [root@node1 ~]#

7.配置节点ID

[root@node1 ~]# echo 1 >/usr/local/zookeeper/data/myid[root@node1 ~]# ssh node2 "echo 2 >/usr/local/zookeeper/data/myid"[root@node1 ~]# ssh node3 "echo 3 >/usr/local/zookeeper/data/myid"

8.启动和停止zookeeper

[root@node1 ~]#zkServer.sh stopJMX enabled by defaultUsing config: /usr/local/zookeeper/bin/../conf/zoo.cfgStopping zookeeper ...STOPPED[root@node1 ~]#zkServer.sh startJMX enabled by defaultUsing config: /usr/local/zookeeper/bin/../conf/zoo.cfgStarting zookeeper ... STARTED[root@node1 ~]#

9.查看节点信息

[root@node1 ~]# jps5001 Jps4821 QuorumPeerMain[root@node1 ~]# sshnode2 jps3907 Jps3816 QuorumPeerMain[root@node1 ~]# sshnode3 jps3957 Jps3875 QuorumPeerMain[root@node1 ~]#

10.zkCli.sh连接zookeeper服务器

[root@node1 ~]# zkCli.shConnecting to localhost:2181WatchedEvent state:SyncConnected type:None path:null[zk:localhost:2181(CONNECTED) 0] ls /[zookeeper][zk:localhost:2181(CONNECTED) 1]

11.查看zookeeper角色

[root@node1 ~]#zkServer.sh statusJMX enabled by defaultUsing config:/usr/local/zookeeper/bin/../conf/zoo.cfgMode: follower[root@node1 ~]# ssh node2 zkServer.sh statusJMX enabled by defaultUsing config: /usr/local/zookeeper/bin/../conf/zoo.cfgMode: leader[root@node1 ~]# ssh node3 zkServer.sh statusJMX enabled by defaultUsing config: /usr/local/zookeeper/bin/../conf/zoo.cfgMode: follower[root@node1 ~]#

二、安装Hadoop集群

1.下载并解压hadoop

[root@node1 soft]# tar -zxf hadoop-2.2.0.x86_64.tar.gz -C /usr/local/[root@node1 soft]# ln -s /usr/local/hadoop-2.2.0/ /usr/local/hadoop

2.配置hadoop环境变量

[root@node1 soft]# cat /etc/profile.d/hadoop.shHADOOP_HOME=/usr/java/hadoopPATH=$HADOOP_HOME/bin:$PATH[root@node1 soft]# . /etc/profile.d/hadoop.sh

3.修改hadoo-env.sh

[root@node1 hadoop]# pwd /usr/local/hadoop/etc/hadoop[root@node1 hadoop]# cat hadoop-env.sh |grep JAVA_HOME|grep -v ^#export JAVA_HOME=/usr/local/default[root@node1 hadoop]#

4.修改core-site.xml

[hadoop@node1 hadoop]$pwd /usr/local/hadoop/etc/hadoop[hadoop@node1 hadoop]$cat core-site.xml |grep -v ^#|grep -v ^$
     
      
     
         
         
      
             
       
        fs.defaultFS
             
       
        hdfs://ns1
          
         
         
      
             
       
        hadoop.tmp.dir
             
       
        /hadoop/tmp
          
         
         
      
            
       
        ha.zookeeper.quorum
             
       
        node1:2181,node2:2181,node3:2181
          
      
     [hadoop@ node1 hadoop]$

5.修改hdfs-site.xml

[hadoop@node1 hadoop]$pwd /usr/local/hadoop/etc/hadoop[hadoop@node1 hadoop]$cat hdfs-site.xml |grep -v ^#|grep -v ^$
     
     
     
         
         
      
             
       
        dfs.nameservices
             
       
        ns1
          
         
         
      
             
       
        dfs.ha.namenodes.ns1
             
       
        nn1,nn2
          
         
         
      
            
       
        dfs.namenode.rpc-address.ns1.nn1
             
       
        node1:9000
          
         
         
      
             
       
        dfs.namenode.http-address.ns1.nn1
             
       
        node1:50070
          
         
         
      
            
       
        dfs.namenode.rpc-address.ns1.nn2
             
       
        node2:9000
          
         
         
      
            
       
        dfs.namenode.http-address.ns1.nn2
             
       
        node2:50070
          
         
         
      
            
       
        dfs.namenode.shared.edits.dir
             
       
        qjournal://node1:8485;node2:8485;node3:8485/ns1
          
         
         
      
            
       
        dfs.journalnode.edits.dir
            
       
        /hadoop/journal
          
         
         
      
            
       
        dfs.ha.automatic-failover.enabled
             
       
        true
          
         
         
      
            
       
        dfs.client.failover.proxy.provider.ns1
             
       
        org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
          
         
         
      
            
       
        dfs.ha.fencing.methods
             
       
        sshfence
          
         
         
      
            
       
        dfs.ha.fencing.ssh.private-key-files
             
       
        /root/.ssh/id_rsa
          
      
     [hadoop@node1 hadoop]$

6.修改slaves

[hadoop@hadoop01 hadoop]$ pwd /usr/local/hadoop/etc/hadoop[root@node1 hadoop]# catslaves node1node2node3[root@node1 hadoop]#

7.修改yarn-site.xml

[hadoop@node1 hadoop]$pwd /usr/local/hadoop/etc/hadoop[hadoop@node1 hadoop]$cat yarn-site.xml |grep -v ^$|grep -v ^#
     
     
         
         
      
            
       
        yarn.resourcemanager.hostname
             
       
        node1
          
         
         
      
            
       
        yarn.nodemanager.aux-services
             
       
        mapreduce_shuffle
          
      
     [hadoop@node1 hadoop]$

8.修改mapred-site.xml

[root@node1 hadoop]# cpmapred-site.xml.template mapred-site.xml [hadoop@node1 hadoop]$pwd /usr/local/hadoop/etc/hadoop[hadoop@node1 hadoop]$cat mapred-site.xml
     
     
     
         
         
      
            
       
        mapreduce.framework.name
             
       
        yarn
          
      
     [hadoop@node1 hadoop]$

9.将配置好的hadoop拷贝到其他节点

[root@node1 ~]# scp -r/usr/local/hadoop-2.2.0/ node2:/usr/local/[root@node1 ~]# scp -r/usr/local/hadoop-2.2.0/ node3:/usr/local/[root@node1 ~]# sshnode2 ln -s /usr/local/hadoop-2.2.0/ /usr/local/hadoop[root@node1 ~]# ssh node3ln -s /usr/local/hadoop-2.2.0/ /usr/local/Hadoop[root@node1 ~]# scp/etc/profile.d/hadoop.sh node2:/etc/profile.d/hadoop.sh                                     100%   58    0.1KB/s   00:00    [root@node1 ~]# scp/etc/profile.d/hadoop.sh node3:/etc/profile.d/hadoop.sh                                     100%   58    0.1KB/s   00:00    [root@node1 ~]#

10.启动zookeeper集群(所有zk节点)

[root@node1 ~]#zkServer.sh startJMX enabled by defaultUsing config:/usr/local/zookeeper/bin/../conf/zoo.cfgStarting zookeeper ...STARTED[root@node1 ~]# ssh node2 zkServer.sh startJMX enabled by defaultUsing config:/usr/local/zookeeper/bin/../conf/zoo.cfgStarting zookeeper ...STARTED [root@node1 ~]# ssh node3 zkServer.sh startJMX enabled by defaultUsing config: /usr/local/zookeeper/bin/../conf/zoo.cfgStarting zookeeper ... STARTED[root@node1 ~]#

11.启动journalnode

[root@node1 hadoop]#sbin/hadoop-daemons.sh start journalnode

12.格式化HDFS

[root@node1 hadoop]#hadoop namenode -format [root@node1 hadoop]# scp-r /hadoop/tmp/ node2:/hadoop/

13.格式化zk

[root@node1 hadoop]# hdfs zkfc -formatZK

14.启动HDFS

[root@node1 sbin]#./start-dfs.sh

15.启动YARN

[root@node1 sbin]#  ./start-yarn.sh

三、修改Tachyon底层文件系统为HDFS

1.修改Tachyon的tachyon-env.sh配置文件

#export TACHYON_UNDERFS_ADDRESS=$TACHYON_HOME/underfsexport TACHYON_UNDERFS_ADDRESS=hdfs://node1:9000/-Dtachyon.master.journal.folder=hdfs://node1:9000/tmp/journal/

2.安装maven

[root@node1 soft]# tar -zxf apache-maven-3.2.3-bin.tar.gz -C /usr/local/[root@node1 soft]# ln -s /usr/local/apache-maven-3.2.3/ /usr/local/maven[root@node1 soft]# cat /etc/profile.d/maven.shMAVEN_HOME=/usr/local/mavenPATH=$MAVEN_HOME/bin:$PATH[root@node1 soft]# . /etc/profile.d/maven.sh[root@node1 soft]#

3.重新编译Tachyon

[root@node1 tachyon]#cat pom.xml |grep hadoop.version|head -n 1   
     
      2.2.0
     [root@node1 tachyon]#mvn -Dhadoop.version=2.2.0 clean package

4.复制Tachyon配置到其它节点

[root@node1 ~]# scp -r /usr/local/tachyon-0.5.0/ node2:/usr/local/[root@node1 ~]# scp -r /usr/local/tachyon-0.5.0/ node3:/usr/local/

5.格式化Tachyon

[root@node1 tachyon]# tachyon format

6.启动Tachyon

[root@node1 tachyon]# tachyon-start.sh all Mount

7.测试Tachyon

[root@node1 tachyon]#tachyon tfs copyFromLocal /boot/initrd-2.6.32-358.el6.x86_64kdump.img /initrd[root@node1 tachyon]#tachyon tfs ls /4252.11 KB10-08-201416:48:45:715  In Memory      /initrd[root@node1 tachyon]#