Linux安装hadoop

1. 作者QQ：67065435 QQ群：669756510

2. 本站内容全部为作者原创，转载请注明出处！

3. 假设三台服务器IP及映射如下

   #/etc/hosts
   192.168.3.11 node01
   192.168.3.12 node02
   192.168.3.13 node03
   

4. 安装准备(三台服务器都执行)

   yum -y --nogpgcheck \
   install \
   vim \
   openssh-server
   

5. 免密通信(三台服务器都执行)

   ssh-keygen -t rsa -b 4096
   
   ssh-copy-id root@node01
   yes
   password
   
   ssh-copy-id root@node02
   yes
   password
   
   ssh-copy-id root@node03
   yes
   password
   

6. 卸载yum安装的java(三台服务器都执行)

   rpm -qa|grep jdk
   
   rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.412.b08-1.el7_9.x86_64
   rpm -e --nodeps java-1.8.0-openjdk-1.8.0.412.b08-1.el7_9.x86_64
   rpm -e --nodeps copy-jdk-configs-3.3-11.el7_9.noarch
   
   rpm -qa|grep jdk
   
   java -version
   

7. 安装java(三台服务器都执行)

8. 安装hadoop(三台服务器都执行)

   cd /root
   wget --no-check-certificate -c https://dlcdn.apache.org/hadoop/common/hadoop-3.4.1/hadoop-3.4.1.tar.gz
   tar -xf hadoop-3.4.1.tar.gz
   mv /root/hadoop-3.4.1 /usr/local/hadoop
   
   vim /etc/profile
   
   export HADOOP_HOME=/usr/local/hadoop
   export PATH=$PATH:$HADOOP_HOME/bin
   export PATH=$PATH:$HADOOP_HOME/sbin
   
   ESC
   :wq
   
   hadoop version
   

9. 配置hadoop(三台服务器都执行) ``` vim /usr/local/hadoop/etc/hadoop/hadoop-env.sh

   export JAVA_HOME=/usr/local/jdk export HDFS_NAMENODE_USER=root export HDFS_DATANODE_USER=root export HDFS_SECONDARYNAMENODE_USER=root export YARN_RESOURCEMANAGER_USER=root export YARN_NODEMANAGER_USER=root

   ESC :wq

   vim /usr/local/hadoop/etc/hadoop/yarn-env.sh

   export YARN_REGISTRYDNS_SECURE_USER=root

   ESC :wq

vim /usr/local/hadoop/etc/hadoop/core-site.xml <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> hadoop.proxyuser.root.hosts * hadoop.proxyuser.root.groups * hadoop.http.staticuser.user root fs.defaultFS hdfs://node01:9870 hadoop.tmp.dir /data/hadoop/tmp ESC :wq

vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> dfs.namenode.http-address 0.0.0.0:50070 dfs.datanode.address 0.0.0.0:50010 dfs.namenode.user.name root dfs.datanode.user.name root dfs.replication 3 ESC :wq

vim /usr/local/hadoop/etc/hadoop/mapred-site.xml <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> mapreduce.framework.name yarn yarn.app.mapreduce.am.env HADOOP_MAPRED_HOME=${HADOOP_HOME} mapreduce.map.env HADOOP_MAPRED_HOME=${HADOOP_HOME} mapreduce.reduce.env HADOOP_MAPRED_HOME=${HADOOP_HOME} ESC :wq

vim /usr/local/hadoop/etc/hadoop/yarn-site.xml <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> yarn.resourcemanager.hostname node01 yarn.nodemanager.aux-services mapreduce_shuffle yarn.nodemanager.pmem-check-enabled true yarn.nodemanager.vmem-check-enabled false ESC :wq

vim /usr/local/hadoop/etc/hadoop/slaves node01 node02 node03 ESC :wq

1. 目录权限(三台服务器都执行)
   ```html
   mkdir -p /data/hadoop
   chown -R root:root /data/hadoop
   chown -R root:root /usr/local/hadoop

1. 格式化文件系统(主服务器执行)

   # 仅首次启动hdfs集群时操作
   hdfs namenode -format
   

2. 启动停止服务(三台服务器都执行)

   # 主服务器执行
   stop-all.sh
   start-all.sh
   
   # 从服务器执行
   hdfs --daemon start datanode
   yarn --daemon start nodemanager
   

3. 查看集群运行状态

   hdfs dfsadmin -report
   jps