[TOC]
准备
文件清单
| 文件 | |
|---|---|
| ambari-2.7.5.0-centos7.tar.gz | √ |
| CentOS-7-x86_64-Everything-2009.iso | |
| HDP-3.1.5.0-centos7-rpm.tar.gz | √ |
| HDP-GPL-3.1.5.0-centos7-gpl.tar.gz | |
| HDP-UTILS-1.1.0.22-centos7.tar.gz | |
| jdk-8u121-linux-x64.tar.gz | |
| mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar | |
| mysql-connector-java-rpm-package.tar.gz |
把安装包上传到 /opt 目录下：
[root@manager opt]# ll /opt/
total 22226212
-rw-r--r--. 1 root root 2062874048 Jul 30 11:18 ambari-2.7.5.0-centos7.tar.gz
-rw-r--r--. 1 root root 10200547328 Jul 30 11:00 CentOS-7-x86_64-Everything-2009.iso
-rw-r--r--. 1 root root 9815239567 Jul 30 11:42 HDP-3.1.5.0-centos7-rpm.tar.gz
-rw-r--r--. 1 root root 161804 Jul 30 11:18 HDP-GPL-3.1.5.0-centos7-gpl.tar.gz
-rw-r--r--. 1 root root 90606616 Jul 30 11:19 HDP-UTILS-1.1.0.22-centos7.tar.gz
-rw-r--r-- 1 root root 183246769 Jul 30 11:06 jdk-8u121-linux-x64.tar.gz
-rw-r--r--. 1 root root 531056640 Jul 30 11:21 mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar
-rw-r--r--. 1 root root 59142788 Jul 30 11:22 mysql-connector-java-rpm-package.tar.gz
Linux
版本
查看
cat /etc/redhat-release
HostName
设置
hostnamectl set-hostname node200.caijidui.top
查看
hostname
语言
vim /etc/locale.conf
LANG=en_US.UTF-8
或者
localectl set-locale LANG=en_US.UTF-8
配置节点间免密
生成秘钥
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
所有节点执行
复制
ssh-copy-id -i ~/.ssh/id_rsa.pub node200.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node201.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node202.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node203.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node204.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node205.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node206.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node207.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node208.caijidui.top
ssh-copy-id -i ~/.ssh/id_rsa.pub node209.caijidui.top
200复制到其他机器
防火墙
关闭防火墙
systemctl stop firewalld
查看防火墙的状态：
systemctl status firewalld
设置开机不启动：
systemctl disable firewalld
查看防火墙服务是否设置开机启动：
systemctl is-enabled firewalld
内存参数
# 编辑配置文件：vim /etc/sysctl.conf，添加如下配置：
vm.swappiness=1
vm.min_free_kbytes=4194304
vm.zone_reclaim_mode=0
# 使修改生效：sysctl -p
最大文件打开数最大进程数
# 编辑配置文件：vim /etc/security/limits.conf，添加如下配置：
* soft nofile 655350
* hard nofile 655350
* soft nproc 204800
* hard nproc 204800
关闭Selinux
# 查看Selinux状态：sestatus
# 编辑配置文件：vim /etc/sysconfig/selinux，修改为以下配置：
SELINUX=disabled
# 重启服务器后才生效：等关闭透明大页之后一并重启
关闭透明大页THP
# 查看透明大页情况：
cat /sys/kernel/mm/transparent_hugepage/defrag
cat /sys/kernel/mm/transparent_hugepage/enabled
# 关闭透明大页：vim /etc/rc.d/rc.local，添加如下配置：
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
# 保存退出，然后赋予rc.local文件执行权限：chmod +x /etc/rc.d/rc.local
# 重启服务器才生效：reboot
基础服务
安装httpd服务（manager节点）
# 安装服务
yum install httpd -y
# 启动httpd服务：systemctl start httpd
# 查看httpd状态：systemctl status httpd
# 设置开启启动：systemctl enable httpd
# 安装完毕httpd服务之后，local.repo即可删除
rm -f /etc/yum.repos.d/local.repo
配置 Ambari yum源（manager节点）
# 解压Ambari压缩包
tar -zxvf /opt/ambari-2.7.5.0-centos7.tar.gz -C /var/www/html/
# 配置Ambari-repo文件
vim /etc/yum.repos.d/Ambari.repo
[Ambari-2.7.5.0]
name=Ambari Version - Ambari-2.7.5.0
baseurl=http://node200.caijidui.top/ambari/centos7/2.7.5.0-72/
gpgcheck=1
gpgkey=http://node200.caijidui.top/ambari/centos7/2.7.5.0-72/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
配置 HDP yum源（manager节点）
# 解压HDP压缩包
tar -zxvf /opt/HDP-3.1.5.0-centos7-rpm.tar.gz -C /var/www/html/
# 配置HDP-repo文件
vim /etc/yum.repos.d/HDP.repo
[HDP-3.1.5.0]
name=HDP Version - HDP-3.1.5.0
baseurl=http://node200.caijidui.top/HDP/centos7/3.1.5.0-152
gpgcheck=1
gpgkey=http://node200.caijidui.top/HDP/centos7/3.1.5.0-152/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
配置 HDP-UTILS yum源（manager节点）
# 解压HDP-UTILS压缩包
tar -zxvf /opt/HDP-UTILS-1.1.0.22-centos7.tar.gz -C /var/www/html/
# 配置HDP-UTILS-repo文件
vim /etc/yum.repos.d/HDP-UTILS.repo
[HDP-UTILS-1.1.0.22]
name=HDP-UTILS Version - HDP-UTILS-1.1.0.22
baseurl=http://node200.caijidui.top/HDP-UTILS/centos7/1.1.0.22
gpgcheck=1
gpgkey=http://node200.caijidui.top/HDP-UTILS/centos7/1.1.0.22/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1
配置 HDP-GPL yum源（manager节点）
# 解压HDP-GPL压缩包
tar -zxvf /opt/HDP-GPL-3.1.5.0-centos7-gpl.tar.gz -C /var/www/html/
# 配置HDP-GPL-repo文件
vim /etc/yum.repos.d/HDP-GPL.repo
[HDP-GPL-3.1.5.0]
name=HDP-UTILS Version - HDP-GPL-3.1.5.0
baseurl=http://node200.caijidui.top/HDP-GPL/centos7/3.1.5.0-152
gpgcheck=1
gpgkey=http://node200.caijidui.top/HDP-GPL/centos7/3.1.5.0-152/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1
分发repo文件到其余节点（manager节点）
# 前提是把master和worker节点 /etc/yum.repos.d 目录下的repo文件删除，再进行拷贝：
cd /etc/yum.repos.d
scp ./*.repo master.bigdata:$PWD
scp ./*.repo worker.bigdata:$PWD
# 拷贝完毕之后，登录所有节点，分别执行yum更新：
yum clean all
yum update
安装时间同步服务NTP（所有节点）
# 集群所有节点，安装ntp服务
yum install ntp* -y
# 配置manager节点：vim /etc/ntp.conf
restrict 192.168.158.2 mask 255.255.255.0 nomodify notrap
server ntp1.aliyun.com prefer
server 127.127.1.0
fudge 127.127.1.0 stratum 8
# 备注
192.168.158.2 和 255.255.255.0是集群所在网段的网关和子网掩码
ntp1.aliyun.com 是主时钟源，我选择的是阿里的时钟源，可根据实际情况选择，如果内网没有时钟服务器，配置为：127.127.1.0
# 在master节点修改：vim /etc/ntp.conf
restrict 192.168.158.2 mask 255.255.255.0 nomodify notrap
server node200.caijidui.top prefer
server 127.127.1.0
fudge 127.127.1.0 stratum 9
# 在worker节点修改：vim /etc/ntp.conf
server node200.caijidui.top prefer
server node200.caijidui.top
# 在manager启动ntp之后：
systemctl start ntpd
systemctl enable ntpd
# 其余各节点启动ntp服务之前，master和worker节点执行命令，同步manager时间：
ntpdate node200.caijidui.top
# 启动master、worker节点ntp服务
systemctl start ntpd
systemctl enable ntpd
# 查看ntp状态
ntpq -p
’*’ 表示当前使用的时钟源，’+’ 表示这些源可作为 NTP 源
# ntpstat查看ntp与上层ntp服务的联通性
删除OpenJDK 安装并配置OracleJDK（所有节点）
# 删除OpenJDK
# 查看openjdk：rpm -qa | grep java
# 删除openjdk：
rpm -e --nodeps java-1.8.0-openjdk-1.8.0.262.b10-1.el7.x86_64
rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.262.b10-1.el7.x86_64
# manager节点分发jdk包到其余节点
cd /opt
scp ./jdk-8u211-linux-x64.tar.gz node201.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node202.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node203.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node204.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node205.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node206.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node207.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node208.caijidui.top:$PWD
scp ./jdk-8u211-linux-x64.tar.gz node209.caijidui.top:$PWD
# 创建jdk目录
mkdir /usr/java
tar -zxvf /opt/jdk-8u281-linux-x64.tar.gz -C /usr/java/
# 配置环境变量：vim /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_281
export PATH=$PATH:$JAVA_HOME/bin
# 使环境变量立即生效命令：source /etc/profile
装数据库
CREATE DATABASE hive CHARACTER SET utf8;
CREATE USER 'hive'@'%'IDENTIFIED BY 'bigdata123';
GRANT ALL PRIVILEGES ON hive.* TO 'hive'@'%';
FLUSH PRIVILEGES;
CREATE DATABASE ranger CHARACTER SET utf8;
CREATE USER 'ranger'@'%'IDENTIFIED BY 'bigdata123';
GRANT ALL PRIVILEGES ON ranger.* TO 'ranger'@'%';
FLUSH PRIVILEGES;
mysql-connector-java-5.1.47.jar
安装Ambari
部署Ambari-Server（manager节点）
# 安装服务
yum install ambari-server -y
[root@manager ~]# ambari-server setup
Using python /usr/bin/python
Setup ambari-server
Checking SELinux...
SELinux status is 'disabled'
Customize user account for ambari-server daemon [y/n] (n)? y
Enter user account for ambari-server daemon (root):root
Adjusting ambari-server permissions and ownership...
Checking firewall status...
Checking JDK...
[1] Oracle JDK 1.8 + Java Cryptography Extension (JCE) Policy Files 8
[2] Custom JDK
==============================================================================
Enter choice (1): 2
WARNING: JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.
WARNING: JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos,please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts.
Path to JAVA_HOME: /usr/java/jdk1.8.0_121
Validating JDK on Ambari Server...done.
Check JDK version for Ambari Server...
JDK version found: 8
Minimum JDK version is 8 for Ambari. Skipping to setup different JDK for Ambari Server.
Checking GPL software agreement...
GPL License for LZO: https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html
Enable Ambari Server to download and install GPL Licensed LZO packages [y/n] (n)? y
Completing setup...
Configuring database...
Enter advanced database configuration [y/n] (n)? y
Configuring database...
==============================================================================
Choose one of the following options:
[1] - PostgreSQL (Embedded)
[2] - Oracle
[3] - MySQL / MariaDB
[4] - PostgreSQL
[5] - Microsoft SQL Server (Tech Preview)
[6] - SQL Anywhere
[7] - BDB
==============================================================================
Enter choice (1): 3
Hostname (localhost): manager.bigdata
Port (3306): 3306
Database name (ambari): ambari
Username (ambari): ambari
Enter Database Password (bigdata):
Re-enter password:
Configuring ambari database...
Enter full path to custom jdbc driver: /opt/mysql-connector-java-5.1.37.jar
Copying /opt/mysql-connector-java-5.1.37.jar to /usr/share/java
Configuring remote database connection properties...
WARNING: Before starting Ambari Server, you must run the following DDL directly from the database shell to create the schema: /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql
Proceed with configuring remote database connection properties [y/n] (y)? y
Extracting system views...
ambari-admin-2.7.5.0.72.jar
....
Ambari repo file doesn't contain latest json url, skipping repoinfos modification
Adjusting ambari-server permissions and ownership...
Ambari Server 'setup' completed successfully.
登录ambari数据库：
mysql -uambari -p
mysql> use ambari
mysql> source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql
启动Ambari-Server服务：
# 启动：ambari-server start
# 查看：ambari-server status
# 停止：ambari-server stop
部署Ambari-Agent（所有节点）
# 安装
yum -y install ambari-agent
# 启动
ambari-agent start
sed -i 's/verify=platform_default/verify=disable/' /etc/python/cert-verification.cfg
cd /var/lib/rpm
rm __db.* -rf
rpm --rebuilddb
yum clean all
ambari-agent stop
yum -y remove ambari-agent
rm -rf /usr/lib/ambari-*
rm -rf /etc/ambari-*
rm -rf `find / -type d -iname ambari*`
yum -y install ambari-agent
ambari-agent start
创建集群
caijidui001
http://node200.caijidui.top/HDP/centos7/3.1.5.0-152/
http://node200.caijidui.top/HDP-GPL/centos7/3.1.5.0-152/
http://node200.caijidui.top/HDP-UTILS/centos7/1.1.0.22/
node200.caijidui.top
node201.caijidui.top
node202.caijidui.top
node203.caijidui.top
node204.caijidui.top
node205.caijidui.top
node206.caijidui.top
node207.caijidui.top
node208.caijidui.top
node209.caijidui.top
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEAvN2J41HjzimmHJVOv2giQ/C5Lc8hYxeFH8XBZY1JC/N4B+Rb
7xPF24I7IRWnHv96ydEiB5u3Bl0nyauGVu0rLDtz4GGTVF5TNY0gQPsdx2A3SqJq
FXsVWhTwpwtkU2TVWuTXjW5JsL2NY/Rnv37oud27Covg8BZTWZj/06zxvDdSYsvK
K97HmBcwONJC/g20T6tzm8pKKcle53DC7LdyL2cQ8Otis8UhX+rK6VFqDZ9IAR22
QPV1AtxYktmT8hpXCu/jg2aTeHu4HiR5gK4pDdJGWREpodC7CkFH3X/XPYhAwU8U
pXry/nk/fOCjb/RXUAasJeGaWLLXzZFdAWwaQQIDAQABAoIBACr9qtOJ4ncdV+iN
vB0p0RLyCtT4nzr1MaJn23wISyZjR2FUg/DJ6XHsFRWU3jPaJ9MkvcBJm8HhFpt7
ZdsHpZot9lSPIYBE8F/yPfNMthyaNDvS+J7nI5cbySPdHfs1qrNEACT7g96tZy/n
hG48rzEP3fGqoYtFKbftcd53MZeMA7x/43kZeSQPKV7A/7Qy/nRlVOT7fmkFkcSI
3PrengI68SRhAkQXVAZbmznjitEmiUeCa7J1I5uQ025W2iUJt+A7qV7bJZDnr3JW
kqf3nzuaCdD5ru4GgOkFRhgoQnmeAbfT/C/62+4wU+u9ZEpONLeq7/ND8229cnrd
tGuFzZkCgYEA8Izxu7DgC76vW3yuR72mz2+gobHy9pGpdeuEo6SNZ+UgcP2AkunZ
ncCV+8Aw1z4mq1gmctLUZlmtQD69sKCHUZXoN4ydVye7sYhj4NCP8GfpTRCQdczp
fMOZS3WrcXMqbCVUQqOdif5isAGWNryzffogmCPqQwZB8prAyhCAu3cCgYEAyP7N
mTkd7WquVgX+L6xl4zQzbxBO6UJSlq4+QrSo4BhbXY0sKjShe1s+remNOLS/B/sZ
316aCZqFf4L8i2ywsXl/56Om34K7+EXRiK9kfXDLMXMALBAd3b1EKVdrDkYwMe5J
WK7tcIhw22X9PHwFuxEOjJLe9hiYXFI3Glm7VgcCgYEA3Ubp/1u/XU51hNT2hKvi
zKLEwrks5MAtgHm8M8RlB5Nv+kBXi81uD31e+oxa4UUc0TVveOST2erYQmb0m4uv
rs8VJDRHcCrur2rlZaK1JdKQfAGmwZZTmh9b3lxJ7zyfY4gbMInNm2jUY0+yXHvN
KS9z2EABeNJCy+arPbVclhUCgYBqsC2r7lB/bEgbq1SwM39lqZi/lFV+pVC3YMf1
mnWldehRcRUHViZ9FwLfR4BJoE9iShwCn/F9tYK1G284BrqHiemlqA10yK1D7LN7
LTZXC18f4V4vBWMFB2i91QQjcIg2iLhwFXI9cH8PRkS0LVTRZg1l+rPUx1tgbdeH
6idxUwKBgQDCEflUzunnuX+0+4NgL5+XyGL0Z07hzX0Lm48aRJ0mhY/oLhT5s9nH
FNHp5MR9D+6yaUfVnWM6xnTk3wUnzilWUt8heHzRhw4o1t3G9a+Fn+kRt9o+z3U/
0aM2x/JY2agQNKkNWHfH3eBhGYcFe0xlspYmu0mDjEfyPZdIU3yMdw==
-----END RSA PRIVATE KEY-----
ambari-server stop
ambari-server reset
ambari-server start
参考：https://blog.csdn.net/Happy_Sunshine_Boy/article/details/119236756
MySQL
删除Mariadb安装MySQL服务
# 查询
rpm -qa | grep mariadb
# 删除
rpm -e --nodeps mariadb-libs-5.5.68-1.el7.x86_64
下载官方yum Repo
wget -i -c http://dev.mysql.com/get/mysql57-community-release-el7-10.noarch.rpm
安装yum Repo
yum -y install mysql57-community-release-el7-10.noarch.rpm
安装MySQL服务
yum -y install mysql-community-server
