1.创建用户组和用户
groupadd hadoops
useradd hadoop -g hadoops
passwd hadoop
2.解压jdk和hadoop
tar -zxvf jdk1.7.0_80.tar.gz -C /usr/local
mv /usr/local/jdk1.7.0_80 /usr/local/java
tar -zxvf hadoop-2.7.3.tar.gz -C /u01
mv /u01/hadoop-2.7.3 /u01/hadoop
chown -R hadoop:hadoop /u01/hadoop/etc/hadoop/**
chown -R hadoop:hadoop /u01/hadoop/bin/**
chown -R hadoop:hadoop /u01/hadoop/sbin/**
3.配置环境变量
vi /etc/profile
export HADOOP_HOME=/u01/hadoop
export PATH=$HADOOP_HOME/bin:$PATH
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME/bin:$PATH
source /etc/profile
4.配置hosts
vi /etc/hosts
192.168.0.9 hadmaster
192.168.0.27 hadslave1
192.168.0.28 hadslave2
5.创建目录
mkdri /u01/hadoopData
chown hadoop:hadoop /u01/hadoopData
vi /u01/hadoop/etc/hadoop/slaves
hadslave1
hadslave2
vi /u01/hadoop/etc/hadoop/masters
hadmaster
6.配置hadoop
-
6.1.编辑hadoop-env.sh
vi /u01/hadoop/etc/hadoop/hadoop-env.sh export JAVA_HOME=/usr/local/java export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true -
6.2.编辑core-site.xml
vi /u01/hadoop/etc/hadoop/core-site.xml <property> <name>hadoop.tmp.dir</name> <value>/u01/hadoopData/tmp</value> </property> <property> <name>fs.default.name</name> <value>hdfs://hadmaster:9000</value> </property> <property> <name>fs.defaultFS</name> <value>hdfs://hadmaster:9000</value> </property> <property> <name>hadoop.proxyuser.hadoop.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hadoop.groups</name> <value>*</value> </property> -
6.3.编辑hdfs-site.xml
vi /u01/hadoop/etc/hadoop/hdfs-site.xml <property> <name>dfs.replication</name> <value>2</value> </property> <property> <name>dfs.namenode.data.dir</name> <value>file:/u01/hadoopData/data,file:/u01/hadoopData/data2</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>file:/u01/hadoopData/name,file:/u01/hadoopData/name2</value> </property> -
6.4.编辑mapred-site.xml
cp /u01/hadoop/etc/hadoop/mapred-site.xml.template /u01/hadoop/etc/hadoop/mapred-site.xml vi /u01/hadoop/etc/hadoop/mapred-site.xml <property> <name>mapred.job.tracker</name> <value>hadmaster:9001</value> </property> <property> <name>mapred.local.dir</name> <value>/u01/hadoopData/tmp</value> </property> -
6.5.编辑yarn-site.xml
vi /u01/hadoop/etc/hadoop/mapred-site.xml <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>hadmaster:8025</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>hadmaster:8035</value> </property> <property> <name>yarn.resourcemanager.address</name> <value>hadmaster:8050</value> </property> <property> <name>yarn.resourcemanager.webapp.address</name> <value>hadmaster:8058</value> </property>
7.ssh免密码登陆
su - hadoop
cd /home/hadoop
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cp id_rsa.pub authorized_keys
cd /home/hadoop/.ssh
scp authorized_keys hadslave1:/home/hadoop/.ssh
scp authorized_keys hadslave2:/home/hadoop/.ssh
ssh hadmaster
exit
ssh hadslave1
exit
ssh hadslave2
exit
rsync -avxP /u01/hadoop/ hadoop@hadslave1:/u01/hadoop/
rsync -avxP /u01/hadoop/ hadoop@hadslave2:/u01/hadoop/
rsync -avxP /u01/hadoop/etc/hadoop/ hadoop@hadslave1:/u01/hadoop/etc/hadoop/
rsync -avxP /u01/hadoop/etc/hadoop/ hadoop@hadslave2:/u01/hadoop/etc/hadoop/
8.hive元数据
create database hive_meta;
create user 'hive' identified by 'hive';
grant all privileges on hive_meta.* to
'hive'@'5'
identified by 'hive' WITH GRANT OPTION;
flush privileges;
9.hive环境变量
tar -zxvf apache-hive-2.1.0-bin.tar.gz -C /u01
mv /u01/apache-hive-2.1.0-bin /u01/hive
chown hadoop:hadoop /u01/hive
vi /etc/profile
export HIVE_HOME=/u01/hive
export PATH=$PATH:$HIVE_HOME/bin
source /etc/profile
10.hive配置
-
10.1.copy配置文件
cd /u01/hive/conf cp hive-env.sh.template hive-env.sh cp hive-default.xml.template hive-site.xml cp hive-log4j2.properties.template hive-log4j2.properties cp hive-exec-log4j2.properties.template hive-exec-log4j2.properties -
10.2.配置hive-env.sh
vi hive-env.sh export JAVA_HOME=/usr/local/java export HADOOP_HOME=/u01/hadoop export HIVE_HOME=/u01/hive export HIVE_CONF_DIR=/u01/hive/conf -
10.3.Mysql驱动
cp mysql-connector-java-5.1.39.jar $HIVE_HOME/lib chown hadoop:hadoop $HIVE_HOME/lib/** -
10.4.配置hive-site.xml
##替换hive-site.xml文件中的 ${system:java.io.tmpdir} 和 ${system:user.name} <property> <name>hive.exec.scratchdir</name> <value>/tmp/hive-${user.name}</value> <description>HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/<username> is created, with ${hive.scratch.dir.permission}. </description> </property> <property> <name>hive.exec.local.scratchdir</name> <value>/tmp/${user.name}</value> <description>Local scratch space for Hive jobs</description> </property> <property> <name>hive.downloaded.resources.dir</name> <value>/tmp/hive/resources</value> <description>Temporary local directory for added resources in the remote file system.</description> </property> <property> <name>hive.querylog.location</name> <value>/tmp/${user.name}</value> <description>Location of Hive run time structured log file</description> </property> <property> <name>hive.server2.logging.operation.log.location</name> <value>/tmp/${user.name}/operation_logs</value> <description>Top level directory where operation logs are stored if logging functionality is enabled </description> </property>##配置Hive Metastore <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://192.168.0.7:3306/hive_meta?createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false</value> </property> <property> <name>javax.jdo.option.ConnectionDriverName</name> <value>com.mysql.jdbc.Driver</value> </property> <property> <name>javax.jdo.option.ConnectionUserName</name> <value>hive</value> </property> <property> <name>javax.jdo.option.ConnectionPassword</name> <value>hive</value> </property>
11.hdfs目录
hdfs dfs -mkdir /tmp
hdfs dfs -mkdir -p /user/hive/warehouse
hdfs dfs -chmod g+w /tmp
hdfs dfs -chmod g+w /user/hive/warehouse
12.初始化hive元数据
cd /u01/hive/bin
./schematool -dbType mysql -initSchema
13.测试hive
hive
show tables
14.连接hive
hive --service metastore >/dev/null 2>/dev/null &
hive --service hiveserver2 10000 >/dev/null 2>/dev/null &
$HIVE_HOME/bin/hive --service hiveserver2 10000 >/dev/null 2>/dev/null &
连接Hive JDBC URL：jdbc:hive2://192.168.6.116:10000/default （Hive默认端口：10000 默认数据库名：default）
beeline
!connect jdbc:hive2://localhost:10000 hadoop hadoop
15.hiveServer2WebUI
<property>
<name>hive.server2.webui.host</name>
<value>192.168.0.9</value>
</property>
<property>
<name>hive.server2.webui.port</name>
<value>10002</value>
</property>
