cdh5部署hadoop环境


2019年5月5日 10:13 ⋅ 阅读: 65


环境

主机名 ip 域名 部署 性能
cdh-1 10.0.1.120 cdh1.ops.net master、agent 8核32G
cdh-2 10.0.1.121 cdh2.ops.net agent 8核16G
cdh-3 10.0.1.122 cdh3.ops.net agent 8核16G
cdh-4 10.0.1.123 cdh3.ops.net agent,sqoop 8核16G

准备

下载包

http://archive.cloudera.com/cm5/cm/5/

cloudera-manager-centos7-cm5.9.0_x86_64.tar.gz

http://archive.cloudera.com/cdh5/parcels/latest/

CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel

CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1

manifest.json

依赖包

yum install -y python-lxml  httpd mod_ssl
yum -y install cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi

rpm jdk包

http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.15.1/RPMS/x86_64/

rpm -vih jdk-6u31-linux-amd64.rpm 
rpm -vih oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm

挂载nfs

mkdir /opt/nfs
mount -t nfs 10.0.1.19:/opt/software /opt/nfs

hosts文件

10.0.1.120 cdh1.ops.net
10.0.1.121 cdh2.ops.net
10.0.1.122 cdh3.ops.net
10.0.1.123 cdh4.ops.net

推送ssh key

## 生成
mkdir /opt/nfs/software/hadoop/key -p
cd  /opt/nfs/software/hadoop/key && ssh-keygen -t rsa -P '' -f hadoop #生成key

## 所有主机的操作
mkdir /root/.ssh
cp /opt/nfs/software/hadoop/key/hadoop.pub /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
cp /opt/nfs/software/hadoop/key/hadoop /root/.ssh/id_rsa

master1 ntp(确保时间一致性)

which ntpd|xargs rpm -qf ||yum install ntp -y

ntp配置文件

restrict 0.0.0.0 mask 0.0.0.0 nomodify notrap noquery
restrict 127.0.0.1
restrict default nomodify notrap

server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
# 阿里云ntp
server 127.127.1.0
fudge 127.127.1.0 stratum 10
# 定义的server都不可用时,将使用local时间作为ntp服务提供给ntp客户端

includefile /etc/ntp/crypto/pw
driftfile /var/lib/ntp/drift
keys /etc/ntp/keys

启用服务

systemctl enable ntpd && systemctl start ntpd

测试命令

ntpstat

synchronised to NTP server (120.25.115.20) at stratum 3 time correct to within 27 ms polling server every 64 s

其他主机同步时间

grep 'cdh-1' /var/spool/cron/root &>/dev/null ||echo '*/5 * * * * ntpdate cdh-1'  > /var/spool/cron/root

禁用Transparent Hugepage

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

cat /sys/kernel/mm/transparent_hugepage/enabled #查看状态

调整vm.swappiness Linux内核参数

grep vm.swappiness /etc/sysctl.conf || echo "vm.swappiness = 1" >> /etc/sysctl.conf
sysctl -p

安装mysql

master1 节点安装mysql

安装jdk

java -version

java version "1.8.0_102"

Java(TM) SE Runtime Environment (build 1.8.0_102-b14)

Java HotSpot(TM) 64-Bit Server VM (build 25.102-b14, mixed mode)

安装MySQL JDBC驱动程序

wget https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz

tar zxvf mysql-connector-java-5.1.46.tar.gz

mkdir -p /usr/share/java/

cd mysql-connector-java-5.1.46

cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

#http://download.softagency.net/MySQL/Downloads/Connector-J/mysql-connector-java-5.1.42.tar.gz

#JDBC目前已经有6.x,但与CDH 5.x存在不兼容

Cloudera Manager 安装与集群配置

所有节点

版本cm-5.15.1

useradd --home=/opt/cm-5.15.1/run/cloudera-scm-server/ --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
tar zxvf cloudera-manager-centos7-cm5.15.1_x86_64.tar.gz -C /opt/
chown -R cloudera-scm:cloudera-scm /opt/cloudera
chown -R cloudera-scm:cloudera-scm /opt/cm-5.15.1

ln -s /opt/cm-5.15.1/ /opt/cm
mkdir  /opt/cm-5.15.1/run/cloudera-scm-agent

vi /opt/cm-5.15.1/etc/cloudera-scm-agent/config.ini

server_host=cdh-1
# 主节点

cdh-1主节点

tar zxvf mysql-connector-java-5.1.42.tar.gz
cp mysql-connector-java-5.1.42/mysql-connector-java-5.1.42-bin.jar /opt/cm-5.15.1/share/cmf/lib/mysql-connector-java.jar

cp  /opt/cm-5.15.1/share/cmf/lib/mysql-connector-java.jar /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/lib/hive/lib
cp /opt/cm-5.15.1/share/cmf/lib/mysql-connector-java.jar /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/lib/oozie/lib
cp /opt/cm-5.15.1/share/cmf/lib/mysql-connector-java.jar /var/lib/oozie/
cp /opt/cm-5.15.1/share/cmf/lib/mysql-connector-java.jar /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/lib/hive/lib/

cdh-1 数据库

测试连通

mysql -h10.0.1.120 -uroot -pblog.attacker.club

生成配置

/opt/cm-5.15.1/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-1 -uroot -pblog.attacker.club --scm-host cdh-1 scm root blog.attacker.club

提示下面这个说明成功

All done, your SCM database is configured correctly!

查看配置

cat  /opt/cm-5.15.1/etc/cloudera-scm-server/db.properties

创建CM用的数据库

--hive数据库
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
--oozie数据库
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
--hue数据库
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

安装CDH5

\cp CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel* /opt/cloudera/parcel-repo
cp manifest.json /opt/cloudera/parcel-repo
mv /opt/cloudera/parcel-repo/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha

服务启动

ln -s /opt/cm-5.15.1/ /opt/cm
yum install psmisc 
# 启动报错 安装pstree
/opt/cm/etc/init.d/cloudera-scm-server restart
#服务端(主节点)
/opt/cm/etc/init.d/cloudera-scm-agent
#客户端 (所有节点)

WEB 登录

http://10.0.1.120:7180/cmf/login admin/admin

当前管理主机

选择cdh-1-4

新管理搜索主机

cdh-[1-4]

图1

图2

安装完成

sqoop部署

wget -c http://archive.cloudera.com/cdh5/cdh/5/sqoop-1.4.6-cdh5.15.1.tar.gz
tar zxvf sqoop-*.tar.gz -C /opt/
cp /usr/share/java/mysql-connector-java.jar /opt/sqoop-*/lib
ln -s /opt/sqoop-1.4.6-cdh5.15.1/ /opt/sqoop

vi /etc/profile

export SQOOP_HOME=/opt/sqoop
export PATH=$PATH:$SQOOP_HOME/bin

vi $SQOOP_HOME/bin/configure-sqoop 注释掉HCatalog,Accumulo检查(除非你准备使用HCatalog,Accumulo等HADOOP上的组件)

## Moved to be a runtime check in sqoop.
#if [ ! -d "${HCAT_HOME}" ]; then
# echo "Warning: $HCAT_HOME does not exist! HCatalog jobs will fail."
# echo 'Please set $HCAT_HOME to the root of your HCatalog installation.'
#fi

#if [ ! -d "${ACCUMULO_HOME}" ]; then
# echo "Warning: $ACCUMULO_HOME does not exist! Accumulo imports will fail."
# echo 'Please set $ACCUMULO_HOME to the root of your Accumulo installation.'
#fi

清理

mysql

drop database oozie;
drop database hive;
drop database hue;
drop database scm;

删除文件

rm -Rf /var/lib/flume-ng/ /var/lib/hadoop* /var/lib/navigator /var/lib/hue /var/lib/oozie /var/lib/solr /var/lib/sqoop* /var/lib/zookeeper /etc/hadoop/ /etc/hbase /etc/hive /dfs /var/lib/hive /usr/lib/hue /var/lib/cloudera* /var/cache/yum/cloudera* /var/log/cloudera* /var/run/cloudera*
rm /opt/cm -rf #软连接
rm -rf /tmp/.scm_prepare_node.lock
rm /opt/cm-5.15.1/lib/cloudera-scm-agent/cm_guid -f
rm /opt/cm/lib/cloudera-scm-agent/cm_guid -f

HBASE 角色添加

java内存堆栈设置