离线安装 cloudera-manager

Posted on Posted in hadoop

离线安装 cloudera-manager

简介

  • 自动化Hadoop安装过程,大幅度缩短部署时间
  • 提供实时的集群概况,例如节点、服务的运行状况
  • 提供了集群的中央控制台对集群的配置进行更改
  • 包含全面的报告和诊断工具,帮助优化性能和效率

系统环境

版本

  • inux内核版本:centos7.5
  • CDH版本信息:5.12.1
  • clouderamanager版本:centos7-cm5.12.1_x86_64
  • mysql版本信息:5.7.22
  • java版本信息:1.8.1

组件

  • manifest.json
  • mysql-connector-java.jar
  • CDH-5.12.1-1.cdh5.12.1.p0.3-el7.parcel.sha

下载

服务规划

  • 172.50.19.21 CDH-master/mysql CDH-master
  • 172.50.19.22 CDH-node CDH-node
  • 172.50.19.23 CDH-node CDH-node

部署安装

基础安装

yum -y install vim  bash-completion iproute net-tools  setuptool  system-config-networktui lsof wget lrzsz ntp -y 
# 修改网卡
mv ifcfg-eno16777729 ifcfg-eth0
sed -i.bak  'a GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 rhgb quiet"'  /etc/sysconfig/grub
grub2-mkconfig -o /boot/grub2/grub.cfg
# 关闭 selinux
grep -i ^selinux /etc/selinux/config
sed -i ‘/^SELINUX/s/enforcing/disabled/g‘ /etc/selinux/config
grep -i ^selinux /etc/selinux/config
getenforce
# 禁止 firewalld
systemctl stop firewalld.service 
systemctl disable firewalld.service
# 更改主机名称
hostnamectl set-hostname CDH-master
hostnamectl set-hostname CDH-node
hostnamectl set-hostname CDH-node
# 开启最大连接数
ulimit -SHn 1024000 
echo "ulimit -SHn 1024000" >> /etc/rc.d/rc.local 
source /etc/rc.d/rc.local
# 重启
reboot

部署安装(所有节点)

# 配置网卡
[root@bogon ~]# vim /etc/sysconfig/network-scripts/ifcfg-eth0
TYPE=Ethernet
BOOTPROT=static
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=eth0
UUID=6f8bae62-6a27-426f-9f4a-983e4b579551
ONBOOT=yes
IPADDR=172.50.19.21
NETMASK=255.255.255.0
GATEWAY=172.50.19.254
DNS1=202.106.196.115
DNS2=202.106.0.20

# 工具存储路径
mkdir /home/tools/
cd /home/tools/

# 修改hosts
sed -i.bak '$a 127.0.0.1 localhost.CDH-master\n127.0.0.1 vm10-12-0-253.ksc.com\n172.50.19.21 CDH-master\n172.50.19.22 CDH-node01\n172.50.19.23 CDH-node02' /etc/hosts 

# 安装java
yum install java -y

# 安装psmisc 
yum install -y psmisc
  • fuser 显示使用指定文件或者文件系统的进程的PID。
  • killall 杀死某个名字的进程,它向运行指定命令的所有进程发出信号。
  • pstree 树型显示当前运行的进程。
# 安装cm
tar -axvf cloudera-manager-centos7-cm5.12.1_x86_64.tar.gz -C /opt/
# 创建用户
useradd --system --home=/opt/cm-5.12.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
# 修改环境变量
sed -i.bak '$a export HADOOP_HOME=/data/software/hadoop/\nexport PATH=$HADOOP_HOME/bin:$PATH\nexport PATH=/data/software/hadoop/:$PATH' /etc/profile
# 创建parcel-repo仓库目录
mkdir -p /opt/cloudera/parcel-repo 
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
# 拷贝软件包
cp CDH-5.7.2-1.cdh5.7.2.p0.18-el7.parcel CDH-5.7.21.cdh5.7.2.p0.18-el7.parcel.sha manifest.json /opt/cloudera/parcelrepo
# 拷贝主节点的jar到各个节点上单独操作
/usr/share/java/mysql-connector-java.jar

部署安装(master)

yum install -y libaio
mkdir -p /data/software/
# 执行写好的脚本
/bin/sh  install_mysql.sh
# 执行sql 命令
# 设置密码
SET PASSWORD = PASSWORD('123456');
ALTER USER 'root'@'localhost' PASSWORD EXPIRE NEVER;
flush privileges;
\q;
# 设置数据库连接权限
use mysql;
update user set host='%' where user='root' and host='localhost'; 
flush privileges;
# 设置开启启动
echo “/data/software/mysql/support-files/mysql.server start”>>/etc/rc.local
# 创建库表
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci; 
create database monitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci; 
# 创建server端目录/var/cloudera-scm-server
chown cloudera-scm:cloudera-scm /var/cloudera-scm-server 
chown -R cloudera-scm:cloudera-scm /opt/
# 主节点创建parcels目录
mkdir -p /opt/cloudera/parcels
chown cloudera-scm:cloudera-scm /opt/cloudera/parcels
# 开启时间同步
# 主:
vim /etc/ntp.conf
server 127.127.1.0 fudge
127.127.1.0 stratum 8
# 开启并自启
systemctl restart ntpd
systemctl enable ntpd
install_mysql.sh
#!/bin/bash
. /etc/init.d/functions
mkdir -p /root/bin/
cp mysql-5.7.22-el7-x86_64.tar /data/software/
cd /data/software
tar xf mysql-5.7.22-el7-x86_64.tar
tar xf mysql-5.7.22-el7-x86_64.tar.gz
mv mysql-5.7.22-el7-x86_64 mysql
cd mysql ; mkdir {data,log,mysqld}
touch /data/software/mysql/log/error.log
touch /data/software/mysql/log/mysqld.log
echo "[mysqld]
port = 3306
server-id = 11
symbolic-links=0
socket = /data/software/mysql/mysqld/mysql.sock
basedir = /data/software/mysql
datadir = /data/software/mysql/data
character-set-server = utf8
relay_log=/data/software/mysql/log/mysql-relay-bin
log-error=/data/software/mysql/log/error.log
datadir = /data/software/mysql/data
slow_query_log=1
slow_query_log_file=/data/software/mysql/log/slow-query.log
explicit_defaults_for_timestamp=true
log-bin=/data/software/mysql/log/mysql-bin.log
long_query_time=3
binlog-format=ROW
binlog-row-image = minimal
back_log = 384
key_buffer_size = 256M
max_allowed_packet = 4000M
thread_stack = 512K
sort_buffer_size = 60M
read_buffer_size = 40M
join_buffer_size = 80M
myisam_sort_buffer_size = 256M
thread_cache_size = 512
query_cache_size = 256M
tmp_table_size = 512M
max_connections = 10000
max_connect_errors = 10000000
wait_timeout = 100
innodb_flush_log_at_trx_commit=1
innodb_log_buffer_size=2M
innodb_thread_concurrency=8
key_buffer_size=256M
tmp_table_size=64M
read_rnd_buffer_size=16M
thread_cache_size=120
query_cache_size=32M
skip_slave_start=1
lower_case_table_names=1
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
max_connections=1200
back_log = 512
max_connect_errors = 50000
tmp_table_size = 64M
max_heap_table_size = 64M
open_files_limit = 8192
key_buffer_size =64M
group_concat_max_len = 102400

innodb_io_capacity = 15000
innodb_io_capacity_max = 20000
innodb_write_io_threads = 8
innodb_read_io_threads = 8
innodb_purge_threads = 4
innodb_page_cleaners = 4
innodb_open_files = 65535
innodb_max_dirty_pages_pct = 50
innodb_flush_method = O_DIRECT
innodb_lru_scan_depth = 4000
innodb_lock_wait_timeout = 60
innodb_rollback_on_timeout = 1
innodb_print_all_deadlocks = 1
innodb_file_per_table = 1
internal_tmp_disk_storage_engine = InnoDB

innodb_buffer_pool_size = 8G
innodb_buffer_pool_instances = 4
innodb_buffer_pool_load_at_startup = 1
innodb_buffer_pool_dump_at_shutdown = 1
innodb_log_buffer_size = 64M
innodb_log_file_size = 1G
innodb_log_files_in_group = 2
innodb_flush_log_at_trx_commit = 2"> /etc/my.cnf
chmod 644 /etc/my.cnf

groupadd mysql
useradd -r -g mysql mysql -d /data/software/mysql
chown -R mysql.mysql /data/software/mysql /etc/my.cnf
#sed -i 's/\/usr\/local\/mysql/\/data\/software\/mysql/g' /data/software/mysql/support-files/mysql.server
echo 'export PATH=$PATH:/data/software/mysql/bin' >> /etc/profile
source /etc/profile
rm -f /data/software/mysql/data/*
mysqld --initialize --user=mysql --basedir=/data/software/mysql/ --datadir=/data/software/mysql/data/
/data/software/mysql/support-files/mysql.server  start
action "mysql-install" /bin/true
echo "/data/software/mysql/support-files/mysql.server  start" > /root/bin/mysql_start.sh
echo "/data/software/mysql/support-files/mysql.server  stop" > /root/bin/mysql_stop.sh
echo "/data/software/mysql/support-files/mysql.server  status" > /root/bin/mysql_status.sh
chmod a+x /data/software/mysql/mysqld/
mysql_passwd=`cat /root/.mysql_secret |tail -1`
ln -s /data/software/mysql/mysqld/mysql.sock /tmp/
echo "/data/software/mysql/support-files/mysql.server start">>/etc/rc.local
mysql -uroot -p'${mysql_passwd}' < /root/.bashell/new-mysql.sql

部署安装(node节点)

# 配置从节点ntp同步master时间
vim /etc/ntp.conf
server 172.50.19.21

# 同步时间
ntpdate -u CDH-master
# 配置从节点cloudera-manger-agent指向主节点服务器
# 将server_host改为CMS所在的主机名即master 
sed -i.bak 's#server_host=localhost#server_host=CDH-master#g'  /opt/cm-5.12.1/etc/cloudera-scm-agent/config.ini

启动顺序

  1. 主节点上出时脚本初始化数据库
/opt/cm-5.12.1/share/cmf/schema/scm_prepare_database.sh mysql -hCDH-master -uroot -p123456 --scm-host CDH-master scmdbn scmdbn scmdbn
  1. 主节点启动服务
/opt/cm-5.12.1/etc/init.d/cloudera-scm-server restart
/opt/cm-5.12.1/etc/init.d/cloudera-scm-agent restart
  1. 从节点启动服务
/opt/cm-5.12.1/etc/init.d/cloudera-scm-agent restart

cloudera manager web管理

访问 http://172.50.19.21:7180, 默认账号密码 admin - admin

签署协议

选择免费版


指定主机

选择版本

读取安装包

勾选安装服务

资源分配

设置默认

根据后面的需求再调整

完成部署

图中所报错误为:master内存不足
» 转载请注明来源:呢喃 » 离线安装 cloudera-manager
相关文章

Leave a Reply

Your email address will not be published. Required fields are marked *

15 − two =