#!/bin/bash #1. 判断参数个数 if [ $# -lt 1 ] then echo Not Enough Arguement! exit; fi #2. 遍历集群所有机器 for host in hadoop102 hadoop103 hadoop104 do echo ==================== $host ==================== #3. 遍历所有目录，挨个发送 for file in $@ do #4. 判断文件是否存在 if [ -e $file ] then #5. 获取父目录 pdir=$(cd -P $(dirname $file); pwd) #6. 获取当前文件的名称 fname=$(basename $file) ssh $host "mkdir -p $pdir" rsync -av $pdir/$fname $host:$pdir else echo $file does not exists! fi done done
sed -i '$a\# JAVA_HOME' /root/.bashrc sed -i '$a\export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera' /root/.bashrc sed -i '$a\export PATH=$JAVA_HOME/bin:$PATH' /root/.bashrc sed -i '$a\export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar' /root/.bashrc
[mysqld] datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock # To prevent deadlocks, set the isolation level to READ-COMMITTED. transaction-isolation = READ-COMMITTED # Disabling symbolic-links is recommended to prevent assorted security risks; # to do so, uncomment this line: symbolic-links = 0
#Allow 100 maximum connections for each database and then add 50extra connections. For example, for two databases, set the maximum connections to 250. If you store five databases on one host(the databases for Cloudera Manager Server, Activity Monitor, Reports Manager, Cloudera Navigator, and Hive metastore), set the maximum connections to 550. max_connections = 550 #expire_logs_days = 10 #max_binlog_size = 100M
#log_bin should be on a disk with enough free space. #Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your #system and chown the specified folder to the mysql user. #如果需要替换mysql的log文件存放位置，需要此处拥有足够的空间并且把文件夹的属主改为mysql用户 log_bin=/var/lib/mysql/mysql_binary_log
#In later versions of MySQL, if you enable the binary log and do not set #a server_id, MySQL will not start. The server_id must be unique within #the replicating group. server_id=1
# InnoDB settings innodb_file_per_table=1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 64M innodb_buffer_pool_size= 4G innodb_thread_concurrency=8 #The default settings in the MySQL installations in most distributions use conservative buffer sizes and memory usage. Cloudera Management Service roles need high write throughput because they might insert many records in the database. Cloudera recommends that you set the innodb_flush_method property to O_DIRECT. innodb_flush_method = O_DIRECT innodb_log_file_size= 512M
Disallow root login remotely 选项需要输入 N , 这样 root 用户才能远程访问
sudo systemctl enable mysqld
sudo systemctl start mysqld
sudo /usr/bin/mysql_secure_installation
[...]这里需要配置密码，密码可以为简单密码，例如123456 Enter current password forroot(enter for none):（这里需要直接回车） OK, successfully used password, moving on... [...] Set root password? [Y/n] Y New password:（这里输入自定义的密码） Re-enter newpassword:（这里输入自定义的密码） Remove anonymous users? [Y/n] Y [...] Disallow root login remotely? [Y/n] N [...] Remove test database and access to it [Y/n] Y [...] Reload privilege tables now? [Y/n] Y All done!
将JDBC的jar放入到/usr/share/java文件夹中
mkdir -p /usr/share/java/
cd /usr/share/java/ wget https://cdh-sgg.oss-cn-shenzhen.aliyuncs.com/fileshare/mysql-connector-java-5.1.46-bin.jar
mysql -u root -p Enter password: # 建表及修改权限的格式： CREATE DATABASE <database> DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; GRANT ALL ON <database>.* TO '<user>'@'%' IDENTIFIED BY '<password>'; # 以下是需要执行的SQL语句 # 替换之后需要执行的SQL为： CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm'; GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'amon'; GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'rman'; GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'hue'; GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'hive'; GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'sentry'; GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'nav'; GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'navms'; GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'oozie'; #####注意此处再授权一个本主机名地址，不然web页面配置很容易出错，注意修改本地主机名hostname GRANT ALL ON amon.* TO 'amon'@'hadoop102' IDENTIFIED BY 'amon';
FLUSH PRIVILEGES;
show grants for'amon'@'%'; show grants for'rman'@'%'; show grants for'hive'@'%'; show grants for'hue'@'%'; show grants for'oozie'@'%';
yum list | grep cloudera-manager yum -y install cloudera-manager-daemons cloudera-manager-agent
[root@hadoop102 ~]# sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm Enter SCM password: (scm)
JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera Verifying that we can write to /etc/cloudera-scm-server Creating SCM configuration file in /etc/cloudera-scm-server Executing: /usr/java/jdk1.8.0_181-cloudera/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cloudera/cm/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db. log4j:ERROR Could not find value for key log4j.appender.Alog4j:ERROR Could not instantiate appender named "A". [2021-06-22 21:14:01,762] INFO 0[main] - com.cloudera.enterprise.dbutil.DbCommandExecutor.testDbConnection(DbCommandExecutor.java) - Successfully connected to database. All done, your SCM database is configured correctly!
# 当你看到以下信息时候 证明启动成功 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server. 下面可能会出现报错信息，这个是因为2021年CDH开始正式收费，从官方网站渠道已经无法拉取到任何安装包和校验文件，所以我们在4.4章节中手动上传文件进行校验。 2022-03-0114:08:23,203 ERROR ParcelUpdateService:com.cloudera.parcel.components.ParcelDownloaderImpl: Failed to download manifest. Status code: 401 URI: https://archive.cloudera.com/p/cdh6/6.3.4/parcels/manifest.json 2022-03-0114:08:23,203 ERROR ParcelUpdateService:com.cloudera.parcel.components.ParcelDownloaderImpl: Could not retrieve repository info for repo https://archive.cloudera.com/cdh6/6.3/parcels/. Got HTTP response code 401
