前提条件：先了解集群搭建流程是什么样，需要改哪些配置，有哪些环境，这个文章目的是简化部署。

clickhouse运维篇（一）：docker-compose 快速部署clickhouse集群
clickhouse运维篇（二）：多机器手动部署ck集群

!https://i-blog.csdnimg.cn/direct/7677be0676904a7687c2f4cc8aef0a46.png

项目目录解析：

 $ tree .
.
├── cluster.conf   #集群配置，  集群包含哪些机器、端口分别为多少
├── config  
│   ├── config_node1.xml  # 生成的ck节点配置文件
│   ├── config_node2.xml
│   ├── config_template.xml  # config模版
│   ├── users_node1.xml     # 生成的ck节点用户配置文件
│   ├── users_node2.xml
│   └── users_template.xml   # users模版
├── dep   #下面包含需要的远程依赖， zookeeper、clickhouse、jdk，根据自己的需求更改
├── gen_cluster_config.sh  
├── gen_login_cmd.sh
├── install.conf  # 远程登录的主机conf，需要在跳板机或者中间机器上去ssh、scp使用
└── main.sh   #启动入口

1、main.sh

CUR_FOLDER=$(cd "/Users/admin/scripts/my_app/ckcluster";pwd). ${CUR_FOLDER}/cluster.conf
. ${CUR_FOLDER}/gen_cluster_config.sh
. ${CUR_FOLDER}/gen_login_cmd.shCONFIG_FOLDER="${CUR_FOLDER}/config"
DEP_FOLDER="${CUR_FOLDER}/dep"
TEMPLATE_FILE="${CONFIG_FOLDER}/config_template.xml"if [ ! -d "$CONFIG_FOLDER" ]; thenmkdir -p "$CONFIG_FOLDER"
fiif [ ! -d "$DEP_FOLDER" ]; thenmkdir -p "$DEP_FOLDER"
fizk_index=1
# 首先读取所有节点信息并根据 shard 进行分组
while true; dohost_var="zk_node_${zk_index}_host"# 检查所有变量是否为空，若有一个为空则跳出循环if [ -z "${!host_var}" ]; thenbreakfi# 远程安装jdk、zookeeperinstall_path=$(get_install_path ${!host_var})echo $(get_scp_command ${!host_var} . "${DEP_FOLDER}/jdk-8u202-nonroot.tar.gz ${DEP_FOLDER}/apache-zookeeper-3.7.2-bin.tar.gz")`get_scp_command ${!host_var} . "${DEP_FOLDER}/jdk-8u202-nonroot.tar.gz ${DEP_FOLDER}/apache-zookeeper-3.7.2-bin.tar.gz"`echo $(get_ssh_command ${!host_var} "cd ${install_path};tar -xzvf ${install_path}/jdk-8u202-nonroot.tar.gz")echo $(get_ssh_command ${!host_var} "cd ${install_path};tar -xzvf ${install_path}/apache-zookeeper-3.7.2-bin.tar.gz")`get_ssh_command ${!host_var} "cd ${install_path};tar -xzvf ${install_path}/jdk-8u202-nonroot.tar.gz"``get_ssh_command ${!host_var} "cd ${install_path};tar -xzvf ${install_path}/apache-zookeeper-3.7.2-bin.tar.gz"`zk_index=$((zk_index + 1))
done# 生成ck集群配置文件
gen_config# 远程安装clickhouse
ck_index=1
# 首先读取所有节点信息并根据 shard 进行分组
while true; dohost_var="ck_node_${ck_index}_host"# 检查所有变量是否为空，若有一个为空则跳出循环if [ -z "${!host_var}" ]; thenbreakfi
#     <users_config>/opt/appaduudit/my_app-2.4/clickhouse-23.4.2.9/ck_node_1/config/users_node.xml</users_config>    install_path=$(get_install_path ${!host_var})    `get_scp_command ${!host_var} . "${DEP_FOLDER}/clickhouse-23.4.2.9.tar.gz"`echo $(get_ssh_command ${!host_var} "cd ${install_path};tar -xzvf ${install_path}/clickhouse-23.4.2.9.tar.gz")NODE_CK_PATH="${BASE_CK_PATH}/ck_node_${ck_index}"echo $(get_scp_command ${!host_var} ${NODE_CK_PATH}/config  "${CONFIG_FOLDER}/user_node${ck_index}.xml")`get_scp_command ${!host_var} ${NODE_CK_PATH}/config "${CONFIG_FOLDER}/config_node${ck_index}.xml"``get_scp_command ${!host_var} ${NODE_CK_PATH}/config  "${CONFIG_FOLDER}/users_node${ck_index}.xml"`echo $(get_ssh_command ${!host_var} "${install_path}/clickhouse-23.4.2.9.tar.gz/bin/clickhouse server --config-file  ${install_path}/config_node${ck_index}.xml")`get_ssh_command ${!host_var} "${install_path}/clickhouse-23.4.2.9.tar.gz/bin/clickhouse server --config-file  ${install_path}/config_node${ck_index}.xml"`    ck_index=$((ck_index + 1))
done

2、cluster.conf

# 定义变量
target_install_path="/opt/app/ck_cluster"
BASE_CK_PATH="/opt/app/my_app-2.4/clickhouse-23.4.2.9"
CK_CLUSTER_NAME="my_ck_cluster_test"#port为zk的clientPort
zk_node_1_host=172.168.1.206
zk_node_1_port=8551zk_node_2_host=172.168.1.207
zk_node_2_port=8551zk_node_3_host=172.168.1.208
zk_node_3_port=8551ck_node_1_host=172.168.1.206
ck_node_1_tcp_port=8601
ck_node_1_http_port=8602
ck_node_1_interserver_http_port=8603
ck_node_1_user=default
ck_node_1_password=password
ck_node_1_shard=01
ck_node_1_replica=replica_63ck_node_2_host=172.168.1.207
ck_node_2_tcp_port=8611
ck_node_2_http_port=8612
ck_node_2_interserver_http_port=8613
ck_node_2_user=default
ck_node_2_password=password
ck_node_2_shard=02
ck_node_2_replica=replica_63#ck_node_3_host=ck_host_3
#ck_node_3_tcp_port=ck_tcp_port_3
#ck_node_3_http_port=ck_http_port_3
#ck_node_3_interserver_http_port=ck_interserver_http_port_3
#ck_node_3_user=user3
#ck_node_3_password=password3
#ck_node_3_shard=02
#ck_node_3_replica=replica_209

3、install.conf

ssh.172.168.1.206=root:app:/opt/app/ck_cluste
ssh.172.168.1.207=admin:adminpass:/opt/app/ck_cluste
ssh.172.168.1.208=admin:adminpass:/opt/app/ck_cluste

4、gen_login_cmd.sh

#!/bin/bash
cd /Users/admin/scripts/my_app/ckcluster
# 读取配置文件并解析
CONFIG_FILE="install.conf"
declare -A HOSTSwhile IFS='=' read -r key user_password || [[ -n "$key" ]]; dokey=$(echo "$key" | xargs)user_password=$(echo "$user_password" | xargs)if [[ $key == ssh.* ]]; thenhost=${key#ssh.}HOSTS["$host"]="$user_password"fi
done < "$CONFIG_FILE"# 获取 SSH 命令
get_ssh_command() {local host=$1local cmd=$2local user_password=${HOSTS["$host"]}IFS=':' read -r user password install_path<<< "$user_password"# echo "sshpass -p '$password' ssh $user@$host $cmd"echo "sshpass -p '$password' ssh $user@$host \"$cmd\""}# 获取 SCP 命令
get_scp_command() {local host=$1local extra_dir=$2local files=$3    local user_password=${HOSTS[$host]}# root:idss:/opt/idss/ck_clusteIFS=':' read -r user password install_path<<< "$user_password"if [ "$extra_dir" != "." ]; theninstall_path=$extra_dirfimkdir_cmd=$(get_ssh_command $host "mkdir -p $install_path")eval "$mkdir_cmd"echo "sshpass -p '$password' scp -r $files $user@$host:$install_path"
}get_install_path() {local host=$1local user_password=${HOSTS[$host]}# root:idss:/opt/idss/ck_clusteIFS=':' read -r user password install_path<<< "$user_password"echo $install_path
}
# ssh_command=$(get_scp_command "10.87.102.206" "/Users/admin/scripts/my_app/ckcluster/dep/jdk-8u202-nonroot.tar.gz /Users/admin/scripts/my_app/ckcluster/dep/apache-zookeeper-3.7.2-bin.tar.gz")
# echo $ssh_command
# scp_command=$(get_ssh_command "10.87.102.206" "ls /opt/idss/ck_cluste")
# echo $scp_command
# `$scp_command`

5、gen_cluster_config.sh

#!/bin/bash
# 定义基础路径和模板文件名function gen_config() {# 生成 zookeeper 配置
ZOOKEEPER_CONFIG="<zookeeper>\n"
# 循环遍历 zk_node_*_host 和 zk_node_*_port 变量
zk_index=1
while true; dohost_var="zk_node_${zk_index}_host"port_var="zk_node_${zk_index}_port"# 检查变量是否已定义，如果未定义则跳出循环if [ -z "${!host_var}" ] || [ -z "${!port_var}" ]; thenbreakfiZOOKEEPER_CONFIG+="    <node index=\"${zk_index}\">\n"ZOOKEEPER_CONFIG+="        <host>${!host_var}</host>\n"ZOOKEEPER_CONFIG+="        <port>${!port_var}</port>\n"ZOOKEEPER_CONFIG+="    </node>\n"zk_index=$((zk_index + 1))
done
ZOOKEEPER_CONFIG+="</zookeeper>"# 生成 remote_servers 配置
declare -A shard_nodes config_contentck_index=1
# 首先读取所有节点信息并根据 shard 进行分组
while true; dohost_var="ck_node_${ck_index}_host"tcp_port_var="ck_node_${ck_index}_tcp_port"http_port_var="ck_node_${ck_index}_http_port"interserver_port_var="ck_node_${ck_index}_interserver_http_port"user_var="ck_node_${ck_index}_user"password_var="ck_node_${ck_index}_password"shard_var="ck_node_${ck_index}_shard"replica_var="ck_node_${ck_index}_replica"# 检查所有变量是否为空，若有一个为空则跳出循环if [ -z "${!host_var}" ] || \[ -z "${!tcp_port_var}" ] || \[ -z "${!http_port_var}" ] || \[ -z "${!interserver_port_var}" ] || \[ -z "${!user_var}" ] || \[ -z "${!password_var}" ] || \[ -z "${!shard_var}" ] || \[ -z "${!replica_var}" ]; thenbreakfi# 将当前节点信息存入以 shard 为 key 的数组shard_nodes["${!shard_var}"]+=$(cat <<-NODE<replica><host>${!host_var}</host><port>${!tcp_port_var}</port><user>${!user_var}</user><password>${!password_var}</password></replica>\n
NODE)NODE_CK_PATH="${BASE_CK_PATH}/ck_node_${ck_index}"config_template=$(cat ${CONFIG_FOLDER}/config_template.xml)config_template=$(echo "$config_template" | sed "s|\${BASE_CK_PATH}|${NODE_CK_PATH}|g")config_template=$(echo "$config_template" | sed "s|\${HTTP_PORT}|${!http_port_var}|g")config_template=$(echo "$config_template" | sed "s|\${TCP_PORT}|${!tcp_port_var}|g")config_template=$(echo "$config_template" | sed "s|\${INTERSERVER_HTTP_PORT}|${!interserver_port_var}|g")config_template=$(echo "$config_template" | sed "s|\${MACROS_SHARD}|${!shard_var}|g")config_template=$(echo "$config_template" | sed "s|\${MACROS_REPLICA}|${!replica_var}|g")config_template=$(echo "$config_template" | sed "s|\${CK_PASSWORD}|${!password_var}|g")    config_template=$(echo "$config_template" | sed "s|\${ck_index}|${!ck_index}|g")        # 将生成的配置内容存入字典config_content["config_node${ck_index}"]="$config_template"# users文件生成users_template=$(cat ${CONFIG_FOLDER}/users_template.xml)users_template=$(echo "$users_template" | sed "s|\${CK_PASSWORD}|${!password_var}|g")    echo -e "$users_template" > "${CONFIG_FOLDER}/users_node${ck_index}.xml"ck_index=$((ck_index + 1))
done# 构建最终的 XML 配置
REMOTE_SERVERS_CONFIG="<remote_servers>\n"
REMOTE_SERVERS_CONFIG+="    <${CK_CLUSTER_NAME}>\n"
for shard in "${!shard_nodes[@]}"; doREMOTE_SERVERS_CONFIG+="        <shard>\n"REMOTE_SERVERS_CONFIG+="            <internal_replication>true</internal_replication>  \n"REMOTE_SERVERS_CONFIG+="${shard_nodes[$shard]}"REMOTE_SERVERS_CONFIG+="        </shard>\n"
done
REMOTE_SERVERS_CONFIG+="    </${CK_CLUSTER_NAME}>\n"
REMOTE_SERVERS_CONFIG+="</remote_servers>"# 动态修改配置文件（插入 Zookeeper\CK集群 配置）
for node_config in "${!config_content[@]}"; doecho "处理 $node_config ..."# 创建临时文件temp_file="${node_config}_tmp.xml"echo -e "${config_content[$node_config]}" > "$temp_file"sed -i '' '/<\/yandex>/d' "$temp_file"# linux下为# sed -i '/<\/yandex>/d' "$temp_file"    # 拼接 Zookeeper 和 Remote Servers 配置combined_insert="${ZOOKEEPER_CONFIG}
${REMOTE_SERVERS_CONFIG}"echo -e "$combined_insert" >> "$temp_file"echo "</yandex>" >> "$temp_file"mv "$temp_file" "$CONFIG_FOLDER/${node_config}.xml"echo "生成 $node_config 完成: ${node_config}.xml"
done}

6、config_template.xml

<?xml version="1.0"?>
<yandex><logger><level>notice</level><log>${BASE_CK_PATH}/log/clickhouse-server.log</log><errorlog>${BASE_CK_PATH}/log/clickhouse-server.err.log</errorlog><size>1000M</size><count>10</count></logger><path>${BASE_CK_PATH}/data/</path><tmp_path>${BASE_CK_PATH}/tmp/</tmp_path><user_files_path>${BASE_CK_PATH}/data/user_files/</user_files_path><users_config>${BASE_CK_PATH}/config/users_node${ck_index}.xml</users_config>    <users><default><password>${CK_PASSWORD}</password> <!-- 空密码 --><networks><ip>::/0</ip> <!-- 允许所有IP访问 --></networks><profile>default</profile><quota>default</quota><access_management>1</access_management></default></users>    <default_profile>default</default_profile><default_database>default</default_database><http><port>${HTTP_PORT}</port><max_connections>1024</max_connections><async_insert>1</async_insert></http><listen_host>0.0.0.0</listen_host><listen_host>::</listen_host><http_port>${HTTP_PORT}</http_port><tcp_port>${TCP_PORT}</tcp_port><interserver_http_port>${INTERSERVER_HTTP_PORT}</interserver_http_port><distributed_ddl><!-- Path in ZooKeeper to queue with DDL queries --><path>/clickhouse/task_queue/ddl</path></distributed_ddl><macros><shard>${MACROS_SHARD}</shard><replica>${MACROS_REPLICA}</replica></macros>
</yandex>

7、users_template.xml

<?xml version="1.0"?>
<yandex><!-- Profiles of settings. --><profiles><!-- Default settings. --><default><!-- Maximum memory usage for processing single query, in bytes. --><max_memory_usage>10000000000</max_memory_usage><load_balancing>random</load_balancing></default><!-- Profile that allows only read queries. --><readonly><readonly>1</readonly></readonly></profiles><users><default><password>${CK_PASSWORD}</password> <networks><ip>::/0</ip> <!-- 允许所有IP访问 --></networks><profile>default</profile><quota>default</quota><access_management>1</access_management></default></users>    <!-- Quotas. --><quotas><!-- Name of quota. --><default><!-- Limits for time interval. You could specify many intervals with different limits. --><interval><!-- Length of interval. --><duration>3600</duration><distributed_product_mode>allow</distributed_product_mode><!-- No limits. Just calculate resource usage for time interval. --><queries>0</queries><errors>0</errors><result_rows>0</result_rows><read_rows>0</read_rows><execution_time>0</execution_time></interval></default></quotas>
</yandex>