1、修改主机名(各个节点)
hostnamectl set-hostname xxx
2、hosts 文件加入主机名(全部节点)
cat /etc/hosts
192.168.88.5 master1
192.168.88.6 master2
192.168.88.7 master3
192.168.88.8 node1
3、关闭防火墙(全部节点)
systemctl stop firewalld && systemctl disable firewalld
4、关闭 selinux(全部节点,需要重启服务器)
sed -i 's/^ *SELINUX=enforcing/SELINUX=disabled/g'
5、关闭各节点 swap 分区(全部节点,需要重启服务器)
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
6、同步时间(全部节点)
yum -y install chrony && systemctl enable chronyd && systemctl start chronyd
timedatectl set-timezone Asia/Shanghai && chronyc -a makestep
7、安装 ipset 服务(全部节点)
yum -y install ipvsadm ipset sysstat conntrack libseccomp
8、内核调整,将桥接的 IPv4 流量传递到 iptables 的链(全部节点)
cat /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
文件立即生效
sysctl -p /etc/sysctl.d/k8s.conf
9、开启 ipvs 模块(全部节点)
cat /etc/sysconfig/modules/ipvs.modules
#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
10、赋予权限并执行脚本,用过滤查看是否生效(全部节点)
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
11、安装 haproxy 和 keepalived(所有节点)
yum -y install keepalived haproxy
12、修改 master 节点 keepalived.conf(全部节点)
cat /etc/keepalived/keepalived.conf(master1 节点)
! Configuration File for keepalived
global_defs {router_id LVS_DEVELscript_user rootenable_script_security
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh" # 脚本检测路径interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {state MASTER # 第一台 master 节点是 MASTER,其他 meater 节点是 BACKUPinterface eno16777736 # 本机网卡名virtual_router_id 51 # 路由都是一样priority 100 # 第一台权重是 100authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.88.3 # VIP IP}track_script {check_apiserver # 虚拟检测模块}
}
cat /etc/keepalived/keepalived.conf (master2 节点)
! Configuration File for keepalived
global_defs {router_id LVS_DEVELscript_user rootenable_script_security
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh" # 脚本检测路径interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {state BACKUP # 第一台 master 节点是 MASTER,其他 meater 节点是 BACKUPinterface eno16777736 # 本机网卡名virtual_router_id 51 # 路由都是一样priority 90 # 第一台权重是 100authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.88.3 # VIP IP}track_script {check_apiserver # 虚拟检测模块}
}
cat /etc/keepalived/keepalived.conf(master3 节点)
! Configuration File for keepalived
global_defs {router_id LVS_DEVELscript_user rootenable_script_security
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh" # 脚本检测路径interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {state BACKUP # 第一台 master 节点是 MASTER,其他 meater 节点是 BACKUPinterface eno16777736 # 本机网卡名virtual_router_id 51 # 路由都是一样priority 80 # 第一台权重是 100authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.88.3 # VIP IP}track_script {check_apiserver # 虚拟检测模块}
}
13、修改 haproxy.cfg 文件(三个 master 节点配置文件一样)
cat /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Example configuration for a possible web application. See the
# full configuration options online.
#
# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global# to have these messages end up in /var/log/haproxy.log you will# need to:## 1) configure syslog to accept network log events. This is done# by adding the '-r' option to the SYSLOGD_OPTIONS in# /etc/sysconfig/syslog## 2) configure local2 events to go to the /var/log/haproxy.log# file. A line like the following can be added to# /etc/sysconfig/syslog## local2.* /var/log/haproxy.log#log 127.0.0.1 local2chroot /var/lib/haproxypidfile /var/run/haproxy.pidmaxconn 4000user haproxygroup haproxydaemon# turn on stats unix socketstats socket /var/lib/haproxy/stats#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaultsmode httplog globaloption httplogoption dontlognulloption http-server-closeoption forwardfor except 127.0.0.0/8option redispatchretries 3timeout http-request 10stimeout queue 1mtimeout connect 10stimeout client 1mtimeout server 1mtimeout http-keep-alive 10stimeout check 10smaxconn 3000#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend kubernetes-apiservermode tcpbind *:8443option tcplogdefault_backend kubernetes-apiserver#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
listen statsbind *:1080stats auth admin:awesomePasswordstats refresh 5sstats realm HAProxy\ Statisticsstats uri /admin?stats#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiservermode tcpbalance roundrobinserver master1 192.168.88.5:6443 checkserver master2 192.168.88.6:6443 checkserver master3 192.168.88.7:6443 check
14、VIP 检测脚本
cat /etc/keepalived/check_apiserver.sh
#!/bin/sh
# HAPROXY down
A=`ps -C haproxy --no-header | wc -l`
if [ $A -eq 0 ]
then
systmectl start haproxyif [ ps -C haproxy --no-header | wc -l -eq 0 ]thenkillall -9 haproxyecho "HAPROXY down" | mail -s "haproxy"sleep 3600
fi
fi
给脚本添加执行权限
chmod +x /etc/keepalived/check_apiserver.sh
15、启动服务(启动之后自行查看服务是否报错)
systemctl enable keepalived && systemctl start keepalived
systemctl enable haproxy && systemctl start haproxy
16、下载 docker 源(全部节点)
wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
17、安装容器运行时(全部节点)
yum -y install docker-ce && systemctl enable docker && systemctl start docker
18、创建 daemon.json 配置文件(全部节点)
cat /etc/docker/daemon.json
{"exec-opts":["native.cgroupdriver=systemd"],"log-driver":"json-file","log-opts":{"max-size":"100m"}
}
重启 docker
systemctl restart docker
19、安装cri-docker
curl -LO https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.13/cri-dockerd-0.3.13-3.el7.x86_64.rpm && rpm -ivh cri-dockerd-0.3.13-3.el7.x86_64.rpm
查看版本
cri-dockerd --version
加入开机启动并启动
systemctl enable cri-docker && systemctl start cri-docker
20、配置 k8s 源
cat /etc/yum.repo.d/k8s.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=0
更新yum缓存
yum clean all && yum makecache
21、安装 kubeadm,kubelet,kubectl
yum -y install kubeadm-1.30.0 kubelet-1.30.0 kubectl-1.30.0 && systemctl enable kubelet
22、整合 kubelet 和 cri-docker
修改/usr/lib/systemd/system/cri-docker.service文件的ExecStart字段(全部节点)
--network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d
重启cri-dockerd服务
systemctl daemon-reload && systemctl restart cri-docker
23、生成初始化配置文件
kubeadm config print init-defaults > kubeadm-init.yaml
24、修改默认配置文件
cat kubeadm-init.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 24h0m0susages:- signing- authentication
kind: InitConfiguration
localAPIEndpoint:advertiseAddress: 192.168.88.5 # 当前主机 ipbindPort: 6443
nodeRegistration:criSocket: unix:///var/run/cri-dockerd.sockimagePullPolicy: IfNotPresentname: master1 # 本机主机名 taints:- effect: "NoSchedule"key: "node-role.kubernetes.io/master"
---
apiServer:timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.88.3:8443 # 重点!! 这里是我们自定义的域名,端口是haproxy的端口,单master可注释掉这个参数
controllerManager: {}
dns: {}
etcd:local:dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.30.0
networking:dnsDomain: cluster.localserviceSubnet: 10.96.0.0/12podSubnet: 10.244.0.0/16 # k8s 网卡网段(flannel,如果是 calico 需要改成别的网段)
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs # k8s 需要 ipvs 模式
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd # k8s 驱动 systemd
25、下载镜像
kubeadm config images pull --config kubeadm-init.yaml
26、初始化集群
kubeadm init --config kubeadm-init.yaml
27、在其他 master 节点创建目录
mkdir -p /etc/kubernetes/pki/etcd/
28、把主 master 节点证书分别复制到从 master 节点
cat cert.sh(自己创建脚本)
#!/bin/sh/
ip="192.168.88.6 192.168.88.7"for host in $ip
do scp /etc/kubernetes/pki/ca.* root@192.168.88.6:/etc/kubernetes/pki/scp /etc/kubernetes/pki/sa.* root@192.168.88.6:/etc/kubernetes/pki/scp /etc/kubernetes/pki/front-proxy-ca.* root@192.168.88.6:/etc/kubernetes/pki/scp /etc/kubernetes/pki/etcd/ca.* root@192.168.88.6:/etc/kubernetes/pki/etcd/scp /etc/kubernetes/admin.conf root@192.168.88.6:/etc/kubernetes/scp /etc/kubernetes/pki/ca.* root@192.168.88.7:/etc/kubernetes/pki/scp /etc/kubernetes/pki/sa.* root@192.168.88.7:/etc/kubernetes/pki/scp /etc/kubernetes/pki/front-proxy-ca.* root@192.168.88.7:/etc/kubernetes/pki/scp /etc/kubernetes/pki/etcd/ca.* root@192.168.88.7:/etc/kubernetes/pki/etcd/scp /etc/kubernetes/admin.conf root@192.168.88.7:/etc/kubernetes/
done
把主节点admin.conf证书复制到其他node节点
scp /etc/kubernetes/admin.conf root@192.168.88.8:/etc/kubernetes/
29、其他 master 节点加入集群命令
kubeadm join 192.168.88.3:6443 --token uozrup.6v02tswc3xk33org \--discovery-token-ca-cert-hash sha256:b5391b689f7986a2ef8aa84665568e9ea25a031c89c31472bb56f14e17c7c471 \--control-plane --certificate-key 3fdfbfe831ebc90939cdbde89d3894cbe527b64f67a4cd90de56018dfae75399 --cri-socket=unix:///var/run/cri-dockerd.sock
node 节点加入集群命令
kubeadm join 192.168.88.3:6443 --token uozrup.6v02tswc3xk33org \--discovery-token-ca-cert-hash sha256:b5391b689f7986a2ef8aa84665568e9ea25a031c89c31472bb56f14e17c7c471 --cri-socket=unix:///var/run/cri-dockerd.sock
30、非 root 用户加入 master 执行以下命令
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
root 用户加入 master 执行以下命令
export KUBECONFIG=/etc/kubernetes/admin.conf
31、安装网络插件
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
查看集群网络是否正常
kubectl get pods --all-namespaces
kubectl get pods --all-namespaces
32、安装 etcdctl 客户端(可以用脚本中的集群查询命令也可以单独用最下面的 etcd 查询命令)
cat etcdctl.sh
#!/bin/sh
# Author: Jacket_San
# Email: xxx@163.com
# Date: 26/11/2024
# Filename: etcd.sh##########################
# etcd 客户端安装 #
# etcdctl 查询数据库信息 #
########################### 定义 etcd 路径
path="/root/etcd/"# 下载 etcd 客户端并加入环境变量
if [ ! -d "$path" ];thenmkdir $path
fi# yum 安装 jq
yum -y install jq# 获取最新版本号
LATEST_VERSION=$(curl -s https://api.github.com/repos/etcd-io/etcd/releases/latest | jq -r .tag_name)# 下载最新版本的 tar.gz 文件
curl -L -O https://github.com/etcd-io/etcd/releases/download/${LATEST_VERSION}/etc-${LATEST_VERSION}-linux-amd64.tar.gz # 移动 etcdctl 到指定目录并授权
mv /root/etcd-*.tar.gz $path && cd $path && tar -zxf *
mv */etcdctl /usr/local/bin && chmod +x /usr/local/bin/# 查看 etcdctl 版本
etcdctl version && etcdctl -h# 查看etcd高可用集群健康状态
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint health# 查看etcd高可用集群列表
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 member list# 查看etcd高可用集群leader
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint status
33、查看etcd高可用集群健康状态
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint health
查看etcd高可用集群列表
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 member list
查看etcd高可用集群leader
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint status