由于centos停更，yum、docker等不支持，采用阿里云仓库搭建K8S

一：准备

服务器信息	主机名	IP地址
Centos7.9	node1-master	192.168.35.130
Centos7.9	node2	192.168.35.131

# 查看系统版本
cat /etc/centos-release
# 查看内核版本
uname -sr

二：服务器前置操作
每个节点都需要操作

#使用 hostnamectl set-hostname设置主机名
hostnamectl set-hostname node01-master

#编辑hosts文件，配置域名映射
vi /etc/hosts 
192.168.35.130  node01-master
192.168.35.131  node02

# 关闭防火墙
systemctl stop firewalld
# 关闭开机自启
systemctl disable firewalld
# 查看防火墙状态 显示not running 表示关闭成功
firewall-cmd --state

#将 SELinux 设置为 permissive 模式（相当于将其禁用）
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

#关闭swap
swapoff -a  # 临时关闭，主机重启后k8s无法自动重启，需要重新关闭swap
vim /etc/fstab  # 永久关闭
注释行 #/dev/mapper/cl_fedora_swap

#允许 iptables 检查桥接流量
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOFcat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOFsudo sysctl --system

由于centos停更，yum无法下载docker，需要更换yum仓库为阿里云仓库

cd /etc/yum.repos.d
vi CentOS-Base.repo
替换全部内容更为下面的
[base]
name=CentOS-$releasever - Base - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7[updates]
name=CentOS-$releasever - Updates - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7[extras]
name=CentOS-$releasever - Extras - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7[centosplus]
name=CentOS-$releasever - Plus - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7保存完成执行
yum clean all
yum makecache
yum update

#下载docker 根据仓库情况建议选择docker版本 
# yum -y install docker-ce 执行不成功选择yum -y install docker
yum -y install docker-ce
systemctl enable docker && systemctl start docker
# 配置阿里云镜像加速 另外k8s 1.2版本以上需要修改native.cgroupdriver=systemd
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{"registry-mirrors": ["https://iedolof4.mirror.aliyuncs.com"],"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
#验证
docker version

三：开始搭建
每个节点都需要操作安装kubeadm、kubelet、kubectl

#添加阿里的yum软件源，这里需要注意baseurl的地址是否是你虚拟机对应的版本  
# centos7.9使用kubernetes-el7-x86_64 可通过uname -sr命令查看
#访问地址 https://mirrors.aliyun.com/kubernetes/yum/repos/ 看对应版本
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#安装 kubelet kubeadm  kubectl 注意版本号
yum install -y kubelet-1.21.0 kubeadm-1.21.0 kubectl-1.21.0
systemctl enable kubelet

只需要在node1-master执行

#执行kubeadm 初始化 注意更改apiserver-advertise-address 的 ip地址为node1-master地址
kubeadm init \--apiserver-advertise-address=192.168.35.130 \--image-repository registry.aliyuncs.com/google_containers \--kubernetes-version v1.21.0 \--service-cidr=10.96.0.0/12 \--pod-network-cidr=10.244.0.0/16 \--ignore-preflight-errors=all      
#-–apiserver-advertise-address 集群通告地址（master内网） 注意修改为master节点的address
#–-image-repository 由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里指定阿里云镜像仓库地址
#–-kubernetes-version K8s版本，与上面安装的一致
#–-service-cidr 集群内部虚拟网络，Pod统一访问入口
#-–pod-network-cidr Pod网络，与下面部署的CNI网络组件yaml中保持一致```

1：执行如果发现报错，docker版本过高，可以根据推荐版本降级，不降级也没多大影响
2：执行如果发现docker pull 拉取不下来镜像 原因是centos停更，docker不支持
需要通过三方进行手动拉取命令行执行 根据缺失的docker镜像执行
假如拉取原始镜像命令如下：docker pull whyour/qinglong:latest
改为 docker pull dockerpull.org/whyour/qinglong:latest
拉去完成后 docker tag 改为的镜像名（dockerpull.org/whyour/qinglong:latest）  需要的镜像名
例如docker tag dockerpull.org/whyour/qinglong:latest whyour/qinglong:latest
执行完成后删除dockerpull.org/whyour/qinglong:latest

#初始化完成后：注意最后一行 保存下
#kubeadm join 192.168.35.130:6443 --token ms6617.5hbjusrd37ovnbh1 \ --discovery-token-ca-cert-hash sha256:1161e6266f482d5fdf436aa115a941657a2aee5ff0cc739d8c58adfc6a8441fb

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=/etc/kubernetes/admin.conf

只需要在node2等子节点执行

#node1-master初始化完成后的最后一行执行
kubeadm join 192.168.35.130:6443 --token ms6617.5hbjusrd37ovnbh1 \ --discovery-token-ca-cert-hash sha256:1161e6266f482d5fdf436aa115a941657a2aee5ff0cc739d8c58adfc6a8441fb

#token有效期为24小时，如果过期，需要重新创建token
kubeadm token create --print-join-command

#验证  能看到所有服务器
kubectl get nodes
kubectl get nodes -owide

只需要在node1-master执行

#本地下载 https://docs.projectcalico.org/manifests/calico.yaml 
#如果404请执行https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/calico.yaml
mkdir -p /z/k8s/calico
cd /z/k8s/calico
#文件calico.yaml 复制到/z/k8s/calico目录下
kubectl apply -f calico.yaml
# 查看运行状态
kubectl get pods -n kube-system
#状态非Runnning的都是没有创建成功的，关注在那个NODE下 查看具体的详细pod信息 ，
#那个NODE有问题去那个NODE执行或所有集群都执行一边
#可以使用kubectl describe pod -n kube-system podname 命令查看容器运行情况，如果是因为镜像下载失败，可以对应的集群节点使用docker pull命令将镜像下载下来
kubectl describe pod -n NAMESPACE名称 name名称
#执行如果发现docker pull 拉取不下来镜像 原因是centos停更，docker不支持
#需要通过三方进行手动拉取命令行执行 根据缺失的docker镜像执行
#假如拉取原始镜像命令如下：docker pull whyour/qinglong:latest
#改为 docker pull dockerpull.org/whyour/qinglong:latest
#拉去完成后 docker tag 改为的镜像名（dockerpull.org/whyour/qinglong:latest）  需要的镜像名
#例如docker tag dockerpull.org/whyour/qinglong:latest whyour/qinglong:latest
#执行完成后删除dockerpull.org/whyour/qinglong:latest
#等待几秒钟等待pod自动刷新，然后再次执行是否状态变为Runnning
kubectl get pods -n kube-system

四：（可选）搭建可视化页面----Dashboard

cd /z/k8s
mkdir dashboard
cd /z/k8s/dashboard
vi recommended.yaml
#编写内容
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.apiVersion: v1
kind: Namespace
metadata:name: kubernetes-dashboard---apiVersion: v1
kind: ServiceAccount
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboardnamespace: kubernetes-dashboard---kind: Service
apiVersion: v1
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboardnamespace: kubernetes-dashboard
spec:ports:- port: 443targetPort: 8443selector:k8s-app: kubernetes-dashboard---apiVersion: v1
kind: Secret
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboard-certsnamespace: kubernetes-dashboard
type: Opaque---apiVersion: v1
kind: Secret
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboard-csrfnamespace: kubernetes-dashboard
type: Opaque
data:csrf: ""---apiVersion: v1
kind: Secret
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboard-key-holdernamespace: kubernetes-dashboard
type: Opaque---kind: ConfigMap
apiVersion: v1
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboard-settingsnamespace: kubernetes-dashboard---kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboardnamespace: kubernetes-dashboard
rules:# Allow Dashboard to get, update and delete Dashboard exclusive secrets.- apiGroups: [""]resources: ["secrets"]resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]verbs: ["get", "update", "delete"]# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.- apiGroups: [""]resources: ["configmaps"]resourceNames: ["kubernetes-dashboard-settings"]verbs: ["get", "update"]# Allow Dashboard to get metrics.- apiGroups: [""]resources: ["services"]resourceNames: ["heapster", "dashboard-metrics-scraper"]verbs: ["proxy"]- apiGroups: [""]resources: ["services/proxy"]resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]verbs: ["get"]---kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboard
rules:# Allow Metrics Scraper to get metrics from the Metrics server- apiGroups: ["metrics.k8s.io"]resources: ["pods", "nodes"]verbs: ["get", "list", "watch"]---apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboardnamespace: kubernetes-dashboard
roleRef:apiGroup: rbac.authorization.k8s.iokind: Rolename: kubernetes-dashboard
subjects:- kind: ServiceAccountname: kubernetes-dashboardnamespace: kubernetes-dashboard---apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:name: kubernetes-dashboard
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: kubernetes-dashboard
subjects:- kind: ServiceAccountname: kubernetes-dashboardnamespace: kubernetes-dashboard---kind: Deployment
apiVersion: apps/v1
metadata:labels:k8s-app: kubernetes-dashboardname: kubernetes-dashboardnamespace: kubernetes-dashboard
spec:replicas: 1revisionHistoryLimit: 10selector:matchLabels:k8s-app: kubernetes-dashboardtemplate:metadata:labels:k8s-app: kubernetes-dashboardspec:containers:- name: kubernetes-dashboardimage: kubernetesui/dashboard:v2.3.1imagePullPolicy: IfNotPresentports:- containerPort: 8443protocol: TCPargs:- --auto-generate-certificates- --namespace=kubernetes-dashboard- --token-ttl=68400- --tls-key-file=apiserver.key- --tls-cert-file=apiserver.crt# Uncomment the following line to manually specify Kubernetes API server Host# If not specified, Dashboard will attempt to auto discover the API server and connect# to it. Uncomment only if the default does not work.# - --apiserver-host=http://my-address:portvolumeMounts:- name: kubernetes-dashboard-certsmountPath: /certs# Create on-disk volume to store exec logs- mountPath: /tmpname: tmp-volumelivenessProbe:httpGet:scheme: HTTPSpath: /port: 8443initialDelaySeconds: 30timeoutSeconds: 30securityContext:allowPrivilegeEscalation: falsereadOnlyRootFilesystem: truerunAsUser: 1001runAsGroup: 2001volumes:- name: kubernetes-dashboard-certssecret:secretName: kubernetes-dashboard-certs- name: tmp-volumeemptyDir: {}serviceAccountName: kubernetes-dashboardnodeSelector:"kubernetes.io/os": linux# Comment the following tolerations if Dashboard must not be deployed on mastertolerations:- key: node-role.kubernetes.io/mastereffect: NoSchedule---kind: Service
apiVersion: v1
metadata:labels:k8s-app: dashboard-metrics-scrapername: dashboard-metrics-scrapernamespace: kubernetes-dashboard
spec:ports:- port: 8000targetPort: 8000selector:k8s-app: dashboard-metrics-scraper---kind: Deployment
apiVersion: apps/v1
metadata:labels:k8s-app: dashboard-metrics-scrapername: dashboard-metrics-scrapernamespace: kubernetes-dashboard
spec:replicas: 1revisionHistoryLimit: 10selector:matchLabels:k8s-app: dashboard-metrics-scrapertemplate:metadata:labels:k8s-app: dashboard-metrics-scraperannotations:seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'spec:containers:- name: dashboard-metrics-scraperimage: kubernetesui/metrics-scraper:v1.0.6imagePullPolicy: IfNotPresentports:- containerPort: 8000protocol: TCPlivenessProbe:httpGet:scheme: HTTPpath: /port: 8000initialDelaySeconds: 30timeoutSeconds: 30volumeMounts:- mountPath: /tmpname: tmp-volumesecurityContext:allowPrivilegeEscalation: falsereadOnlyRootFilesystem: truerunAsUser: 1001runAsGroup: 2001serviceAccountName: kubernetes-dashboardnodeSelector:"kubernetes.io/os": linux# Comment the following tolerations if Dashboard must not be deployed on mastertolerations:- key: node-role.kubernetes.io/mastereffect: NoSchedulevolumes:- name: tmp-volumeemptyDir: {}

#方案一：
#由于docker问题需要手动拉取镜像
docker pull kubernetesui/metrics-scraper:v1.0.6
docker pull kubernetesui/dashboard:v2.3.1
#需要通过三方进行手动拉取命令行执行 根据缺失的docker镜像执行
#假如拉取原始镜像命令如下：docker pull whyour/qinglong:latest
#改为 docker pull dockerpull.org/whyour/qinglong:latest
#拉去完成后 docker tag 改为的镜像名（dockerpull.org/whyour/qinglong:latest）  需要的镜像名
#例如docker tag dockerpull.org/whyour/qinglong:latest whyour/qinglong:latest
#执行完成后删除dockerpull.org/whyour/qinglong:latest#方案二（需要建立到方案一的基础上，如果方案一执行完成，还是启动无法加载镜像采用方案二继续执行）：
docker save kubernetesui/dashboard:v2.3.1 -o dashboard.tar
docker save kubernetesui/metrics-scraper:v1.0.6 -o metrics-scraper.tar
sudo ctr -n k8s.io images import dashboard.tar
sudo ctr -n k8s.io images import metrics-scraper.tar

#部署：
kubectl apply -f recommended.yaml
#更改外部访问
kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
#type: ClusterIP 改为 type: NodePort

kubectl get pod,svc -n kubernetes-dashboard
#pod如果非RUNNING都是没有成功，可以使用kubectl describe pod -n kube-system podname 命令查看容器运行情况
#可以看到443映射给新端口，通过master ip进行访问
#注意一定是https://ip:端口
#由于dashboard证书问题，需要临时使用火狐浏览器进行访问

#创建用户
cd /z/k8s/dashboard
mkdir user
cd /z/k8s/dashboard/user
#创建用户yaml
vi dashboard-root.yaml
#编写内容
apiVersion: v1
kind: ServiceAccount
metadata:name: dashboard-rootnamespace: kube-system
#创建用户角色yaml
vi dashboard-root-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:name: dashboard-admin-binding
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: cluster-admin
subjects:
- kind: ServiceAccountname: dashboard-rootnamespace: kube-system

#部署：
kubectl apply -f dashboard-root.yaml
kubectl apply -f dashboard-root-role.yaml
#获取token
kubectl -n kube-system get secret $(kubectl -n kube-system get sa dashboard-root -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode
#注意保存token
eyJhbGciOiJSUzI1NiIsImtpZCI6ImRvRXBBcS1hTGVOb1VjTF95YzliSUtleDlOOEpSZndDSTE5ajBXX2xXNlUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtcm9vdC10b2tlbi10cmt4dCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJkYXNoYm9hcmQtcm9vdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjkwYzdkYjZlLTRmNDEtNGRlNy1iMzc5LWU2YzIyODczYjE3NSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTpkYXNoYm9hcmQtcm9vdCJ9.C59IlVhFAd8k9t7TqR6z9mwexzkBh86-peJLda2ELYbn72mS8gBsRkfIzfSA64qHW2ajMPFgFes-RZDSjsuekNg2xJe-k952lmJqUdThxtauGzemK0OCCy6UKSHPkkWzXaHtH6juMOGPX4k5C0tUv_9ro5LfnotjPNvCrUsk-BqJssT6Pe62jFHcxpxLzaM50IIpbR1gTMU9OKaN80cb-e4HXx633oXCsO3zy_8tUELX3jDmW7NTIq-7CACMGXZW6xChgSGtBxbueYENUJ06X1Z7p8nix7uOhoxYVw0QaUBc6sgBP0HMqbLPg9HWqvJe_XqZTp2GkgjBOgrFLCyC_g

#注意：kubernetes-dashboard可视化页面使用token进行登录, 观看右上角是否有消息，如果为空代表成功，如果不为空，根据情况自行修改yaml文件

#登录提示401 token有问题
kubectl get secret -n kube-system 
找到对应的用户名
kubectl describe secret -n kube-system 
#获取里面的token

 #注意：可视化访问目前支持火狐访问，如果需要支持谷歌访问，可以使用自带的k8s证书实现
#删除默认的secret，用自签证书创建新的secret
kubectl delete secret kubernetes-dashboard-certs -n kubernetes-dashboard
kubectl create secret generic kubernetes-dashboard-certs \
--from-file=/etc/kubernetes/pki/apiserver.key --from-file=/etc/kubernetes/pki/apiserver.crt -n kubernetes-dashboard
#在recommended.yaml文件中args位置添加（如果使用上面文件，博主已加）
cd /z/k8s/dashboard
vi recommended.yaml
#添加下面内容
args:# PLATFORM-SPECIFIC ARGS HERE- --auto-generate-certificates- --tls-key-file=apiserver.key- --tls-cert-file=apiserver.crt
#重新部署
kubectl apply -f recommended.yaml
#清理旧pod
kubectl get pod -n kubernetes-dashboard | grep -v NAME | awk '{print "kubectl delete po " $1 " -n kubernetes-dashboard"}' | sh
#可以通过谷歌浏览器进行访问，如果访问不了，端口应该发生了改变 https://ip:端口
kubectl get pod,svc -n kubernetes-dashboard
#如443无映射重新执行更改外部访问
kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
#type: ClusterIP 改为 type: NodePort
#再次执行获取新端口
kubectl get pod,svc -n kubernetes-dashboard
#通过谷歌浏览器进行访问，如果访问不了，端口应该发生了改变 https://ip:端口
#原token无法登录执行重新绑定用户
cd /z/k8s/dashboard/user
kubectl apply -f dashboard-root.yaml
kubectl apply -f dashboard-root-role.yaml
#重新获取token
kubectl -n kube-system get secret $(kubectl -n kube-system get sa dashboard-root -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode