记录一次k8s pod之间ip无法访问，问题排查与定位

在这里插入图片描述

问题展现现象

node之间通信正常
部分node上的pod无法通信

排查有问题node

使用启动网络测试工具

环境准备

docker

数据库mysql

使用有状态副本集合

---
apiVersion: apps/v1
kind: StatefulSet
metadata:annotations:k8s.kuboard.cn/displayName: 有状态内网穿透集群labels:k8s.kuboard.cn/layer: svck8s.kuboard.cn/name: network-server-cluster-startname: network-server-cluster-startnamespace: default
spec:podManagementPolicy: OrderedReadyreplicas: 10revisionHistoryLimit: 10selector:matchLabels:k8s.kuboard.cn/layer: svck8s.kuboard.cn/name: network-server-cluster-startserviceName: network-server-cluster-starttemplate:metadata:labels:k8s.kuboard.cn/layer: svck8s.kuboard.cn/name: network-server-cluster-startspec:affinity:podAntiAffinity:requiredDuringSchedulingIgnoredDuringExecution:- labelSelector:matchLabels:k8s.kuboard.cn/layer: svck8s.kuboard.cn/name: network-server-cluster-starttopologyKey: kubernetes.io/hostnamecontainers:- env:- name: spring.datasource.urlvalue: >-jdbc:mysql://mysql-host:mysql-port/wu_lazy_cloud_netty_server_cluster?allowMultiQueries=true&useUnicode=true&autoReconnect=true&useAffectedRows=true&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&databaseTerm=SCHEMA- name: JAVA_OPTSvalue: '-Xms64m -Xmx128m'- name: spring.datasource.usernamevalue: root- name: spring.datasource.passwordvalue: laihui- name: spring.lazy.netty.server.node-idvalueFrom:fieldRef:apiVersion: v1fieldPath: spec.nodeName- name: spring.lazy.netty.server.node-portvalue: '7101'envFrom:- configMapRef:name: wu-smart-agent-network-heartbeat-server-cluster-start-confimage: >-registry.cn-hangzhou.aliyuncs.com/wu-lazy/wu-smart-agent-network-heartbeat-server-cluster-start:1.2.5-JDK17-NATIVE-SNAPSHOTimagePullPolicy: Alwaysname: network-server-cluster-startports:- containerPort: 7101hostPort: 7101name: tcp7101protocol: TCPresources: {}terminationMessagePath: /dev/termination-logterminationMessagePolicy: FilednsPolicy: ClusterFirstrestartPolicy: AlwaysschedulerName: default-schedulersecurityContext: {}terminationGracePeriodSeconds: 30updateStrategy:rollingUpdate:partition: 0type: RollingUpdate---
apiVersion: v1
kind: Service
metadata:annotations: {}labels:k8s.kuboard.cn/layer: svck8s.kuboard.cn/name: network-server-cluster-startname: network-server-cluster-startnamespace: default
spec:ipFamilyPolicy: SingleStackports:- name: 6eqe4dport: 7101protocol: TCPtargetPort: 7101selector:k8s.kuboard.cn/layer: svck8s.kuboard.cn/name: network-server-cluster-startsessionAffinity: Nonetype: ClusterIP---
apiVersion: v1
data:spring.lazy.netty.server.node-host: '${HOSTNAME}.network-server-cluster-start.default.svc.cluster.local'
kind: ConfigMap
metadata:name: wu-smart-agent-network-heartbeat-server-cluster-start-confnamespace: default---
apiVersion: v1
kind: Service
metadata:annotations: {}name: network-server-cluster-start-webnamespace: default
spec:ports:- name: sjmxmanodePort: 33201port: 6101protocol: TCPtargetPort: 6101selector:k8s.kuboard.cn/name: network-server-cluster-startsessionAffinity: Nonetype: NodePort