在k8s中部署一个可外部访问的Redis Sentinel

1.前提条件：

1.部署了multus
想要k8s外部能访问k8s内部的redis，redis-server启动时必须使用multus的IP
2.helm客户端安装

2.开始安装

准备3个multus ip

10.10.10.130
10.10.10.131
10.10.10.132

apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:name: net10-130-132namespace: default
spec:config: |-{"cniVersion": "0.3.1","name": "net10-130-132","type": "macvlan","master": "ens224","mode": "bridge","ipam": {"type": "whereabouts","range": "10.10.10.0/24", "range_start": "10.10.10.130","range_end": "10.10.10.132","routes": [{ "dst": "10.206.0.0/16", "gw": "10.10.10.1" }]}}

通过helm部署redis

helm pull bitnami/redis --version=16.13.2
helm install redis -f values.yaml .

修改values.yaml

global:imageRegistry: ""imagePullSecrets: []storageClass: "vsan-csi" ##设置storage classredis:password: "password"  ##设置redis密码
kubeVersion: ""
nameOverride: ""
fullnameOverride: ""
commonLabels: {}
commonAnnotations: {}
secretAnnotations: {}
clusterDomain: cluster.local
extraDeploy: []
diagnosticMode:enabled: falsecommand:- sleepargs:- infinity
image:registry: docker.iorepository: bitnami/redistag: 6.2.7-debian-11-r11pullPolicy: IfNotPresentpullSecrets: []debug: falsearchitecture: replication
auth:enabled: true #开启redis密码sentinel: false ##关闭sentinel密码password: "password"existingSecret: ""existingSecretPasswordKey: ""usePasswordFiles: falsecommonConfiguration: |-# Enable AOF https://redis.io/topics/persistence#append-only-fileappendonly yes# Disable RDB persistence, AOF persistence already enabled.save ""
existingConfigmap: ""master:count: 1configuration: ""disableCommands:- FLUSHDB- FLUSHALLcommand: []args: []preExecCmds: []extraFlags: []extraEnvVars: []extraEnvVarsCM: ""extraEnvVarsSecret: ""containerPorts:redis: 6379startupProbe:enabled: falseinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 5successThreshold: 1failureThreshold: 5livenessProbe:enabled: trueinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 5successThreshold: 1failureThreshold: 5readinessProbe:enabled: trueinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 1successThreshold: 1failureThreshold: 5customStartupProbe: {}customLivenessProbe: {}customReadinessProbe: {}resources: ##设置master资源大小limits:cpu: 1000mmemory: 4096MiBrequests: cpu: 1000mmemory: 4096MiBpodSecurityContext:enabled: truefsGroup: 1001containerSecurityContext:enabled: truerunAsUser: 1001kind: StatefulSetschedulerName: ""updateStrategy:type: RollingUpdaterollingUpdate: {}priorityClassName: ""hostAliases: []podLabels: {}podAnnotations: {}shareProcessNamespace: falsepodAffinityPreset: ""podAntiAffinityPreset: softnodeAffinityPreset:type: ""key: ""values: []affinity: {}nodeSelector: {}tolerations: []topologySpreadConstraints: []dnsPolicy: ""dnsConfig: {}lifecycleHooks: {}extraVolumes: []extraVolumeMounts: []sidecars: []initContainers: []persistence:enabled: truemedium: ""sizeLimit: ""path: /datasubPath: ""storageClass: "vsan-csi"accessModes:- ReadWriteOncesize: 10Giannotations: {}selector: {}dataSource: {}existingClaim: ""service:type: ClusterIPports:redis: 6379nodePorts:redis: ""externalTrafficPolicy: ClusterextraPorts: []internalTrafficPolicy: ClusterclusterIP: ""loadBalancerIP: ""loadBalancerSourceRanges: []annotations: {}sessionAffinity: NonesessionAffinityConfig: {}terminationGracePeriodSeconds: 30replica:replicaCount: 3configuration: ""disableCommands:- FLUSHDB- FLUSHALLcommand: []args: []preExecCmds: []extraFlags: []extraEnvVars: []extraEnvVarsCM: ""extraEnvVarsSecret: ""externalMaster:enabled: falsehost: ""port: 6379containerPorts:redis: 6379startupProbe:enabled: trueinitialDelaySeconds: 10periodSeconds: 10timeoutSeconds: 5successThreshold: 1failureThreshold: 22livenessProbe:enabled: trueinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 5successThreshold: 1failureThreshold: 5readinessProbe:enabled: trueinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 1successThreshold: 1failureThreshold: 5customStartupProbe: {}customLivenessProbe: {}customReadinessProbe: {}resources:limits: cpu: 250mmemory: 256Mirequests: cpu: 250mmemory: 256MipodSecurityContext:enabled: truefsGroup: 1001containerSecurityContext:enabled: truerunAsUser: 1001schedulerName: ""updateStrategy:type: RollingUpdaterollingUpdate: {}priorityClassName: ""podManagementPolicy: ""hostAliases: []podLabels: {}podAnnotations: {}shareProcessNamespace: falsepodAffinityPreset: ""podAntiAffinityPreset: softnodeAffinityPreset:type: ""key: ""values: []affinity: {}nodeSelector: {}tolerations: []topologySpreadConstraints: []dnsPolicy: ""dnsConfig: {}lifecycleHooks: {}extraVolumes: []extraVolumeMounts: []sidecars: []initContainers: []persistence:enabled: truemedium: ""sizeLimit: ""path: /datasubPath: ""storageClass: "vsan-csi"accessModes:- ReadWriteOncesize: 10Giannotations: {}selector: {}dataSource: {}existingClaim: ""service:type: ClusterIPports:redis: 6379nodePorts:redis: ""externalTrafficPolicy: ClusterinternalTrafficPolicy: ClusterextraPorts: []clusterIP: ""loadBalancerIP: ""loadBalancerSourceRanges: []annotations: {}sessionAffinity: NonesessionAffinityConfig: {}terminationGracePeriodSeconds: 30autoscaling:enabled: falseminReplicas: 1maxReplicas: 11targetCPU: ""targetMemory: ""sentinel:enabled: trueimage:registry: docker.iorepository: bitnami/redis-sentineltag: 6.2.7-debian-11-r12pullPolicy: IfNotPresentpullSecrets: []debug: falsemasterSet: mymasterquorum: 2getMasterTimeout: 220automateClusterRecovery: falsedownAfterMilliseconds: 60000failoverTimeout: 18000parallelSyncs: 1configuration: ""command: []args: []preExecCmds: []extraEnvVars: []extraEnvVarsCM: ""extraEnvVarsSecret: ""externalMaster:enabled: falsehost: ""port: 6379containerPorts:sentinel: 26379startupProbe:enabled: trueinitialDelaySeconds: 10periodSeconds: 10timeoutSeconds: 5successThreshold: 1failureThreshold: 22livenessProbe:enabled: trueinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 5successThreshold: 1failureThreshold: 5readinessProbe:enabled: trueinitialDelaySeconds: 20periodSeconds: 5timeoutSeconds: 1successThreshold: 1failureThreshold: 5customStartupProbe: {}customLivenessProbe: {}customReadinessProbe: {}persistence:enabled: falsestorageClass: ""accessModes:- ReadWriteOncesize: 100Miannotations: {}selector: {}dataSource: {}medium: ""resources:limits: cpu: 250mmemory: 256Mirequests: cpu: 250mmemory: 256MicontainerSecurityContext:enabled: truerunAsUser: 1001lifecycleHooks: {}extraVolumes: []extraVolumeMounts: []service:type: ClusterIPports:redis: 6379sentinel: 26379nodePorts:redis: ""sentinel: ""externalTrafficPolicy: ClusterextraPorts: []clusterIP: ""loadBalancerIP: ""loadBalancerSourceRanges: []annotations: {}sessionAffinity: NonesessionAffinityConfig: {}terminationGracePeriodSeconds: 30networkPolicy:enabled: falseallowExternal: trueextraIngress: []extraEgress: []ingressNSMatchLabels: {}ingressNSPodMatchLabels: {}
podSecurityPolicy:create: falseenabled: false
rbac:create: falserules: []
serviceAccount:create: truename: ""automountServiceAccountToken: trueannotations: {}
pdb:create: falseminAvailable: 1maxUnavailable: ""
tls:enabled: falseauthClients: trueautoGenerated: falseexistingSecret: ""certificatesSecret: ""certFilename: ""certKeyFilename: ""certCAFilename: ""dhParamsFilename: ""metrics:enabled: trueimage:registry: docker.iorepository: bitnami/redis-exportertag: 1.43.0-debian-11-r4pullPolicy: IfNotPresentpullSecrets: []command: []redisTargetHost: "localhost"extraArgs: {}extraEnvVars: []containerSecurityContext:enabled: truerunAsUser: 1001extraVolumes: []extraVolumeMounts: []resources:limits: cpu: 250mmemory: 256Mirequests: cpu: 250mmemory: 256MipodLabels: {}podAnnotations:prometheus.io/scrape: "true"prometheus.io/port: "9121"service:type: ClusterIPport: 9121externalTrafficPolicy: ClusterextraPorts: []loadBalancerIP: ""loadBalancerSourceRanges: []annotations: {}serviceMonitor:enabled: truenamespace: ""interval: 30sscrapeTimeout: ""relabellings: []metricRelabelings: []honorLabels: falseadditionalLabels: {}prometheusRule:enabled: truenamespace: ""additionalLabels: {}rules:- alert: RedisDownexpr: redis_up{service="{{ template "common.names.fullname" . }}-metrics"} == 0for: 2mlabels:severity: errorannotations:summary: Redis&reg; instance {{ "{{ $labels.instance }}" }} downdescription: Redis&reg; instance {{ "{{ $labels.instance }}" }} is down- alert: RedisMemoryHighexpr: >redis_memory_used_bytes{service="{{ template "common.names.fullname" . }}-metrics"} * 100/redis_memory_max_bytes{service="{{ template "common.names.fullname" . }}-metrics"}> 90for: 2mlabels:severity: errorannotations:summary: Redis&reg; instance {{ "{{ $labels.instance }}" }} is using too much memorydescription: |Redis&reg; instance {{ "{{ $labels.instance }}" }} is using {{ "{{ $value }}" }}% of its available memory.- alert: RedisKeyEvictionexpr: |increase(redis_evicted_keys_total{service="{{ template "common.names.fullname" . }}-metrics"}[5m]) > 0for: 1slabels:severity: errorannotations:summary: Redis&reg; instance {{ "{{ $labels.instance }}" }} has evicted keysdescription: |Redis&reg; instance {{ "{{ $labels.instance }}" }} has evicted {{ "{{ $value }}" }} keys in the last 5 minutes.
volumePermissions:enabled: falseimage:registry: docker.iorepository: bitnami/bitnami-shelltag: 11-debian-11-r11pullPolicy: IfNotPresentpullSecrets: []resources:limits: {}requests: {}containerSecurityContext:runAsUser: 0sysctl:enabled: falseimage:registry: docker.iorepository: bitnami/bitnami-shelltag: 11-debian-11-r11pullPolicy: IfNotPresentpullSecrets: []command: []mountHostSys: falseresources:limits: {}requests: {}useExternalDNS:enabled: falsesuffix: ""annotationKey: external-dns.alpha.kubernetes.io/additionalAnnotations: {}

修改configMap redis-scripts
将

get_full_hostname() {hostname="$1"echo "${hostname}.${HEADLESS_SERVICE}"
}

替换为

get_full_hostname() {hostname="$1"if [[ "${hostname}" =~ 0$ ]]; thenecho "10.10.10.130"fiif [[ "${hostname}" =~ 1$ ]]; thenecho "10.10.10.131"fiif [[ "${hostname}" =~ 2$ ]]; thenecho "10.10.10.132"fi
}

然后将statefulset的replica从3改为0，

spec:template:metadata:annotations:k8s.v1.cni.cncf.io/networks: default/net10-130-132

再从0改为3。

3.测试连接：

测试工具 Another Redis Desktop Manager
在这里插入图片描述

连接OK

4.Springboot 连接redis

新建一个springboot项目
关键信息如下

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.xxxx</groupId><artifactId>redis-test</artifactId><version>0.0.1-SNAPSHOT</version><name>redis-test</name><description>redis-test</description><properties><java.version>1.8</java.version><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding><spring-boot.version>2.6.13</spring-boot.version></properties><dependencies><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-data-redis</artifactId></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-web</artifactId></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-test</artifactId><scope>test</scope></dependency><dependency><groupId>org.projectlombok</groupId><artifactId>lombok</artifactId></dependency><!--redis连接池--><dependency><groupId>org.apache.commons</groupId><artifactId>commons-pool2</artifactId></dependency></dependencies><dependencyManagement><dependencies><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-dependencies</artifactId><version>${spring-boot.version}</version><type>pom</type><scope>import</scope></dependency></dependencies></dependencyManagement><build><plugins><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><version>3.8.1</version><configuration><source>1.8</source><target>1.8</target><encoding>UTF-8</encoding></configuration></plugin><plugin><groupId>org.springframework.boot</groupId><artifactId>spring-boot-maven-plugin</artifactId><version>${spring-boot.version}</version>
<!--                <configuration>-->
<!--                    <mainClass>com.aecqauto.redistest.RedisTestApplication</mainClass>-->
<!--                    <skip>true</skip>-->
<!--                </configuration>--><executions><execution><id>repackage</id><goals><goal>repackage</goal></goals></execution></executions></plugin></plugins></build>
</project>

application.yml

server:port: 8080spring:redis:#password: passwordlettuce:pool:# 连接池最大连接数(使用负值表示没有限制) 默认为8max-active: 8# 连接池中的最大空闲连接 默认为8max-idle: 8# 连接池最大阻塞等待时间(使用负值表示没有限制) 默认为-1max-wait: -1ms# 连接池中的最小空闲连接 默认为 0min-idle: 0sentinel:# 主节点的别名master: mymasterpassword: password# sentinel服务的ip和端口nodes:- 10.10.10.130:26379- 10.10.10.131:26379- 10.10.10.132:26379

RedisController.java

package com.aecqauto.redistest.demos.web;import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;@RestController
@RequestMapping("/redis")
public class RedisController {// 使用SpringBoot封装的RestTemplate对象@AutowiredRedisTemplate<String, String> redisTemplate;@RequestMapping("/get")public String get(String key) {String value = redisTemplate.opsForValue().get(key);return value;}@RequestMapping("/set")public String set(String key, String value) {redisTemplate.opsForValue().set(key, value);return "success";}
}