使用Prometheus对微服务性能自定义指标监控

背景

随着云计算和容器化技术的不断发展，微服务架构逐渐成为现代软件开发的主流趋势。微服务架构将大型应用程序拆分成多个小型、独立的服务，每个服务都可以独立开发、部署和扩展。这种架构模式提高了系统的可伸缩性、灵活性和可靠性，但同时也带来了服务监控和管理的挑战。

在微服务架构中，服务之间的依赖关系变得复杂，服务数量众多，因此需要一种有效的监控和管理工具来确保系统的稳定性和可靠性。监控工具可以帮助开发人员实时了解服务的运行状态、性能指标和异常情况，从而及时发现问题并进行处理。同时，管理工具还可以提供自动化的部署、配置和扩展功能，提高开发效率和运维质量。

Prometheus的优势

请求、数据库查询、消息队列等应用指标。
高效数据存储：Prometheus采用时间序列数据库（TSDB）来存储监控数据，具有高效的数据压缩和查询性能。
丰富查询语言：Prometheus提供了强大的数据查询语言PromQL，可以方便地对监控数据进行过滤、聚合和计算。
灵活告警机制：Prometheus支持基于规则的告警机制，可以根据监控数据的阈值触发告警通知，支持多种告警方式，如邮件、短信、Slack等。

springBoot集成Prometheus

导入Pom依赖

<dependency><groupId>org.mybatis.spring.boot</groupId><artifactId>spring-boot-starter-actuator</artifactId><version>2.2.1.RELEASE</version>
</dependency>
<dependency><groupId>io.micrometer</groupId><artifactId>micrometer-registry-prometheus</artifactId><version>1.3.1</version>
</dependency>

修改springBoot配置文件

开启prometheus监控配置

management:endpoint:prometheus:enabled: trueendpoints:web:exposure:include: 'prometheus'

修改默认的Prometheus监控度量名称

prometheus默认指标中有个http.server.requests的度量名称，记录了http请求调用情况；现在以这个为例，修改名称

新建一个@Configuration 类 PrometheusConfig

import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.config.NamingConvention;
import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Collectors;@Configuration
public class PrometheusConfig {/** 用于替换 Prometheus中 公共的 http.server.requests度量名替换* @return*/@BeanMeterRegistryCustomizer<MeterRegistry> metricsConfig() {return registry -> registry.config().namingConvention(new NamingConvention() {@Overridepublic String name(String name, Meter.Type type, String baseUnit) {String collect = "";if(name.contains("http.server.requests")){collect = Arrays.stream(name.replaceAll("http.server.requests", "jiang.xiao.yu.http").split("\\.")).filter(Objects::nonNull).collect(Collectors.joining("_"));}else {collect = Arrays.stream(name.split("\\.")).filter(Objects::nonNull).collect(Collectors.joining("_"));}return collect;}});}
}

自定义Prometheus监控指标

使用拦截器监控指标

利用拦截器实现所有HTTP接口的监控

利用HTTP的拦截器添加Prometheus的监控指标，首先创建一个拦截器CustomInterceptor 实现HandlerInterceptor接口，然后重写里面的前置处理、后置处理；

import io.micrometer.core.instrument.*;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.ModelAndView;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.ToDoubleFunction;public class CustomInterceptor implements HandlerInterceptor {private static final String CUSTOM_KPI_NAME_TIMER = "custom.kpi.timer"; //耗时private static final String CUSTOM_KPI_NAME_COUNTER = "custom.kpi.counter"; //api调用次数。private static final String CUSTOM_KPI_NAME_SUMMARY = "custom.kpi.summary"; //汇总率private static MeterRegistry registry;private long startTime;private GaugeNumber gaugeNumber = new GaugeNumber();void getRegistry(){if(registry == null){//这里使用的时SpringUtil获取Bean，没有用@Autowired注解，Autowired会因为加载时机问题导致拿不到；SpringUtil.getBean网上实现有很多，可以自行搜索；registry = SpringUtil.getBean(MeterRegistry.class);}}@Overridepublic boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {getRegistry();//记录接口开始调用的时间startTime = System.currentTimeMillis();return HandlerInterceptor.super.preHandle(request, response, handler);}@Overridepublic void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception {HandlerInterceptor.super.postHandle(request, response, handler, modelAndView);}@Overridepublic void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {//统计调用次数registry.counter(CUSTOM_KPI_NAME_COUNTER,"uri", request.getRequestURI(), "method", request.getMethod(),"status", response.getStatus() + "", "exception", ex == null ? "" : ex.getMessage(), "outcome", response.getStatus() == 200 ? "SUCCESS" : "CLIENT_ERROR").increment();//统计单次耗时registry.timer(CUSTOM_KPI_NAME_TIMER,"uri", request.getRequestURI(), "method", request.getMethod(),"status", response.getStatus() + "", "exception", ex == null ? "" : ex.getMessage(), "outcome", response.getStatus() == 200 ? "SUCCESS" : "CLIENT_ERROR").record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);//统计调用成功率，根据过滤Counter对象，获取计数Collection<Meter> meters = registry.get(CUSTOM_KPI_NAME_COUNTER).tag("uri", request.getRequestURI()).tag("method", request.getMethod()).meters();double total = 0;double success = 0;for (Meter meter : meters) {Counter counter = (Counter) meter;total += counter.count();String status = meter.getId().getTag("status");if (status.equals("200")){success+= counter.count();}}//保存对应的成功率到Map中String key = request.getMethod() + request.getRequestURI();gaugeNumber.setPercent(key, Double.valueOf(success / total * 100L));registry.gauge(CUSTOM_KPI_NAME_SUMMARY, Tags.of("uri", request.getRequestURI(), "method", request.getMethod()), gaugeNumber, new ToDoubleFunction<GaugeNumber>() {@Overridepublic double applyAsDouble(GaugeNumber value) {return value.getPercent(key);}});HandlerInterceptor.super.afterCompletion(request, response, handler, ex);}// gauge监控某个对象，所以用内部类替代，然后根据tag标签区分对应的成功率；key 为 method + uriclass GaugeNumber {Map<String,Double> map = new HashMap<>();public Double getPercent(String key) {return map.get(key);}public void setPercent(String key, Double percent) {map.put(key, percent);}}
}

注册自定义拦截器给Spring

import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;@Configuration
public class CustomInterceptors implements WebMvcConfigurer {@Overridepublic void addInterceptors(InterceptorRegistry registry) {registry.addInterceptor(new CustomInterceptor()).addPathPatterns("/**");}
}

大功告成，启动程序测试吧

使用AOP记录监控指标

自定义指标注解

@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface MethodMetrics {String name() default "";String desc() default "";String[] tags() default {};//是否记录时间间隔boolean withoutDuration() default false;
}

切面实现

@Aspect
public class PrometheusAnnotationAspect {@Autowiredprivate MeterRegistry meterRegistry;@Pointcut("@annotation(com.smac.prometheus.annotation.MethodMetrics)")public void pointcut() {}@Around(value = "pointcut()")public Object process(ProceedingJoinPoint joinPoint) throws Throwable {Method targetMethod = ((MethodSignature) joinPoint.getSignature()).getMethod();Method currentMethod = ClassUtils.getUserClass(joinPoint.getTarget().getClass()).getDeclaredMethod(targetMethod.getName(), targetMethod.getParameterTypes());if (currentMethod.isAnnotationPresent(MethodMetrics.class)) {MethodMetrics methodMetrics = currentMethod.getAnnotation(MethodMetrics.class);return processMetric(joinPoint, currentMethod, methodMetrics);} else {return joinPoint.proceed();}}private Object processMetric(ProceedingJoinPoint joinPoint, Method currentMethod, MethodMetrics methodMetrics) {String name = methodMetrics.name();if (!StringUtils.hasText(name)) {name = currentMethod.getName();}String desc = methodMetrics.desc();if (!StringUtils.hasText(desc)) {desc = currentMethod.getName();}//不需要记录时间if (methodMetrics.withoutDuration()) {Counter counter = Counter.builder(name).tags(methodMetrics.tags()).description(desc).register(meterRegistry);try {return joinPoint.proceed();} catch (Throwable e) {throw new IllegalStateException(e);} finally {counter.increment();}}//需要记录时间（默认）Timer timer = Timer.builder(name).tags(methodMetrics.tags()).description(desc).register(meterRegistry);return timer.record(() -> {try {return joinPoint.proceed();} catch (Throwable e) {throw new IllegalStateException(e);}});}
}

在需要记监控的地方加上这个注解

@MethodMetrics(name="sms_send",tags = {"vendor"})
public void send(String mobile, SendMessage message) throws Exception {//do something
}