一、需求：错误日志Top10告警发送
二、需求分解
- jsoup实现登录，获取到cookie和token等用户鉴权信息
- 获取接口相应的key值
- 调用日志平台错误日志Top榜接口，查询到结果集
- 调用企业微信机器人发送消息接口
- 加上定时任务，可以实现定时发送错误日志告警的功能
定时任务高度设置1小时（根据实际需要）执行一次，可以实现自动巡检错误日志的功能。以便及时发现生产问题
jsoup是java的爬虫框架，可以爬取网页数据，这里没有重点使用，只是做了个登录功能。后续可以专门它写一份爬虫的程序。。
<dependency><groupId>org.jsoup</groupId><artifactId>jsoup</artifactId><version>1.12.1</version></dependency>package com.smy.cbs.task;import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.smy.cbs.util.RetryUtil;
import com.smy.smyx.scheduler.dto.TaskRequest;
import com.smy.smyx.scheduler.dto.TaskResponse;
import com.smy.smyx.scheduler.service.TaskService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;import javax.annotation.Resource;
import javax.net.ssl.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.stream.Collectors;/*** @author youlu* @ClassName logCenterTask* @Date 2023/11/22 17:14* @Version V1.0**/
@Slf4j
@Service("logCenterTask")
public class LogCenterTask implements TaskService {@Resourceprivate RetryUtil retryUtil;@Value("${log.usename:aaaa}")public String USER_NAME;@Value("${log.password:bbbbbb}")public String PASSWORD;@Value("${log.content.warn.max:24000000}")public int maxCount;//达到阈值则告警@Value("${log.content.filter.content:输入要过滤的内容有英文逗号隔开}'.split(',')}")public List<String> filterContent = Lists.newArrayList();//过滤内容@Value("${log.content.length:240}")public int SUB_CONTENT_LENGTH = 240;//截取报错内容字符串长度@Value("${log.title.length:20}")public int SUB_TITLE_LENGTH = 20;//截取报错类型字符串长度@Value("${log.period.hour:24}")public int PERIOD_HOUR;//24小时内的错误日志@Value("#{'${log.search.system:cbs_core,rls_core,uts_core,adv_core}'.split(',')}")public List<String> SEARCH_SYSTEM;//查询的系统@Value("#{'${log.wx.url:https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=aaaaaa,https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=bbbbbbbb3}'.split(',')}")public String WX_ROBOT_URL;public static final String LOG_CENTER_BASE_URL = "https://log_center.xxx.com/api/";public static String X_Csrf_Token = "";public static String COOKIE = "";public static String id = "";public static final String TEMPLATE = "{\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"%s\"}}";public static final String QUERY_TEMPLATE = "{\n" +" \"app\": \"system\",\n" +" \"source\": \"other\",\n" +" \"query\": \"repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n| where level=\\\"ERROR\\\"\\n| eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n| stats count() as num by err_type\\n| sort 10 by num\\n| join type=inner err_type [\\n repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n | where level=\\\"ERROR\\\"\\n | eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n | fields + err_type, _raw\\n | dedup err_type\\n]\\n| rename _raw as 原始日志, num as 统计, err_type as 错误类型\",\n" +" \"mode\": \"smart\",\n" +" \"preview\": false,\n" +" \"collectSize\": -1,\n" +" \"timeout\": 1000,\n" +" \"sorts\": []\n" +"}";@SneakyThrows@Overridepublic TaskResponse execute(TaskRequest request) {//1.模拟登录 jsoupjsoupLogin();for (String systemName : SEARCH_SYSTEM) {try {Thread.sleep(1000);//2.获取keyString id = getKey(systemName);Thread.sleep(8000);List<Map<String, Object>> logCenterList = retryUtil.doRetry(4, () -> {//3.获取查询结果List<Map<String, Object>> contentList = getContentList(systemName, id);if (CollectionUtils.isEmpty(contentList)) {Thread.sleep(8000);throw new Exception(systemName + "未查询到数据，需要重试！");}return contentList;}, systemName + "获取日志数据");//4.调微信发送短信sendWxMessage(systemName, id, logCenterList);} catch (Exception e) {log.error("{}发送错误日志告警出现异常e:{}", systemName, e);}}return new TaskResponse(TaskResponse.SUCCESS, "logCenter任务完成");}public String getKey(String systemName) {Date endDate = new Date();Date startDate = DateUtil.offsetHour(endDate, -PERIOD_HOUR);//请求参数JSONObject paramJson = JSON.parseObject(String.format(QUERY_TEMPLATE, systemName, systemName));paramJson.put("startTime", startDate.getTime());paramJson.put("endTime", endDate.getTime());HttpResponse execute = HttpRequest.post(LOG_CENTER_BASE_URL + "jobs")//设置请求头(可任意加).header("X-Csrf-Token", X_Csrf_Token).header("Cookie", COOKIE).header("Content-Type", "application/json").header("Connection", "keep-alive")//请求参数.body(paramJson.toJSONString()).timeout(40000).execute();String body1 = execute.body();String id = JSON.parseObject(body1).getString("id");return id;}public List<Map<String, Object>> getContentList(String systemName, String id) {String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";HttpResponse execute = HttpRequest.get(url)//设置请求头(可任意加).header("X-Csrf-Token", X_Csrf_Token).header("Cookie", COOKIE)//.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36").timeout(40000).execute();JSONArray rows = JSON.parseObject(execute.body()).getJSONArray("rows");System.err.println(rows.toString());List<Map<String, Object>> list = Lists.newArrayList();for (Object row : rows) {try {JSONArray jsonArray = JSON.parseArray(row.toString());String title = jsonArray.get(0).toString();Integer count = Integer.valueOf(jsonArray.get(1).toString());String content = jsonArray.get(2).toString();if (count >= maxCount) {continue;}if (filterContent.stream().anyMatch(k -> content.contains(k))) {continue;}int subContentLen = content.length() <= SUB_CONTENT_LENGTH ? content.length() : SUB_CONTENT_LENGTH;String subContent = content.substring(0, subContentLen);int subTitleLen = title.length() <= SUB_TITLE_LENGTH ? title.length() : SUB_TITLE_LENGTH;String subTitle = content.substring(0, subTitleLen);Map<String, Object> map = new HashMap<>();map.put("title", subTitle);map.put("count", count);map.put("content", subContent);list.add(map);} catch (Exception e) {log.error("{}获取日志统计数据出现异常e:{}", systemName, e);}}if (CollectionUtils.isEmpty(list)) {log.info("{}未获取到数据，请求链接url：{},token:{},cookie:{}", systemName, url, X_Csrf_Token, COOKIE);}return list;}public void sendWxMessage(String systemName, String id, List<Map<String, Object>> contentList) {if (CollectionUtils.isEmpty(contentList)) {log.error("{}发送内容为空，不发送机器人微信消息,id:{}", systemName, id);return;}String periodDesc = getPeriodDesc();final int[] topNum = {1};String StringContent = contentList.stream().map(k -> {String title = (String) k.get("title");Integer count = (Integer) k.get("count");String content = (String) k.get("content");return String.format("> top-%d：出现次数：%d\n> 错误详情描述: %s", topNum[0]++, count, content.trim());}).collect(Collectors.joining("\n\n"));String content = String.format("%s近%s内错误日志Top10\n%s", systemName, periodDesc, StringContent);String sendContent = String.format(TEMPLATE, content);//Arrays.stream(WX_ROBOT_URL.split(",")).forEach(wx -> {String post2 = HttpUtil.post(wx, sendContent);String errcode = JSON.parseObject(post2).getString("errcode");if (!"0".equals(errcode)) {log.error("{}发送微信不成功，返回数据：{}", systemName, post2);}});}private String getPeriodDesc() {if (PERIOD_HOUR <= 24) {return PERIOD_HOUR + "小时";}BigDecimal dayBigDecimal = BigDecimal.valueOf(PERIOD_HOUR).divide(BigDecimal.valueOf(24), 2, BigDecimal.ROUND_HALF_UP);String s = StrUtil.removeSuffix(StrUtil.removeSuffix(String.valueOf(dayBigDecimal), "00"), "0");String[] split = s.split("\\.");if (split.length == 1) {return split[0] + "天";}return s + "天";}/*** Jsoup 模拟登录 访问个人中心* 在登录时先输入一个错误的账号密码，查看到登录所需要的参数* 先构造登录请求参数，成功后获取到cookies* 设置request cookies，再次请求** @throws IOException*/public void jsoupLogin() throws IOException {//Jsoup加这个，避免请求https报证书问题trustEveryone();// 构造登陆参数Map<String, String> data = new HashMap<>();data.put("username", USER_NAME);data.put("password", PASSWORD);Connection.Response response = Jsoup.connect(LOG_CENTER_BASE_URL + "account/ldap/login").ignoreContentType(true) // 忽略类型验证.ignoreHttpErrors(true).followRedirects(false) // 禁止重定向.postDataCharset("utf-8").header("Upgrade-Insecure-Requests", "1").header("Accept", "application/json").header("Content-Type", "application/json").header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36").requestBody(JSON.toJSONString(data)).method(Connection.Method.POST).execute();response.charset("UTF-8");// login 中已经获取到登录成功之后的cookies// 构造访问个人中心的请求Map<String, String> cookies = response.cookies();cookies.forEach((k, v) -> COOKIE = k + "=" + v);String body = response.body();X_Csrf_Token = JSON.parseObject(body).getString("X-Csrf-Token");System.err.println("COOKIE:" + COOKIE);System.err.println("X_Csrf_Token:" + X_Csrf_Token);}public static void jsoupHandle(String id) throws IOException {String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";Document document = Jsoup.connect(url).header("X-Csrf-Token", X_Csrf_Token).header("Cookie", COOKIE).ignoreContentType(true) // 忽略类型验证.ignoreHttpErrors(true).method(Connection.Method.GET).get();System.err.println(url);String s = JSON.toJSONString(document);System.err.println(s);}public static void trustEveryone() {try {HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {public boolean verify(String hostname, SSLSession session) {return true;}});SSLContext context = SSLContext.getInstance("TLS");context.init(null, new X509TrustManager[]{new X509TrustManager() {public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {}public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {}public X509Certificate[] getAcceptedIssuers() {return new X509Certificate[0];}}}, new SecureRandom());HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());} catch (Exception e) {// e.printStackTrace();}}}三、实现效果
企业微信机器狗开发者文档：群机器人配置说明 - 接口文档 - 企业微信开发者中心
