172.16.10.21 prometheus
172.16.10.33 altermanager
172.16.10.59 mysql服务，node探针以及mysql的探针
[root@k8s-node02 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
d0a03819e7f8 harbor.jettech.com/prometheus/mysqld-exporter:latest "/bin/mysqld_exporte…" 10 minutes ago Up 10 minutes 0.0.0.0:9104->9104/tcp, :::9104->9104/tcp mysql-export
b62689186d4c harbor.jettech.com/prometheus/node-exporter:latest "/bin/node_exporter …" 20 hours ago Up 2 hours node
09f094a92ef1 harbor.jettech.com/jettechtools/mysql:8.0.28 "docker-entrypoint.s…" 21 hours ago Up 5 minutes
1.prometheus 172.16.10.21
1.1）配置文件
[root@nginx conf]# cat prometheus.yml
global:scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.scrape_timeout: 10s
alerting:alertmanagers:- static_configs:- targets:- 172.16.10.33:9093
rule_files:- "rules/*_rules.yml"
scrape_configs:- job_name: "jettech-prod-node-59"static_configs:- targets: - 172.16.10.59:9100labels:instance: "172.16.10.59-wubo"ip: "172.16.10.59-wubo"env: "prod-wubo-59"team: "jettopro-wubo-59"group1: g1operations: wubostatus: 'High'#file ds- job_name: "jettech-prod-mysql8"static_configs:- targets: ["172.16.10.59:9104"]labels:instance: mysqld-exporteroperations: wubostatus: 'Warning'1.2）规则mysql
[root@nginx conf]# cat rules/mysql_rules.yml
groups:
- name: 数据库资源监控rules:
#mysql状态检测- alert: MySQL Statusexpr: mysql_up == 0for: 10slabels:severity: warningannotations:summary: "{{ $labels.instance }} Mysql服务 !!!"description: "{{ $labels.instance }} Mysql服务不可用 请检查!"#mysql主从IO线程停止时触发告警- alert: MySQL Slave IO Thread Statusexpr: mysql_slave_status_slave_io_running == 0for: 5slabels:severity: warningannotations:summary: "{{ $labels.instance }} Mysql从节点IO线程"description: "Mysql主从IO线程故障，请检测!"#mysql主从sql线程停止时触发告警- alert: MySQL Slave SQL Thread Status expr: mysql_slave_status_slave_sql_running == 0for: 5s labels:severity: errorannotations: summary: "{{$labels.instance}}: MySQL Slave SQL Thread has stop !!!"description: "检测MySQL主从SQL线程运行状态"#mysql主从延时状态告警- alert: MySQL Slave Delay Status expr: mysql_slave_status_sql_delay == 30for: 5s labels:severity: warningannotations: summary: "{{$labels.instance}}: MySQL 主从延迟超过 30s !!!"description: "检测MySQL主从延时状态"#mysql连接数告警- alert: Mysql_Too_Many_Connectionsexpr: rate(mysql_global_status_threads_connected[5m]) > 200for: 2mlabels:severity: warningannotations:summary: "{{$labels.instance}}: 连接数过多"description: "{{$labels.instance}}: 连接数过多，请处理 ,(current value is: {{ $value }})！" #mysql慢查询有点多告警- alert: Mysql_Too_Many_slow_queriesexpr: rate(mysql_global_status_slow_queries[5m]) > 3for: 2mlabels:severity: warningannotations:summary: "{{$labels.instance}}: 慢查询有点多，请检查处理！"description: "{{$labels.instance}}: Mysql slow_queries is more than 3 per second ,(current value is: {{ $value }})"1.3）node规则
[root@nginx conf]# cat rules/node_rules.yml
groups:
- name: 服务器资源监控rules:- alert: 内存使用率过高expr: 100 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 80for: 3m labels:severity: 严重告警annotations:summary: "{{ $labels.instance }} 内存使用率过高, 请尽快处理！"description: "{{ $labels.instance }}内存使用率超过80%,当前使用率{{ $value }}%."- alert: 服务器宕机expr: up == 0for: 5slabels:severity: 严重告警annotations:summary: "{{$labels.instance}} 服务器宕机, 请尽快处理!"description: "{{$labels.instance}} 服务器延时超过3分钟,当前状态{{ $value }}. "- alert: CPU高负荷expr: 100 - (avg by (instance,job)(irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90for: 5mlabels:severity: 严重告警annotations:summary: "{{$labels.instance}} CPU使用率过高,请尽快处理！"description: "{{$labels.instance}} CPU使用大于90%,当前使用率{{ $value }}%. "- alert: 磁盘IO性能expr: avg(irate(node_disk_io_time_seconds_total[1m])) by(instance,job)* 100 > 90for: 5mlabels:severity: 严重告警annotations:summary: "{{$labels.instance}} 流入磁盘IO使用率过高,请尽快处理！"description: "{{$labels.instance}} 流入磁盘IO大于90%,当前使用率{{ $value }}%."- alert: 网络流入expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400for: 5mlabels:severity: 严重告警annotations:summary: "{{$labels.instance}} 流入网络带宽过高，请尽快处理！"description: "{{$labels.instance}} 流入网络带宽持续5分钟高于100M. RX带宽使用量{{$value}}."- alert: 网络流出expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400for: 5mlabels:severity: 严重告警annotations:summary: "{{$labels.instance}} 流出网络带宽过高,请尽快处理！"description: "{{$labels.instance}} 流出网络带宽持续5分钟高于100M. RX带宽使用量{$value}}."- alert: TCP连接数expr: node_netstat_Tcp_CurrEstab > 10000for: 2mlabels:severity: 严重告警annotations:summary: " TCP_ESTABLISHED过高！"description: "{{$labels.instance}} TCP_ESTABLISHED大于100%,当前使用率{{ $value }}%."- alert: 磁盘容量#expr: 100 - round(node_filesystem_free_bytes{fstype=~"ext3|ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 30expr: 100 - round(node_filesystem_free_bytes{device="/dev/mapper/centos-root",fstype=~"ext3|ext4|xfs"}/node_filesystem_size_bytes {device="/dev/mapper/centos-root",fstype=~"ext4|xfs"}*100) > 10for: 1mlabels:severity: 严重告警annotations:summary: "{{$labels.mountpoint}} 磁盘分区使用率过高，请尽快处理！"description: "{{$labels.instance}} 磁盘分区使用大于10%，当前使用率{{ $value }}%."
2.altermanger 172.16.10.33
2.1）配置
[root@k8s-node03 config]# cat alertmanager.yml
global:resolve_timeout: 5msmtp_smarthost: 'smtp.qq.com:465'smtp_hello: 'localhost'smtp_from: '459097610@qq.com'smtp_auth_username: '459097610@qq.com'smtp_auth_password: 'oqiapxzubiajbgjh'smtp_require_tls: false
templates:- 'templates/email.tmpl'- 'templates/email_wubo.tmpl'- 'templates/email_wuqi.tmpl'
route:group_by: ['alertname']group_wait: 10sgroup_interval: 10srepeat_interval: 1hreceiver: 'email'routes:- receiver: 'email_wubo'continue: false#group_by: [group1]#match: # name: jettech#team: jettopro-wubo-33#env: prod-wubo-33#- receiver: 'email_wuqi'# continue: false# group_by: [group1]# #match: # # name: jettech# #team: jettopro-wuqi-65# #env: prod-wuqi-65receivers:- name: 'email'email_configs:- to: '{{ template "email.to" }}'html: '{{ template "email.to.html" . }}'send_resolved: true #故障恢复后通知- name: 'email_wubo'email_configs:- to: '{{ template "email_wubo.to" }}'html: '{{ template "email_wubo.to.html" . }}'send_resolved: true #故障恢复后通知- name: 'email_wuqi'email_configs:- to: '{{ template "email_wuqi.to" }}'html: '{{ template "email_wuqi.to.html" . }}'send_resolved: true #故障恢复后通知inhibit_rules:- source_match:status: 'High'target_match:status: 'Warning'equal:- operations2.2）模版
[root@k8s-node03 config]# cat templates/email.tmpl
{{ define "email.to" }}459097610@qq.com{{ end }}
{{ define "email.to.html" }}
{{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }}
<h2>@告警通知</h2>
告警程序: prometheus_alertmanager <br>
告警级别: {{ .Labels.severity }} 级 <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
故障主机所属环境: {{ .Labels.env }} <br>
故障主机所属团队: {{ .Labels.team }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} <br>
{{ end }}{{ end -}}
{{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }}
<h2>@告警恢复</h2>
告警程序: prometheus_alert <br>
故障主机: {{ .Labels.instance }}<br>
故障主机所属环境: {{ .Labels.env }} <br>
故障主机所属团队: {{ .Labels.team }} <br>
故障主题: {{ .Annotations.summary }}<br>
告警详情: {{ .Annotations.description }}<br>
告警时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}<br>
恢复时间: {{ .EndsAt.Local.Format "2006-01-02 15:04:05" }}<br>
{{ end }}{{ end -}}
{{- end }}3 被检测服务 172.16.10.59
3.1）探针mysql
[root@k8s-node02 ~]# docker run -d --name mysql-export -p 9104:9104 -v /opt/export/mysql/conf/my.cnf:/etc/mysql/my.cnf -d harbor.jettech.com/prometheus/mysqld-exporter:latest --collect.info_schema.processlist --collect.info_schema.innodb_tablespaces --collect.info_schema.innodb_metrics --collect.perf_schema.tableiowaits --collect.perf_schema.indexiowaits --collect.perf_schema.tablelocks --collect.engine_innodb_status --collect.perf_schema.file_events --collect.binlog_size --collect.info_schema.clientstats --collect.perf_schema.eventswaits --config.my-cnf=/etc/mysql/my.cnf3.2）探针node
[root@k8s-node02 ~]# docker run --name node --net="host" --pid="host" -v "/:/host:ro,rslave" -d harbor.jettech.com/prometheus/node-exporter:latest --path.rootfs=/host3.3）被检测服务：mysql服务
[root@k8s-node02 ~]# docker run --name mysql8 --net host -e MYSQL_ROOT_PASSWORD=123456aA -d harbor.jettech.com/jettechtools/mysql:8.0.284开始测试：
4.1）alertmanager，不加抑制inhibit_rules配置
[root@k8s-node03 config]# cat alertmanager.yml
global:resolve_timeout: 5msmtp_smarthost: 'smtp.qq.com:465'smtp_hello: 'localhost'smtp_from: '459097610@qq.com'smtp_auth_username: '459097610@qq.com'smtp_auth_password: 'oqiapxzubiajbgjh'smtp_require_tls: false
templates:- 'templates/email.tmpl'- 'templates/email_wubo.tmpl'- 'templates/email_wuqi.tmpl'
route:group_by: ['alertname']group_wait: 10sgroup_interval: 10srepeat_interval: 1hreceiver: 'email'routes:- receiver: 'email_wubo'continue: false#group_by: [group1]#match: # name: jettech#team: jettopro-wubo-33#env: prod-wubo-33#- receiver: 'email_wuqi'# continue: false# group_by: [group1]# #match: # # name: jettech# #team: jettopro-wuqi-65# #env: prod-wuqi-65receivers:- name: 'email'email_configs:- to: '{{ template "email.to" }}'html: '{{ template "email.to.html" . }}'send_resolved: true #故障恢复后通知- name: 'email_wubo'email_configs:- to: '{{ template "email_wubo.to" }}'html: '{{ template "email_wubo.to.html" . }}'send_resolved: true #故障恢复后通知- name: 'email_wuqi'email_configs:- to: '{{ template "email_wuqi.to" }}'html: '{{ template "email_wuqi.to.html" . }}'send_resolved: true #故障恢复后通知4.1.1）172.16.10.59直接关机，看看发几次报警
4.1.2）这样prometheus会触发两次报警,就应该触发两次没问题
4.1.3）alertmanager也会收到2次prometheus传过来的报警，也不会合并抑制之类的操作
4.1.4）进而alertmanager也会发两次邮件
4.1.5）加上抑制
inhibit_rules:- source_match: #源标签警报触发时抑制含有目标标签的警报，在当前警报匹配 status: 'High'status: 'High'target_match:status: 'Warning' equal:- operations # 确保这个配置下的标签内容相同才会抑制，也就是说警报中必须有这三个标签值才会被抑制
上图中 就是prometheus中的被处罚的两个job里面的标签
source_match：意思是prometheus中第一次进来的一个job的标签匹配上了status: 'High'。则抑制target_match中标签是status: 'Warning' ，而status: 'Warning' 是prometheus中另一个触发的job中的标签且确保equal这个配置下的operations 标签内容相同才会抑制，也就是说警报中的两个job的都有这个标签而且值还要相同本案例中是【operations =wubo】才会被抑制
加上抑制的完整配置
[root@k8s-node03 config]# cat alertmanager.yml
global:resolve_timeout: 5msmtp_smarthost: 'smtp.qq.com:465'smtp_hello: 'localhost'smtp_from: '459097610@qq.com'smtp_auth_username: '459097610@qq.com'smtp_auth_password: 'oqiapxzubiajbgjh'smtp_require_tls: false
templates:- 'templates/email.tmpl'- 'templates/email_wubo.tmpl'- 'templates/email_wuqi.tmpl'
route:group_by: ['alertname']group_wait: 10sgroup_interval: 10srepeat_interval: 1hreceiver: 'email'routes:- receiver: 'email_wubo'continue: false#group_by: [group1]#match: # name: jettech#team: jettopro-wubo-33#env: prod-wubo-33#- receiver: 'email_wuqi'# continue: false# group_by: [group1]# #match: # # name: jettech# #team: jettopro-wuqi-65# #env: prod-wuqi-65receivers:- name: 'email'email_configs:- to: '{{ template "email.to" }}'html: '{{ template "email.to.html" . }}'send_resolved: true #故障恢复后通知- name: 'email_wubo'email_configs:- to: '{{ template "email_wubo.to" }}'html: '{{ template "email_wubo.to.html" . }}'send_resolved: true #故障恢复后通知- name: 'email_wuqi'email_configs:- to: '{{ template "email_wuqi.to" }}'html: '{{ template "email_wuqi.to.html" . }}'send_resolved: true #故障恢复后通知inhibit_rules:- source_match:status: 'High'target_match:status: 'Warning'equal:- operations然后再次172.16.10.59 关机测试发几次邮件
4.1.6)prometheus 依旧是触发两次报警，这个没问题
4.1.7）但是alertmanager只处理了关机的报警，没有处理mysql服务器停掉的报警
4.1.8）邮件之后一封，邮件的个数由alertmanager处理个数决定
4.1.8）恢复机器开机
prometheus 没问题
alertmanager也没有要处理的警报信息
邮件：依旧是一封邮件但是是两段内容
4.1.9）测试一下equal这个配置下的operations不同会发生什么
[root@nginx conf]# cat prometheus.yml
global:scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.scrape_timeout: 10s
alerting:alertmanagers:- static_configs:- targets:- 172.16.10.33:9093
rule_files:- "rules/*_rules.yml"
scrape_configs:- job_name: "jettech-prod-node-59"static_configs:- targets: - 172.16.10.59:9100labels:instance: "172.16.10.59-wubo"ip: "172.16.10.59-wubo"env: "prod-wubo-59"team: "jettopro-wubo-59"group1: g1operations: wubostatus: 'High'#file ds- job_name: "jettech-prod-mysql8"static_configs:- targets: ["172.16.10.59:9104"]labels:instance: mysqld-exporteroperations: wuqistatus: 'Warning'两个job，一个job：operations: wuqi，另一个job：operations: wubo
然后抑制的equal: ["operations"]依然和之前一样
inhibit_rules:- source_match:status: 'High'target_match:status: 'Warning'equal:- operations再次关机prometheus 中发现两个job的operations标签的值不一样了一个是wubo一个是wuqi
alertmanager：处理两个警报了
有间个数:
说明 “确保这个配置下的标签内容相同才会抑制，也就是说警报中必须有这三个标签值才会被抑制“句话起作用了
临时静默
除了基于抑制机制可以控制告警通知的行为以外，用户或者管理员还可以直接通过Alertmanager的UI临时屏蔽特定的告警通知。通过定义标签的匹配规则(字符串或者正则表达式)，如果新的告警通知满足静默规则的设置，则停止向receiver发送通知。
进入Alertmanager UI，点击"New Silence"显示如下内容：
