1、部署VictoriaMetrics
cd /usr/local
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.65.0/victoria-metrics-amd64-v1.65.0.tar.gz
mkdir victoria-metrics && tar -xvzf victoria-metrics-amd64-v1.65.0.tar.gz && \
mv victoria-metrics-prod victoria-metrics/victoria-metrics && cd victoria-metrics
nohup ./victoria-metrics -retentionPeriod=30d -storageDataPath=data &2、配置Prometheus
# my global config
global:scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.# scrape_timeout is set to the global default (10s).external_labels:datacenter: "victoria-1"
# 远程写入victoria
remote_write:- url: "http://127.0.0.1:8428/api/v1/write"queue_config:max_samples_per_send: 10000
# Alertmanager configuration
alerting:alertmanagers:- static_configs:- targets:- localhost:9093# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:- "/usr/local/prometheus/rules/rule_node_down.yml"- "/usr/local/prometheus/rules/rule_disk_over.yml"- "/usr/local/prometheus/rules/rule_cpu_over.yml"- "/usr/local/prometheus/rules/rule_memory_over.yml"# - "second_rules.yml"# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:# The job name is added as a label to any timeseries scraped from this config.- job_name: 'prometheus'static_configs:- targets: ['localhost:9090']
# 用于配置victoria- job_name: 'victoria'static_configs:- targets: ['47.105.38.224:8480']- targets: ['47.105.38.224:8481']- targets: ['47.105.38.224:8482']# metrics_path defaults to '/metrics'# # scheme defaults to 'http'.- job_name: 'consul-prometheus'# metrics_path: "/v1/agent/metrics"scrape_interval: 60sscrape_timeout: 10sscheme: httpparams:format: ['prometheus']#static_configs:# - targets:# - 47.105.38.224:8500consul_sd_configs:- server: '47.105.38.224:8500'services: []relabel_configs:- source_labels: [__metrics_path__]separator: ;regex: /metricstarget_label: __metrics_path__replacement: /actuator/prometheusaction: replace- source_labels: ['__meta_consul_tags']regex: '^.*,metrics=true,.*$'action: keep- job_name: "node20_exporter"static_configs:- targets: ["localhost:9100"]- job_name: "node21_exporter"static_configs:- targets: ["172.16.17.21:9100"] #监控主机- job_name: "node22_exporter"static_configs:- targets: ["172.16.17.22:9100"]- job_name: "node23_exporter"static_configs:- targets: ["172.16.17.23:9100"] #监控主机- job_name: "alertmanager"static_configs:- targets: ["localhost:9093"]
3、配置Grafana数据源
4、构建vmalert
从源代码构建vmalert：
git clone https://github.com/VictoriaMetrics/VictoriaMetrics
cd VictoriaMetrics
make vmalert构建二进制文件将放置在VictoriaMetrics/bin文件夹中。
5、添加alert.rules
vim alert.rules#rule示例
groups:- name: test-rulerules:- alert: 主机状态expr: up == 0for: 2mlabels:status: warningannotations:summary: "{{$labels.instance}}:服务器关闭"description: "{{$labels.instance}}:服务器关闭"6、修改钉钉prometheus-webhook-dingtalk配置文件
vim /usr/local/prometheus-webhook-dingtalk/config.example.yml## Request timeout
# timeout: 5s## Uncomment following line in order to write template from scratch (be careful!)
#no_builtin_template: true## Customizable templates path
templates:- '/usr/local/alertmanager/template/default.tmpl'## You can also override default template using `default_message`
## The following example to use the 'legacy' template from v0.3.0
#default_message:
# title: '{{ template "legacy.title" . }}'
# text: '{{ template "legacy.content" . }}'## Targets, previously was known as "profiles"
targets:webhook1:url: https://oapi.dingtalk.com/robot/send?access_token=XXXXXXXX# secret for signaturesecret: SEC000000000000000000000
# webhook2:
# url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
# webhook_legacy:# url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx# Customize template contentmessage:# Use legacy template# title: '{{ template "legacy.title" . }}'text: '{{ template "wechat.default.message" . }}'webhook_mention_all:url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxxmention:all: truewebhook_mention_users:url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxxmention:mobiles: ['156xxxx8827', '189xxxx8325']
7、修改alertmanager配置文件
vim /usr/local/alertmanager/alertmanager.ymlglobal:resolve_timeout: 5m # 处理超时时间，默认为5mintemplates: # 指定邮件模板的路径，可以使用相对路径，template/*.tmpl的方式- '/usr/local/alertmanager/template/default.tmpl'
# 定义路由树信息
route:group_by: [alertname] # 报警分组依据receiver: ops_notify # 设置默认接收人group_wait: 30s # 最初即第一次等待多久时间发送一组警报的通知group_interval: 60s # 在发送新警报前的等待时间repeat_interval: 1h # 重复发送告警时间。默认1hroutes:- receiver: ops_notify # 基础告警通知人group_wait: 10smatch_re:alertname: 实例存活告警|磁盘使用率告警 # 匹配告警规则中的名称发送- receiver: info_notify # 消息告警通知人group_wait: 10smatch_re:alertname: 内存使用率告警|CPU使用率告警|目录大小告警# 定义基础告警接收者
receivers:
- name: ops_notifywebhook_configs:- url: http://localhost:8060/dingtalk/webhook1/sendsend_resolved: true # 警报被解决之后是否通知
# message: '{{ template "wechat.default.message" . }}'# 定义消息告警接收者
- name: info_notifywebhook_configs:- url: http://localhost:8060/dingtalk/webhook1/sendsend_resolved: true# message: '{{ template "wechat.default.message" . }}'# 一个inhibition规则是在与另一组匹配器匹配的警报存在的条件下，使匹配一组匹配器的#警报失效的规则。两个警报必须具有一组相同的标签。
inhibit_rules:- source_match:severity: 'critical'target_match:severity: 'warning'equal: ['alertname', 'dev', 'instance']
8、启动vmalert
./bin/vmalert -rule=alert.rules \-datasource.url=http://localhost:8428 \-notifier.url=http://localhost:9093 &9、查看钉钉告警
