[root@node1 ~]# tree prom
prom
├── docker-compose.yml  #docker-compose文件
├── grafana  #grafana數據掛載
├── prometheus_data  #Prometheus數據掛載
├── rules    #報警規則文件
│   ├── cpu_over.yml
│   ├── disk_over.yml
│   ├── memory_over.yml
│   └── node_alived.yml
└── yml
    ├── alertmanager.yml   alertmanager配置
    ├── config.yml   釘釘機器人配置 
    └── prometheus.yml   Prometheus配置

[root@node1 prom]# cat docker-compose.yml 
version: "3.7"
services:
  node-exporter:
    image: prom/node-exporter:latest
    container_name: "node-exporter"
    ports:
      - "9100:9100"
    restart: always
  cadvisor:
    image: google/cadvisor:latest
    container_name: cadvisor
    restart: always
    ports:
      - '8080:8080'
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    ports:
      - "9090:9090"
    restart: always
    volumes:
      - "./yml/prometheus.yml:/etc/prometheus/prometheus.yml"
      - "./prometheus_data:/prometheus"
      - "./rules:/etc/prometheus/rules"
  grafana:
    image: grafana/grafana
    container_name: "grafana"
    ports:
      - "3000:3000"
    restart: always
    volumes:
      - "./grafana:/var/lib/grafana"
  alertmanager:
    image: prom/alertmanager:latest
    restart: "always"
    ports:
      - 9093:9093
    container_name: "alertmanager"
    volumes:
      - "./yml/alertmanager.yml:/etc/alertmanager/alertmanager.yml"
  webhook:
    image: timonwong/prometheus-webhook-dingtalk
    restart: "always"
    ports:
      - 8060:8060
    container_name: "webhook"         
    volumes:
      - "./yml/config.yml:/etc/prometheus-webhook-dingtalk/config.yml"

[root@node1 prom]# cat yml/prometheus.yml 
# my global config
global:  # 此片段指定的是prometheus的全局配置， 比如采集間隔，抓取超時時間等.
  scrape_interval: 1m  # 抓取間隔 默認1m
  evaluation_interval: 1m   # 評估規則間隔 默認1m
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
 # 此片段指定報警配置， 這里主要是指定prometheus將報警規則推送到指定的alertmanager實例地址
alerting: 
  alertmanagers:
    - static_configs:
        - targets:
           - 192.168.10.10:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
   - "/etc/prometheus/rules/*.yml"   #報警規則文件
#  - "cpu_over.yml"
#  - "disk_over.yml"
#  - "memory_over.yml"
#  - "node_alived.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
# 抓取配置列表
scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"] 
  - job_name: "linux"
    static_configs:
      - targets: ["192.168.10.10:9100","192.168.10.10:8080","192.168.10.20:9100","192.168.10.20:8080"]

[root@node1 prom]#cat alertmanager.yml
global:
  resolve_timeout: 5m  #在指定時間內沒有新的事件就發送恢復通知
route:
  receiver: webhook  #設置接收人
  group_wait: 1m  #組告警等待時間。在等待時間結束后，如果有同組告警一起發出
  group_interval: 1m  #兩組告警間隔時間。
  repeat_interval: 1m  #重復告警間隔時間，減少相同郵件的發送頻率。
  group_by: [alertname] #采用那個標簽來作為分組。
receivers:   #通知接收者列表
- name: webhook
  webhook_configs:
  - url: http://192.168.10.10:8060/dingtalk/webhook1/send             
    send_resolved: true

#########################################################

[root@node1 prom]# cat yml/config.yml 
targets:
  webhook1:
    url: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX    #webhook
    secret: SEC000000    #加簽

[root@node1 prom]#cat alertmanager.yml
global:
  resolve_timeout: 5m  #在指定時間內沒有新的事件就發送恢復通知
route:
  receiver: webhook  #設置接收人
  group_wait: 1m  #組告警等待時間。在等待時間結束后，如果有同組告警一起發出
  group_interval: 1m  #兩組告警間隔時間。
  repeat_interval: 1m  #重復告警間隔時間，減少相同郵件的發送頻率。
  group_by: [alertname] #采用那個標簽來作為分組。
receivers:   #通知接收者列表
- name: webhook
  webhook_configs:
  - url: http://192.168.10.10:8060/dingtalk/webhook1/send             
    send_resolved: true

#########################################################

[root@node1 prom]# cat yml/config.yml 
targets:
  webhook1:
    url: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX    #webhook
    secret: SEC000000    #加簽