容器监控告警方案（cAdvisor + nodeExporter + alertmanager + prometheus +grafana）

version: '2.1'
services:
  node-exporter:
    image: prom/node-exporter
    container_name: prometheus_node-exporter
    restart: always
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "5"
    ports:
      - 9100:9100
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/devicemapper|rootfs/var/lib/docker/aufs)($$|/)'
  cadvisor:
    image: google/cadvisor
    container_name: prometheus_cadvisor
    restart: always
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "5"
    ports:
      - 9101:8080
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro

cd /opt/container-monitor/nodeexporter-cadvisor
docker-compose up -d

# 全局配置项
 global: 
   resolve_timeout: 5m  #处理超时时间，默认为5min
   smtp_smarthost: 'smtp.163.com:25'# 邮箱smtp服务器代理
   smtp_from: 'xxx@163.com'# 发送邮箱名称
   smtp_auth_username: 'xxx@163.com'# 邮箱名称
   smtp_auth_password: 'xxx'#邮箱密码
# 定义模板信心
 templates:
   - '/etc/alertmanager/templates/*.html'
# 定义路由树信息
 route:
   group_by: ['alertname'] # 报警分组依据
   group_wait: 10s # 最初即第一次等待多久时间发送一组警报的通知
   group_interval: 10s # 在发送新警报前的等待时间
   repeat_interval: 1m # 发送重复警报的周期 对于email配置中，此项不可以设置过低，否则将会由于邮件发送太多频繁，被smtp服务器拒绝
   receiver: 'email'# 发送警报的接收者的名称，以下receivers name的名称
# 定义警报接收者信息
 receivers:
   - name: 'email'# 警报
     email_configs: # 邮箱配置
     - to: 'xxxx@qq.com'# 接收警报的email配置
       html: '{{ template "test.html" . }}'# 设定邮箱的内容模板
       headers: { Subject: "[WARN] 报警邮件"} # 接收邮件的标题
     webhook_configs: # webhook配置
     - url: 'http://127.0.0.1:5001'

version: '2.1'
services:
  alertmanager:
    image: prom/alertmanager
    container_name: alertmanager
    restart: always
    network_mode: "host"
    ports:
      - 9093:9093
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "5"
    volumes:
      - ./config.yml:/etc/alertmanager/config.yml
      - ./test.html:/etc/alertmanager/templates/test.html
    command:
      - '--config.file=/etc/alertmanager/config.yml'
      - '--storage.path=/alertmanager'
      - '--log.level=debug'

{{ define "test.html"}}
<table border="1">
        <tr>
                <td>报警项</td>
                <td>实例</td>
                <td>报警内容</td>
                <td>开始时间</td>
        </tr>
        {{ range $i, $alert :=.Alerts }}
                <tr>
                        <td>{{ index $alert.Labels "alertname" }}</td>
                        <td>{{ index $alert.Labels "instance" }}</td>
                        <td>{{ index $alert.Annotations "description" }}</td>
                        <td>{{ $alert.StartsAt }}</td>
                </tr>
        {{ end }}
</table>
{{ end }}

cd /opt/container-monitor/alertmanager
docker-compose up -d

groups:
- name: test-rule
  rules:
  - alert: "服务运行"
    expr: container_tasks_state{name="mysql",state="running"} == 0
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "服务名:{{$labels.alertname}}"
      description: "容器： {{ $labels.name }} 处于运行中"

# my globalconfig
global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1minute.
  external_labels:
      monitor: 'codelab-monitor'
# Alertmanager configuration
alerting:
  alertmanagers:
  -static_configs:
    - targets:
       - 172.16.101.250:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
   - running-rule.yml
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from thisconfig.
  - job_name: 'prometheus'
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.
    static_configs:
    - targets: ['172.16.101.250:9090','172.16.101.250:9100','172.16.101.250:9101','172.16.101.251:9100','172.16.101.251:9101']

version: '2.1'
services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    restart: always
    network_mode: "host"
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "5"
    volumes:
      - ./running-rule.yml:/etc/prometheus/running-rule.yml:z
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:z

cd /opt/container-monitor/prometheus
docker-compose up -d

version: '2.1'
services:
  grafana:
    image: grafana/grafana
    container_name: grafana
    restart: always
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "5"
    ports:
      - 3000:3000
    environment:
      - GF_SERVER_ROOT_URL=http://grafana.server.name
      - GF_SECURITY_ADMIN_PASSWORD=123456

cd /opt/container-monitor/grafana
docker-compose up -d