• Мониторинг сервисов с Prometheus
• Полное руководство по Prometheus в 2019 году
• Prometheus: Удаление Метрик Временных Рядов
• Человеческим языком про метрики
• Мониторинг в K8s с помощью Prometheus

# apt install prometheus

# less /etc/prometheus/prometheus.yml

...
global:
  scrape_interval:     15s
...

# promtool check config /etc/prometheus/prometheus.yml

# service prometheus restart

!!! Ссылки содержат DNS имена

• http://192.168.X.10:9090/
• http://server.corpX.un:9090/
• http://server:9090/classic/targets

# /usr/share/prometheus/install-ui.sh

• Exporters

• Awesome Prometheus alerts
• https://github.com/samber/awesome-prometheus-alerts/tree/master/dist/rules

• Установка и настройка MTA на обработку почты домена hostname
• Сервис adnanh/webhook

# apt install prometheus-alertmanager

# cat /etc/prometheus/alertmanager.yml

global:
  smtp_smarthost: 'server.corpX.un:25'
  smtp_from: 'alertmanager@corpX.un'
  smtp_require_tls: false
templates:
- '/etc/prometheus/alertmanager_templates/*.tmpl'

route:
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 3h
  receiver: team-X-mails

#  routes:
#    - matchers:
#        - job="roundcube_monitor"
#      receiver: script-receiver

receivers:
- name: 'team-X-mails'
  email_configs:
  - to: 'student@corpX.un'
    send_resolved: true

#  - name: 'script-receiver'
#    webhook_configs:
#    - url: 'http://localhost:9000/hooks/my-alert-script'

# amtool check-config /etc/prometheus/alertmanager.yml

# service prometheus-alertmanager restart

$ curl -H "Content-Type: application/json" -d '[
  {
    "labels": {
      "alertname": "TestAlert",
      "severity": "critical",
      "instance": "test-host"
    },
    "annotations": {
      "summary": "This is a test alert",
      "description": "Triggered manually via curl to test Alertmanager routing."
    }
  }
]' http://localhost:9093/api/v2/alerts

• Управление юнитами Systemd reset-failed

# wget -qO - https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/master/dist/rules/host-and-hardware/node-exporter.yml | tee /etc/prometheus/node_exporter_rules.yml

# cat /etc/prometheus/first_rules.yml

groups:
- name: alert.rules
  rules:
  - alert: InstanceDown
    expr: up == 0
    for: 1m
    labels:
      severity: "critical"
    annotations:
      summary: "Endpoint {{ $labels.instance }} down"
      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."

  - alert: EndpointDown
    expr: probe_success == 0
    for: 1m
    labels:
      severity: "critical"
    annotations:
      summary: "Endpoint {{ $labels.instance }} down"

  - alert: CriticalTraffic
    expr: rate(ifInOctets{instance="router"}[1m])>125000
    for: 1m
    labels:
      severity: "critical"
    annotations:
      summary: "CriticalTraffic {{ $labels.instance }}"

#  - alert: RoundcubeDown
#    expr: roundcube_login_logout_runtime == 0
#    for: 1m
#    labels:
#      severity: "critical"
#    annotations:
#      summary: "Endpoint {{ $labels.instance }} down"

# cat /etc/prometheus/prometheus.yml

...
# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets: ['localhost:9093']

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - "first_rules.yml"
  # - "second_rules.yml"

...

• Проверка конфигурации и перезапуск

...
Checking /etc/prometheus/first_rules.yml
  SUCCESS: N rules found
...

• http://server.corpX.un:9090/classic/alerts

# cat /etc/default/prometheus-alertmanager
ARGS="--log.level=debug"

• В Debian/Ubuntu ставится как зависимость к пакету prometheus и добавлен в конфигурацию
• http://server.corpX.un:9100/metrics

$ df /
...
/dev/.....  15662008 1877488  12969212  13% /
...

# TYPE node_filesystem_free_bytes gauge
node_filesystem_free_bytes{fstype="ext4",mountpoint="/"} = (15662008 - 1877488) * 1024 

  или

{__name__="node_filesystem_free_bytes",fstype="ext4",mountpoint="/"}

$ cat /sys/class/net/eth0/statistics/rx_bytes
  или
$ cat /sys/class/net/bond0/statistics/rx_bytes

# TYPE node_network_receive_bytes_total counter
node_network_receive_bytes_total{device="bond0"}

# less /etc/prometheus/prometheus.yml
...
  - job_name: node
    # If prometheus-node-exporter is installed, grab stats about the local
    # machine by default.
    static_configs:
      - targets: ['localhost:9100']

8*rate(node_network_receive_bytes_total[1m])

8*rate(node_network_receive_bytes_total{device="bond0"}[1m])

8*rate(node_network_receive_bytes_total{device="eth0",instance="localhost:9100",job="node"}[1m])

• How to Monitor Website Performance with Blackbox Exporter and Grafana?
• Prometheus: мониторинг HTTP через Blackbox экспортер

# apt install prometheus-blackbox-exporter

# cat /etc/prometheus/blackbox.yml

...
  http_2xx:
    prober: http
    http:
      preferred_ip_protocol: "ip4"
...

# service prometheus-blackbox-exporter restart

# cat /etc/prometheus/prometheus.yml

...
  - job_name: check_http
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - https://val.bmstu.ru
        - https://ya.ru
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115

  - job_name: check_ssh
    metrics_path: /probe
    params:
      module: [ssh_banner]
    static_configs:
      - targets:
        - switch1:22
        - switch2:22
        - switch3:22
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115

• Проверка конфигурации и перезапуск
• http://server.corpX.un:9115/ Blackbox Exporter→Logs и http://server.corpX.un:9090/graph

probe_success...

probe_duration_seconds...

probe_http_duration_seconds...

• ICMP Pings with the Blackbox exporter, github blackbox_exporter permissions, POSIX capabilities или Переменные ядра

# cat /etc/prometheus/prometheus.yml

...
  - job_name: check_ping
    metrics_path: /probe
    params:
      module: [icmp]
    file_sd_configs:
      - files:
#        - switchs.yml
#        - switchs.json
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115

# cat /etc/prometheus/switchs.json

[
  {
    "targets": [ "switch1", "switch2", "switch3" ]
  }
]

# cat /etc/prometheus/switchs.yml

- targets:
  - switch1
  - switch2
  - switch3

• Проверка конфигурации и перезапуск

• Grafana + Prometheus мониторинг траффика Cisco SNMP
• SNMP monitoring and easing it with Prometheus
• An advanced guide to network monitoring with Grafana and Prometheus

# apt install prometheus-snmp-exporter

# cat /etc/prometheus/snmp.yml

auths:
  public_v2:
    community: public
    version: 2
modules:
  if_mib:
    walk:
    - 1.3.6.1.2.1.2.2.1.10
    - 1.3.6.1.2.1.2.2.1.16
    - 1.3.6.1.2.1.2.2.1.2
    metrics:
    - name: ifInOctets
      oid: 1.3.6.1.2.1.2.2.1.10
      type: counter
      indexes:
      - labelname: ifIndex
        type: Integer
      lookups:
      - labels:
        - ifIndex
        labelname: ifDescr
        oid: 1.3.6.1.2.1.2.2.1.2
        type: DisplayString
    - name: ifOutOctets
      oid: 1.3.6.1.2.1.2.2.1.16
      type: counter
      indexes:
      - labelname: ifIndex
        type: Integer
      lookups:
      - labels:
        - ifIndex
        labelname: ifDescr
        oid: 1.3.6.1.2.1.2.2.1.2
        type: DisplayString

# cp /usr/share/doc/prometheus-snmp-exporter/examples/generator.yml .
  может понадобиться удалить все modules, кроме if_mib

# prometheus-snmp-generator generate

# cp snmp.yml /etc/prometheus/snmp.yml

# prometheus-snmp-exporter --dry-run

# service prometheus-snmp-exporter restart

• http://server.corpX.un:9116/

# curl 'http://127.0.0.1:9116/snmp?target=router'

# cat /etc/prometheus/prometheus.yml

...
  - job_name: 'snmp'
    static_configs:
      - targets:
        - router
    metrics_path: /snmp
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9116

• Проверка конфигурации и перезапуск

rate(ifInOctets{ifDescr="FastEthernet0/0",ifIndex="1",instance="router",job="snmp"}[1m])
  или
rate(ifOutOctets{ifIndex="5",instance="router",job="snmp"}[1m])

8*rate(ifInOctets{ifDescr="FastEthernet0/0",instance="router"}[1m])
  или
8*rate(ifOutOctets{ifDescr="Port-channel1",instance="router"}[1m])

root@server:~# apt install prometheus-apache-exporter

root@server:~# prometheus-apache-exporter --scrape_uri="http://localhost/server-status?auto"

root@server:~# cat /etc/default/prometheus-apache-exporter
...
ARGS=""

# apt install prometheus-pushgateway

# cat /etc/prometheus/prometheus.yml

...
  - job_name: 'pushgateway'
    honor_labels: true
    static_configs:
    - targets: ['localhost:9091']

• Проверка конфигурации и перезапуск

• Scrape data using Bash script and push it to Prometheus using PushGateway

# cat ip_dhcp_binding.sh

#!/bin/sh

unset http_proxy
DHCP_SERVER=router
NET=192.168

COUNT=`rsh ${DHCP_SERVER} show ip dhcp binding | grep ${NET} | wc -l`

cat << EOF | curl --data-binary @- http://127.0.0.1:9091/metrics/job/cisco_dhcp/dhcp_server/${DHCP_SERVER}/net/${NET}
  ip_dhcp_binding ${COUNT}
EOF

ip_dhcp_binding{dhcp_server="router",job="cisco_dhcp",net="192.168"}

# crontab -l

* * * * * /root/ip_dhcp_binding.sh

• Web интерфейс к почте Roundcube
• Тестирование Web приложения на shell
• Технология cgroup

root@server:~# cat /usr/local/bin/roundcube_monitor_stat_log.sh

#!/bin/bash

start=$(date +%s.%N)
/usr/local/bin/roundcube_monitor.sh 2>&1 | logger -t roundcube_monitor
test ${PIPESTATUS[0]} -eq 0 && end=$(date +%s.%N) || end=$start

runtime=$(echo "$end - $start" | bc)
echo "roundcube_login_logout_runtime ${runtime}" | \
  curl --data-binary @- http://localhost:9091/metrics/job/roundcube_monitor/host/$(hostname)
echo "Выполнено за $runtime сек."

root@server:~# crontab -l

* * * * * /usr/local/bin/roundcube_monitor_stat_log.sh >/dev/null