261 lines
8.3 KiB
Go
Executable file
261 lines
8.3 KiB
Go
Executable file
package devopsmodules
|
||
|
||
import (
|
||
"rubric/utils"
|
||
)
|
||
|
||
var MonitoringModule = Module{
|
||
Key: "monitoring",
|
||
Aliases: []string{"mon", "prometheus", "grafana", "observability"},
|
||
Title: "Мониторинг и наблюдаемость",
|
||
Description: "Prometheus, Grafana, AlertManager, Loki, трассировка, метрики",
|
||
Run: showMonitoring,
|
||
}
|
||
|
||
func showMonitoring() {
|
||
utils.Header("DevOps — Мониторинг и наблюдаемость")
|
||
|
||
utils.Section("Три столпа наблюдаемости")
|
||
utils.Code(`Metrics — числовые измерения во времени (CPU, RPS, latency)
|
||
Logs — события с временными метками и контекстом
|
||
Traces — путь запроса через микросервисы (span, trace)
|
||
|
||
Популярный стек:
|
||
Prometheus + Grafana — метрики и визуализация
|
||
Loki — логи (как Prometheus, но для логов)
|
||
Tempo / Jaeger — распределённая трассировка
|
||
AlertManager — маршрутизация алертов
|
||
|
||
Облачные альтернативы:
|
||
Datadog, New Relic, Dynatrace — коммерческие
|
||
AWS CloudWatch, GCP Monitoring, Azure Monitor`)
|
||
|
||
utils.Section("Prometheus")
|
||
utils.Code(`Prometheus — pull-based система мониторинга.
|
||
Сам опрашивает (scrape) цели по HTTP /metrics.
|
||
|
||
Типы метрик:
|
||
Counter — только растёт (запросы, ошибки)
|
||
Gauge — любое значение (CPU%, размер очереди)
|
||
Histogram — распределение значений (latency, размер запроса)
|
||
Summary — квантили (p50, p95, p99 latency)
|
||
|
||
# prometheus.yml — конфигурация:
|
||
global:
|
||
scrape_interval: 15s
|
||
evaluation_interval: 15s
|
||
|
||
alerting:
|
||
alertmanagers:
|
||
- static_configs:
|
||
- targets: ['alertmanager:9093']
|
||
|
||
rule_files:
|
||
- "alerts/*.yml"
|
||
|
||
scrape_configs:
|
||
- job_name: 'prometheus'
|
||
static_configs:
|
||
- targets: ['localhost:9090']
|
||
|
||
- job_name: 'node-exporter'
|
||
static_configs:
|
||
- targets: ['node1:9100', 'node2:9100']
|
||
|
||
- job_name: 'kubernetes-pods'
|
||
kubernetes_sd_configs:
|
||
- role: pod
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||
action: keep
|
||
regex: "true"`)
|
||
|
||
utils.Println(utils.ColorBold + "PromQL — язык запросов:" + utils.ColorReset)
|
||
utils.Code(`# Мгновенный вектор:
|
||
http_requests_total # все метки
|
||
http_requests_total{job="api", status="200"} # с фильтром
|
||
http_requests_total{status=~"5.."} # regex: 5xx
|
||
|
||
# Функции:
|
||
rate(http_requests_total[5m]) # скорость за 5 минут
|
||
irate(http_requests_total[1m]) # мгновенная скорость
|
||
increase(http_requests_total[1h]) # прирост за час
|
||
sum(rate(http_requests_total[5m])) # сумма
|
||
sum by (status)(rate(http_requests_total[5m])) # группировка
|
||
|
||
# Полезные запросы:
|
||
# Загрузка CPU:
|
||
100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
|
||
|
||
# Свободная RAM:
|
||
node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100
|
||
|
||
# Latency p99 (если histogram):
|
||
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
|
||
|
||
# Error rate:
|
||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||
/ sum(rate(http_requests_total[5m])) * 100`)
|
||
|
||
utils.Println(utils.ColorBold + "Exporters — источники метрик:" + utils.ColorReset)
|
||
utils.Code(`node_exporter — метрики хоста (CPU, RAM, диск, сеть) :9100
|
||
blackbox_exporter — проверка доступности (HTTP, TCP, ICMP) :9115
|
||
mysqld_exporter — MySQL :9104
|
||
postgres_exporter — PostgreSQL :9187
|
||
nginx-vts-exporter — Nginx :9913
|
||
redis_exporter — Redis :9121
|
||
cadvisor — Docker/cgroup метрики :8080
|
||
|
||
# Kubernetes: kube-state-metrics, metrics-server`)
|
||
|
||
utils.Section("AlertManager — маршрутизация алертов")
|
||
utils.Code(`# Правила алертинга (alerts/nodes.yml):
|
||
groups:
|
||
- name: nodes
|
||
rules:
|
||
- alert: HighCPU
|
||
expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Высокая загрузка CPU на {{ $labels.instance }}"
|
||
description: "CPU загружен на {{ $value | printf \"%.1f\" }}%"
|
||
|
||
- alert: DiskRunningOut
|
||
expr: node_filesystem_avail_bytes / node_filesystem_size_bytes * 100 < 10
|
||
for: 10m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Заканчивается место на {{ $labels.instance }}"
|
||
|
||
- alert: ServiceDown
|
||
expr: up == 0
|
||
for: 1m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Сервис {{ $labels.job }} недоступен"
|
||
|
||
# alertmanager.yml:
|
||
global:
|
||
slack_api_url: 'https://hooks.slack.com/services/...'
|
||
|
||
route:
|
||
group_by: ['alertname', 'instance']
|
||
group_wait: 30s
|
||
group_interval: 5m
|
||
repeat_interval: 4h
|
||
receiver: 'slack-warnings'
|
||
routes:
|
||
- match:
|
||
severity: critical
|
||
receiver: 'pagerduty'
|
||
|
||
receivers:
|
||
- name: 'slack-warnings'
|
||
slack_configs:
|
||
- channel: '#alerts'
|
||
title: '{{ .GroupLabels.alertname }}'
|
||
text: '{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}'
|
||
|
||
- name: 'pagerduty'
|
||
pagerduty_configs:
|
||
- routing_key: '<PD_ROUTING_KEY>'`)
|
||
|
||
utils.Section("Grafana")
|
||
utils.Code(`# Запуск (Docker Compose):
|
||
services:
|
||
grafana:
|
||
image: grafana/grafana:latest
|
||
ports:
|
||
- "3000:3000"
|
||
environment:
|
||
GF_SECURITY_ADMIN_PASSWORD: secret
|
||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||
volumes:
|
||
- grafana-data:/var/lib/grafana
|
||
- ./grafana/provisioning:/etc/grafana/provisioning
|
||
|
||
# Provisioning — автоматическая настройка:
|
||
# datasources/prometheus.yml:
|
||
apiVersion: 1
|
||
datasources:
|
||
- name: Prometheus
|
||
type: prometheus
|
||
url: http://prometheus:9090
|
||
isDefault: true
|
||
|
||
# dashboards.yml — загрузка дашбордов из файлов
|
||
# Готовые дашборды: grafana.com/grafana/dashboards
|
||
# ID 1860 — Node Exporter Full
|
||
# ID 13770 — Kubernetes Pods
|
||
# ID 9628 — PostgreSQL`)
|
||
|
||
utils.Section("Loki — логи как Prometheus")
|
||
utils.Code(`# Loki собирает логи, Promtail — агент отправки.
|
||
|
||
# promtail-config.yml:
|
||
server:
|
||
http_listen_port: 9080
|
||
|
||
clients:
|
||
- url: http://loki:3100/loki/api/v1/push
|
||
|
||
scrape_configs:
|
||
- job_name: system
|
||
static_configs:
|
||
- targets: [localhost]
|
||
labels:
|
||
job: varlogs
|
||
__path__: /var/log/**/*.log
|
||
|
||
- job_name: containers
|
||
docker_sd_configs:
|
||
- host: unix:///var/run/docker.sock
|
||
relabel_configs:
|
||
- source_labels: [__meta_docker_container_name]
|
||
target_label: container
|
||
|
||
# LogQL — язык запросов Loki (похож на PromQL):
|
||
{job="nginx"} |= "error" # поиск строки
|
||
{container="api"} | json | level="error" # парсинг JSON
|
||
{job="app"} | logfmt | duration > 1s # фильтр по полю
|
||
|
||
rate({job="nginx"} |= "error" [5m]) # скорость ошибок`)
|
||
|
||
utils.Section("Kubernetes мониторинг")
|
||
utils.Code(`# kube-prometheus-stack (Helm) — полный стек:
|
||
helm repo add prometheus-community \
|
||
https://prometheus-community.github.io/helm-charts
|
||
helm repo update
|
||
|
||
helm install kube-prometheus prometheus-community/kube-prometheus-stack \
|
||
--namespace monitoring --create-namespace \
|
||
--set grafana.adminPassword=secret \
|
||
--set prometheus.prometheusSpec.retention=30d
|
||
|
||
# Компоненты стека:
|
||
# - Prometheus Operator
|
||
# - Prometheus
|
||
# - Alertmanager
|
||
# - Grafana
|
||
# - node-exporter
|
||
# - kube-state-metrics
|
||
|
||
# ServiceMonitor — автоматический scraping:
|
||
apiVersion: monitoring.coreos.com/v1
|
||
kind: ServiceMonitor
|
||
metadata:
|
||
name: myapp
|
||
spec:
|
||
selector:
|
||
matchLabels:
|
||
app: myapp
|
||
endpoints:
|
||
- port: metrics
|
||
interval: 30s`)
|
||
|
||
utils.Footer()
|
||
}
|