535 lines
No EOL
15 KiB
JSON
535 lines
No EOL
15 KiB
JSON
{
|
|
"version": "1.3.0",
|
|
"description": "Шаблон правил алертинга Grafana для PTAF",
|
|
"defaults": {
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"receiver": "Telegram PTAF Grafana",
|
|
"folder": "WAF - PTAF",
|
|
"group": "PTAF Grafana",
|
|
"group_interval": "1m"
|
|
},
|
|
"rps_alert": {
|
|
"relative_time_range_from": 1800,
|
|
"keep_firing_for": "",
|
|
"annotation_summary": "Превышен порог в {rps_limit} RPS в {client_title}",
|
|
"queries": {
|
|
"data_query": {
|
|
"refId": "A",
|
|
"queryType": "lucene",
|
|
"datasource_uid": "",
|
|
"model": {
|
|
"bucketAggs": [
|
|
{
|
|
"field": "@timestamp",
|
|
"id": "2",
|
|
"settings": {
|
|
"interval": "1m",
|
|
"min_doc_count": "0",
|
|
"trimEdges": "0"
|
|
},
|
|
"type": "date_histogram"
|
|
}
|
|
],
|
|
"format": "table",
|
|
"instant": false,
|
|
"luceneQueryType": "Metric",
|
|
"metrics": [
|
|
{
|
|
"hide": false,
|
|
"id": "1",
|
|
"type": "count"
|
|
}
|
|
],
|
|
"queryType": "lucene",
|
|
"range": true,
|
|
"timeField": "@timestamp"
|
|
}
|
|
},
|
|
"math_expression": {
|
|
"refId": "RPS",
|
|
"expression": "$A / 60",
|
|
"type": "math"
|
|
},
|
|
"reduce_expression": {
|
|
"refId": "Текущий RPS",
|
|
"reducer": "median",
|
|
"type": "reduce",
|
|
"source_ref": "RPS"
|
|
},
|
|
"threshold_expression": {
|
|
"refId": "Превышение",
|
|
"type": "threshold",
|
|
"source_ref": "Текущий RPS",
|
|
"evaluator_type": "gt"
|
|
}
|
|
}
|
|
},
|
|
"static_alerts": [
|
|
{
|
|
"uid": "ef4ohmap00utcd",
|
|
"title": "Use in /",
|
|
"condition": "Порог",
|
|
"for": "3m",
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"data": [
|
|
{
|
|
"refId": "Занято в /",
|
|
"relativeTimeRange": {
|
|
"from": 86400,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "deibaq3pwytxce",
|
|
"model": {
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "deibaq3pwytxce"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n )\n)",
|
|
"format": "time_series",
|
|
"instant": true,
|
|
"intervalMs": 15000,
|
|
"legendFormat": "{{mountpoint}} - Available",
|
|
"maxDataPoints": 43200,
|
|
"range": false,
|
|
"refId": "Занято в /"
|
|
}
|
|
},
|
|
{
|
|
"refId": "Порог",
|
|
"relativeTimeRange": {
|
|
"from": 0,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {
|
|
"params": [
|
|
75
|
|
],
|
|
"type": "gt"
|
|
},
|
|
"operator": {
|
|
"type": "and"
|
|
},
|
|
"query": {
|
|
"params": [
|
|
"E"
|
|
]
|
|
},
|
|
"reducer": {
|
|
"params": [],
|
|
"type": "last"
|
|
},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "__expr__",
|
|
"uid": "__expr__"
|
|
},
|
|
"expression": "Занято в /",
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": "Порог",
|
|
"type": "threshold"
|
|
}
|
|
}
|
|
],
|
|
"annotations": {
|
|
"summary": "🖥 {{ $labels.instance }}: диск / заполнен на {{ $values.\"Занято в /\".Value | printf \"%.1f\" }}% (порог 75%)"
|
|
}
|
|
},
|
|
{
|
|
"uid": "af4ok3zmdvny8f",
|
|
"title": "Use in /var/log",
|
|
"condition": "Порог",
|
|
"for": "5m",
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"data": [
|
|
{
|
|
"refId": "Занято в /var/log",
|
|
"relativeTimeRange": {
|
|
"from": 86400,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "deibaq3pwytxce",
|
|
"model": {
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "deibaq3pwytxce"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n )\n)",
|
|
"format": "time_series",
|
|
"instant": true,
|
|
"intervalMs": 15000,
|
|
"legendFormat": "{{mountpoint}} - Available",
|
|
"maxDataPoints": 43200,
|
|
"range": false,
|
|
"refId": "Занято в /var/log"
|
|
}
|
|
},
|
|
{
|
|
"refId": "Порог",
|
|
"relativeTimeRange": {
|
|
"from": 0,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {
|
|
"params": [
|
|
75
|
|
],
|
|
"type": "gt"
|
|
},
|
|
"operator": {
|
|
"type": "and"
|
|
},
|
|
"query": {
|
|
"params": [
|
|
"E"
|
|
]
|
|
},
|
|
"reducer": {
|
|
"params": [],
|
|
"type": "last"
|
|
},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "__expr__",
|
|
"uid": "__expr__"
|
|
},
|
|
"expression": "Занято в /var/log",
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": "Порог",
|
|
"type": "threshold"
|
|
}
|
|
}
|
|
],
|
|
"annotations": {
|
|
"summary": "🖥 {{ $labels.instance }}: диск /var/log заполнен на {{ $values.\"Занято в /var/log\".Value | printf \"%.1f\" }}% (порог 75%)"
|
|
}
|
|
},
|
|
{
|
|
"uid": "ef4spdg2qcge8c",
|
|
"title": "CPU Busy",
|
|
"condition": "C",
|
|
"for": "3m",
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"data": [
|
|
{
|
|
"refId": "Занято CPU",
|
|
"relativeTimeRange": {
|
|
"from": 600,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "deibaq3pwytxce",
|
|
"model": {
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "deibaq3pwytxce"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)",
|
|
"instant": true,
|
|
"intervalMs": 1000,
|
|
"legendFormat": "__auto",
|
|
"maxDataPoints": 43200,
|
|
"range": false,
|
|
"refId": "Занято CPU"
|
|
}
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"relativeTimeRange": {
|
|
"from": 0,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {
|
|
"params": [
|
|
75
|
|
],
|
|
"type": "gt"
|
|
},
|
|
"operator": {
|
|
"type": "and"
|
|
},
|
|
"query": {
|
|
"params": [
|
|
"C"
|
|
]
|
|
},
|
|
"reducer": {
|
|
"params": [],
|
|
"type": "last"
|
|
},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "__expr__",
|
|
"uid": "__expr__"
|
|
},
|
|
"expression": "Занято CPU",
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": "C",
|
|
"type": "threshold"
|
|
}
|
|
}
|
|
],
|
|
"annotations": {
|
|
"summary": "🔥 {{ $labels.instance }}: CPU загружен на {{ $values.\"Занято CPU\".Value | printf \"%.1f\" }}% (порог 75%)"
|
|
}
|
|
},
|
|
{
|
|
"uid": "ef4spmgmowe80e",
|
|
"title": "RAM Busy",
|
|
"condition": "Превышение потребления RAM",
|
|
"for": "3m",
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"data": [
|
|
{
|
|
"refId": "Занято RAM",
|
|
"relativeTimeRange": {
|
|
"from": 600,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "deibaq3pwytxce",
|
|
"model": {
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "deibaq3pwytxce"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)",
|
|
"instant": true,
|
|
"intervalMs": 1000,
|
|
"legendFormat": "__auto",
|
|
"maxDataPoints": 43200,
|
|
"range": false,
|
|
"refId": "Занято RAM"
|
|
}
|
|
},
|
|
{
|
|
"refId": "Превышение потребления RAM",
|
|
"relativeTimeRange": {
|
|
"from": 0,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {
|
|
"params": [
|
|
75
|
|
],
|
|
"type": "gt"
|
|
},
|
|
"operator": {
|
|
"type": "and"
|
|
},
|
|
"query": {
|
|
"params": [
|
|
"C"
|
|
]
|
|
},
|
|
"reducer": {
|
|
"params": [],
|
|
"type": "last"
|
|
},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "__expr__",
|
|
"uid": "__expr__"
|
|
},
|
|
"expression": "Занято RAM",
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": "Превышение потребления RAM",
|
|
"type": "threshold"
|
|
}
|
|
}
|
|
],
|
|
"annotations": {
|
|
"summary": "💾 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 75%)"
|
|
}
|
|
},
|
|
{
|
|
"uid": "cfc4iwprdvn5se",
|
|
"title": "Состояние контейнеров",
|
|
"condition": "Целевое значение",
|
|
"for": "2m",
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"annotations": {
|
|
"summary": "🐳 {{ $labels.instance }}: контейнер {{ $labels.name }} упал (статус: {{ $values.\"Текущее количество\".Value }})"
|
|
},
|
|
"data": [
|
|
{
|
|
"refId": "Текущее количество",
|
|
"relativeTimeRange": {
|
|
"from": 600,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "deibaq3pwytxce",
|
|
"model": {
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "deibaq3pwytxce"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "docker_container_status",
|
|
"instant": true,
|
|
"intervalMs": 1000,
|
|
"legendFormat": "__auto",
|
|
"maxDataPoints": 43200,
|
|
"range": false,
|
|
"refId": "Текущее количество"
|
|
}
|
|
},
|
|
{
|
|
"refId": "Целевое значение",
|
|
"relativeTimeRange": {
|
|
"from": 0,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {
|
|
"params": [
|
|
0
|
|
],
|
|
"type": "eq"
|
|
},
|
|
"operator": {
|
|
"type": "and"
|
|
},
|
|
"query": {
|
|
"params": [
|
|
"C"
|
|
]
|
|
},
|
|
"reducer": {
|
|
"params": [],
|
|
"type": "last"
|
|
},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "__expr__",
|
|
"uid": "__expr__"
|
|
},
|
|
"expression": "Текущее количество",
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": "Целевое значение",
|
|
"type": "threshold"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"uid": "efcxg0oa65gcgd",
|
|
"title": "Упала Angie",
|
|
"condition": "C",
|
|
"for": "1m",
|
|
"no_data_state": "NoData",
|
|
"exec_err_state": "Error",
|
|
"data": [
|
|
{
|
|
"refId": "A",
|
|
"relativeTimeRange": {
|
|
"from": 600,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "deibaq3pwytxce",
|
|
"model": {
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "deibaq3pwytxce"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "node_systemd_unit_state{name=\"angie.service\", state=\"active\",instance=~\"PTAF-TB-.*\"}",
|
|
"hide": false,
|
|
"instant": true,
|
|
"intervalMs": 1000,
|
|
"legendFormat": "__auto",
|
|
"maxDataPoints": 43200,
|
|
"range": false,
|
|
"refId": "A"
|
|
}
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"relativeTimeRange": {
|
|
"from": 0,
|
|
"to": 0
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {
|
|
"params": [
|
|
0
|
|
],
|
|
"type": "eq"
|
|
},
|
|
"operator": {
|
|
"type": "and"
|
|
},
|
|
"query": {
|
|
"params": [
|
|
"C"
|
|
]
|
|
},
|
|
"reducer": {
|
|
"params": [],
|
|
"type": "last"
|
|
},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "__expr__",
|
|
"uid": "__expr__"
|
|
},
|
|
"expression": "A",
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": "C",
|
|
"type": "threshold"
|
|
}
|
|
}
|
|
],
|
|
"annotations": {
|
|
"summary": "🔴 {{ $labels.instance }}: служба angie.service не активна"
|
|
}
|
|
}
|
|
]
|
|
} |