Added alerts

This commit is contained in:
Magnus Root 2026-04-23 11:20:05 +03:00
parent 625caf195d
commit c1ba2d0570
2 changed files with 520 additions and 30 deletions

View file

@ -4,7 +4,7 @@
"defaults": { "defaults": {
"no_data_state": "NoData", "no_data_state": "NoData",
"exec_err_state": "Error", "exec_err_state": "Error",
"receiver": "Telegram PTAF Grafana", "receiver": "tg+mail+mattermost PTAF Grafana",
"folder": "WAF - PTAF", "folder": "WAF - PTAF",
"group": "PTAF Grafana", "group": "PTAF Grafana",
"group_interval": "1m" "group_interval": "1m"
@ -68,7 +68,7 @@
"static_alerts": [ "static_alerts": [
{ {
"uid": "ef4ohmap00utcd", "uid": "ef4ohmap00utcd",
"title": "Use in /", "title": "[SYS] Use in / 75%",
"condition": "Порог", "condition": "Порог",
"for": "3m", "for": "3m",
"no_data_state": "NoData", "no_data_state": "NoData",
@ -146,7 +146,7 @@
}, },
{ {
"uid": "af4ok3zmdvny8f", "uid": "af4ok3zmdvny8f",
"title": "Use in /var/log", "title": "[SYS] Use in /var/log 75%",
"condition": "Порог", "condition": "Порог",
"for": "5m", "for": "5m",
"no_data_state": "NoData", "no_data_state": "NoData",
@ -224,7 +224,7 @@
}, },
{ {
"uid": "ef4spdg2qcge8c", "uid": "ef4spdg2qcge8c",
"title": "CPU Busy", "title": "[SYS] CPU Busy 75%",
"condition": "C", "condition": "C",
"for": "3m", "for": "3m",
"no_data_state": "NoData", "no_data_state": "NoData",
@ -301,7 +301,7 @@
}, },
{ {
"uid": "ef4spmgmowe80e", "uid": "ef4spmgmowe80e",
"title": "RAM Busy", "title": "[SYS] RAM Busy 75%",
"condition": "Превышение потребления RAM", "condition": "Превышение потребления RAM",
"for": "3m", "for": "3m",
"no_data_state": "NoData", "no_data_state": "NoData",
@ -376,15 +376,325 @@
"summary": "💾 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 75%)" "summary": "💾 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 75%)"
} }
}, },
{
"uid": "ef4ohmap00utcd90",
"title": "[SYS] Use in / 90%",
"condition": "Порог",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято в /",
"relativeTimeRange": {
"from": 86400,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n )\n)",
"format": "time_series",
"instant": true,
"intervalMs": 15000,
"legendFormat": "{{mountpoint}} - Available",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято в /"
}
},
{
"refId": "Порог",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"E"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято в /",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "Порог",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: диск / заполнен на {{ $values.\"Занято в /\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "af4ok3zmdvny8f90",
"title": "[SYS] Use in /var/log 90%",
"condition": "Порог",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято в /var/log",
"relativeTimeRange": {
"from": 86400,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n )\n)",
"format": "time_series",
"instant": true,
"intervalMs": 15000,
"legendFormat": "{{mountpoint}} - Available",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято в /var/log"
}
},
{
"refId": "Порог",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"E"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято в /var/log",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "Порог",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: диск /var/log заполнен на {{ $values.\"Занято в /var/log\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "ef4spdg2qcge8c90",
"title": "[SYS] CPU Busy 90%",
"condition": "C",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято CPU",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято CPU"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"C"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято CPU",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "C",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: CPU загружен на {{ $values.\"Занято CPU\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "ef4spmgmowe80e90",
"title": "[SYS] RAM Busy 90%",
"condition": "Превышение потребления RAM",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято RAM",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято RAM"
}
},
{
"refId": "Превышение потребления RAM",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"C"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято RAM",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "Превышение потребления RAM",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{ {
"uid": "cfc4iwprdvn5se", "uid": "cfc4iwprdvn5se",
"title": "Состояние контейнеров", "title": "[SYS] Состояние контейнеров",
"condition": "Целевое значение", "condition": "Целевое значение",
"for": "2m", "for": "2m",
"no_data_state": "NoData", "no_data_state": "NoData",
"exec_err_state": "Error", "exec_err_state": "Error",
"annotations": { "annotations": {
"summary": "🐳 {{ $labels.instance }}: контейнер {{ $labels.name }} упал (статус: {{ $values.\"Текущее количество\".Value }})" "summary": "🐳 Контейнер {{ index $labels `name` }} на хосте {{ index $labels `instance` }} упал. Текущее значение: {{ $values.A.Value }}"
}, },
"data": [ "data": [
{ {
@ -455,7 +765,7 @@
}, },
{ {
"uid": "efcxg0oa65gcgd", "uid": "efcxg0oa65gcgd",
"title": "Упала Angie", "title": "[SYS] Упала Angie",
"condition": "C", "condition": "C",
"for": "1m", "for": "1m",
"no_data_state": "NoData", "no_data_state": "NoData",
@ -533,7 +843,7 @@
} }
], ],
"contact_point": { "contact_point": {
"receiver_name": "Telegram PTAF Grafana", "receiver_name": "tg+mail+mattermost PTAF Grafana",
"message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 *Firing*{{ else }}✅ *Resolved*{{ end }}\n{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}\n\n{{ if index .Values \"Значение\" }}*Текущее значение:* {{ index .Values \"Значение\" | printf \"%.2f\" }}\n{{ end }}\n*Labels:*\n{{ range .Labels.SortedPairs }} • {{ .Name }} = {{ .Value }}\n{{ end }}{{ end }}" "message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 Firing{{ else }}✅ Resolved{{ end }}\n{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}\n\n{{ if index .Values \"Значение\" }}Текущее значение: {{ index .Values \"Значение\" | printf \"%.2f\" }}\n{{ end }}Labels:\n{{ range .Labels.SortedPairs }} • {{ .Name }} = {{ .Value }}\n{{ end }}{{ end }}"
} }
} }

View file

@ -133,6 +133,12 @@
"panel_key": "overview_errors_by_container", "panel_key": "overview_errors_by_container",
"width": 24, "width": 24,
"x_offset": 0 "x_offset": 0
},
{
"panel_key": "overview_nginx_vs_angie",
"width": 24,
"x_offset": 0,
"title_format": "Nginx vs Angie Status Codes SID: ${SID:raw} / app: ${app_name:raw}"
} }
], ],
"inter_domain_panels": [], "inter_domain_panels": [],
@ -591,7 +597,7 @@
}, },
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:access AND SID:{sid}", "base_query": "log_type:nginx-access-PTAF AND SID:{sid}",
"metric": "count", "metric": "count",
"bucket_aggs": [ "bucket_aggs": [
{ {
@ -779,7 +785,7 @@
}, },
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:access AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}", "base_query": "log_type:nginx-access-PTAF AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "response_status_code", "field": "response_status_code",
@ -887,7 +893,7 @@
}, },
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:access AND TENANT:${tenant:lucene}", "base_query": "log_type:nginx-access-PTAF AND TENANT:${tenant:lucene}",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "node_name.keyword", "field": "node_name.keyword",
@ -910,7 +916,7 @@
"type": "date_histogram" "type": "date_histogram"
} }
], ],
"vl_base_query": "log_type:access TENANT:in(${tenant:csv})" "vl_base_query": "log_type:nginx-access-PTAF TENANT:in(${tenant:csv})"
} }
}, },
"overview_tenant_per_node": { "overview_tenant_per_node": {
@ -994,7 +1000,7 @@
}, },
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:access AND node_name:${node_name:lucene}", "base_query": "log_type:nginx-access-PTAF AND node_name:${node_name:lucene}",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "TENANT.keyword", "field": "TENANT.keyword",
@ -1017,7 +1023,7 @@
"type": "date_histogram" "type": "date_histogram"
} }
], ],
"vl_base_query": "log_type:access node_name:in(${node_name:csv})" "vl_base_query": "log_type:nginx-access-PTAF node_name:in(${node_name:csv})"
} }
}, },
"overview_uri_top15": { "overview_uri_top15": {
@ -1106,7 +1112,7 @@
}, },
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:access AND SID:(${SID:lucene}) AND app_name:${app_name:lucene} AND response_status_code:${response_status_code:lucene}", "base_query": "log_type:nginx-access-PTAF AND SID:(${SID:lucene}) AND app_name:${app_name:lucene} AND response_status_code:${response_status_code:lucene}",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "uri.keyword", "field": "uri.keyword",
@ -1366,7 +1372,7 @@
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene}", "base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene}",
"vl_base_query": "log_type:angie-error-PTAF", "vl_base_query": "log_type:angie-error-PTAF error_server:!=\"\"",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "error_server", "field": "error_server",
@ -1388,7 +1394,8 @@
}, },
"type": "date_histogram" "type": "date_histogram"
} }
] ],
"unknown_label": "(unknown)"
}, },
"vl_interval": "1m", "vl_interval": "1m",
"interval": "1m" "interval": "1m"
@ -1476,7 +1483,7 @@
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene} AND error_server:${error_server:lucene}", "base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene} AND error_server:${error_server:lucene}",
"vl_base_query": "log_type:angie-error-PTAF error_server:${error_server:raw}", "vl_base_query": "log_type:angie-error-PTAF error_upstream_path:!=\"\" error_server:${error_server:raw}",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "error_upstream_path", "field": "error_upstream_path",
@ -1498,7 +1505,8 @@
}, },
"type": "date_histogram" "type": "date_histogram"
} }
] ],
"unknown_label": "(unknown path)"
}, },
"vl_interval": "1m", "vl_interval": "1m",
"interval": "1m" "interval": "1m"
@ -1775,8 +1783,8 @@
}, },
"query_mode": "bucket_logs", "query_mode": "bucket_logs",
"query_config": { "query_config": {
"base_query": "log_type:access AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}", "base_query": "log_type:nginx-access-PTAF AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}",
"vl_base_query": "log_type:access SID:in(${SID:csv}) response_status_code:${response_status_code:raw}", "vl_base_query": "log_type:nginx-access-PTAF SID:in(${SID:csv}) response_status_code:${response_status_code:raw}",
"bucket_aggs": [ "bucket_aggs": [
{ {
"field": "agent_name", "field": "agent_name",
@ -1800,6 +1808,168 @@
} }
] ]
} }
},
"overview_nginx_vs_angie": {
"type": "timeseries",
"description": "Сравнение кодов ответа nginx (log_type:access) и angie (log_type:angie-access-PTAF)",
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "smooth",
"lineWidth": 1,
"showPoints": "never",
"spanNulls": true
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "2**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "3**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "4**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single"
}
},
"query_mode": "multi_target",
"query_config": {
"base_query": "SID:{sid}",
"metric": "count",
"targets": [
{
"alias": "nginx 2**",
"query_suffix": "AND response_status_code:[200 TO 299]",
"ref_id": "A",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=200 response_status_code:<=299"
},
{
"alias": "nginx 3**",
"query_suffix": "AND response_status_code:[300 TO 399]",
"ref_id": "B",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=300 response_status_code:<=399"
},
{
"alias": "nginx 4**",
"query_suffix": "AND response_status_code:[400 TO 499]",
"ref_id": "C",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=400 response_status_code:<=499"
},
{
"alias": "nginx 5**",
"query_suffix": "AND response_status_code:[500 TO 599]",
"ref_id": "D",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=500 response_status_code:<=599"
},
{
"alias": "angie 2**",
"query_suffix": "AND response_status_code:[200 TO 299]",
"ref_id": "E",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=200 response_status_code:<=299"
},
{
"alias": "angie 3**",
"query_suffix": "AND response_status_code:[300 TO 399]",
"ref_id": "F",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=300 response_status_code:<=399"
},
{
"alias": "angie 4**",
"query_suffix": "AND response_status_code:[400 TO 499]",
"ref_id": "G",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=400 response_status_code:<=499"
},
{
"alias": "angie 5**",
"query_suffix": "AND response_status_code:[500 TO 599]",
"ref_id": "H",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=500 response_status_code:<=599"
}
]
},
"vl_interval": "1m",
"title": "Nginx vs Angie Status Codes SID: ${SID:raw} / app: ${app_name:raw}"
} }
}, },
"queries": { "queries": {
@ -1944,20 +2114,30 @@
"name": "error_server", "name": "error_server",
"label": "Error App", "label": "Error App",
"description": "Введите имя приложения (error_server) или * для всех", "description": "Введите имя приложения (error_server) или * для всех",
"type": "textbox", "type": "query",
"datasource": null, "datasource": {
"definition": "", "type": "victoriametrics-logs-datasource",
"query": "", "uid": "efewdonokxybkf"
},
"definition": "log_type:angie-error-PTAF",
"refresh": 1, "refresh": 1,
"includeAll": false, "includeAll": true,
"multi": false, "multi": false,
"allValue": "*", "allValue": "*",
"sort": 1, "sort": 1,
"regex": "", "regex": "",
"allowCustomValue": false, "allowCustomValue": false,
"current": { "current": {
"text": "*", "text": "All",
"value": "*" "value": "$__all"
},
"options": [],
"query": {
"field": "error_server",
"limit": 25,
"query": "log_type:angie-error-PTAF",
"refId": "VictoriaLogsVariableQueryEditor-VariableQuery",
"type": "fieldValue"
} }
} }
] ]