mroot/grafana_gen_template
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added alerts

Browse source
This commit is contained in:
Magnus Root 2026-04-23 11:20:05 +03:00
parent 625caf195d
commit c1ba2d0570
2 changed files with 520 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

330
alert_rules_template.json
View file

@ -4,7 +4,7 @@
"defaults": {
"no_data_state": "NoData",
"exec_err_state": "Error",
"receiver": "Telegram PTAF Grafana",
"receiver": "tg+mail+mattermost PTAF Grafana",
"folder": "WAF - PTAF",
"group": "PTAF Grafana",
"group_interval": "1m"
@ -68,7 +68,7 @@
"static_alerts": [
{
"uid": "ef4ohmap00utcd",
"title": "Use in /",
"title": "[SYS] Use in / 75%",
"condition": "Порог",
"for": "3m",
"no_data_state": "NoData",
@ -146,7 +146,7 @@
},
{
"uid": "af4ok3zmdvny8f",
"title": "Use in /var/log",
"title": "[SYS] Use in /var/log 75%",
"condition": "Порог",
"for": "5m",
"no_data_state": "NoData",
@ -224,7 +224,7 @@
},
{
"uid": "ef4spdg2qcge8c",
"title": "CPU Busy",
"title": "[SYS] CPU Busy 75%",
"condition": "C",
"for": "3m",
"no_data_state": "NoData",
@ -301,7 +301,7 @@
},
{
"uid": "ef4spmgmowe80e",
"title": "RAM Busy",
"title": "[SYS] RAM Busy 75%",
"condition": "Превышение потребления RAM",
"for": "3m",
"no_data_state": "NoData",
@ -376,15 +376,325 @@
"summary": "💾 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 75%)"
}
},
{
"uid": "ef4ohmap00utcd90",
"title": "[SYS] Use in / 90%",
"condition": "Порог",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято в /",
"relativeTimeRange": {
"from": 86400,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n )\n)",
"format": "time_series",
"instant": true,
"intervalMs": 15000,
"legendFormat": "{{mountpoint}} - Available",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято в /"
}
},
{
"refId": "Порог",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"E"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято в /",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "Порог",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: диск / заполнен на {{ $values.\"Занято в /\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "af4ok3zmdvny8f90",
"title": "[SYS] Use in /var/log 90%",
"condition": "Порог",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято в /var/log",
"relativeTimeRange": {
"from": 86400,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n )\n)",
"format": "time_series",
"instant": true,
"intervalMs": 15000,
"legendFormat": "{{mountpoint}} - Available",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято в /var/log"
}
},
{
"refId": "Порог",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"E"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято в /var/log",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "Порог",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: диск /var/log заполнен на {{ $values.\"Занято в /var/log\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "ef4spdg2qcge8c90",
"title": "[SYS] CPU Busy 90%",
"condition": "C",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято CPU",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято CPU"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"C"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято CPU",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "C",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: CPU загружен на {{ $values.\"Занято CPU\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "ef4spmgmowe80e90",
"title": "[SYS] RAM Busy 90%",
"condition": "Превышение потребления RAM",
"for": "1m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "Занято RAM",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "deibaq3pwytxce",
"model": {
"datasource": {
"type": "prometheus",
"uid": "deibaq3pwytxce"
},
"editorMode": "code",
"expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "Занято RAM"
}
},
{
"refId": "Превышение потребления RAM",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
90
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"C"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "Занято RAM",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "Превышение потребления RAM",
"type": "threshold"
}
}
],
"annotations": {
"summary": "🚨 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 90%)"
}
},
{
"uid": "cfc4iwprdvn5se",
"title": "Состояние контейнеров",
"title": "[SYS] Состояние контейнеров",
"condition": "Целевое значение",
"for": "2m",
"no_data_state": "NoData",
"exec_err_state": "Error",
"annotations": {
"summary": "🐳 {{ $labels.instance }}: контейнер {{ $labels.name }} упал (статус: {{ $values.\"Текущее количество\".Value }})"
"summary": "🐳 Контейнер {{ index $labels `name` }} на хосте {{ index $labels `instance` }} упал. Текущее значение: {{ $values.A.Value }}"
},
"data": [
{
@ -455,7 +765,7 @@
},
{
"uid": "efcxg0oa65gcgd",
"title": "Упала Angie",
"title": "[SYS] Упала Angie",
"condition": "C",
"for": "1m",
"no_data_state": "NoData",
@ -533,7 +843,7 @@
}
],
"contact_point": {
"receiver_name": "Telegram PTAF Grafana",
"message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 *Firing*{{ else }}✅ *Resolved*{{ end }}\n{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}\n\n{{ if index .Values \"Значение\" }}*Текущее значение:* {{ index .Values \"Значение\" | printf \"%.2f\" }}\n{{ end }}\n*Labels:*\n{{ range .Labels.SortedPairs }} • {{ .Name }} = {{ .Value }}\n{{ end }}{{ end }}"
"receiver_name": "tg+mail+mattermost PTAF Grafana",
"message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 Firing{{ else }}✅ Resolved{{ end }}\n{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}\n\n{{ if index .Values \"Значение\" }}Текущее значение: {{ index .Values \"Значение\" | printf \"%.2f\" }}\n{{ end }}Labels:\n{{ range .Labels.SortedPairs }} • {{ .Name }} = {{ .Value }}\n{{ end }}{{ end }}"
}
}

220
dashboard_template.json
View file

@ -133,6 +133,12 @@
"panel_key": "overview_errors_by_container",
"width": 24,
"x_offset": 0
},
{
"panel_key": "overview_nginx_vs_angie",
"width": 24,
"x_offset": 0,
"title_format": "Nginx vs Angie Status Codes SID: ${SID:raw} / app: ${app_name:raw}"
}
],
"inter_domain_panels": [],
@ -591,7 +597,7 @@
},
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:access AND SID:{sid}",
"base_query": "log_type:nginx-access-PTAF AND SID:{sid}",
"metric": "count",
"bucket_aggs": [
{
@ -779,7 +785,7 @@
},
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:access AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}",
"base_query": "log_type:nginx-access-PTAF AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}",
"bucket_aggs": [
{
"field": "response_status_code",
@ -887,7 +893,7 @@
},
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:access AND TENANT:${tenant:lucene}",
"base_query": "log_type:nginx-access-PTAF AND TENANT:${tenant:lucene}",
"bucket_aggs": [
{
"field": "node_name.keyword",
@ -910,7 +916,7 @@
"type": "date_histogram"
}
],
"vl_base_query": "log_type:access TENANT:in(${tenant:csv})"
"vl_base_query": "log_type:nginx-access-PTAF TENANT:in(${tenant:csv})"
}
},
"overview_tenant_per_node": {
@ -994,7 +1000,7 @@
},
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:access AND node_name:${node_name:lucene}",
"base_query": "log_type:nginx-access-PTAF AND node_name:${node_name:lucene}",
"bucket_aggs": [
{
"field": "TENANT.keyword",
@ -1017,7 +1023,7 @@
"type": "date_histogram"
}
],
"vl_base_query": "log_type:access node_name:in(${node_name:csv})"
"vl_base_query": "log_type:nginx-access-PTAF node_name:in(${node_name:csv})"
}
},
"overview_uri_top15": {
@ -1106,7 +1112,7 @@
},
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:access AND SID:(${SID:lucene}) AND app_name:${app_name:lucene} AND response_status_code:${response_status_code:lucene}",
"base_query": "log_type:nginx-access-PTAF AND SID:(${SID:lucene}) AND app_name:${app_name:lucene} AND response_status_code:${response_status_code:lucene}",
"bucket_aggs": [
{
"field": "uri.keyword",
@ -1366,7 +1372,7 @@
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene}",
"vl_base_query": "log_type:angie-error-PTAF",
"vl_base_query": "log_type:angie-error-PTAF error_server:!=\"\"",
"bucket_aggs": [
{
"field": "error_server",
@ -1388,7 +1394,8 @@
},
"type": "date_histogram"
}
]
],
"unknown_label": "(unknown)"
},
"vl_interval": "1m",
"interval": "1m"
@ -1476,7 +1483,7 @@
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene} AND error_server:${error_server:lucene}",
"vl_base_query": "log_type:angie-error-PTAF error_server:${error_server:raw}",
"vl_base_query": "log_type:angie-error-PTAF error_upstream_path:!=\"\" error_server:${error_server:raw}",
"bucket_aggs": [
{
"field": "error_upstream_path",
@ -1498,7 +1505,8 @@
},
"type": "date_histogram"
}
]
],
"unknown_label": "(unknown path)"
},
"vl_interval": "1m",
"interval": "1m"
@ -1775,8 +1783,8 @@
},
"query_mode": "bucket_logs",
"query_config": {
"base_query": "log_type:access AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}",
"vl_base_query": "log_type:access SID:in(${SID:csv}) response_status_code:${response_status_code:raw}",
"base_query": "log_type:nginx-access-PTAF AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}",
"vl_base_query": "log_type:nginx-access-PTAF SID:in(${SID:csv}) response_status_code:${response_status_code:raw}",
"bucket_aggs": [
{
"field": "agent_name",
@ -1800,6 +1808,168 @@
}
]
}
},
"overview_nginx_vs_angie": {
"type": "timeseries",
"description": "Сравнение кодов ответа nginx (log_type:access) и angie (log_type:angie-access-PTAF)",
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "smooth",
"lineWidth": 1,
"showPoints": "never",
"spanNulls": true
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "2**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "3**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "4**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5**"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single"
}
},
"query_mode": "multi_target",
"query_config": {
"base_query": "SID:{sid}",
"metric": "count",
"targets": [
{
"alias": "nginx 2**",
"query_suffix": "AND response_status_code:[200 TO 299]",
"ref_id": "A",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=200 response_status_code:<=299"
},
{
"alias": "nginx 3**",
"query_suffix": "AND response_status_code:[300 TO 399]",
"ref_id": "B",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=300 response_status_code:<=399"
},
{
"alias": "nginx 4**",
"query_suffix": "AND response_status_code:[400 TO 499]",
"ref_id": "C",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=400 response_status_code:<=499"
},
{
"alias": "nginx 5**",
"query_suffix": "AND response_status_code:[500 TO 599]",
"ref_id": "D",
"vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=500 response_status_code:<=599"
},
{
"alias": "angie 2**",
"query_suffix": "AND response_status_code:[200 TO 299]",
"ref_id": "E",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=200 response_status_code:<=299"
},
{
"alias": "angie 3**",
"query_suffix": "AND response_status_code:[300 TO 399]",
"ref_id": "F",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=300 response_status_code:<=399"
},
{
"alias": "angie 4**",
"query_suffix": "AND response_status_code:[400 TO 499]",
"ref_id": "G",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=400 response_status_code:<=499"
},
{
"alias": "angie 5**",
"query_suffix": "AND response_status_code:[500 TO 599]",
"ref_id": "H",
"vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}",
"vl_suffix": "response_status_code:>=500 response_status_code:<=599"
}
]
},
"vl_interval": "1m",
"title": "Nginx vs Angie Status Codes SID: ${SID:raw} / app: ${app_name:raw}"
}
},
"queries": {
@ -1944,20 +2114,30 @@
"name": "error_server",
"label": "Error App",
"description": "Введите имя приложения (error_server) или * для всех",
"type": "textbox",
"datasource": null,
"definition": "",
"query": "",
"type": "query",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "efewdonokxybkf"
},
"definition": "log_type:angie-error-PTAF",
"refresh": 1,
"includeAll": false,
"includeAll": true,
"multi": false,
"allValue": "*",
"sort": 1,
"regex": "",
"allowCustomValue": false,
"current": {
"text": "*",
"value": "*"
"text": "All",
"value": "$__all"
},
"options": [],
"query": {
"field": "error_server",
"limit": 25,
"query": "log_type:angie-error-PTAF",
"refId": "VictoriaLogsVariableQueryEditor-VariableQuery",
"type": "fieldValue"
}
}
]