{ "version": "1.4.3", "description": "Шаблон правил алертинга Grafana для PTAF", "defaults": { "no_data_state": "NoData", "exec_err_state": "Error", "receiver": "tg+mail+mattermost PTAF Grafana", "folder": "WAF - PTAF", "group": "PTAF Grafana", "group_interval": "1m" }, "rps_alert": { "relative_time_range_from": 1800, "keep_firing_for": "", "annotation_summary": "🔴 {client_title} инфраструктурный RPS > {rps_limit} RPS.", "queries": { "data_query": { "refId": "A", "queryType": "lucene", "datasource_uid": "", "model": { "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "1m", "min_doc_count": "0", "trimEdges": "0" }, "type": "date_histogram" } ], "format": "table", "instant": false, "luceneQueryType": "Metric", "metrics": [ { "hide": false, "id": "1", "type": "count" } ], "queryType": "lucene", "range": true, "timeField": "@timestamp" } }, "math_expression": { "refId": "RPS", "expression": "$A / 60", "type": "math" }, "reduce_expression": { "refId": "Текущий RPS", "reducer": "median", "type": "reduce", "source_ref": "RPS" }, "threshold_expression": { "refId": "Превышение", "type": "threshold", "source_ref": "Текущий RPS", "evaluator_type": "gt" } } }, "static_alerts": [ { "uid": "ef4ohmap00utcd", "title": "[SYS] Use in / 75%", "condition": "Порог", "for": "3m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято в /", "relativeTimeRange": { "from": 86400, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n )\n)", "format": "time_series", "instant": true, "intervalMs": 15000, "legendFormat": "{{mountpoint}} - Available", "maxDataPoints": 43200, "range": false, "refId": "Занято в /" } }, { "refId": "Порог", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "E" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято в /", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Порог", "type": "threshold" } } ], "annotations": { "summary": "🟡 {{ $labels.instance }}: диск / заполнен {{ (index $values \"Занято в /\").Value | printf \"%.1f\" }}% (порог 75%)", "circle": "🟡", "alert_text": "диск / заполнен", "value_key": "Занято в /", "threshold": "75%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "af4ok3zmdvny8f", "title": "[SYS] Use in /var/log 75%", "condition": "Порог", "for": "5m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято в /var/log", "relativeTimeRange": { "from": 86400, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n )\n)", "format": "time_series", "instant": true, "intervalMs": 15000, "legendFormat": "{{mountpoint}} - Available", "maxDataPoints": 43200, "range": false, "refId": "Занято в /var/log" } }, { "refId": "Порог", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "E" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято в /var/log", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Порог", "type": "threshold" } } ], "annotations": { "summary": "🟡 {{ $labels.instance }}: диск /var/log заполнен {{ (index $values \"Занято в /var/log\").Value | printf \"%.1f\" }}% (порог 75%)", "circle": "🟡", "alert_text": "диск /var/log заполнен", "value_key": "Занято в /var/log", "threshold": "75%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4ohmap00utcd90", "title": "[SYS] Use in / 90%", "condition": "Порог", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято в /", "relativeTimeRange": { "from": 86400, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n )\n)", "format": "time_series", "instant": true, "intervalMs": 15000, "legendFormat": "{{mountpoint}} - Available", "maxDataPoints": 43200, "range": false, "refId": "Занято в /" } }, { "refId": "Порог", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 90 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "E" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято в /", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Порог", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: диск / заполнен {{ (index $values \"Занято в /\").Value | printf \"%.1f\" }}% (порог 90%)", "circle": "🔴", "alert_text": "диск / заполнен", "value_key": "Занято в /", "threshold": "90%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "af4ok3zmdvny8f90", "title": "[SYS] Use in /var/log 90%", "condition": "Порог", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято в /var/log", "relativeTimeRange": { "from": 86400, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n )\n)", "format": "time_series", "instant": true, "intervalMs": 15000, "legendFormat": "{{mountpoint}} - Available", "maxDataPoints": 43200, "range": false, "refId": "Занято в /var/log" } }, { "refId": "Порог", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 90 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "E" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято в /var/log", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Порог", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: диск /var/log заполнен {{ (index $values \"Занято в /var/log\").Value | printf \"%.1f\" }}% (порог 90%)", "circle": "🔴", "alert_text": "диск /var/log заполнен", "value_key": "Занято в /var/log", "threshold": "90%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spdg2qcge35", "title": "[SYS] CPU Busy 35%", "condition": "C", "for": "5m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято CPU", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято CPU" } }, { "refId": "C", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 35 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято CPU", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "C", "type": "threshold" } } ], "annotations": { "summary": "⚪ {{ $labels.instance }}: CPU загружен {{ (index $values \"Занято CPU\").Value | printf \"%.1f\" }}% (порог 35%)", "circle": "⚪", "alert_text": "CPU загружен", "value_key": "Занято CPU", "threshold": "35%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spdg2qcge45", "title": "[SYS] CPU Busy 45%", "condition": "C", "for": "5m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято CPU", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято CPU" } }, { "refId": "C", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 45 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято CPU", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "C", "type": "threshold" } } ], "annotations": { "summary": "🟡 {{ $labels.instance }}: CPU загружен {{ (index $values \"Занято CPU\").Value | printf \"%.1f\" }}% (порог 45%)", "circle": "🟡", "alert_text": "CPU загружен", "value_key": "Занято CPU", "threshold": "45%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spdg2qcge8c", "title": "[SYS] CPU Busy 75%", "condition": "C", "for": "3m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято CPU", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято CPU" } }, { "refId": "C", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято CPU", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "C", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: CPU загружен {{ (index $values \"Занято CPU\").Value | printf \"%.1f\" }}% (порог 75%)", "circle": "🔴", "alert_text": "CPU загружен", "value_key": "Занято CPU", "threshold": "75%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spmgmowe35", "title": "[SYS] RAM Busy 35%", "condition": "Превышение потребления RAM", "for": "5m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято RAM", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято RAM" } }, { "refId": "Превышение потребления RAM", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 35 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято RAM", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение потребления RAM", "type": "threshold" } } ], "annotations": { "summary": "⚪ {{ $labels.instance }}: RAM занята {{ (index $values \"Занято RAM\").Value | printf \"%.1f\" }}% (порог 35%)", "circle": "⚪", "alert_text": "RAM занята", "value_key": "Занято RAM", "threshold": "35%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spmgmowe45", "title": "[SYS] RAM Busy 45%", "condition": "Превышение потребления RAM", "for": "5m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято RAM", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято RAM" } }, { "refId": "Превышение потребления RAM", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 45 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято RAM", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение потребления RAM", "type": "threshold" } } ], "annotations": { "summary": "🟡 {{ $labels.instance }}: RAM занята {{ (index $values \"Занято RAM\").Value | printf \"%.1f\" }}% (порог 45%)", "circle": "🟡", "alert_text": "RAM занята", "value_key": "Занято RAM", "threshold": "45%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spmgmowe80e", "title": "[SYS] RAM Busy 75%", "condition": "Превышение потребления RAM", "for": "3m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято RAM", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято RAM" } }, { "refId": "Превышение потребления RAM", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято RAM", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение потребления RAM", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: RAM занята {{ (index $values \"Занято RAM\").Value | printf \"%.1f\" }}% (порог 75%)", "circle": "🔴", "alert_text": "RAM занята", "value_key": "Занято RAM", "threshold": "75%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spdg2qcge8c90", "title": "[SYS] CPU Busy 90%", "condition": "C", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято CPU", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято CPU" } }, { "refId": "C", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 90 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято CPU", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "C", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: CPU загружен {{ (index $values \"Занято CPU\").Value | printf \"%.1f\" }}% (порог 90%)", "circle": "🔴", "alert_text": "CPU загружен", "value_key": "Занято CPU", "threshold": "90%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "ef4spmgmowe80e90", "title": "[SYS] RAM Busy 90%", "condition": "Превышение потребления RAM", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "Занято RAM", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Занято RAM" } }, { "refId": "Превышение потребления RAM", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 90 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Занято RAM", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение потребления RAM", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: RAM занята {{ (index $values \"Занято RAM\").Value | printf \"%.1f\" }}% (порог 90%)", "circle": "🔴", "alert_text": "RAM занята", "value_key": "Занято RAM", "threshold": "90%" }, "ruleGroup": "[SYSTEM]" }, { "uid": "net_total_35_bond0", "title": "[SYS] Network bond0 35%", "condition": "Превышение", "for": "5m", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": false, "folderUID": "", "ruleGroup": "[SYSTEM]", "annotations": { "summary": "⚪ {{ $labels.instance }}: нагрузка на bond0 {{ (index $values \"Total\").Value | printf \"%.1f\" }}% (порог 35% от 25 Гбит/с)", "circle": "⚪", "alert_text": "нагрузка на bond0", "value_key": "Total", "threshold": "35% от 25 Гбит/с" }, "data": [ { "refId": "Total", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "(rate(node_network_receive_bytes_total{instance=~\"PTAF-TB-.*\", device=\"bond0\"}[5m]) + rate(node_network_transmit_bytes_total{instance=~\"PTAF-TB-.*\", device=\"bond0\"}[5m])) * 8 / 1000000000", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Total" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 8.75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Total", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "net_total_45_bond0", "title": "[SYS] Network bond0 45%", "condition": "Превышение", "for": "3m", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": false, "folderUID": "", "ruleGroup": "[SYSTEM]", "annotations": { "summary": "🟡 {{ $labels.instance }}: нагрузка на bond0 {{ (index $values \"Total\").Value | printf \"%.1f\" }}% (порог 45% от 25 Гбит/с)", "circle": "🟡", "alert_text": "нагрузка на bond0", "value_key": "Total", "threshold": "45% от 25 Гбит/с" }, "data": [ { "refId": "Total", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "(rate(node_network_receive_bytes_total{instance=~\"PTAF-TB-.*\", device=\"bond0\"}[5m]) + rate(node_network_transmit_bytes_total{instance=~\"PTAF-TB-.*\", device=\"bond0\"}[5m])) * 8 / 1000000000", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Total" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 11.25 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Total", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "net_total_75_bond0", "title": "[SYS] Network bond0 75%", "condition": "Превышение", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": false, "folderUID": "", "ruleGroup": "[SYSTEM]", "annotations": { "summary": "🔴 {{ $labels.instance }}: нагрузка на bond0 {{ (index $values \"Total\").Value | printf \"%.1f\" }}% (порог 75% от 25 Гбит/с)", "circle": "🔴", "alert_text": "нагрузка на bond0", "value_key": "Total", "threshold": "75% от 25 Гбит/с" }, "data": [ { "refId": "Total", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "(rate(node_network_receive_bytes_total{instance=~\"PTAF-TB-.*\", device=\"bond0\"}[5m]) + rate(node_network_transmit_bytes_total{instance=~\"PTAF-TB-.*\", device=\"bond0\"}[5m])) * 8 / 1000000000", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Total" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 18.75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Total", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "cfc4iwprdvn5se", "title": "[SYS] Состояние контейнеров", "condition": "Целевое значение", "for": "2m", "no_data_state": "NoData", "exec_err_state": "Error", "annotations": { "summary": "🔴 {{ $labels.instance }}: контейнер упал {{ (index $values \"A\").Value | printf \"%.1f\" }}% (порог )", "circle": "🔴", "alert_text": "контейнер упал", "value_key": "A", "threshold": "" }, "data": [ { "refId": "Текущее количество", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "docker_container_status", "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "Текущее количество" } }, { "refId": "Целевое значение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 0 ], "type": "eq" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "Текущее количество", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Целевое значение", "type": "threshold" } } ], "ruleGroup": "[Docker]" }, { "uid": "efcxg0oa65gcgd", "title": "[SYS] Упала Angie", "condition": "C", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "data": [ { "refId": "A", "relativeTimeRange": { "from": 600, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "node_systemd_unit_state{name=\"angie.service\", state=\"active\",instance=~\"PTAF-TB-.*\"}", "hide": false, "instant": true, "intervalMs": 1000, "legendFormat": "__auto", "maxDataPoints": 43200, "range": false, "refId": "A" } }, { "refId": "C", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 0 ], "type": "eq" }, "operator": { "type": "and" }, "query": { "params": [ "C" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "A", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "C", "type": "threshold" } } ], "annotations": { "summary": "🔴 {{ $labels.instance }}: служба angie.service не активна", "circle": "🔴", "alert_text": "служба angie.service не активна", "value_key": "", "threshold": "" }, "ruleGroup": "[SYSTEM]" }, { "uid": "blackbox_tcp_80", "title": "[SYS] TCP Port 80 Unavailable", "condition": "Превышение", "for": "15s", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": true, "folderUID": "", "ruleGroup": "[SYSTEM]", "annotations": { "summary": "🔴 Хост {{ $labels.host }} ({{ $labels.instance }}): недоступен TCP порт 80" }, "data": [ { "refId": "A", "relativeTimeRange": { "from": 60, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "probe_success{job=\"blackbox_tcp\",instance=~\".*:80\"}", "instant": true, "intervalMs": 1000, "legendFormat": "{{host}}", "maxDataPoints": 43200, "range": false, "refId": "A" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 1 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "A", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "blackbox_tcp_443", "title": "[SYS] TCP Port 443 Unavailable", "condition": "Превышение", "for": "15s", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": true, "folderUID": "", "ruleGroup": "[SYSTEM]", "annotations": { "summary": "🔴 Хост {{ $labels.host }} ({{ $labels.instance }}): недоступен TCP порт 443" }, "data": [ { "refId": "A", "relativeTimeRange": { "from": 60, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "probe_success{job=\"blackbox_tcp\",instance=~\".*:443\"}", "instant": true, "intervalMs": 1000, "legendFormat": "{{host}}", "maxDataPoints": 43200, "range": false, "refId": "A" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 1 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "A", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "blackbox_icmp", "title": "[SYS] ICMP Unavailable", "condition": "Превышение", "for": "15s", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": true, "folderUID": "", "ruleGroup": "[SYSTEM]", "annotations": { "summary": "🔴 Хост {{ $labels.host }} ({{ $labels.instance }}): недоступен по ICMP (ping)" }, "data": [ { "refId": "A", "relativeTimeRange": { "from": 60, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "probe_success{job=\"blackbox_icmp\",host=~\".+\"}", "instant": true, "intervalMs": 1000, "legendFormat": "{{host}}", "maxDataPoints": 43200, "range": false, "refId": "A" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 1 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "A", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "shm_usage_75pct", "title": "[Docker] SHM Usage 75%", "condition": "Превышение", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": false, "folderUID": "", "ruleGroup": "[Docker]", "annotations": { "summary": "🟡 Использование Shared memory контейнером {{ $labels.name }} > 75% > 1 минуты.\nТекущее значение: {{ $values.B.Value | printf \"%.2f\" }} ГБ" }, "data": [ { "refId": "A", "relativeTimeRange": { "from": 300, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "container_shm_usage_bytes / container_shm_size_bytes * 100", "instant": true, "intervalMs": 1000, "legendFormat": "{{name}}", "maxDataPoints": 43200, "range": false, "refId": "A" } }, { "refId": "B", "relativeTimeRange": { "from": 300, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "container_shm_usage_bytes / 1073741824", "instant": true, "intervalMs": 1000, "legendFormat": "{{name}}", "maxDataPoints": 43200, "range": false, "refId": "B" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 75 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "A", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] }, { "uid": "shm_usage_90pct", "title": "[Docker] SHM Usage 90%", "condition": "Превышение", "for": "1m", "no_data_state": "NoData", "exec_err_state": "Error", "isPaused": false, "folderUID": "", "ruleGroup": "[Docker]", "annotations": { "summary": "🔴 Использование Shared memory контейнером {{ $labels.name }} > 90% > 1 минуты.\nТекущее значение: {{ $values.B.Value | printf \"%.2f\" }} ГБ\nВнимание! Контейнер может упасть" }, "data": [ { "refId": "A", "relativeTimeRange": { "from": 300, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "container_shm_usage_bytes / container_shm_size_bytes * 100", "instant": true, "intervalMs": 1000, "legendFormat": "{{name}}", "maxDataPoints": 43200, "range": false, "refId": "A" } }, { "refId": "B", "relativeTimeRange": { "from": 300, "to": 0 }, "datasourceUid": "deibaq3pwytxce", "model": { "datasource": { "type": "prometheus", "uid": "deibaq3pwytxce" }, "editorMode": "code", "expr": "container_shm_usage_bytes / 1073741824", "instant": true, "intervalMs": 1000, "legendFormat": "{{name}}", "maxDataPoints": 43200, "range": false, "refId": "B" } }, { "refId": "Превышение", "relativeTimeRange": { "from": 0, "to": 0 }, "datasourceUid": "__expr__", "model": { "conditions": [ { "evaluator": { "params": [ 90 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "Превышение" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "A", "intervalMs": 1000, "maxDataPoints": 43200, "refId": "Превышение", "type": "threshold" } } ] } ], "contact_point": { "receiver_name": "tg+mail+mattermost PTAF Grafana", "message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 Firing{{ else }}✅ Resolved{{ end }}\n{{ if .Annotations.client_domain }}{{ if .Annotations.value_unit }}🟡 {{ .Annotations.client_domain }}{{ if index .Values \"_Reduce\" }} — {{ .Annotations.code_field }} кодов ответа на WAF {{ index .Values \"_Reduce\" | printf \"%.0f\" }} {{ .Annotations.value_unit }}{{ end }} {{ if .Annotations.field_info }}{{ .Annotations.field_info }}{{ end }}\n{{ if .Annotations.threshold_count }}Пороговое значение {{ .Annotations.threshold_count }}\n{{ end }}{{ else }}{{ if eq .Status \"firing\" }}{{ if eq .Annotations.severity \"warning\" }}🟡{{ else }}🔴{{ end }} {{ .Annotations.client_domain }} — {{ .Annotations.code_field }} ошибок > {{ .Annotations.threshold_value }}% держится > {{ .Annotations.for_duration }}.\nТекущее значение: {{ if index .Values \"Значение\" }}{{ index .Values \"Значение\" | printf \"%.2f\" }}%{{ end }}{{ if index .Values \"_Reduce\" }}{{ index .Values \"_Reduce\" | printf \"%.2f\" }}%{{ end }} {{ if .Annotations.field_info }}{{ .Annotations.field_info }}{{ end }}\n{{ else }}{{ if eq .Annotations.severity \"warning\" }}🟡{{ else }}🔴{{ end }} {{ .Annotations.client_domain }} — {{ .Annotations.code_field }} ошибок < {{ .Annotations.threshold_value }}%.\n{{ end }}{{ end }}{{ else }}{{ if .Annotations.circle }}{{ .Annotations.circle }} {{ .Labels.instance }}: {{ .Annotations.alert_text }}{{ if .Annotations.value_key }} {{ index .Values .Annotations.value_key | printf \"%.1f\" }}% (порог {{ .Annotations.threshold }}){{ end }}\n{{ else }}{{ if .Annotations.summary }}{{ .Annotations.summary }}\n{{ end }}{{ end }}{{ end }}{{ if eq .Status \"firing\" }}Время сработки алерта: {{ .StartsAt.Format \"2006-01-02 15:04:05 UTC\" }}{{ else }}Время разрешения алерта: {{ .EndsAt.Format \"2006-01-02 15:04:05 UTC\" }}{{ end }}\n{{ if eq .Status \"firing\" }}{{ if .Annotations.runbook }}\nНеобходимо {{ .Annotations.runbook }}\n{{ end }}{{ end }}{{ end }}" } }