diff --git a/alert_rules_template.json b/alert_rules_template.json index 67d3643..f3909bc 100644 --- a/alert_rules_template.json +++ b/alert_rules_template.json @@ -4,7 +4,7 @@ "defaults": { "no_data_state": "NoData", "exec_err_state": "Error", - "receiver": "Telegram PTAF Grafana", + "receiver": "tg+mail+mattermost PTAF Grafana", "folder": "WAF - PTAF", "group": "PTAF Grafana", "group_interval": "1m" @@ -68,7 +68,7 @@ "static_alerts": [ { "uid": "ef4ohmap00utcd", - "title": "Use in /", + "title": "[SYS] Use in / 75%", "condition": "Порог", "for": "3m", "no_data_state": "NoData", @@ -146,7 +146,7 @@ }, { "uid": "af4ok3zmdvny8f", - "title": "Use in /var/log", + "title": "[SYS] Use in /var/log 75%", "condition": "Порог", "for": "5m", "no_data_state": "NoData", @@ -224,7 +224,7 @@ }, { "uid": "ef4spdg2qcge8c", - "title": "CPU Busy", + "title": "[SYS] CPU Busy 75%", "condition": "C", "for": "3m", "no_data_state": "NoData", @@ -301,7 +301,7 @@ }, { "uid": "ef4spmgmowe80e", - "title": "RAM Busy", + "title": "[SYS] RAM Busy 75%", "condition": "Превышение потребления RAM", "for": "3m", "no_data_state": "NoData", @@ -376,15 +376,325 @@ "summary": "💾 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 75%)" } }, + { + "uid": "ef4ohmap00utcd90", + "title": "[SYS] Use in / 90%", + "condition": "Порог", + "for": "1m", + "no_data_state": "NoData", + "exec_err_state": "Error", + "data": [ + { + "refId": "Занято в /", + "relativeTimeRange": { + "from": 86400, + "to": 0 + }, + "datasourceUid": "deibaq3pwytxce", + "model": { + "datasource": { + "type": "prometheus", + "uid": "deibaq3pwytxce" + }, + "editorMode": "code", + "expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/\",\n device!~'rootfs|tmpfs|sysfs|proc|devpts|overlay|squashfs'\n }\n )\n)", + "format": "time_series", + "instant": true, + "intervalMs": 15000, + "legendFormat": "{{mountpoint}} - Available", + "maxDataPoints": 43200, + "range": false, + "refId": "Занято в /" + } + }, + { + "refId": "Порог", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 90 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "E" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "Занято в /", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "Порог", + "type": "threshold" + } + } + ], + "annotations": { + "summary": "🚨 {{ $labels.instance }}: диск / заполнен на {{ $values.\"Занято в /\".Value | printf \"%.1f\" }}% (порог 90%)" + } + }, + { + "uid": "af4ok3zmdvny8f90", + "title": "[SYS] Use in /var/log 90%", + "condition": "Порог", + "for": "1m", + "no_data_state": "NoData", + "exec_err_state": "Error", + "data": [ + { + "refId": "Занято в /var/log", + "relativeTimeRange": { + "from": 86400, + "to": 0 + }, + "datasourceUid": "deibaq3pwytxce", + "model": { + "datasource": { + "type": "prometheus", + "uid": "deibaq3pwytxce" + }, + "editorMode": "code", + "expr": "100 - (\n 100 * (\n node_filesystem_avail_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n /\n on(instance, device, mountpoint) group_left\n node_filesystem_size_bytes{\n job=\"node_exporter\",\n mountpoint=\"/var/log\",\n device!~'rootfs'\n }\n )\n)", + "format": "time_series", + "instant": true, + "intervalMs": 15000, + "legendFormat": "{{mountpoint}} - Available", + "maxDataPoints": 43200, + "range": false, + "refId": "Занято в /var/log" + } + }, + { + "refId": "Порог", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 90 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "E" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "Занято в /var/log", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "Порог", + "type": "threshold" + } + } + ], + "annotations": { + "summary": "🚨 {{ $labels.instance }}: диск /var/log заполнен на {{ $values.\"Занято в /var/log\".Value | printf \"%.1f\" }}% (порог 90%)" + } + }, + { + "uid": "ef4spdg2qcge8c90", + "title": "[SYS] CPU Busy 90%", + "condition": "C", + "for": "1m", + "no_data_state": "NoData", + "exec_err_state": "Error", + "data": [ + { + "refId": "Занято CPU", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "deibaq3pwytxce", + "model": { + "datasource": { + "type": "prometheus", + "uid": "deibaq3pwytxce" + }, + "editorMode": "code", + "expr": "100 - (\n avg by(instance) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"PTAF-TB-.*\"}[5m])\n ) * 100\n)", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "Занято CPU" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 90 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "Занято CPU", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "annotations": { + "summary": "🚨 {{ $labels.instance }}: CPU загружен на {{ $values.\"Занято CPU\".Value | printf \"%.1f\" }}% (порог 90%)" + } + }, + { + "uid": "ef4spmgmowe80e90", + "title": "[SYS] RAM Busy 90%", + "condition": "Превышение потребления RAM", + "for": "1m", + "no_data_state": "NoData", + "exec_err_state": "Error", + "data": [ + { + "refId": "Занято RAM", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "deibaq3pwytxce", + "model": { + "datasource": { + "type": "prometheus", + "uid": "deibaq3pwytxce" + }, + "editorMode": "code", + "expr": "100 - (\n (node_memory_MemAvailable_bytes{instance=~\"PTAF-TB-.*\"} / \n node_memory_MemTotal_bytes{instance=~\"PTAF-TB-.*\"}) * 100\n)", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "Занято RAM" + } + }, + { + "refId": "Превышение потребления RAM", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 90 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "Занято RAM", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "Превышение потребления RAM", + "type": "threshold" + } + } + ], + "annotations": { + "summary": "🚨 {{ $labels.instance }}: RAM занята на {{ $values.\"Занято RAM\".Value | printf \"%.1f\" }}% (порог 90%)" + } + }, { "uid": "cfc4iwprdvn5se", - "title": "Состояние контейнеров", + "title": "[SYS] Состояние контейнеров", "condition": "Целевое значение", "for": "2m", "no_data_state": "NoData", "exec_err_state": "Error", "annotations": { - "summary": "🐳 {{ $labels.instance }}: контейнер {{ $labels.name }} упал (статус: {{ $values.\"Текущее количество\".Value }})" + "summary": "🐳 Контейнер {{ index $labels `name` }} на хосте {{ index $labels `instance` }} упал. Текущее значение: {{ $values.A.Value }}" }, "data": [ { @@ -455,7 +765,7 @@ }, { "uid": "efcxg0oa65gcgd", - "title": "Упала Angie", + "title": "[SYS] Упала Angie", "condition": "C", "for": "1m", "no_data_state": "NoData", @@ -533,7 +843,7 @@ } ], "contact_point": { - "receiver_name": "Telegram PTAF Grafana", - "message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 *Firing*{{ else }}✅ *Resolved*{{ end }}\n{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}\n\n{{ if index .Values \"Значение\" }}*Текущее значение:* {{ index .Values \"Значение\" | printf \"%.2f\" }}\n{{ end }}\n*Labels:*\n{{ range .Labels.SortedPairs }} • {{ .Name }} = {{ .Value }}\n{{ end }}{{ end }}" + "receiver_name": "tg+mail+mattermost PTAF Grafana", + "message_template": "{{ range .Alerts }}{{ if eq .Status \"firing\" }}🔴 Firing{{ else }}✅ Resolved{{ end }}\n{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}\n\n{{ if index .Values \"Значение\" }}Текущее значение: {{ index .Values \"Значение\" | printf \"%.2f\" }}\n{{ end }}Labels:\n{{ range .Labels.SortedPairs }} • {{ .Name }} = {{ .Value }}\n{{ end }}{{ end }}" } } \ No newline at end of file diff --git a/dashboard_template.json b/dashboard_template.json index 49b1648..588564f 100644 --- a/dashboard_template.json +++ b/dashboard_template.json @@ -133,6 +133,12 @@ "panel_key": "overview_errors_by_container", "width": 24, "x_offset": 0 + }, + { + "panel_key": "overview_nginx_vs_angie", + "width": 24, + "x_offset": 0, + "title_format": "Nginx vs Angie Status Codes SID: ${SID:raw} / app: ${app_name:raw}" } ], "inter_domain_panels": [], @@ -591,7 +597,7 @@ }, "query_mode": "bucket_logs", "query_config": { - "base_query": "log_type:access AND SID:{sid}", + "base_query": "log_type:nginx-access-PTAF AND SID:{sid}", "metric": "count", "bucket_aggs": [ { @@ -779,7 +785,7 @@ }, "query_mode": "bucket_logs", "query_config": { - "base_query": "log_type:access AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}", + "base_query": "log_type:nginx-access-PTAF AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}", "bucket_aggs": [ { "field": "response_status_code", @@ -887,7 +893,7 @@ }, "query_mode": "bucket_logs", "query_config": { - "base_query": "log_type:access AND TENANT:${tenant:lucene}", + "base_query": "log_type:nginx-access-PTAF AND TENANT:${tenant:lucene}", "bucket_aggs": [ { "field": "node_name.keyword", @@ -910,7 +916,7 @@ "type": "date_histogram" } ], - "vl_base_query": "log_type:access TENANT:in(${tenant:csv})" + "vl_base_query": "log_type:nginx-access-PTAF TENANT:in(${tenant:csv})" } }, "overview_tenant_per_node": { @@ -994,7 +1000,7 @@ }, "query_mode": "bucket_logs", "query_config": { - "base_query": "log_type:access AND node_name:${node_name:lucene}", + "base_query": "log_type:nginx-access-PTAF AND node_name:${node_name:lucene}", "bucket_aggs": [ { "field": "TENANT.keyword", @@ -1017,7 +1023,7 @@ "type": "date_histogram" } ], - "vl_base_query": "log_type:access node_name:in(${node_name:csv})" + "vl_base_query": "log_type:nginx-access-PTAF node_name:in(${node_name:csv})" } }, "overview_uri_top15": { @@ -1106,7 +1112,7 @@ }, "query_mode": "bucket_logs", "query_config": { - "base_query": "log_type:access AND SID:(${SID:lucene}) AND app_name:${app_name:lucene} AND response_status_code:${response_status_code:lucene}", + "base_query": "log_type:nginx-access-PTAF AND SID:(${SID:lucene}) AND app_name:${app_name:lucene} AND response_status_code:${response_status_code:lucene}", "bucket_aggs": [ { "field": "uri.keyword", @@ -1366,7 +1372,7 @@ "query_mode": "bucket_logs", "query_config": { "base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene}", - "vl_base_query": "log_type:angie-error-PTAF", + "vl_base_query": "log_type:angie-error-PTAF error_server:!=\"\"", "bucket_aggs": [ { "field": "error_server", @@ -1388,7 +1394,8 @@ }, "type": "date_histogram" } - ] + ], + "unknown_label": "(unknown)" }, "vl_interval": "1m", "interval": "1m" @@ -1476,7 +1483,7 @@ "query_mode": "bucket_logs", "query_config": { "base_query": "log_type:angie-error-PTAF AND SID:${SID:lucene} AND error_server:${error_server:lucene}", - "vl_base_query": "log_type:angie-error-PTAF error_server:${error_server:raw}", + "vl_base_query": "log_type:angie-error-PTAF error_upstream_path:!=\"\" error_server:${error_server:raw}", "bucket_aggs": [ { "field": "error_upstream_path", @@ -1498,7 +1505,8 @@ }, "type": "date_histogram" } - ] + ], + "unknown_label": "(unknown path)" }, "vl_interval": "1m", "interval": "1m" @@ -1775,8 +1783,8 @@ }, "query_mode": "bucket_logs", "query_config": { - "base_query": "log_type:access AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}", - "vl_base_query": "log_type:access SID:in(${SID:csv}) response_status_code:${response_status_code:raw}", + "base_query": "log_type:nginx-access-PTAF AND SID:${SID:lucene} AND response_status_code:${response_status_code:lucene}", + "vl_base_query": "log_type:nginx-access-PTAF SID:in(${SID:csv}) response_status_code:${response_status_code:raw}", "bucket_aggs": [ { "field": "agent_name", @@ -1800,6 +1808,168 @@ } ] } + }, + "overview_nginx_vs_angie": { + "type": "timeseries", + "description": "Сравнение кодов ответа nginx (log_type:access) и angie (log_type:angie-access-PTAF)", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 1, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2**" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3**" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4**" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5**" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "query_mode": "multi_target", + "query_config": { + "base_query": "SID:{sid}", + "metric": "count", + "targets": [ + { + "alias": "nginx 2**", + "query_suffix": "AND response_status_code:[200 TO 299]", + "ref_id": "A", + "vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=200 response_status_code:<=299" + }, + { + "alias": "nginx 3**", + "query_suffix": "AND response_status_code:[300 TO 399]", + "ref_id": "B", + "vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=300 response_status_code:<=399" + }, + { + "alias": "nginx 4**", + "query_suffix": "AND response_status_code:[400 TO 499]", + "ref_id": "C", + "vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=400 response_status_code:<=499" + }, + { + "alias": "nginx 5**", + "query_suffix": "AND response_status_code:[500 TO 599]", + "ref_id": "D", + "vl_base": "log_type:nginx-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=500 response_status_code:<=599" + }, + { + "alias": "angie 2**", + "query_suffix": "AND response_status_code:[200 TO 299]", + "ref_id": "E", + "vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=200 response_status_code:<=299" + }, + { + "alias": "angie 3**", + "query_suffix": "AND response_status_code:[300 TO 399]", + "ref_id": "F", + "vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=300 response_status_code:<=399" + }, + { + "alias": "angie 4**", + "query_suffix": "AND response_status_code:[400 TO 499]", + "ref_id": "G", + "vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=400 response_status_code:<=499" + }, + { + "alias": "angie 5**", + "query_suffix": "AND response_status_code:[500 TO 599]", + "ref_id": "H", + "vl_base": "log_type:angie-access-PTAF SID:in(${SID:csv}) app_name:${app_name:raw}", + "vl_suffix": "response_status_code:>=500 response_status_code:<=599" + } + ] + }, + "vl_interval": "1m", + "title": "Nginx vs Angie Status Codes SID: ${SID:raw} / app: ${app_name:raw}" } }, "queries": { @@ -1944,20 +2114,30 @@ "name": "error_server", "label": "Error App", "description": "Введите имя приложения (error_server) или * для всех", - "type": "textbox", - "datasource": null, - "definition": "", - "query": "", + "type": "query", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "efewdonokxybkf" + }, + "definition": "log_type:angie-error-PTAF", "refresh": 1, - "includeAll": false, + "includeAll": true, "multi": false, "allValue": "*", "sort": 1, "regex": "", "allowCustomValue": false, "current": { - "text": "*", - "value": "*" + "text": "All", + "value": "$__all" + }, + "options": [], + "query": { + "field": "error_server", + "limit": 25, + "query": "log_type:angie-error-PTAF", + "refId": "VictoriaLogsVariableQueryEditor-VariableQuery", + "type": "fieldValue" } } ]