From dbe38ec81633878348c5ab63c251aa9582073c40 Mon Sep 17 00:00:00 2001 From: Magnus Root Date: Wed, 25 Feb 2026 14:47:07 +0300 Subject: [PATCH] Added alert rules --- alerts.go | 746 ++++++++++++++++++++++++++++++++++++++++++++++++++++ config.go | 64 +++-- database.go | 49 ++-- main.go | 47 +++- panels.go | 74 ++++++ state.go | 80 +++++- template.go | 27 ++ 7 files changed, 1036 insertions(+), 51 deletions(-) create mode 100644 alerts.go diff --git a/alerts.go b/alerts.go new file mode 100644 index 0000000..a0eb83b --- /dev/null +++ b/alerts.go @@ -0,0 +1,746 @@ +package main + +import ( + "bytes" + "crypto/md5" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "sort" + "strings" + "sync" +) + +// AlertRulesTemplate — структура шаблона алертов из alert_rules_template.json +type AlertRulesTemplate struct { + Version string `json:"version"` + Description string `json:"description"` + Defaults AlertDefaults `json:"defaults"` + RPSAlert map[string]interface{} `json:"rps_alert"` + StaticAlerts []map[string]interface{} `json:"static_alerts"` // Алерты без привязки к БД +} + +type AlertDefaults struct { + NoDataState string `json:"no_data_state"` + ExecErrState string `json:"exec_err_state"` + Receiver string `json:"receiver"` + Folder string `json:"folder"` + Group string `json:"group"` + GroupInterval string `json:"group_interval"` +} + +// generateAlertUID генерирует детерминированный UID алерта из имени клиента и типа. +// Одинаковый UID при повторном запуске = Grafana обновит существующий алерт, не создаст новый. +func generateAlertUID(clientTitle, alertType string) string { + hash := md5.Sum([]byte(clientTitle + ":" + alertType)) + return fmt.Sprintf("%x", hash[:8]) +} + +// buildRPSAlertRule строит JSON правила алерта для одного клиента по шаблону +func buildRPSAlertRule(client ClientData, tmpl AlertRulesTemplate, config Config) map[string]interface{} { + datasourceUID := config.AlertsDatasourceUID + + // SID-запрос: один SID или несколько через OR + sids := make([]string, 0, len(client.Domains)) + for _, d := range client.Domains { + sids = append(sids, d.SID) + } + // Убираем дубликаты и сортируем для детерминированности + sidMap := make(map[string]bool) + for _, s := range sids { + sidMap[s] = true + } + uniqueSIDs := make([]string, 0, len(sidMap)) + for s := range sidMap { + uniqueSIDs = append(uniqueSIDs, s) + } + sort.Strings(uniqueSIDs) + + sidParts := make([]string, len(uniqueSIDs)) + for i, s := range uniqueSIDs { + sidParts[i] = "SID:" + s + } + sidQuery := strings.Join(sidParts, " OR ") + + // Получаем настройки из шаблона + defaults := tmpl.Defaults + receiver := orDefault(config.AlertsReceiver, defaults.Receiver) + group := orDefault(config.AlertsGroup, defaults.Group) + + timeRangeFrom := int64(1800) + if v, ok := tmpl.RPSAlert["relative_time_range_from"].(float64); ok { + timeRangeFrom = int64(v) + } + + summaryTemplate := "Превышен порог в {rps_limit} RPS в {client_title}" + if v, ok := tmpl.RPSAlert["annotation_summary"].(string); ok { + summaryTemplate = v + } + summary := strings.NewReplacer( + "{rps_limit}", fmt.Sprintf("%d", client.RPSLimit), + "{client_title}", client.ClientTitle, + ).Replace(summaryTemplate) + + uid := generateAlertUID(client.ClientTitle, "rps") + dataRefID := "A" + rpsRefID := "RPS " + client.ClientTitle + reduceRefID := "Текущий RPS" + thresholdRefID := "Превышение" + + rule := map[string]interface{}{ + "uid": uid, + "title": fmt.Sprintf("RPS %s", client.ClientTitle), + "condition": thresholdRefID, + "noDataState": defaults.NoDataState, + "execErrState": defaults.ExecErrState, + "isPaused": false, + "folderUID": "", // будет заполнен при отправке + "ruleGroup": group, + "annotations": map[string]string{ + "summary": summary, + }, + "notification_settings": map[string]string{ + "receiver": receiver, + }, + "data": []interface{}{ + // Шаг 1: count-запрос по SID + map[string]interface{}{ + "refId": dataRefID, + "queryType": "lucene", + "relativeTimeRange": map[string]interface{}{ + "from": timeRangeFrom, + "to": 0, + }, + "datasourceUid": datasourceUID, + "model": map[string]interface{}{ + "alias": fmt.Sprintf("RPS %s", client.ClientTitle), + "bucketAggs": []interface{}{ + map[string]interface{}{ + "field": "@timestamp", + "id": "2", + "settings": map[string]interface{}{ + "interval": "1m", + "min_doc_count": "0", + "trimEdges": "0", + }, + "type": "date_histogram", + }, + }, + "datasource": map[string]interface{}{ + "type": "grafana-opensearch-datasource", + "uid": datasourceUID, + }, + "format": "table", + "instant": false, + "intervalMs": 1000, + "luceneQueryType": "Metric", + "maxDataPoints": 43200, + "metrics": []interface{}{ + map[string]interface{}{"hide": false, "id": "1", "type": "count"}, + }, + "query": sidQuery, + "queryType": "lucene", + "range": true, + "refId": dataRefID, + "timeField": "@timestamp", + }, + }, + // Шаг 2: math $A / 60 → RPS + map[string]interface{}{ + "refId": rpsRefID, + "relativeTimeRange": map[string]interface{}{ + "from": timeRangeFrom, + "to": 0, + }, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "datasource": map[string]interface{}{ + "name": "Expression", + "type": "__expr__", + "uid": "__expr__", + }, + "expression": fmt.Sprintf("$%s / 60", dataRefID), + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": rpsRefID, + "type": "math", + "window": "", + }, + }, + // Шаг 3: reduce median + map[string]interface{}{ + "refId": reduceRefID, + "relativeTimeRange": map[string]interface{}{ + "from": 0, + "to": 0, + }, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "evaluator": map[string]interface{}{"params": []int{0, 0}, "type": "gt"}, + "operator": map[string]interface{}{"type": "and"}, + "query": map[string]interface{}{"params": []string{}}, + "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, + "type": "query", + }, + }, + "datasource": map[string]interface{}{ + "name": "Expression", + "type": "__expr__", + "uid": "__expr__", + }, + "expression": rpsRefID, + "intervalMs": 1000, + "maxDataPoints": 43200, + "reducer": "median", + "refId": reduceRefID, + "settings": map[string]interface{}{"mode": ""}, + "type": "reduce", + }, + }, + // Шаг 4: threshold > rps_limit + map[string]interface{}{ + "refId": thresholdRefID, + "relativeTimeRange": map[string]interface{}{ + "from": 0, + "to": 0, + }, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "evaluator": map[string]interface{}{ + "params": []interface{}{client.RPSLimit, 0}, + "type": "gt", + }, + "operator": map[string]interface{}{"type": "and"}, + "query": map[string]interface{}{"params": []string{}}, + "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, + "type": "query", + }, + }, + "datasource": map[string]interface{}{ + "name": "Expression", + "type": "__expr__", + "uid": "__expr__", + }, + "expression": reduceRefID, + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": thresholdRefID, + "type": "threshold", + }, + }, + }, + } + + return rule +} + +// alertTask описывает одно правило алерта для отправки +type alertTask struct { + rule map[string]interface{} + label string // для логирования +} + +// parallelUpsert отправляет список алертов параллельно с ограничением concurrency. +// dryRun=true только логирует без отправки. +// Возвращает количество успешных и неуспешных отправок. +func parallelUpsert(tasks []alertTask, config Config, dryRun bool, concurrency int) (sent, failed int) { + sem := make(chan struct{}, concurrency) + var mu sync.Mutex + var wg sync.WaitGroup + + for _, task := range tasks { + wg.Add(1) + sem <- struct{}{} // захватить слот + go func(t alertTask) { + defer wg.Done() + defer func() { <-sem }() // освободить слот + + if dryRun { + data, _ := json.MarshalIndent(t.rule, "", " ") + log.Printf("DRY RUN: Alert rule '%s':\n%s", t.label, string(data)) + mu.Lock() + sent++ + mu.Unlock() + return + } + + if err := upsertAlertRule(t.rule, config); err != nil { + log.Printf("Warning: failed to upsert alert '%s': %v", t.label, err) + mu.Lock() + failed++ + mu.Unlock() + } else { + log.Printf("Alert upserted: %s", t.label) + mu.Lock() + sent++ + mu.Unlock() + } + }(task) + } + + wg.Wait() + return +} + +// generateAndSendAlerts генерирует и отправляет RPS-алерты для клиентов с rps_limit > 0 +func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplate, config Config, dryRun bool) error { + // Собираем клиентов с лимитом, сортируем для детерминированности + type clientWithLimit struct { + title string + client ClientData + } + var targets []clientWithLimit + for _, client := range clients { + if client.RPSLimit > 0 { + targets = append(targets, clientWithLimit{client.ClientTitle, client}) + } + } + + if len(targets) == 0 { + log.Printf("Alerts: no clients with rps_limit set, skipping") + return nil + } + + sort.Slice(targets, func(i, j int) bool { + return targets[i].title < targets[j].title + }) + + log.Printf("Alerts: generating RPS rules for %d clients", len(targets)) + + // Получить UID папки для алертов + folderUID := "" + if !dryRun { + uid, err := ensureAlertFolder(config) + if err != nil { + return fmt.Errorf("failed to ensure alerts folder: %w", err) + } + folderUID = uid + } + + // Собираем все динамические задачи: RPS + 4xx + 5xx + var tasks []alertTask + for _, t := range targets { + if t.client.RPSLimit > 0 { + rule := buildRPSAlertRule(t.client, tmpl, config) + rule["folderUID"] = folderUID + tasks = append(tasks, alertTask{ + rule: rule, + label: fmt.Sprintf("RPS %s (limit: %d)", t.client.ClientTitle, t.client.RPSLimit), + }) + } + if t.client.Limit4xx > 0 { + rule := buildErrorRateAlertRule(t.client, 4, t.client.Limit4xx, tmpl, config) + rule["folderUID"] = folderUID + tasks = append(tasks, alertTask{ + rule: rule, + label: fmt.Sprintf("4xx %s (limit: %d%%)", t.client.ClientTitle, t.client.Limit4xx), + }) + } + if t.client.Limit5xx > 0 { + rule := buildErrorRateAlertRule(t.client, 5, t.client.Limit5xx, tmpl, config) + rule["folderUID"] = folderUID + tasks = append(tasks, alertTask{ + rule: rule, + label: fmt.Sprintf("5xx %s (limit: %d%%)", t.client.ClientTitle, t.client.Limit5xx), + }) + } + } + + log.Printf("Alerts: sending %d dynamic rules (concurrency: 5)", len(tasks)) + sent, failed := parallelUpsert(tasks, config, dryRun, 5) + log.Printf("Alerts: sent=%d, failed=%d", sent, failed) + if failed > 0 { + return fmt.Errorf("%d alert rules failed to upsert", failed) + } + + // Статические алерты — отправляются всегда вместе с динамическими + if err := sendStaticAlerts(tmpl, config, dryRun); err != nil { + log.Printf("Warning: static alerts error: %v", err) + } + + return nil +} + +// buildErrorRateAlertRule строит алерт на процент 4xx или 5xx ответов для клиента. +// Структура: два count-запроса (total + errors) → reduce → math (%) → threshold +func buildErrorRateAlertRule(client ClientData, errCode, limitPct int, tmpl AlertRulesTemplate, config Config) map[string]interface{} { + datasourceUID := config.AlertsDatasourceUID + defaults := tmpl.Defaults + receiver := orDefault(config.AlertsReceiver, defaults.Receiver) + group := orDefault(config.AlertsGroup, defaults.Group) + + // SID-запрос + sids := make(map[string]bool) + for _, d := range client.Domains { + sids[d.SID] = true + } + unique := make([]string, 0, len(sids)) + for s := range sids { + unique = append(unique, s) + } + sort.Strings(unique) + parts := make([]string, len(unique)) + for i, s := range unique { + parts[i] = "SID:" + s + } + sidQuery := strings.Join(parts, " OR ") + + codeRange := "[400 TO 499]" + codeName := "4xx" + if errCode == 5 { + codeRange = "[500 TO 599]" + codeName = "5xx" + } + + uid := generateAlertUID(client.ClientTitle, codeName) + totalRefID := "Total" + errRefID := "Errors" + totalReduceID := "TotalReduce" + errReduceID := "ErrorsReduce" + percentRefID := "ErrorPercent" + thresholdRefID := "Превышение" + + summary := fmt.Sprintf("🚨 %s — %s ошибок >%d%% трафика", client.ClientTitle, codeName, limitPct) + + relFrom := int64(900) + + return map[string]interface{}{ + "uid": uid, + "title": fmt.Sprintf("%s %s", codeName, client.ClientTitle), + "condition": thresholdRefID, + "noDataState": defaults.NoDataState, + "execErrState": defaults.ExecErrState, + "isPaused": false, + "folderUID": "", + "ruleGroup": group, + "for": "1m", + "annotations": map[string]string{"summary": summary}, + "notification_settings": map[string]string{"receiver": receiver}, + "data": []interface{}{ + // Total запрос + map[string]interface{}{ + "refId": totalRefID, + "queryType": "lucene", + "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, + "datasourceUid": datasourceUID, + "model": map[string]interface{}{ + "alias": "Total", + "bucketAggs": []interface{}{map[string]interface{}{"field": "@timestamp", "id": "2", "settings": map[string]interface{}{"interval": "1m"}, "type": "date_histogram"}}, + "datasource": map[string]interface{}{"type": "grafana-opensearch-datasource", "uid": datasourceUID}, + "format": "table", + "instant": false, + "intervalMs": 1000, + "luceneQueryType": "Metric", + "maxDataPoints": 43200, + "metrics": []interface{}{map[string]interface{}{"id": "1", "type": "count"}}, + "query": fmt.Sprintf("response_status_code:[0 TO 599] AND (%s)", sidQuery), + "queryType": "lucene", + "range": true, + "refId": totalRefID, + "timeField": "@timestamp", + }, + }, + // Error запрос + map[string]interface{}{ + "refId": errRefID, + "queryType": "lucene", + "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, + "datasourceUid": datasourceUID, + "model": map[string]interface{}{ + "alias": codeName, + "bucketAggs": []interface{}{map[string]interface{}{"field": "@timestamp", "id": "2", "settings": map[string]interface{}{"interval": "1m"}, "type": "date_histogram"}}, + "datasource": map[string]interface{}{"type": "grafana-opensearch-datasource", "uid": datasourceUID}, + "format": "table", + "instant": false, + "intervalMs": 1000, + "luceneQueryType": "Metric", + "maxDataPoints": 43200, + "metrics": []interface{}{map[string]interface{}{"id": "1", "type": "count"}}, + "query": fmt.Sprintf("response_status_code:%s AND (%s)", codeRange, sidQuery), + "queryType": "lucene", + "range": true, + "refId": errRefID, + "timeField": "@timestamp", + }, + }, + // Reduce total + map[string]interface{}{ + "refId": totalReduceID, + "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, + "expression": totalRefID, + "intervalMs": 1000, + "maxDataPoints": 43200, + "reducer": "last", + "refId": totalReduceID, + "type": "reduce", + }, + }, + // Reduce errors + map[string]interface{}{ + "refId": errReduceID, + "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, + "expression": errRefID, + "intervalMs": 1000, + "maxDataPoints": 43200, + "reducer": "last", + "refId": errReduceID, + "type": "reduce", + }, + }, + // Math: процент ошибок + map[string]interface{}{ + "refId": percentRefID, + "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, + "expression": fmt.Sprintf("($%s / $%s) * 100", errReduceID, totalReduceID), + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": percentRefID, + "type": "math", + }, + }, + // Threshold + map[string]interface{}{ + "refId": thresholdRefID, + "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, + "datasourceUid": "__expr__", + "model": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "evaluator": map[string]interface{}{"params": []interface{}{limitPct, 0}, "type": "gt"}, + "operator": map[string]interface{}{"type": "and"}, + "query": map[string]interface{}{"params": []string{}}, + "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, + "type": "query", + }, + }, + "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, + "expression": percentRefID, + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": thresholdRefID, + "type": "threshold", + }, + }, + }, + } +} + +// sendStaticAlerts отправляет статические алерты из шаблона (не зависят от БД) +func sendStaticAlerts(tmpl AlertRulesTemplate, config Config, dryRun bool) error { + if len(tmpl.StaticAlerts) == 0 { + return nil + } + + defaults := tmpl.Defaults + folderUID := "" + + if !dryRun { + uid, err := ensureAlertFolder(config) + if err != nil { + return fmt.Errorf("failed to ensure alerts folder for static alerts: %w", err) + } + folderUID = uid + } + + group := orDefault(config.AlertsGroup, defaults.Group) + receiver := orDefault(config.AlertsReceiver, defaults.Receiver) + + log.Printf("Alerts: sending %d static alert rules (concurrency: 5)", len(tmpl.StaticAlerts)) + + var tasks []alertTask + for _, alertDef := range tmpl.StaticAlerts { + rule := deepCopyMap(alertDef) + + // Применяем defaults если не заданы в шаблоне + if _, ok := rule["noDataState"]; !ok { + rule["noDataState"] = defaults.NoDataState + } + if _, ok := rule["execErrState"]; !ok { + rule["execErrState"] = defaults.ExecErrState + } + rule["folderUID"] = folderUID + rule["ruleGroup"] = group + if _, ok := rule["notification_settings"]; !ok { + rule["notification_settings"] = map[string]string{"receiver": receiver} + } + + title, _ := rule["title"].(string) + tasks = append(tasks, alertTask{rule: rule, label: title}) + } + + sent, failed := parallelUpsert(tasks, config, dryRun, 5) + log.Printf("Static alerts: sent=%d, failed=%d", sent, failed) + return nil +} + +// deepCopyMap делает глубокую копию map через JSON +func deepCopyMap(src map[string]interface{}) map[string]interface{} { + data, _ := json.Marshal(src) + var dst map[string]interface{} + json.Unmarshal(data, &dst) + return dst +} + +// upsertAlertRule создаёт или обновляет правило алерта по UID +func upsertAlertRule(rule map[string]interface{}, config Config) error { + uid, _ := rule["uid"].(string) + baseURL := strings.TrimRight(config.GrafanaURL, "/") + + // Проверяем существует ли алерт с таким UID + existing, err := getAlertRule(uid, config) + if err == nil && existing != nil { + // Обновляем — PUT /api/v1/provisioning/alert-rules/{uid} + // Сохраняем version из существующего для оптимистичной блокировки + if v, ok := existing["version"]; ok { + rule["version"] = v + } + return putAlertRule(uid, rule, baseURL, config.GrafanaAPIKey) + } + + // Создаём — POST /api/v1/provisioning/alert-rules + return postAlertRule(rule, baseURL, config.GrafanaAPIKey) +} + +func getAlertRule(uid string, config Config) (map[string]interface{}, error) { + url := fmt.Sprintf("%s/api/v1/provisioning/alert-rules/%s", + strings.TrimRight(config.GrafanaURL, "/"), uid) + + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Authorization", "Bearer "+config.GrafanaAPIKey) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == 404 { + return nil, fmt.Errorf("not found") + } + + body, _ := io.ReadAll(resp.Body) + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return result, nil +} + +func postAlertRule(rule map[string]interface{}, baseURL, apiKey string) error { + return sendAlertRequest("POST", + baseURL+"/api/v1/provisioning/alert-rules", + rule, apiKey) +} + +func putAlertRule(uid string, rule map[string]interface{}, baseURL, apiKey string) error { + return sendAlertRequest("PUT", + fmt.Sprintf("%s/api/v1/provisioning/alert-rules/%s", baseURL, uid), + rule, apiKey) +} + +func sendAlertRequest(method, url string, payload map[string]interface{}, apiKey string) error { + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshal error: %w", err) + } + + req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+apiKey) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Disable-Provenance", "true") // разрешить редактирование в UI + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("status %d: %s", resp.StatusCode, string(body)) + } + return nil +} + +// ensureAlertFolder возвращает UID папки для алертов, создаёт если нет +func ensureAlertFolder(config Config) (string, error) { + folder := orDefault(config.AlertsFolder, DefaultAlertsFolder) + if folder == "" { + return "", nil + } + baseURL := strings.TrimRight(config.GrafanaURL, "/") + + // Ищем существующую папку + req, _ := http.NewRequest("GET", baseURL+"/api/folders", nil) + req.Header.Set("Authorization", "Bearer "+config.GrafanaAPIKey) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + var folders []map[string]interface{} + if err := json.Unmarshal(body, &folders); err != nil { + return "", err + } + + for _, f := range folders { + if title, _ := f["title"].(string); title == folder { + if uid, _ := f["uid"].(string); uid != "" { + return uid, nil + } + } + } + + // Создаём папку + payload := map[string]interface{}{"title": folder} + data, _ := json.Marshal(payload) + req, _ = http.NewRequest("POST", baseURL+"/api/folders", bytes.NewBuffer(data)) + req.Header.Set("Authorization", "Bearer "+config.GrafanaAPIKey) + req.Header.Set("Content-Type", "application/json") + + resp, err = http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, _ = io.ReadAll(resp.Body) + var created map[string]interface{} + if err := json.Unmarshal(body, &created); err != nil { + return "", err + } + if uid, _ := created["uid"].(string); uid != "" { + log.Printf("Created alerts folder: %s (uid: %s)", folder, uid) + return uid, nil + } + return "", fmt.Errorf("failed to get folder UID") +} + +// orDefault возвращает val если не пустой, иначе def +func orDefault(val, def string) string { + if val != "" { + return val + } + return def +} diff --git a/config.go b/config.go index 5758c46..e502356 100644 --- a/config.go +++ b/config.go @@ -5,13 +5,13 @@ const ( // Рекомендуется задавать через .env файл или переменные окружения DefaultDBUser = "" // Задать через DB_USER в .env DefaultDBPassword = "" // Задать через DB_PASSWORD в .env - DefaultDBName = "test_info" + DefaultDBName = "waf_info" DefaultDBPort = 5432 - DefaultPrimaryDBHost = "10.10.10.8" - DefaultSecondaryDBHost = "10.10.10.5" + DefaultPrimaryDBHost = "10.100.10.8" + DefaultSecondaryDBHost = "10.100.13.5" // Git repository settings - DefaultTemplatesRepo = "" + DefaultTemplatesRepo = "https://svc-git.cirex.ru/cloudstack/grafana_gen_template.git" DefaultTemplatesBranch = "master" DefaultTemplatesPath = "/etc/grafana_gen/templates" // Системный путь DefaultStateFile = "/var/lib/grafana_gen/state.json" // Системный путь для state @@ -59,6 +59,12 @@ const ( // Можно переопределить под ваши нужды: // DefaultGrafanaFolder = "WAF - PTAF" + // Alerts settings + DefaultAlertsFolder = "WAF - PTAF" + DefaultAlertsReceiver = "Telegram PTAF Grafana" + DefaultAlertsGroup = "PTAF Grafana" + DefaultAlertsDatasource = "af84zsvlp9blsa" // UID datasource OpenSearch + // DefaultDashboardTitle - название единого дашборда со всеми клиентами // Пример результата: Home -> Dashboards -> WAF - PTAF -> PT AF Nodes DefaultDashboardTitle = "PT AF Nodes" @@ -70,29 +76,36 @@ const ( ) type Config struct { - DBHost string - DBPort int - DBUser string - DBPassword string - DBName string - GrafanaURL string - GrafanaAPIKey string - GrafanaFolder string // Папка в Grafana для дашбордов - DashboardTitle string // Название единого дашборда - UseManualInfo bool - DryRun bool - TemplatesRepo string - TemplatesBranch string - TemplatesPath string - SkipGitPull bool - StateFile string - ForceRun bool - GitToken string // GitLab token для приватного репозитория + DBHost string + DBPort int + DBUser string + DBPassword string + DBName string + GrafanaURL string + GrafanaAPIKey string + GrafanaFolder string // Папка в Grafana для дашбордов + DashboardTitle string // Название единого дашборда + DryRun bool + TemplatesRepo string + TemplatesBranch string + TemplatesPath string + SkipGitPull bool + StateFile string + ForceRun bool + GitToken string // GitLab token для приватного репозитория + AlertsFolder string // Папка в Grafana для алертов + AlertsReceiver string // Receiver (канал уведомлений) для алертов + AlertsGroup string // Имя группы алертов в Grafana + AlertsDatasourceUID string // UID datasource для алертов OpenSearch } type ClientData struct { - ClientTitle string - Domains []DomainInfo + ClientTitle string + Domains []DomainInfo + RPSLimit int // 0 = алерт не создавать (NULL или не задан в БД) + RPSCommercialLimit int // коммерческий лимит, при NULL берётся 100 + Limit4xx int // порог 4xx ответов в % (0 = алерт не создавать) + Limit5xx int // порог 5xx ответов в % (0 = алерт не создавать) } type DomainInfo struct { @@ -113,12 +126,15 @@ type DashboardTemplate struct { QueryTemplates map[string]interface{} `json:"query_templates"` Panels map[string]interface{} `json:"panels"` Queries map[string]interface{} `json:"queries"` + Templating map[string]interface{} `json:"templating"` // Переменные дашборда Grafana } type LayoutConfig struct { TenantPanels []PanelLayout `json:"tenant_panels"` DomainPanels []PanelLayout `json:"domain_panels"` UseCollapsedRows bool `json:"use_collapsed_rows"` + OverviewPanels []PanelLayout `json:"overview_panels"` // Панели обзора перед клиентскими строками + OverviewRowTitle string `json:"overview_row_title"` // Заголовок collapsed row обзора } type PanelLayout struct { diff --git a/database.go b/database.go index 8e4e6a5..2273cfa 100644 --- a/database.go +++ b/database.go @@ -26,24 +26,29 @@ func connectDB(config Config) (*sql.DB, error) { return db, nil } -func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error) { - tableName := "sp_info" - if useManual { - tableName = "manual_info" - } - +func fetchClientsData(db *sql.DB) (map[string]ClientData, error) { // aliases это JSON массив в БД, конвертируем в текст через ::text - query := fmt.Sprintf(` - SELECT DISTINCT + // rps_limit берётся из client_info — NULL означает "алерт не нужен" + // Данные всегда берутся из обеих таблиц: sp_info + manual_info (UNION) + query := ` + SELECT DISTINCT COALESCE(a.client_title, 'Unknown') as client_title, s.sid, s.domain_name, - COALESCE(s.aliases::text, 'null') as aliases - FROM %s s + COALESCE(s.aliases::text, 'null') as aliases, + COALESCE(MAX(ci.rps_limit) OVER (PARTITION BY a.client_title), 0) as rps_limit, + COALESCE(MAX(ci.rps_commercial_limit) OVER (PARTITION BY a.client_title), 100) as rps_commercial_limit, + MAX(ci.four_hundred) OVER (PARTITION BY a.client_title) as limit_4xx, + MAX(ci.five_hundred) OVER (PARTITION BY a.client_title) as limit_5xx + FROM ( + SELECT sid, domain_name, aliases FROM sp_info WHERE sid IS NOT NULL + UNION + SELECT sid, domain_name, aliases FROM manual_info WHERE sid IS NOT NULL + ) s LEFT JOIN apps_settings a ON s.sid = a.l7resourceid - WHERE s.sid IS NOT NULL + LEFT JOIN client_info ci ON a.client_title = ci.client_title ORDER BY COALESCE(a.client_title, 'Unknown'), s.sid - `, tableName) + ` rows, err := db.Query(query) if err != nil { @@ -55,8 +60,10 @@ func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error) for rows.Next() { var clientTitle, sid, domainName, aliasesJSON string + var rpsLimit, rpsCommercialLimit int + var limit4xx, limit5xx sql.NullInt64 - if err := rows.Scan(&clientTitle, &sid, &domainName, &aliasesJSON); err != nil { + if err := rows.Scan(&clientTitle, &sid, &domainName, &aliasesJSON, &rpsLimit, &rpsCommercialLimit, &limit4xx, &limit5xx); err != nil { return nil, fmt.Errorf("scan failed: %w", err) } @@ -65,8 +72,12 @@ func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error) if _, exists := clients[clientTitle]; !exists { clients[clientTitle] = ClientData{ - ClientTitle: clientTitle, - Domains: []DomainInfo{}, + ClientTitle: clientTitle, + Domains: []DomainInfo{}, + RPSLimit: rpsLimit, + RPSCommercialLimit: rpsCommercialLimit, + Limit4xx: nullIntToInt(limit4xx), + Limit5xx: nullIntToInt(limit5xx), } } @@ -82,6 +93,14 @@ func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error) return clients, rows.Err() } +// nullIntToInt конвертирует sql.NullInt64 в int (0 если NULL) +func nullIntToInt(n sql.NullInt64) int { + if n.Valid { + return int(n.Int64) + } + return 0 +} + // parseAliasesJSON конвертирует JSON массив aliases в строку // Input: ["domain1.com", "domain2.com"] или null // Output: "domain1.com, domain2.com" или "" diff --git a/main.go b/main.go index f30bbd3..43df148 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,13 @@ func main() { log.Printf("Loaded template version: %s", template.Version) + // Load alert rules template + alertTmpl, err := loadAlertTemplate(config.TemplatesPath) + if err != nil { + log.Printf("Warning: Failed to load alert template: %v", err) + log.Printf("Alert rules will not be generated") + } + // Connect to database db, err := connectDB(config) if err != nil { @@ -56,7 +63,7 @@ func main() { defer db.Close() // Fetch clients data - clients, err := fetchClientsData(db, config.UseManualInfo) + clients, err := fetchClientsData(db) if err != nil { log.Fatalf("Failed to fetch clients data: %v", err) } @@ -87,10 +94,14 @@ func main() { os.Exit(0) } + // Хэш шаблона алертов — считается независимо от дашборда + alertTemplateHash := hashAlertTemplate(config.TemplatesPath) + alertsChanged := stateManager.AlertsNeedUpdate(validClients) || stateManager.AlertTemplateChanged(alertTemplateHash) + // Check for changes hasChanges, changeReason := stateManager.CheckChanges(templateCommit, template.Version, clients) - if !hasChanges && !config.ForceRun { + if !hasChanges && !alertsChanged && !config.ForceRun { log.Printf("=== No Changes Detected ===") log.Printf("No changes in templates or database since last run.") log.Printf("Skipping dashboard generation.") @@ -103,6 +114,9 @@ func main() { log.Printf("=== Changes Detected ===") log.Printf("%s", changeReason) log.Printf("") + } else if alertsChanged { + log.Printf("=== Alert Rules Changed ===") + log.Printf("") } if config.ForceRun { @@ -117,6 +131,10 @@ func main() { if config.DryRun { logDryRun(config.DashboardTitle, dashboard) + if alertTmpl != nil { + log.Printf("DRY RUN: Generating alert rules preview...") + _ = generateAndSendAlerts(validClients, *alertTmpl, config, true) + } log.Printf("=== Summary ===") log.Printf("Clients: %d", len(validClients)) log.Printf("Skipped: %d", skippedCount) @@ -136,6 +154,20 @@ func main() { log.Printf("Successfully created/updated dashboard: %s", config.DashboardTitle) + // Generate and send alert rules + if alertTmpl != nil { + if alertsChanged || config.ForceRun { + if err := generateAndSendAlerts(validClients, *alertTmpl, config, false); err != nil { + log.Printf("Warning: alerts generation failed: %v", err) + } else { + stateManager.UpdateAlertsHash(validClients) + stateManager.UpdateAlertTemplateHash(alertTemplateHash) + } + } else { + log.Printf("Alerts: no changes in rps_limit, skipping") + } + } + // Summary log.Printf("=== Summary ===") log.Printf("Dashboard: %s", config.DashboardTitle) @@ -179,7 +211,6 @@ func parseFlags() Config { flag.StringVar(&config.DBUser, "db-user", dbUser, "PostgreSQL user") flag.StringVar(&config.DBPassword, "db-password", dbPassword, "PostgreSQL password") flag.StringVar(&config.DBName, "db-name", DefaultDBName, "PostgreSQL database name") - flag.BoolVar(&config.UseManualInfo, "use-manual", false, "Use manual_info table instead of sp_info") // Grafana flags var grafanaURLFlag, grafanaAPIKeyFlag string @@ -196,6 +227,16 @@ func parseFlags() Config { flag.StringVar(&config.TemplatesPath, "templates-path", DefaultTemplatesPath, "Local path for templates") flag.BoolVar(&config.SkipGitPull, "skip-git-pull", false, "Skip updating templates from Git") + // Alerts flags + alertsFolder := getEnvOrDefault("ALERTS_FOLDER", DefaultAlertsFolder) + alertsReceiver := getEnvOrDefault("ALERTS_RECEIVER", DefaultAlertsReceiver) + alertsGroup := getEnvOrDefault("ALERTS_GROUP", DefaultAlertsGroup) + alertsDatasource := getEnvOrDefault("ALERTS_DATASOURCE_UID", DefaultAlertsDatasource) + flag.StringVar(&config.AlertsFolder, "alerts-folder", alertsFolder, "Grafana folder for alert rules") + flag.StringVar(&config.AlertsReceiver, "alerts-receiver", alertsReceiver, "Grafana notification receiver for alerts") + flag.StringVar(&config.AlertsGroup, "alerts-group", alertsGroup, "Grafana alert rule group name") + flag.StringVar(&config.AlertsDatasourceUID, "alerts-datasource-uid", alertsDatasource, "OpenSearch datasource UID for alerts") + // Other flags flag.BoolVar(&config.DryRun, "dry-run", false, "Generate JSON but don't send to Grafana") flag.StringVar(&config.StateFile, "state-file", DefaultStateFile, "Path to state file") diff --git a/panels.go b/panels.go index 2e33672..e1037d1 100644 --- a/panels.go +++ b/panels.go @@ -19,6 +19,7 @@ var queryModeRegistry = map[string]targetBuilder{ "multi_sum_expression": buildMultiSumExpressionTargets, "sum_expression": buildSumExpressionTargets, "bucket_logs": buildBucketLogsTargets, + "static": buildStaticTargets, } // conditionEvaluator — тип функции-вычислителя условия отображения панели. @@ -73,6 +74,7 @@ func buildPanel( queryConfigRaw, _ := panelMap["query_config"].(map[string]interface{}) panel := copyPanelBase(panelMap, id, yPos, xPos, width) + applyRPSThreshold(panel, panelKey, client.RPSLimit, client.RPSCommercialLimit) sidQuery := buildSIDQuery(domain.SID) panel["targets"] = builder(queryConfigRaw, sidQuery, datasourceType, datasourceUID, template, domain, client) @@ -85,6 +87,55 @@ func buildPanel( } // evaluateCondition проверяет условие отображения панели через реестр +// applyRPSThreshold подставляет rps_limit и rps_commercial_limit клиента в threshold панели rps. +// Красная линия — rps_limit (если 0, берётся из шаблона). +// Синяя линия — rps_commercial_limit (если 0, ставится 100). +func applyRPSThreshold(panel map[string]interface{}, panelKey string, rpsLimit, rpsCommercialLimit int) { + if panelKey != "rps" { + return + } + // Ничего менять не нужно если оба лимита не заданы + if rpsLimit == 0 && rpsCommercialLimit == 0 { + return + } + + fc, ok := panel["fieldConfig"].(map[string]interface{}) + if !ok { + return + } + defaults, ok := fc["defaults"].(map[string]interface{}) + if !ok { + return + } + thresholds, ok := defaults["thresholds"].(map[string]interface{}) + if !ok { + return + } + + // Коммерческий лимит: если не задан — 100 + commercial := rpsCommercialLimit + if commercial == 0 { + commercial = 100 + } + + steps := []interface{}{ + map[string]interface{}{"color": "green", "value": nil}, + map[string]interface{}{"color": "blue", "value": commercial}, + } + if rpsLimit > 0 { + steps = append(steps, map[string]interface{}{"color": "dark-red", "value": rpsLimit}) + } + thresholds["steps"] = steps + + // Включаем отображение линий если ещё не задано + custom, ok := defaults["custom"].(map[string]interface{}) + if ok { + if _, hasStyle := custom["thresholdsStyle"]; !hasStyle { + custom["thresholdsStyle"] = map[string]interface{}{"mode": "line"} + } + } +} + func evaluateCondition(condition *PanelCondition, client ClientData, domain *DomainInfo) bool { if condition == nil { return true @@ -295,6 +346,29 @@ func buildSumExpressionTargets(cfg map[string]interface{}, sidQuery, datasourceT } } +// buildStaticTargets: панель без подстановки SID из БД. +// targets берутся из query_config.targets как есть — для глобальных/обзорных панелей. +// Поддерживает произвольные bucket_aggs из шаблона. +func buildStaticTargets(cfg map[string]interface{}, _ string, datasourceType, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { + targetsRaw, ok := cfg["targets"].([]interface{}) + if !ok || len(targetsRaw) == 0 { + return []interface{}{} + } + // Копируем targets как есть, только проставляем datasource + targets := []interface{}{} + for _, t := range targetsRaw { + target := deepCopy(t).(map[string]interface{}) + if _, hasDatasource := target["datasource"]; !hasDatasource { + target["datasource"] = map[string]interface{}{ + "type": datasourceType, + "uid": datasourceUID, + } + } + targets = append(targets, target) + } + return targets +} + func buildBucketLogsTargets(cfg map[string]interface{}, sidQuery, datasourceType, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { query := strings.ReplaceAll(getString(cfg, "base_query"), "{sid}", sidQuery) return []interface{}{ diff --git a/state.go b/state.go index 7f85415..9236319 100644 --- a/state.go +++ b/state.go @@ -11,14 +11,16 @@ import ( ) type State struct { - LastRun time.Time `json:"last_run"` - TemplateCommit string `json:"template_commit"` - TemplateVersion string `json:"template_version"` - ClientsHash string `json:"clients_hash"` - ClientCount int `json:"client_count"` - DomainCount int `json:"domain_count"` - SIDs []string `json:"sids"` - LastChangeReason string `json:"last_change_reason,omitempty"` + LastRun time.Time `json:"last_run"` + TemplateCommit string `json:"template_commit"` + TemplateVersion string `json:"template_version"` + ClientsHash string `json:"clients_hash"` + ClientCount int `json:"client_count"` + DomainCount int `json:"domain_count"` + SIDs []string `json:"sids"` + LastChangeReason string `json:"last_change_reason,omitempty"` + AlertsHash string `json:"alerts_hash,omitempty"` // Хэш лимитов алертов из БД + AlertTemplateHash string `json:"alert_template_hash,omitempty"` // Хэш файла alert_rules_template.json } type StateManager struct { @@ -151,6 +153,7 @@ func calculateClientsHash(clients map[string]ClientData) string { // Create a deterministic representation type hashData struct { ClientTitle string + RPSLimit int Domains []struct { SID string DomainName string @@ -160,7 +163,7 @@ func calculateClientsHash(clients map[string]ClientData) string { var data []hashData for _, client := range clients { - hd := hashData{ClientTitle: client.ClientTitle} + hd := hashData{ClientTitle: client.ClientTitle, RPSLimit: client.RPSLimit} for _, domain := range client.Domains { hd.Domains = append(hd.Domains, struct { SID string @@ -190,6 +193,65 @@ func calculateClientsHash(clients map[string]ClientData) string { return fmt.Sprintf("%x", hash) } +// calculateAlertsHash считает хэш только от данных нужных для алертов (client_title + rps_limit) +func calculateAlertsHash(clients map[string]ClientData) string { + type alertData struct { + ClientTitle string + RPSLimit int + Limit4xx int + Limit5xx int + } + var data []alertData + for _, client := range clients { + if client.RPSLimit > 0 || client.Limit4xx > 0 || client.Limit5xx > 0 { + data = append(data, alertData{ + ClientTitle: client.ClientTitle, + RPSLimit: client.RPSLimit, + Limit4xx: client.Limit4xx, + Limit5xx: client.Limit5xx, + }) + } + } + sort.Slice(data, func(i, j int) bool { + return data[i].ClientTitle < data[j].ClientTitle + }) + jsonData, _ := json.Marshal(data) + hash := sha256.Sum256(jsonData) + return fmt.Sprintf("%x", hash) +} + +// AlertTemplateChanged проверяет изменился ли файл alert_rules_template.json +func (sm *StateManager) AlertTemplateChanged(templateHash string) bool { + if sm.state == nil { + return true + } + return sm.state.AlertTemplateHash != templateHash +} + +// UpdateAlertTemplateHash сохраняет хэш шаблона алертов +func (sm *StateManager) UpdateAlertTemplateHash(templateHash string) { + if sm.state == nil { + sm.state = &State{} + } + sm.state.AlertTemplateHash = templateHash +} + +// AlertsNeedUpdate проверяет изменился ли хэш алертов +func (sm *StateManager) AlertsNeedUpdate(clients map[string]ClientData) bool { + if sm.state == nil { + return true + } + return sm.state.AlertsHash != calculateAlertsHash(clients) +} + +// UpdateAlertsHash сохраняет хэш алертов в state +func (sm *StateManager) UpdateAlertsHash(clients map[string]ClientData) { + if sm.state == nil { + sm.state = &State{} + } + sm.state.AlertsHash = calculateAlertsHash(clients) +} + func extractSIDs(clients map[string]ClientData) []string { sidMap := make(map[string]bool) for _, client := range clients { diff --git a/template.go b/template.go index 1239a9e..9668455 100644 --- a/template.go +++ b/template.go @@ -1,6 +1,7 @@ package main import ( + "crypto/sha256" "encoding/json" "fmt" "os" @@ -23,6 +24,32 @@ func loadTemplate(templatesPath string) (*DashboardTemplate, error) { return &template, nil } +func loadAlertTemplate(templatesPath string) (*AlertRulesTemplate, error) { + templateFile := filepath.Join(templatesPath, "alert_rules_template.json") + + data, err := os.ReadFile(templateFile) + if err != nil { + return nil, fmt.Errorf("failed to read alert template file: %w", err) + } + + var tmpl AlertRulesTemplate + if err := json.Unmarshal(data, &tmpl); err != nil { + return nil, fmt.Errorf("failed to parse alert template JSON: %w", err) + } + + return &tmpl, nil +} + +// hashAlertTemplate считает SHA256 от содержимого alert_rules_template.json +func hashAlertTemplate(templatesPath string) string { + data, err := os.ReadFile(filepath.Join(templatesPath, "alert_rules_template.json")) + if err != nil { + return "" + } + hash := sha256.Sum256(data) + return fmt.Sprintf("%x", hash) +} + func deepCopy(src interface{}) interface{} { data, _ := json.Marshal(src) var dst interface{}