package main import ( "fmt" "log" "sort" "strings" ) // buildRPSAlertRuleVL строит правило алерта на RPS для VictoriaLogs datasource. func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Config) map[string]interface{} { datasourceUID := config.VLDatasourceUID datasourceType := DefaultVLDatasourceType // SID-запрос в формате LogsQL sidMap := make(map[string]bool) for _, d := range client.Domains { sidMap[d.SID] = true } uniqueSIDs := make([]string, 0, len(sidMap)) for s := range sidMap { uniqueSIDs = append(uniqueSIDs, s) } sort.Strings(uniqueSIDs) sidQuery := buildTenantSIDQueryVL(client.Domains) defaults := tmpl.Defaults receiver := orDefault(config.AlertsReceiver, defaults.Receiver) group := client.ClientTitle timeRangeFrom := int64(1800) if v, ok := tmpl.RPSAlert["relative_time_range_from"].(float64); ok { timeRangeFrom = int64(v) } summaryTemplate := "Превышен порог в {rps_limit} RPS в {client_title}" if v, ok := tmpl.RPSAlert["annotation_summary"].(string); ok { summaryTemplate = v } summary := strings.NewReplacer( "{rps_limit}", fmt.Sprintf("%d", client.RPSLimit), "{client_title}", client.ClientTitle, ).Replace(summaryTemplate) _ = uniqueSIDs // используется через sidQuery uid := generateAlertUID(client.ClientTitle+"_vl", "rps") dataRefID := "A" rpsRefID := "RPS" reduceRefID := "_Reduce" thresholdRefID := "Превышение" // LogsQL запрос: count за минуту expr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() rps", sidQuery) return map[string]interface{}{ "uid": uid, "title": fmt.Sprintf("VL RPS %s", client.ClientTitle), "condition": thresholdRefID, "noDataState": "NoData", // VL: нет данных = нет алерта "execErrState": "Error", // VL: ошибка datasource = Error, не Alerting "isPaused": false, "folderUID": "", "ruleGroup": group, "annotations": map[string]string{"summary": summary}, "notification_settings": map[string]string{ "receiver": receiver, }, "data": []interface{}{ // Шаг 1: VL statsRange запрос map[string]interface{}{ "refId": dataRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{ "from": timeRangeFrom, "to": 0, }, "datasourceUid": datasourceUID, "model": map[string]interface{}{ "datasource": map[string]interface{}{ "type": datasourceType, "uid": datasourceUID, }, "editorMode": "code", "expr": expr, "legendFormat": fmt.Sprintf("RPS %s", client.ClientTitle), "maxLines": 1000, "queryType": "statsRange", "refId": dataRefID, "intervalMs": 60000, "maxDataPoints": 43200, }, }, // Шаг 2: math $A / 60 → RPS (интервал 1m = 60 секунд) map[string]interface{}{ "refId": rpsRefID, "relativeTimeRange": map[string]interface{}{ "from": timeRangeFrom, "to": 0, }, "datasourceUid": "__expr__", "model": map[string]interface{}{ "datasource": map[string]interface{}{ "name": "Expression", "type": "__expr__", "uid": "__expr__", }, "expression": fmt.Sprintf("$%s / 60", dataRefID), "intervalMs": 1000, "maxDataPoints": 43200, "refId": rpsRefID, "type": "math", "window": "", }, }, // Шаг 3: reduce median map[string]interface{}{ "refId": reduceRefID, "relativeTimeRange": map[string]interface{}{ "from": 0, "to": 0, }, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{"params": []int{0, 0}, "type": "gt"}, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{ "name": "Expression", "type": "__expr__", "uid": "__expr__", }, "expression": rpsRefID, "intervalMs": 1000, "maxDataPoints": 43200, "reducer": "median", "refId": reduceRefID, "settings": map[string]interface{}{"mode": ""}, "type": "reduce", }, }, // Шаг 4: threshold > rps_limit map[string]interface{}{ "refId": thresholdRefID, "relativeTimeRange": map[string]interface{}{ "from": 0, "to": 0, }, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{ "params": []interface{}{client.RPSLimit, 0}, "type": "gt", }, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{ "name": "Expression", "type": "__expr__", "uid": "__expr__", }, "expression": reduceRefID, "intervalMs": 1000, "maxDataPoints": 43200, "refId": thresholdRefID, "type": "threshold", }, }, }, } } // buildErrorRateAlertRulesVL строит алерт на процент ошибок для конкретного домена (VictoriaLogs). // statusField: "response_status_code" или "upstream_status" func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} { datasourceUID := config.VLDatasourceUID datasourceType := DefaultVLDatasourceType defaults := tmpl.Defaults receiver := orDefault(config.AlertsReceiver, defaults.Receiver) group := client.ClientTitle codeFrom, codeTo := 400, 499 codeName := "4xx" if errCode == 5 { codeFrom, codeTo = 500, 599 codeName = "5xx" } forDuration := "1m" if errCode == 4 { forDuration = "3m" } relFrom := int64(900) var rules []map[string]interface{} { sidQuery := fmt.Sprintf("SID:%s", domain.SID) // Короткое имя поля для использования в названии алерта fieldShort := "response" if statusField == "upstream_status" { fieldShort = "upstream" } uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID+"_"+fieldShort, codeName) title := fmt.Sprintf("VL %s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID) fieldDesc := "ответ от PTAF" if statusField == "upstream_status" { fieldDesc = "ответ от origin" } summary := fmt.Sprintf( "🔴 %s / %s (SID:%s) — %s %s ошибок трафика держится > %s.", client.ClientTitle, domain.DomainName, domain.SID, codeName, fieldShort, forDuration, ) thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc) fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc) thresholdValueAnnotation := fmt.Sprintf("%d", limitPct) totalRefID := "Total" errRefID := "Errors" totalReduceID := "_TotalReduce" errReduceID := "_ErrorsReduce" percentRefID := "Значение" thresholdRefID := "Превышение" totalExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() hits", sidQuery) errExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF %s:>=%d %s:<=%d | stats by (_time:1m) count() hits", sidQuery, statusField, codeFrom, statusField, codeTo) makeVLModel := func(refID, expr, legend string) map[string]interface{} { return map[string]interface{}{ "datasource": map[string]interface{}{ "type": datasourceType, "uid": datasourceUID, }, "editorMode": "code", "expr": expr, "legendFormat": legend, "maxLines": 1000, "queryType": "statsRange", "refId": refID, "intervalMs": 60000, "maxDataPoints": 43200, } } makeReduce := func(refID, expression, reducer string) map[string]interface{} { return map[string]interface{}{ "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": expression, "intervalMs": 1000, "maxDataPoints": 43200, "reducer": reducer, "refId": refID, "type": "reduce", } } rule := map[string]interface{}{ "uid": uid, "title": title, "condition": thresholdRefID, "noDataState": "NoData", // VL: нет данных = нет алерта "execErrState": "Error", // VL: ошибка datasource = Error, не Alerting "isPaused": false, "folderUID": "", "ruleGroup": group, "for": forDuration, "annotations": map[string]string{ "summary": summary, "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), "code_field": codeName, "for_duration": forDuration, "threshold": thresholdAnnotation, "threshold_value": thresholdValueAnnotation, "field_info": fieldInfoAnnotation, }, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{ map[string]interface{}{ "refId": totalRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, "datasourceUid": datasourceUID, "model": makeVLModel(totalRefID, totalExpr, "Total"), }, map[string]interface{}{ "refId": errRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, "datasourceUid": datasourceUID, "model": makeVLModel(errRefID, errExpr, codeName), }, map[string]interface{}{ "refId": totalReduceID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": makeReduce(totalReduceID, totalRefID, "sum"), }, map[string]interface{}{ "refId": errReduceID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": makeReduce(errReduceID, errRefID, "sum"), }, map[string]interface{}{ "refId": percentRefID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": map[string]interface{}{ "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": fmt.Sprintf("($%s / $%s) * 100", errReduceID, totalReduceID), "intervalMs": 1000, "maxDataPoints": 43200, "refId": percentRefID, "type": "math", }, }, map[string]interface{}{ "refId": thresholdRefID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{"params": []interface{}{limitPct, 0}, "type": "gt"}, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": percentRefID, "intervalMs": 1000, "maxDataPoints": 43200, "refId": thresholdRefID, "type": "threshold", }, }, }, } rules = append(rules, rule) } return rules } // generateAndSendAlertsVL генерирует и отправляет алерты для VictoriaLogs. func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTemplate, config Config, dryRun bool) ([]string, error) { if config.VLDatasourceUID == "" { log.Printf("Skipping VL alerts: VLDatasourceUID not configured") return nil, nil } folderUID, err := ensureAlertFolder(config) if err != nil { return nil, fmt.Errorf("failed to ensure alert folder: %w", err) } var tasks []alertTask for _, client := range clients { if len(client.Domains) == 0 { continue } // RPS алерт if client.RPSLimit > 0 { rule := buildRPSAlertRuleVL(client, tmpl, config) rule["folderUID"] = folderUID tasks = append(tasks, alertTask{ rule: rule, label: fmt.Sprintf("VL RPS %s", client.ClientTitle), }) } // WAF block алерт — по одному на каждый SID (пропускаем если ptaf_fallback_code = "pass" или пустой) if client.PtafFallbackCode != "" && strings.ToLower(client.PtafFallbackCode) != "pass" { for _, domain := range client.Domains { rule := buildWAFBlockAlertRuleVL(client, domain, tmpl, config) rule["folderUID"] = folderUID tasks = append(tasks, alertTask{ rule: rule, label: fmt.Sprintf("VL %s WAF %s / %s SID:%s", client.PtafFallbackCode, client.ClientTitle, domain.DomainName, domain.SID), }) } } // Error rate алерты — пороги берутся из каждого домена (apps_settings) // Генерируем для response_status_code и upstream_status_code for _, domain := range client.Domains { for _, statusField := range []string{"response_status_code", "upstream_status"} { if domain.Limit4xx > 0 { for _, rule := range buildErrorRateAlertRulesVL(client, domain, 4, domain.Limit4xx, statusField, tmpl, config) { rule["folderUID"] = folderUID tasks = append(tasks, alertTask{ rule: rule, label: fmt.Sprintf("VL 4xx %s %s / %s SID:%s", statusField, client.ClientTitle, domain.DomainName, domain.SID), }) } } if domain.Limit5xx > 0 { for _, rule := range buildErrorRateAlertRulesVL(client, domain, 5, domain.Limit5xx, statusField, tmpl, config) { rule["folderUID"] = folderUID tasks = append(tasks, alertTask{ rule: rule, label: fmt.Sprintf("VL 5xx %s %s / %s SID:%s", statusField, client.ClientTitle, domain.DomainName, domain.SID), }) } } } } } // Обновляем шаблон contact point (если OS алерты отключены — делаем здесь) if !config.AlertsOSEnabled { if err := upsertContactPoint(tmpl, config, dryRun); err != nil { log.Printf("Warning: failed to update contact point: %v", err) } } sent, failed := parallelUpsert(tasks, config, dryRun, 20) log.Printf("VL Alerts: sent=%d failed=%d", sent, failed) // Статические алерты отправляются отдельно из main.go чтобы избежать // дублирования при вызове функции для разных провайдеров (spik/sp) // Собираем активные UID для возврата activeUIDs := []string{} for _, task := range tasks { if uid, ok := task.rule["uid"].(string); ok { activeUIDs = append(activeUIDs, uid) } } return activeUIDs, nil } // buildWAFBlockAlertRuleVL строит алерт на количество 418 ответов за минуту (VictoriaLogs). func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRulesTemplate, config Config) map[string]interface{} { datasourceUID := config.VLDatasourceUID datasourceType := DefaultVLDatasourceType defaults := tmpl.Defaults receiver := orDefault(config.AlertsReceiver, defaults.Receiver) group := client.ClientTitle uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID, "waf_block") title := fmt.Sprintf("VL %s WAF %s / %s SID:%s", client.PtafFallbackCode, client.ClientTitle, domain.DomainName, domain.SID) clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID) summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode) thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount) thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount) expr := fmt.Sprintf("SID:%s log_type:nginx-access-PTAF response_status_code:=%s | stats by (_time:1m) count() hits", domain.SID, client.PtafFallbackCode) dataRefID := "A" reduceRefID := "_Reduce" thresholdRefID := "Превышение" relFrom := int64(120) return map[string]interface{}{ "uid": uid, "title": title, "condition": thresholdRefID, "noDataState": "NoData", "execErrState": "Error", "isPaused": false, "folderUID": "", "ruleGroup": group, "for": "1m", "annotations": map[string]string{ "summary": summary, "client_domain": clientDomain, "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), "for_duration": "1m", "threshold": thresholdAnnotation, "field_info": "для response_status_code (ответ от PTAF, не от origin)", "runbook": "проверить логи docker и эскалировать на аналитика.", "severity": "warning", "value_unit": "шт.", "threshold_count": thresholdCountAnnotation, }, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{ map[string]interface{}{ "refId": dataRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, "datasourceUid": datasourceUID, "model": map[string]interface{}{ "datasource": map[string]interface{}{"type": datasourceType, "uid": datasourceUID}, "editorMode": "code", "expr": expr, "legendFormat": fmt.Sprintf("418 %s", domain.DomainName), "maxLines": 1000, "queryType": "statsRange", "refId": dataRefID, "intervalMs": 60000, "maxDataPoints": 43200, }, }, map[string]interface{}{ "refId": reduceRefID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": map[string]interface{}{ "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": dataRefID, "intervalMs": 1000, "maxDataPoints": 43200, "reducer": "last", "refId": reduceRefID, "type": "reduce", }, }, map[string]interface{}{ "refId": thresholdRefID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{"params": []interface{}{domain.PtafFallbackCodeAlertCount, 0}, "type": "gt"}, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": reduceRefID, "intervalMs": 1000, "maxDataPoints": 43200, "refId": thresholdRefID, "type": "threshold", }, }, }, } }