package main import ( "fmt" "log" "sort" "strings" ) // buildRPSAlertRuleVL строит правило алерта на RPS для VictoriaLogs datasource. func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Config) map[string]interface{} { datasourceUID := config.VLDatasourceUID datasourceType := DefaultVLDatasourceType // SID-запрос в формате LogsQL sidMap := make(map[string]bool) for _, d := range client.Domains { sidMap[d.SID] = true } uniqueSIDs := make([]string, 0, len(sidMap)) for s := range sidMap { uniqueSIDs = append(uniqueSIDs, s) } sort.Strings(uniqueSIDs) sidQuery := buildTenantSIDQueryVL(client.Domains) defaults := tmpl.Defaults receiver := orDefault(config.AlertsReceiver, defaults.Receiver) group := orDefault(config.AlertsGroup, defaults.Group) timeRangeFrom := int64(1800) if v, ok := tmpl.RPSAlert["relative_time_range_from"].(float64); ok { timeRangeFrom = int64(v) } summaryTemplate := "Превышен порог в {rps_limit} RPS в {client_title}" if v, ok := tmpl.RPSAlert["annotation_summary"].(string); ok { summaryTemplate = v } summary := strings.NewReplacer( "{rps_limit}", fmt.Sprintf("%d", client.RPSLimit), "{client_title}", client.ClientTitle, ).Replace(summaryTemplate) _ = uniqueSIDs // используется через sidQuery uid := generateAlertUID(client.ClientTitle+"_vl", "rps") dataRefID := "A" rpsRefID := "RPS" reduceRefID := "_Reduce" thresholdRefID := "Превышение" // LogsQL запрос: count за минуту expr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() rps", sidQuery) return map[string]interface{}{ "uid": uid, "title": fmt.Sprintf("VL RPS %s", client.ClientTitle), "condition": thresholdRefID, "noDataState": "NoData", // VL: нет данных = нет алерта "execErrState": "Error", // VL: ошибка datasource = Error, не Alerting "isPaused": false, "folderUID": "", "ruleGroup": group, "annotations": map[string]string{"summary": summary}, "notification_settings": map[string]string{ "receiver": receiver, }, "data": []interface{}{ // Шаг 1: VL statsRange запрос map[string]interface{}{ "refId": dataRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{ "from": timeRangeFrom, "to": 0, }, "datasourceUid": datasourceUID, "model": map[string]interface{}{ "datasource": map[string]interface{}{ "type": datasourceType, "uid": datasourceUID, }, "editorMode": "code", "expr": expr, "legendFormat": fmt.Sprintf("RPS %s", client.ClientTitle), "maxLines": 1000, "queryType": "statsRange", "refId": dataRefID, "intervalMs": 60000, "maxDataPoints": 43200, }, }, // Шаг 2: math $A / 60 → RPS (интервал 1m = 60 секунд) map[string]interface{}{ "refId": rpsRefID, "relativeTimeRange": map[string]interface{}{ "from": timeRangeFrom, "to": 0, }, "datasourceUid": "__expr__", "model": map[string]interface{}{ "datasource": map[string]interface{}{ "name": "Expression", "type": "__expr__", "uid": "__expr__", }, "expression": fmt.Sprintf("$%s / 60", dataRefID), "intervalMs": 1000, "maxDataPoints": 43200, "refId": rpsRefID, "type": "math", "window": "", }, }, // Шаг 3: reduce median map[string]interface{}{ "refId": reduceRefID, "relativeTimeRange": map[string]interface{}{ "from": 0, "to": 0, }, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{"params": []int{0, 0}, "type": "gt"}, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{ "name": "Expression", "type": "__expr__", "uid": "__expr__", }, "expression": rpsRefID, "intervalMs": 1000, "maxDataPoints": 43200, "reducer": "median", "refId": reduceRefID, "settings": map[string]interface{}{"mode": ""}, "type": "reduce", }, }, // Шаг 4: threshold > rps_limit map[string]interface{}{ "refId": thresholdRefID, "relativeTimeRange": map[string]interface{}{ "from": 0, "to": 0, }, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{ "params": []interface{}{client.RPSLimit, 0}, "type": "gt", }, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{ "name": "Expression", "type": "__expr__", "uid": "__expr__", }, "expression": reduceRefID, "intervalMs": 1000, "maxDataPoints": 43200, "refId": thresholdRefID, "type": "threshold", }, }, }, } } // buildErrorRateAlertRuleVL строит правило алерта на процент ошибок для VictoriaLogs. func buildErrorRateAlertRuleVL(client ClientData, errCode, limitPct int, tmpl AlertRulesTemplate, config Config) map[string]interface{} { datasourceUID := config.VLDatasourceUID datasourceType := DefaultVLDatasourceType defaults := tmpl.Defaults receiver := orDefault(config.AlertsReceiver, defaults.Receiver) group := orDefault(config.AlertsGroup, defaults.Group) sidQuery := buildTenantSIDQueryVL(client.Domains) codeFrom, codeTo := 400, 499 codeName := "4xx" if errCode == 5 { codeFrom, codeTo = 500, 599 codeName = "5xx" } uid := generateAlertUID(client.ClientTitle+"_vl", codeName) totalRefID := "Total" errRefID := "Errors" totalReduceID := "_TotalReduce" errReduceID := "_ErrorsReduce" percentRefID := "Значение" thresholdRefID := "Превышение" summary := fmt.Sprintf("🚨 %s — %s ошибок >%d%% трафика", client.ClientTitle, codeName, limitPct) relFrom := int64(900) forDuration := "1m" if errCode == 4 { forDuration = "3m" } totalExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() hits", sidQuery) errExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF response_status_code:>=%d response_status_code:<=%d | stats by (_time:1m) count() hits", sidQuery, codeFrom, codeTo) makeVLModel := func(refID, expr, legend string) map[string]interface{} { return map[string]interface{}{ "datasource": map[string]interface{}{ "type": datasourceType, "uid": datasourceUID, }, "editorMode": "code", "expr": expr, "legendFormat": legend, "maxLines": 1000, "queryType": "statsRange", "refId": refID, "intervalMs": 60000, "maxDataPoints": 43200, } } makeReduce := func(refID, expression, reducer string) map[string]interface{} { return map[string]interface{}{ "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": expression, "intervalMs": 1000, "maxDataPoints": 43200, "reducer": reducer, "refId": refID, "type": "reduce", } } return map[string]interface{}{ "uid": uid, "title": fmt.Sprintf("VL %s %s", codeName, client.ClientTitle), "condition": thresholdRefID, "noDataState": "NoData", // VL: нет данных = нет алерта "execErrState": "Error", // VL: ошибка datasource = Error, не Alerting "isPaused": false, "folderUID": "", "ruleGroup": group, "for": forDuration, "annotations": map[string]string{"summary": summary}, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{ map[string]interface{}{ "refId": totalRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, "datasourceUid": datasourceUID, "model": makeVLModel(totalRefID, totalExpr, "Total"), }, map[string]interface{}{ "refId": errRefID, "queryType": "statsRange", "relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0}, "datasourceUid": datasourceUID, "model": makeVLModel(errRefID, errExpr, codeName), }, map[string]interface{}{ "refId": totalReduceID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": makeReduce(totalReduceID, totalRefID, "sum"), }, map[string]interface{}{ "refId": errReduceID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": makeReduce(errReduceID, errRefID, "sum"), }, map[string]interface{}{ "refId": percentRefID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": map[string]interface{}{ "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": fmt.Sprintf("($%s / $%s) * 100", errReduceID, totalReduceID), "intervalMs": 1000, "maxDataPoints": 43200, "refId": percentRefID, "type": "math", }, }, map[string]interface{}{ "refId": thresholdRefID, "relativeTimeRange": map[string]interface{}{"from": 0, "to": 0}, "datasourceUid": "__expr__", "model": map[string]interface{}{ "conditions": []interface{}{ map[string]interface{}{ "evaluator": map[string]interface{}{"params": []interface{}{limitPct, 0}, "type": "gt"}, "operator": map[string]interface{}{"type": "and"}, "query": map[string]interface{}{"params": []string{}}, "reducer": map[string]interface{}{"params": []string{}, "type": "avg"}, "type": "query", }, }, "datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"}, "expression": percentRefID, "intervalMs": 1000, "maxDataPoints": 43200, "refId": thresholdRefID, "type": "threshold", }, }, }, } } // generateAndSendAlertsVL генерирует и отправляет алерты для VictoriaLogs. func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTemplate, config Config, dryRun bool) error { if config.VLDatasourceUID == "" { log.Printf("Skipping VL alerts: VLDatasourceUID not configured") return nil } folderUID, err := ensureAlertFolder(config) if err != nil { return fmt.Errorf("failed to ensure alert folder: %w", err) } var tasks []alertTask for _, client := range clients { if len(client.Domains) == 0 { continue } // RPS алерт if client.RPSLimit > 0 { rule := buildRPSAlertRuleVL(client, tmpl, config) rule["folderUID"] = folderUID tasks = append(tasks, alertTask{ rule: rule, label: fmt.Sprintf("VL RPS %s", client.ClientTitle), }) } // Error rate алерты (4xx и 5xx) for _, errCode := range []int{4, 5} { limitPct := client.Limit4xx if errCode == 5 { limitPct = client.Limit5xx } if limitPct <= 0 { continue } rule := buildErrorRateAlertRuleVL(client, errCode, limitPct, tmpl, config) rule["folderUID"] = folderUID tasks = append(tasks, alertTask{ rule: rule, label: fmt.Sprintf("VL %dxx %s", errCode, client.ClientTitle), }) } } sent, failed := parallelUpsert(tasks, config, dryRun, 5) log.Printf("VL Alerts: sent=%d failed=%d", sent, failed) // Статические алерты (CPU, RAM, диск, контейнеры, Angie) // Отправляем только если OS алерты отключены, иначе они уже отправлены там if !config.AlertsOSEnabled { if err := sendStaticAlerts(tmpl, config, dryRun); err != nil { log.Printf("Warning: VL static alerts error: %v", err) } } return nil }