diff --git a/alerts.go b/alerts.go index 765124a..6b71813 100644 --- a/alerts.go +++ b/alerts.go @@ -356,7 +356,7 @@ func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplat // Error rate алерты — пороги берутся из каждого домена (apps_settings) // Генерируем для response_status_code и upstream_status_code for _, domain := range t.client.Domains { - for _, statusField := range []string{"response_status_code", "upstream_status_code"} { + for _, statusField := range []string{"response_status_code", "upstream_status"} { if domain.Limit4xx > 0 { for _, rule := range buildErrorRateAlertRules(t.client, domain, 4, domain.Limit4xx, statusField, tmpl, config) { rule["folderUID"] = folderUID @@ -400,7 +400,7 @@ func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplat } // buildErrorRateAlertRules строит алерт на процент 4xx или 5xx ответов для конкретного домена. -// statusField: "response_status_code" или "upstream_status_code" +// statusField: "response_status_code" или "upstream_status" func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} { datasourceUID := config.AlertsDatasourceUID defaults := tmpl.Defaults @@ -426,13 +426,13 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim { sidQuery := "SID:" + domain.SID fieldShort := "response" - if statusField == "upstream_status_code" { + if statusField == "upstream_status" { fieldShort = "upstream" } uid := generateAlertUID(client.ClientTitle+"_"+domain.SID+"_"+fieldShort, codeName) title := fmt.Sprintf("%s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID) - fieldDesc := "ответ от PTAF, не от origin" - if statusField == "upstream_status_code" { + fieldDesc := "ответ от PTAF" + if statusField == "upstream_status" { fieldDesc = "ответ от origin" } summary := fmt.Sprintf( @@ -441,6 +441,7 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim ) thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc) fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc) + thresholdValueAnnotation := fmt.Sprintf("%d", limitPct) totalRefID := "Total" errRefID := "Errors" @@ -460,12 +461,13 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim "ruleGroup": group, "for": forDuration, "annotations": map[string]string{ - "summary": summary, - "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), - "code_field": fmt.Sprintf("%s %s", codeName, fieldShort), - "for_duration": forDuration, - "threshold": thresholdAnnotation, - "field_info": fieldInfoAnnotation, + "summary": summary, + "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), + "code_field": codeName, + "for_duration": forDuration, + "threshold": thresholdAnnotation, + "threshold_value": thresholdValueAnnotation, + "field_info": fieldInfoAnnotation, }, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{ @@ -823,16 +825,30 @@ func upsertContactPoint(tmpl AlertRulesTemplate, config Config, dryRun bool) err // Обновляем все типы интеграций (telegram, email, slack и т.д.) uid, _ := cp["uid"].(string) cpType, _ := cp["type"].(string) + log.Printf("Contact point integration found: name=%s type=%s uid=%s", name, cpType, uid) settings, ok := cp["settings"].(map[string]interface{}) if !ok { settings = make(map[string]interface{}) cp["settings"] = settings } + + // Удаляем секретные поля которые Grafana вернула как "configured" + // чтобы не затереть реальные значения при PUT + for _, secretField := range []string{"token", "bot_token", "api_token", "password", "secret", "url", "recipient"} { + if val, exists := settings[secretField]; exists { + if strVal, ok := val.(string); ok && (strVal == "configured" || strVal == "") { + delete(settings, secretField) + } + } + } + // Поле для шаблона зависит от типа интеграции + // Slack/Mattermost не поддерживают кастомный шаблон через API — пропускаем switch cpType { case "slack": settings["text"] = tmpl.ContactPoint.MessageTemplate - default: // telegram, email, oncall, mattermost и др. + settings["title"] = "" // очищаем автоматический заголовок + default: // telegram, email и др. settings["message"] = tmpl.ContactPoint.MessageTemplate } @@ -938,6 +954,7 @@ func buildWAFBlockAlertRule(client ClientData, domain DomainInfo, tmpl AlertRule clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID) summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode) thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount) + thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount) sidQuery := "SID:" + domain.SID dataRefID := "A" @@ -956,15 +973,16 @@ func buildWAFBlockAlertRule(client ClientData, domain DomainInfo, tmpl AlertRule "ruleGroup": group, "for": "1m", "annotations": map[string]string{ - "summary": summary, - "client_domain": clientDomain, - "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), - "for_duration": "1m", - "threshold": thresholdAnnotation, - "field_info": "для response_status_code (ответ от PTAF, не от origin)", - "runbook": "Проверить логи docker, процитировать ошибку и эскалировать на аналитика.", - "severity": "warning", - "value_unit": "шт.", + "summary": summary, + "client_domain": clientDomain, + "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), + "for_duration": "1m", + "threshold": thresholdAnnotation, + "field_info": "для response_status_code (ответ от PTAF, не от origin)", + "runbook": "проверить логи docker, процитировать ошибку и эскалировать на аналитика.", + "severity": "warning", + "value_unit": "шт.", + "threshold_count": thresholdCountAnnotation, }, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{ diff --git a/alerts_vl.go b/alerts_vl.go index 2b8d4c7..f0f6c64 100644 --- a/alerts_vl.go +++ b/alerts_vl.go @@ -183,7 +183,7 @@ func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Conf } // buildErrorRateAlertRulesVL строит алерт на процент ошибок для конкретного домена (VictoriaLogs). -// statusField: "response_status_code" или "upstream_status_code" +// statusField: "response_status_code" или "upstream_status" func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} { datasourceUID := config.VLDatasourceUID datasourceType := DefaultVLDatasourceType @@ -212,13 +212,13 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l sidQuery := fmt.Sprintf("SID:%s", domain.SID) // Короткое имя поля для использования в названии алерта fieldShort := "response" - if statusField == "upstream_status_code" { + if statusField == "upstream_status" { fieldShort = "upstream" } uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID+"_"+fieldShort, codeName) title := fmt.Sprintf("VL %s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID) - fieldDesc := "ответ от PTAF, не от origin" - if statusField == "upstream_status_code" { + fieldDesc := "ответ от PTAF" + if statusField == "upstream_status" { fieldDesc = "ответ от origin" } summary := fmt.Sprintf( @@ -227,6 +227,7 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l ) thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc) fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc) + thresholdValueAnnotation := fmt.Sprintf("%d", limitPct) totalRefID := "Total" errRefID := "Errors" @@ -278,12 +279,13 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l "ruleGroup": group, "for": forDuration, "annotations": map[string]string{ - "summary": summary, - "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), - "code_field": fmt.Sprintf("%s %s", codeName, fieldShort), - "for_duration": forDuration, - "threshold": thresholdAnnotation, - "field_info": fieldInfoAnnotation, + "summary": summary, + "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), + "code_field": codeName, + "for_duration": forDuration, + "threshold": thresholdAnnotation, + "threshold_value": thresholdValueAnnotation, + "field_info": fieldInfoAnnotation, }, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{ @@ -400,7 +402,7 @@ func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTempl // Error rate алерты — пороги берутся из каждого домена (apps_settings) // Генерируем для response_status_code и upstream_status_code for _, domain := range client.Domains { - for _, statusField := range []string{"response_status_code", "upstream_status_code"} { + for _, statusField := range []string{"response_status_code", "upstream_status"} { if domain.Limit4xx > 0 { for _, rule := range buildErrorRateAlertRulesVL(client, domain, 4, domain.Limit4xx, statusField, tmpl, config) { rule["folderUID"] = folderUID @@ -462,6 +464,7 @@ func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRu clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID) summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode) thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount) + thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount) expr := fmt.Sprintf("SID:%s log_type:nginx-access-PTAF response_status_code:=%s | stats by (_time:1m) count() hits", domain.SID, client.PtafFallbackCode) @@ -481,15 +484,16 @@ func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRu "ruleGroup": group, "for": "1m", "annotations": map[string]string{ - "summary": summary, - "client_domain": clientDomain, - "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), - "for_duration": "1m", - "threshold": thresholdAnnotation, - "field_info": "для response_status_code (ответ от PTAF, не от origin)", - "runbook": "Проверить логи docker, процитировать ошибку и эскалировать на аналитика.", - "severity": "warning", - "value_unit": "шт.", + "summary": summary, + "client_domain": clientDomain, + "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), + "for_duration": "1m", + "threshold": thresholdAnnotation, + "field_info": "для response_status_code (ответ от PTAF, не от origin)", + "runbook": "проверить логи docker, процитировать ошибку и эскалировать на аналитика.", + "severity": "warning", + "value_unit": "шт.", + "threshold_count": thresholdCountAnnotation, }, "notification_settings": map[string]string{"receiver": receiver}, "data": []interface{}{