mroot/grafana_gen
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Change alerts

Browse source
This commit is contained in:
Magnus Root 2026-04-28 17:57:56 +03:00
parent 1aa91ba296
commit 77db1d29f1
2 changed files with 63 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
alerts.go
View file

@ -356,7 +356,7 @@ func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplat
// Error rate алерты — пороги берутся из каждого домена (apps_settings) // Error rate алерты — пороги берутся из каждого домена (apps_settings)
// Генерируем для response_status_code и upstream_status_code // Генерируем для response_status_code и upstream_status_code
for _, domain := range t.client.Domains { for _, domain := range t.client.Domains {
for _, statusField := range []string{"response_status_code", "upstream_status_code"} { for _, statusField := range []string{"response_status_code", "upstream_status"} {
if domain.Limit4xx > 0 { if domain.Limit4xx > 0 {
for _, rule := range buildErrorRateAlertRules(t.client, domain, 4, domain.Limit4xx, statusField, tmpl, config) { for _, rule := range buildErrorRateAlertRules(t.client, domain, 4, domain.Limit4xx, statusField, tmpl, config) {
rule["folderUID"] = folderUID rule["folderUID"] = folderUID
@ -400,7 +400,7 @@ func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplat
} }
// buildErrorRateAlertRules строит алерт на процент 4xx или 5xx ответов для конкретного домена. // buildErrorRateAlertRules строит алерт на процент 4xx или 5xx ответов для конкретного домена.
// statusField: "response_status_code" или "upstream_status_code" // statusField: "response_status_code" или "upstream_status"
func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} { func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} {
datasourceUID := config.AlertsDatasourceUID datasourceUID := config.AlertsDatasourceUID
defaults := tmpl.Defaults defaults := tmpl.Defaults
@ -426,13 +426,13 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim
{ {
sidQuery := "SID:" + domain.SID sidQuery := "SID:" + domain.SID
fieldShort := "response" fieldShort := "response"
if statusField == "upstream_status_code" { if statusField == "upstream_status" {
fieldShort = "upstream" fieldShort = "upstream"
} }
uid := generateAlertUID(client.ClientTitle+"_"+domain.SID+"_"+fieldShort, codeName) uid := generateAlertUID(client.ClientTitle+"_"+domain.SID+"_"+fieldShort, codeName)
title := fmt.Sprintf("%s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID) title := fmt.Sprintf("%s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID)
fieldDesc := "ответ от PTAF, не от origin" fieldDesc := "ответ от PTAF"
if statusField == "upstream_status_code" { if statusField == "upstream_status" {
fieldDesc = "ответ от origin" fieldDesc = "ответ от origin"
} }
summary := fmt.Sprintf( summary := fmt.Sprintf(
@ -441,6 +441,7 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim
) )
thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc) thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc)
fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc) fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc)
thresholdValueAnnotation := fmt.Sprintf("%d", limitPct)
totalRefID := "Total" totalRefID := "Total"
errRefID := "Errors" errRefID := "Errors"
@ -460,12 +461,13 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim
"ruleGroup": group, "ruleGroup": group,
"for": forDuration, "for": forDuration,
"annotations": map[string]string{ "annotations": map[string]string{
"summary": summary, "summary": summary,
"client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID),
"code_field": fmt.Sprintf("%s %s", codeName, fieldShort), "code_field": codeName,
"for_duration": forDuration, "for_duration": forDuration,
"threshold": thresholdAnnotation, "threshold": thresholdAnnotation,
"field_info": fieldInfoAnnotation, "threshold_value": thresholdValueAnnotation,
"field_info": fieldInfoAnnotation,
}, },
"notification_settings": map[string]string{"receiver": receiver}, "notification_settings": map[string]string{"receiver": receiver},
"data": []interface{}{ "data": []interface{}{
@ -823,16 +825,30 @@ func upsertContactPoint(tmpl AlertRulesTemplate, config Config, dryRun bool) err
// Обновляем все типы интеграций (telegram, email, slack и т.д.) // Обновляем все типы интеграций (telegram, email, slack и т.д.)
uid, _ := cp["uid"].(string) uid, _ := cp["uid"].(string)
cpType, _ := cp["type"].(string) cpType, _ := cp["type"].(string)
log.Printf("Contact point integration found: name=%s type=%s uid=%s", name, cpType, uid)
settings, ok := cp["settings"].(map[string]interface{}) settings, ok := cp["settings"].(map[string]interface{})
if !ok { if !ok {
settings = make(map[string]interface{}) settings = make(map[string]interface{})
cp["settings"] = settings cp["settings"] = settings
} }
// Удаляем секретные поля которые Grafana вернула как "configured"
// чтобы не затереть реальные значения при PUT
for _, secretField := range []string{"token", "bot_token", "api_token", "password", "secret", "url", "recipient"} {
if val, exists := settings[secretField]; exists {
if strVal, ok := val.(string); ok && (strVal == "configured" || strVal == "") {
delete(settings, secretField)
}
}
}
// Поле для шаблона зависит от типа интеграции // Поле для шаблона зависит от типа интеграции
// Slack/Mattermost не поддерживают кастомный шаблон через API — пропускаем
switch cpType { switch cpType {
case "slack": case "slack":
settings["text"] = tmpl.ContactPoint.MessageTemplate settings["text"] = tmpl.ContactPoint.MessageTemplate
default: // telegram, email, oncall, mattermost и др. settings["title"] = "" // очищаем автоматический заголовок
default: // telegram, email и др.
settings["message"] = tmpl.ContactPoint.MessageTemplate settings["message"] = tmpl.ContactPoint.MessageTemplate
} }
@ -938,6 +954,7 @@ func buildWAFBlockAlertRule(client ClientData, domain DomainInfo, tmpl AlertRule
clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID) clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID)
summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode) summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode)
thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount) thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount)
thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount)
sidQuery := "SID:" + domain.SID sidQuery := "SID:" + domain.SID
dataRefID := "A" dataRefID := "A"
@ -956,15 +973,16 @@ func buildWAFBlockAlertRule(client ClientData, domain DomainInfo, tmpl AlertRule
"ruleGroup": group, "ruleGroup": group,
"for": "1m", "for": "1m",
"annotations": map[string]string{ "annotations": map[string]string{
"summary": summary, "summary": summary,
"client_domain": clientDomain, "client_domain": clientDomain,
"code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode),
"for_duration": "1m", "for_duration": "1m",
"threshold": thresholdAnnotation, "threshold": thresholdAnnotation,
"field_info": "для response_status_code (ответ от PTAF, не от origin)", "field_info": "для response_status_code (ответ от PTAF, не от origin)",
"runbook": "Проверить логи docker, процитировать ошибку и эскалировать на аналитика.", "runbook": "проверить логи docker, процитировать ошибку и эскалировать на аналитика.",
"severity": "warning", "severity": "warning",
"value_unit": "шт.", "value_unit": "шт.",
"threshold_count": thresholdCountAnnotation,
}, },
"notification_settings": map[string]string{"receiver": receiver}, "notification_settings": map[string]string{"receiver": receiver},
"data": []interface{}{ "data": []interface{}{

44
alerts_vl.go
View file

@ -183,7 +183,7 @@ func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Conf
} }
// buildErrorRateAlertRulesVL строит алерт на процент ошибок для конкретного домена (VictoriaLogs). // buildErrorRateAlertRulesVL строит алерт на процент ошибок для конкретного домена (VictoriaLogs).
// statusField: "response_status_code" или "upstream_status_code" // statusField: "response_status_code" или "upstream_status"
func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} { func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} {
datasourceUID := config.VLDatasourceUID datasourceUID := config.VLDatasourceUID
datasourceType := DefaultVLDatasourceType datasourceType := DefaultVLDatasourceType
@ -212,13 +212,13 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l
sidQuery := fmt.Sprintf("SID:%s", domain.SID) sidQuery := fmt.Sprintf("SID:%s", domain.SID)
// Короткое имя поля для использования в названии алерта // Короткое имя поля для использования в названии алерта
fieldShort := "response" fieldShort := "response"
if statusField == "upstream_status_code" { if statusField == "upstream_status" {
fieldShort = "upstream" fieldShort = "upstream"
} }
uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID+"_"+fieldShort, codeName) uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID+"_"+fieldShort, codeName)
title := fmt.Sprintf("VL %s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID) title := fmt.Sprintf("VL %s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID)
fieldDesc := "ответ от PTAF, не от origin" fieldDesc := "ответ от PTAF"
if statusField == "upstream_status_code" { if statusField == "upstream_status" {
fieldDesc = "ответ от origin" fieldDesc = "ответ от origin"
} }
summary := fmt.Sprintf( summary := fmt.Sprintf(
@ -227,6 +227,7 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l
) )
thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc) thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc)
fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc) fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc)
thresholdValueAnnotation := fmt.Sprintf("%d", limitPct)
totalRefID := "Total" totalRefID := "Total"
errRefID := "Errors" errRefID := "Errors"
@ -278,12 +279,13 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l
"ruleGroup": group, "ruleGroup": group,
"for": forDuration, "for": forDuration,
"annotations": map[string]string{ "annotations": map[string]string{
"summary": summary, "summary": summary,
"client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID), "client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID),
"code_field": fmt.Sprintf("%s %s", codeName, fieldShort), "code_field": codeName,
"for_duration": forDuration, "for_duration": forDuration,
"threshold": thresholdAnnotation, "threshold": thresholdAnnotation,
"field_info": fieldInfoAnnotation, "threshold_value": thresholdValueAnnotation,
"field_info": fieldInfoAnnotation,
}, },
"notification_settings": map[string]string{"receiver": receiver}, "notification_settings": map[string]string{"receiver": receiver},
"data": []interface{}{ "data": []interface{}{
@ -400,7 +402,7 @@ func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTempl
// Error rate алерты — пороги берутся из каждого домена (apps_settings) // Error rate алерты — пороги берутся из каждого домена (apps_settings)
// Генерируем для response_status_code и upstream_status_code // Генерируем для response_status_code и upstream_status_code
for _, domain := range client.Domains { for _, domain := range client.Domains {
for _, statusField := range []string{"response_status_code", "upstream_status_code"} { for _, statusField := range []string{"response_status_code", "upstream_status"} {
if domain.Limit4xx > 0 { if domain.Limit4xx > 0 {
for _, rule := range buildErrorRateAlertRulesVL(client, domain, 4, domain.Limit4xx, statusField, tmpl, config) { for _, rule := range buildErrorRateAlertRulesVL(client, domain, 4, domain.Limit4xx, statusField, tmpl, config) {
rule["folderUID"] = folderUID rule["folderUID"] = folderUID
@ -462,6 +464,7 @@ func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRu
clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID) clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID)
summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode) summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode)
thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount) thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount)
thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount)
expr := fmt.Sprintf("SID:%s log_type:nginx-access-PTAF response_status_code:=%s | stats by (_time:1m) count() hits", domain.SID, client.PtafFallbackCode) expr := fmt.Sprintf("SID:%s log_type:nginx-access-PTAF response_status_code:=%s | stats by (_time:1m) count() hits", domain.SID, client.PtafFallbackCode)
@ -481,15 +484,16 @@ func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRu
"ruleGroup": group, "ruleGroup": group,
"for": "1m", "for": "1m",
"annotations": map[string]string{ "annotations": map[string]string{
"summary": summary, "summary": summary,
"client_domain": clientDomain, "client_domain": clientDomain,
"code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode), "code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode),
"for_duration": "1m", "for_duration": "1m",
"threshold": thresholdAnnotation, "threshold": thresholdAnnotation,
"field_info": "для response_status_code (ответ от PTAF, не от origin)", "field_info": "для response_status_code (ответ от PTAF, не от origin)",
"runbook": "Проверить логи docker, процитировать ошибку и эскалировать на аналитика.", "runbook": "проверить логи docker, процитировать ошибку и эскалировать на аналитика.",
"severity": "warning", "severity": "warning",
"value_unit": "шт.", "value_unit": "шт.",
"threshold_count": thresholdCountAnnotation,
}, },
"notification_settings": map[string]string{"receiver": receiver}, "notification_settings": map[string]string{"receiver": receiver},
"data": []interface{}{ "data": []interface{}{