Change alerts
This commit is contained in:
parent
1aa91ba296
commit
77db1d29f1
2 changed files with 63 additions and 41 deletions
34
alerts.go
34
alerts.go
|
|
@ -356,7 +356,7 @@ func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplat
|
|||
// Error rate алерты — пороги берутся из каждого домена (apps_settings)
|
||||
// Генерируем для response_status_code и upstream_status_code
|
||||
for _, domain := range t.client.Domains {
|
||||
for _, statusField := range []string{"response_status_code", "upstream_status_code"} {
|
||||
for _, statusField := range []string{"response_status_code", "upstream_status"} {
|
||||
if domain.Limit4xx > 0 {
|
||||
for _, rule := range buildErrorRateAlertRules(t.client, domain, 4, domain.Limit4xx, statusField, tmpl, config) {
|
||||
rule["folderUID"] = folderUID
|
||||
|
|
@ -400,7 +400,7 @@ func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplat
|
|||
}
|
||||
|
||||
// buildErrorRateAlertRules строит алерт на процент 4xx или 5xx ответов для конкретного домена.
|
||||
// statusField: "response_status_code" или "upstream_status_code"
|
||||
// statusField: "response_status_code" или "upstream_status"
|
||||
func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} {
|
||||
datasourceUID := config.AlertsDatasourceUID
|
||||
defaults := tmpl.Defaults
|
||||
|
|
@ -426,13 +426,13 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim
|
|||
{
|
||||
sidQuery := "SID:" + domain.SID
|
||||
fieldShort := "response"
|
||||
if statusField == "upstream_status_code" {
|
||||
if statusField == "upstream_status" {
|
||||
fieldShort = "upstream"
|
||||
}
|
||||
uid := generateAlertUID(client.ClientTitle+"_"+domain.SID+"_"+fieldShort, codeName)
|
||||
title := fmt.Sprintf("%s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID)
|
||||
fieldDesc := "ответ от PTAF, не от origin"
|
||||
if statusField == "upstream_status_code" {
|
||||
fieldDesc := "ответ от PTAF"
|
||||
if statusField == "upstream_status" {
|
||||
fieldDesc = "ответ от origin"
|
||||
}
|
||||
summary := fmt.Sprintf(
|
||||
|
|
@ -441,6 +441,7 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim
|
|||
)
|
||||
thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc)
|
||||
fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc)
|
||||
thresholdValueAnnotation := fmt.Sprintf("%d", limitPct)
|
||||
|
||||
totalRefID := "Total"
|
||||
errRefID := "Errors"
|
||||
|
|
@ -462,9 +463,10 @@ func buildErrorRateAlertRules(client ClientData, domain DomainInfo, errCode, lim
|
|||
"annotations": map[string]string{
|
||||
"summary": summary,
|
||||
"client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID),
|
||||
"code_field": fmt.Sprintf("%s %s", codeName, fieldShort),
|
||||
"code_field": codeName,
|
||||
"for_duration": forDuration,
|
||||
"threshold": thresholdAnnotation,
|
||||
"threshold_value": thresholdValueAnnotation,
|
||||
"field_info": fieldInfoAnnotation,
|
||||
},
|
||||
"notification_settings": map[string]string{"receiver": receiver},
|
||||
|
|
@ -823,16 +825,30 @@ func upsertContactPoint(tmpl AlertRulesTemplate, config Config, dryRun bool) err
|
|||
// Обновляем все типы интеграций (telegram, email, slack и т.д.)
|
||||
uid, _ := cp["uid"].(string)
|
||||
cpType, _ := cp["type"].(string)
|
||||
log.Printf("Contact point integration found: name=%s type=%s uid=%s", name, cpType, uid)
|
||||
settings, ok := cp["settings"].(map[string]interface{})
|
||||
if !ok {
|
||||
settings = make(map[string]interface{})
|
||||
cp["settings"] = settings
|
||||
}
|
||||
|
||||
// Удаляем секретные поля которые Grafana вернула как "configured"
|
||||
// чтобы не затереть реальные значения при PUT
|
||||
for _, secretField := range []string{"token", "bot_token", "api_token", "password", "secret", "url", "recipient"} {
|
||||
if val, exists := settings[secretField]; exists {
|
||||
if strVal, ok := val.(string); ok && (strVal == "configured" || strVal == "") {
|
||||
delete(settings, secretField)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Поле для шаблона зависит от типа интеграции
|
||||
// Slack/Mattermost не поддерживают кастомный шаблон через API — пропускаем
|
||||
switch cpType {
|
||||
case "slack":
|
||||
settings["text"] = tmpl.ContactPoint.MessageTemplate
|
||||
default: // telegram, email, oncall, mattermost и др.
|
||||
settings["title"] = "" // очищаем автоматический заголовок
|
||||
default: // telegram, email и др.
|
||||
settings["message"] = tmpl.ContactPoint.MessageTemplate
|
||||
}
|
||||
|
||||
|
|
@ -938,6 +954,7 @@ func buildWAFBlockAlertRule(client ClientData, domain DomainInfo, tmpl AlertRule
|
|||
clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID)
|
||||
summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode)
|
||||
thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount)
|
||||
thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount)
|
||||
|
||||
sidQuery := "SID:" + domain.SID
|
||||
dataRefID := "A"
|
||||
|
|
@ -962,9 +979,10 @@ func buildWAFBlockAlertRule(client ClientData, domain DomainInfo, tmpl AlertRule
|
|||
"for_duration": "1m",
|
||||
"threshold": thresholdAnnotation,
|
||||
"field_info": "для response_status_code (ответ от PTAF, не от origin)",
|
||||
"runbook": "Проверить логи docker, процитировать ошибку и эскалировать на аналитика.",
|
||||
"runbook": "проверить логи docker, процитировать ошибку и эскалировать на аналитика.",
|
||||
"severity": "warning",
|
||||
"value_unit": "шт.",
|
||||
"threshold_count": thresholdCountAnnotation,
|
||||
},
|
||||
"notification_settings": map[string]string{"receiver": receiver},
|
||||
"data": []interface{}{
|
||||
|
|
|
|||
18
alerts_vl.go
18
alerts_vl.go
|
|
@ -183,7 +183,7 @@ func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Conf
|
|||
}
|
||||
|
||||
// buildErrorRateAlertRulesVL строит алерт на процент ошибок для конкретного домена (VictoriaLogs).
|
||||
// statusField: "response_status_code" или "upstream_status_code"
|
||||
// statusField: "response_status_code" или "upstream_status"
|
||||
func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} {
|
||||
datasourceUID := config.VLDatasourceUID
|
||||
datasourceType := DefaultVLDatasourceType
|
||||
|
|
@ -212,13 +212,13 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l
|
|||
sidQuery := fmt.Sprintf("SID:%s", domain.SID)
|
||||
// Короткое имя поля для использования в названии алерта
|
||||
fieldShort := "response"
|
||||
if statusField == "upstream_status_code" {
|
||||
if statusField == "upstream_status" {
|
||||
fieldShort = "upstream"
|
||||
}
|
||||
uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID+"_"+fieldShort, codeName)
|
||||
title := fmt.Sprintf("VL %s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID)
|
||||
fieldDesc := "ответ от PTAF, не от origin"
|
||||
if statusField == "upstream_status_code" {
|
||||
fieldDesc := "ответ от PTAF"
|
||||
if statusField == "upstream_status" {
|
||||
fieldDesc = "ответ от origin"
|
||||
}
|
||||
summary := fmt.Sprintf(
|
||||
|
|
@ -227,6 +227,7 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l
|
|||
)
|
||||
thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc)
|
||||
fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc)
|
||||
thresholdValueAnnotation := fmt.Sprintf("%d", limitPct)
|
||||
|
||||
totalRefID := "Total"
|
||||
errRefID := "Errors"
|
||||
|
|
@ -280,9 +281,10 @@ func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, l
|
|||
"annotations": map[string]string{
|
||||
"summary": summary,
|
||||
"client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID),
|
||||
"code_field": fmt.Sprintf("%s %s", codeName, fieldShort),
|
||||
"code_field": codeName,
|
||||
"for_duration": forDuration,
|
||||
"threshold": thresholdAnnotation,
|
||||
"threshold_value": thresholdValueAnnotation,
|
||||
"field_info": fieldInfoAnnotation,
|
||||
},
|
||||
"notification_settings": map[string]string{"receiver": receiver},
|
||||
|
|
@ -400,7 +402,7 @@ func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTempl
|
|||
// Error rate алерты — пороги берутся из каждого домена (apps_settings)
|
||||
// Генерируем для response_status_code и upstream_status_code
|
||||
for _, domain := range client.Domains {
|
||||
for _, statusField := range []string{"response_status_code", "upstream_status_code"} {
|
||||
for _, statusField := range []string{"response_status_code", "upstream_status"} {
|
||||
if domain.Limit4xx > 0 {
|
||||
for _, rule := range buildErrorRateAlertRulesVL(client, domain, 4, domain.Limit4xx, statusField, tmpl, config) {
|
||||
rule["folderUID"] = folderUID
|
||||
|
|
@ -462,6 +464,7 @@ func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRu
|
|||
clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID)
|
||||
summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode)
|
||||
thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF, не от origin)", domain.PtafFallbackCodeAlertCount)
|
||||
thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount)
|
||||
|
||||
expr := fmt.Sprintf("SID:%s log_type:nginx-access-PTAF response_status_code:=%s | stats by (_time:1m) count() hits", domain.SID, client.PtafFallbackCode)
|
||||
|
||||
|
|
@ -487,9 +490,10 @@ func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRu
|
|||
"for_duration": "1m",
|
||||
"threshold": thresholdAnnotation,
|
||||
"field_info": "для response_status_code (ответ от PTAF, не от origin)",
|
||||
"runbook": "Проверить логи docker, процитировать ошибку и эскалировать на аналитика.",
|
||||
"runbook": "проверить логи docker, процитировать ошибку и эскалировать на аналитика.",
|
||||
"severity": "warning",
|
||||
"value_unit": "шт.",
|
||||
"threshold_count": thresholdCountAnnotation,
|
||||
},
|
||||
"notification_settings": map[string]string{"receiver": receiver},
|
||||
"data": []interface{}{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue