389 lines
13 KiB
Go
389 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// buildRPSAlertRuleVL строит правило алерта на RPS для VictoriaLogs datasource.
|
|
func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Config) map[string]interface{} {
|
|
datasourceUID := config.VLDatasourceUID
|
|
datasourceType := DefaultVLDatasourceType
|
|
|
|
// SID-запрос в формате LogsQL
|
|
sidMap := make(map[string]bool)
|
|
for _, d := range client.Domains {
|
|
sidMap[d.SID] = true
|
|
}
|
|
uniqueSIDs := make([]string, 0, len(sidMap))
|
|
for s := range sidMap {
|
|
uniqueSIDs = append(uniqueSIDs, s)
|
|
}
|
|
sort.Strings(uniqueSIDs)
|
|
sidQuery := buildTenantSIDQueryVL(client.Domains)
|
|
|
|
defaults := tmpl.Defaults
|
|
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
|
group := orDefault(config.AlertsGroup, defaults.Group)
|
|
|
|
timeRangeFrom := int64(1800)
|
|
if v, ok := tmpl.RPSAlert["relative_time_range_from"].(float64); ok {
|
|
timeRangeFrom = int64(v)
|
|
}
|
|
|
|
summaryTemplate := "Превышен порог в {rps_limit} RPS в {client_title}"
|
|
if v, ok := tmpl.RPSAlert["annotation_summary"].(string); ok {
|
|
summaryTemplate = v
|
|
}
|
|
summary := strings.NewReplacer(
|
|
"{rps_limit}", fmt.Sprintf("%d", client.RPSLimit),
|
|
"{client_title}", client.ClientTitle,
|
|
).Replace(summaryTemplate)
|
|
|
|
_ = uniqueSIDs // используется через sidQuery
|
|
|
|
uid := generateAlertUID(client.ClientTitle+"_vl", "rps")
|
|
dataRefID := "A"
|
|
rpsRefID := "RPS"
|
|
reduceRefID := "_Reduce"
|
|
thresholdRefID := "Превышение"
|
|
|
|
// LogsQL запрос: count за минуту
|
|
expr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() rps", sidQuery)
|
|
|
|
return map[string]interface{}{
|
|
"uid": uid,
|
|
"title": fmt.Sprintf("VL RPS %s", client.ClientTitle),
|
|
"condition": thresholdRefID,
|
|
"noDataState": "NoData", // VL: нет данных = нет алерта
|
|
"execErrState": "Error", // VL: ошибка datasource = Error, не Alerting
|
|
"isPaused": false,
|
|
"folderUID": "",
|
|
"ruleGroup": group,
|
|
"annotations": map[string]string{"summary": summary},
|
|
"notification_settings": map[string]string{
|
|
"receiver": receiver,
|
|
},
|
|
"data": []interface{}{
|
|
// Шаг 1: VL statsRange запрос
|
|
map[string]interface{}{
|
|
"refId": dataRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": timeRangeFrom,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": datasourceUID,
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{
|
|
"type": datasourceType,
|
|
"uid": datasourceUID,
|
|
},
|
|
"editorMode": "code",
|
|
"expr": expr,
|
|
"legendFormat": fmt.Sprintf("RPS %s", client.ClientTitle),
|
|
"maxLines": 1000,
|
|
"queryType": "statsRange",
|
|
"refId": dataRefID,
|
|
"intervalMs": 60000,
|
|
"maxDataPoints": 43200,
|
|
},
|
|
},
|
|
// Шаг 2: math $A / 60 → RPS (интервал 1m = 60 секунд)
|
|
map[string]interface{}{
|
|
"refId": rpsRefID,
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": timeRangeFrom,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{
|
|
"name": "Expression",
|
|
"type": "__expr__",
|
|
"uid": "__expr__",
|
|
},
|
|
"expression": fmt.Sprintf("$%s / 60", dataRefID),
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": rpsRefID,
|
|
"type": "math",
|
|
"window": "",
|
|
},
|
|
},
|
|
// Шаг 3: reduce median
|
|
map[string]interface{}{
|
|
"refId": reduceRefID,
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": 0,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{"params": []int{0, 0}, "type": "gt"},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{
|
|
"name": "Expression",
|
|
"type": "__expr__",
|
|
"uid": "__expr__",
|
|
},
|
|
"expression": rpsRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"reducer": "median",
|
|
"refId": reduceRefID,
|
|
"settings": map[string]interface{}{"mode": ""},
|
|
"type": "reduce",
|
|
},
|
|
},
|
|
// Шаг 4: threshold > rps_limit
|
|
map[string]interface{}{
|
|
"refId": thresholdRefID,
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": 0,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{
|
|
"params": []interface{}{client.RPSLimit, 0},
|
|
"type": "gt",
|
|
},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{
|
|
"name": "Expression",
|
|
"type": "__expr__",
|
|
"uid": "__expr__",
|
|
},
|
|
"expression": reduceRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": thresholdRefID,
|
|
"type": "threshold",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// buildErrorRateAlertRuleVL строит правило алерта на процент ошибок для VictoriaLogs.
|
|
func buildErrorRateAlertRuleVL(client ClientData, errCode, limitPct int, tmpl AlertRulesTemplate, config Config) map[string]interface{} {
|
|
datasourceUID := config.VLDatasourceUID
|
|
datasourceType := DefaultVLDatasourceType
|
|
|
|
defaults := tmpl.Defaults
|
|
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
|
group := orDefault(config.AlertsGroup, defaults.Group)
|
|
|
|
sidQuery := buildTenantSIDQueryVL(client.Domains)
|
|
|
|
codeFrom, codeTo := 400, 499
|
|
codeName := "4xx"
|
|
if errCode == 5 {
|
|
codeFrom, codeTo = 500, 599
|
|
codeName = "5xx"
|
|
}
|
|
|
|
uid := generateAlertUID(client.ClientTitle+"_vl", codeName)
|
|
totalRefID := "Total"
|
|
errRefID := "Errors"
|
|
totalReduceID := "_TotalReduce"
|
|
errReduceID := "_ErrorsReduce"
|
|
percentRefID := "Значение"
|
|
thresholdRefID := "Превышение"
|
|
|
|
summary := fmt.Sprintf("🚨 %s — %s ошибок >%d%% трафика", client.ClientTitle, codeName, limitPct)
|
|
|
|
relFrom := int64(900)
|
|
|
|
forDuration := "1m"
|
|
if errCode == 4 {
|
|
forDuration = "3m"
|
|
}
|
|
|
|
totalExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() hits", sidQuery)
|
|
errExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF response_status_code:>=%d response_status_code:<=%d | stats by (_time:1m) count() hits", sidQuery, codeFrom, codeTo)
|
|
|
|
makeVLModel := func(refID, expr, legend string) map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"datasource": map[string]interface{}{
|
|
"type": datasourceType,
|
|
"uid": datasourceUID,
|
|
},
|
|
"editorMode": "code",
|
|
"expr": expr,
|
|
"legendFormat": legend,
|
|
"maxLines": 1000,
|
|
"queryType": "statsRange",
|
|
"refId": refID,
|
|
"intervalMs": 60000,
|
|
"maxDataPoints": 43200,
|
|
}
|
|
}
|
|
|
|
makeReduce := func(refID, expression, reducer string) map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": expression,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"reducer": reducer,
|
|
"refId": refID,
|
|
"type": "reduce",
|
|
}
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"uid": uid,
|
|
"title": fmt.Sprintf("VL %s %s", codeName, client.ClientTitle),
|
|
"condition": thresholdRefID,
|
|
"noDataState": "NoData", // VL: нет данных = нет алерта
|
|
"execErrState": "Error", // VL: ошибка datasource = Error, не Alerting
|
|
"isPaused": false,
|
|
"folderUID": "",
|
|
"ruleGroup": group,
|
|
"for": forDuration,
|
|
"annotations": map[string]string{"summary": summary},
|
|
"notification_settings": map[string]string{"receiver": receiver},
|
|
"data": []interface{}{
|
|
map[string]interface{}{
|
|
"refId": totalRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
|
"datasourceUid": datasourceUID,
|
|
"model": makeVLModel(totalRefID, totalExpr, "Total"),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": errRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
|
"datasourceUid": datasourceUID,
|
|
"model": makeVLModel(errRefID, errExpr, codeName),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": totalReduceID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": makeReduce(totalReduceID, totalRefID, "sum"),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": errReduceID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": makeReduce(errReduceID, errRefID, "sum"),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": percentRefID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": fmt.Sprintf("($%s / $%s) * 100", errReduceID, totalReduceID),
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": percentRefID,
|
|
"type": "math",
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"refId": thresholdRefID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{"params": []interface{}{limitPct, 0}, "type": "gt"},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": percentRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": thresholdRefID,
|
|
"type": "threshold",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// generateAndSendAlertsVL генерирует и отправляет алерты для VictoriaLogs.
|
|
func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTemplate, config Config, dryRun bool) error {
|
|
if config.VLDatasourceUID == "" {
|
|
log.Printf("Skipping VL alerts: VLDatasourceUID not configured")
|
|
return nil
|
|
}
|
|
|
|
folderUID, err := ensureAlertFolder(config)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to ensure alert folder: %w", err)
|
|
}
|
|
|
|
var tasks []alertTask
|
|
|
|
for _, client := range clients {
|
|
if len(client.Domains) == 0 {
|
|
continue
|
|
}
|
|
|
|
// RPS алерт
|
|
if client.RPSLimit > 0 {
|
|
rule := buildRPSAlertRuleVL(client, tmpl, config)
|
|
rule["folderUID"] = folderUID
|
|
tasks = append(tasks, alertTask{
|
|
rule: rule,
|
|
label: fmt.Sprintf("VL RPS %s", client.ClientTitle),
|
|
})
|
|
}
|
|
|
|
// Error rate алерты (4xx и 5xx)
|
|
for _, errCode := range []int{4, 5} {
|
|
limitPct := client.Limit4xx
|
|
if errCode == 5 {
|
|
limitPct = client.Limit5xx
|
|
}
|
|
if limitPct <= 0 {
|
|
continue
|
|
}
|
|
rule := buildErrorRateAlertRuleVL(client, errCode, limitPct, tmpl, config)
|
|
rule["folderUID"] = folderUID
|
|
tasks = append(tasks, alertTask{
|
|
rule: rule,
|
|
label: fmt.Sprintf("VL %dxx %s", errCode, client.ClientTitle),
|
|
})
|
|
}
|
|
}
|
|
|
|
sent, failed := parallelUpsert(tasks, config, dryRun, 5)
|
|
log.Printf("VL Alerts: sent=%d failed=%d", sent, failed)
|
|
|
|
// Статические алерты (CPU, RAM, диск, контейнеры, Angie)
|
|
// Отправляем только если OS алерты отключены, иначе они уже отправлены там
|
|
if !config.AlertsOSEnabled {
|
|
if err := sendStaticAlerts(tmpl, config, dryRun); err != nil {
|
|
log.Printf("Warning: VL static alerts error: %v", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|