Added alert rules
This commit is contained in:
parent
5dfc842b5e
commit
dbe38ec816
7 changed files with 1036 additions and 51 deletions
746
alerts.go
Normal file
746
alerts.go
Normal file
|
|
@ -0,0 +1,746 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// AlertRulesTemplate — структура шаблона алертов из alert_rules_template.json
|
||||
type AlertRulesTemplate struct {
|
||||
Version string `json:"version"`
|
||||
Description string `json:"description"`
|
||||
Defaults AlertDefaults `json:"defaults"`
|
||||
RPSAlert map[string]interface{} `json:"rps_alert"`
|
||||
StaticAlerts []map[string]interface{} `json:"static_alerts"` // Алерты без привязки к БД
|
||||
}
|
||||
|
||||
type AlertDefaults struct {
|
||||
NoDataState string `json:"no_data_state"`
|
||||
ExecErrState string `json:"exec_err_state"`
|
||||
Receiver string `json:"receiver"`
|
||||
Folder string `json:"folder"`
|
||||
Group string `json:"group"`
|
||||
GroupInterval string `json:"group_interval"`
|
||||
}
|
||||
|
||||
// generateAlertUID генерирует детерминированный UID алерта из имени клиента и типа.
|
||||
// Одинаковый UID при повторном запуске = Grafana обновит существующий алерт, не создаст новый.
|
||||
func generateAlertUID(clientTitle, alertType string) string {
|
||||
hash := md5.Sum([]byte(clientTitle + ":" + alertType))
|
||||
return fmt.Sprintf("%x", hash[:8])
|
||||
}
|
||||
|
||||
// buildRPSAlertRule строит JSON правила алерта для одного клиента по шаблону
|
||||
func buildRPSAlertRule(client ClientData, tmpl AlertRulesTemplate, config Config) map[string]interface{} {
|
||||
datasourceUID := config.AlertsDatasourceUID
|
||||
|
||||
// SID-запрос: один SID или несколько через OR
|
||||
sids := make([]string, 0, len(client.Domains))
|
||||
for _, d := range client.Domains {
|
||||
sids = append(sids, d.SID)
|
||||
}
|
||||
// Убираем дубликаты и сортируем для детерминированности
|
||||
sidMap := make(map[string]bool)
|
||||
for _, s := range sids {
|
||||
sidMap[s] = true
|
||||
}
|
||||
uniqueSIDs := make([]string, 0, len(sidMap))
|
||||
for s := range sidMap {
|
||||
uniqueSIDs = append(uniqueSIDs, s)
|
||||
}
|
||||
sort.Strings(uniqueSIDs)
|
||||
|
||||
sidParts := make([]string, len(uniqueSIDs))
|
||||
for i, s := range uniqueSIDs {
|
||||
sidParts[i] = "SID:" + s
|
||||
}
|
||||
sidQuery := strings.Join(sidParts, " OR ")
|
||||
|
||||
// Получаем настройки из шаблона
|
||||
defaults := tmpl.Defaults
|
||||
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
||||
group := orDefault(config.AlertsGroup, defaults.Group)
|
||||
|
||||
timeRangeFrom := int64(1800)
|
||||
if v, ok := tmpl.RPSAlert["relative_time_range_from"].(float64); ok {
|
||||
timeRangeFrom = int64(v)
|
||||
}
|
||||
|
||||
summaryTemplate := "Превышен порог в {rps_limit} RPS в {client_title}"
|
||||
if v, ok := tmpl.RPSAlert["annotation_summary"].(string); ok {
|
||||
summaryTemplate = v
|
||||
}
|
||||
summary := strings.NewReplacer(
|
||||
"{rps_limit}", fmt.Sprintf("%d", client.RPSLimit),
|
||||
"{client_title}", client.ClientTitle,
|
||||
).Replace(summaryTemplate)
|
||||
|
||||
uid := generateAlertUID(client.ClientTitle, "rps")
|
||||
dataRefID := "A"
|
||||
rpsRefID := "RPS " + client.ClientTitle
|
||||
reduceRefID := "Текущий RPS"
|
||||
thresholdRefID := "Превышение"
|
||||
|
||||
rule := map[string]interface{}{
|
||||
"uid": uid,
|
||||
"title": fmt.Sprintf("RPS %s", client.ClientTitle),
|
||||
"condition": thresholdRefID,
|
||||
"noDataState": defaults.NoDataState,
|
||||
"execErrState": defaults.ExecErrState,
|
||||
"isPaused": false,
|
||||
"folderUID": "", // будет заполнен при отправке
|
||||
"ruleGroup": group,
|
||||
"annotations": map[string]string{
|
||||
"summary": summary,
|
||||
},
|
||||
"notification_settings": map[string]string{
|
||||
"receiver": receiver,
|
||||
},
|
||||
"data": []interface{}{
|
||||
// Шаг 1: count-запрос по SID
|
||||
map[string]interface{}{
|
||||
"refId": dataRefID,
|
||||
"queryType": "lucene",
|
||||
"relativeTimeRange": map[string]interface{}{
|
||||
"from": timeRangeFrom,
|
||||
"to": 0,
|
||||
},
|
||||
"datasourceUid": datasourceUID,
|
||||
"model": map[string]interface{}{
|
||||
"alias": fmt.Sprintf("RPS %s", client.ClientTitle),
|
||||
"bucketAggs": []interface{}{
|
||||
map[string]interface{}{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": map[string]interface{}{
|
||||
"interval": "1m",
|
||||
"min_doc_count": "0",
|
||||
"trimEdges": "0",
|
||||
},
|
||||
"type": "date_histogram",
|
||||
},
|
||||
},
|
||||
"datasource": map[string]interface{}{
|
||||
"type": "grafana-opensearch-datasource",
|
||||
"uid": datasourceUID,
|
||||
},
|
||||
"format": "table",
|
||||
"instant": false,
|
||||
"intervalMs": 1000,
|
||||
"luceneQueryType": "Metric",
|
||||
"maxDataPoints": 43200,
|
||||
"metrics": []interface{}{
|
||||
map[string]interface{}{"hide": false, "id": "1", "type": "count"},
|
||||
},
|
||||
"query": sidQuery,
|
||||
"queryType": "lucene",
|
||||
"range": true,
|
||||
"refId": dataRefID,
|
||||
"timeField": "@timestamp",
|
||||
},
|
||||
},
|
||||
// Шаг 2: math $A / 60 → RPS
|
||||
map[string]interface{}{
|
||||
"refId": rpsRefID,
|
||||
"relativeTimeRange": map[string]interface{}{
|
||||
"from": timeRangeFrom,
|
||||
"to": 0,
|
||||
},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"datasource": map[string]interface{}{
|
||||
"name": "Expression",
|
||||
"type": "__expr__",
|
||||
"uid": "__expr__",
|
||||
},
|
||||
"expression": fmt.Sprintf("$%s / 60", dataRefID),
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"refId": rpsRefID,
|
||||
"type": "math",
|
||||
"window": "",
|
||||
},
|
||||
},
|
||||
// Шаг 3: reduce median
|
||||
map[string]interface{}{
|
||||
"refId": reduceRefID,
|
||||
"relativeTimeRange": map[string]interface{}{
|
||||
"from": 0,
|
||||
"to": 0,
|
||||
},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"conditions": []interface{}{
|
||||
map[string]interface{}{
|
||||
"evaluator": map[string]interface{}{"params": []int{0, 0}, "type": "gt"},
|
||||
"operator": map[string]interface{}{"type": "and"},
|
||||
"query": map[string]interface{}{"params": []string{}},
|
||||
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
||||
"type": "query",
|
||||
},
|
||||
},
|
||||
"datasource": map[string]interface{}{
|
||||
"name": "Expression",
|
||||
"type": "__expr__",
|
||||
"uid": "__expr__",
|
||||
},
|
||||
"expression": rpsRefID,
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"reducer": "median",
|
||||
"refId": reduceRefID,
|
||||
"settings": map[string]interface{}{"mode": ""},
|
||||
"type": "reduce",
|
||||
},
|
||||
},
|
||||
// Шаг 4: threshold > rps_limit
|
||||
map[string]interface{}{
|
||||
"refId": thresholdRefID,
|
||||
"relativeTimeRange": map[string]interface{}{
|
||||
"from": 0,
|
||||
"to": 0,
|
||||
},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"conditions": []interface{}{
|
||||
map[string]interface{}{
|
||||
"evaluator": map[string]interface{}{
|
||||
"params": []interface{}{client.RPSLimit, 0},
|
||||
"type": "gt",
|
||||
},
|
||||
"operator": map[string]interface{}{"type": "and"},
|
||||
"query": map[string]interface{}{"params": []string{}},
|
||||
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
||||
"type": "query",
|
||||
},
|
||||
},
|
||||
"datasource": map[string]interface{}{
|
||||
"name": "Expression",
|
||||
"type": "__expr__",
|
||||
"uid": "__expr__",
|
||||
},
|
||||
"expression": reduceRefID,
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"refId": thresholdRefID,
|
||||
"type": "threshold",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return rule
|
||||
}
|
||||
|
||||
// alertTask описывает одно правило алерта для отправки
|
||||
type alertTask struct {
|
||||
rule map[string]interface{}
|
||||
label string // для логирования
|
||||
}
|
||||
|
||||
// parallelUpsert отправляет список алертов параллельно с ограничением concurrency.
|
||||
// dryRun=true только логирует без отправки.
|
||||
// Возвращает количество успешных и неуспешных отправок.
|
||||
func parallelUpsert(tasks []alertTask, config Config, dryRun bool, concurrency int) (sent, failed int) {
|
||||
sem := make(chan struct{}, concurrency)
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, task := range tasks {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{} // захватить слот
|
||||
go func(t alertTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }() // освободить слот
|
||||
|
||||
if dryRun {
|
||||
data, _ := json.MarshalIndent(t.rule, "", " ")
|
||||
log.Printf("DRY RUN: Alert rule '%s':\n%s", t.label, string(data))
|
||||
mu.Lock()
|
||||
sent++
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
if err := upsertAlertRule(t.rule, config); err != nil {
|
||||
log.Printf("Warning: failed to upsert alert '%s': %v", t.label, err)
|
||||
mu.Lock()
|
||||
failed++
|
||||
mu.Unlock()
|
||||
} else {
|
||||
log.Printf("Alert upserted: %s", t.label)
|
||||
mu.Lock()
|
||||
sent++
|
||||
mu.Unlock()
|
||||
}
|
||||
}(task)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return
|
||||
}
|
||||
|
||||
// generateAndSendAlerts генерирует и отправляет RPS-алерты для клиентов с rps_limit > 0
|
||||
func generateAndSendAlerts(clients map[string]ClientData, tmpl AlertRulesTemplate, config Config, dryRun bool) error {
|
||||
// Собираем клиентов с лимитом, сортируем для детерминированности
|
||||
type clientWithLimit struct {
|
||||
title string
|
||||
client ClientData
|
||||
}
|
||||
var targets []clientWithLimit
|
||||
for _, client := range clients {
|
||||
if client.RPSLimit > 0 {
|
||||
targets = append(targets, clientWithLimit{client.ClientTitle, client})
|
||||
}
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
log.Printf("Alerts: no clients with rps_limit set, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
sort.Slice(targets, func(i, j int) bool {
|
||||
return targets[i].title < targets[j].title
|
||||
})
|
||||
|
||||
log.Printf("Alerts: generating RPS rules for %d clients", len(targets))
|
||||
|
||||
// Получить UID папки для алертов
|
||||
folderUID := ""
|
||||
if !dryRun {
|
||||
uid, err := ensureAlertFolder(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to ensure alerts folder: %w", err)
|
||||
}
|
||||
folderUID = uid
|
||||
}
|
||||
|
||||
// Собираем все динамические задачи: RPS + 4xx + 5xx
|
||||
var tasks []alertTask
|
||||
for _, t := range targets {
|
||||
if t.client.RPSLimit > 0 {
|
||||
rule := buildRPSAlertRule(t.client, tmpl, config)
|
||||
rule["folderUID"] = folderUID
|
||||
tasks = append(tasks, alertTask{
|
||||
rule: rule,
|
||||
label: fmt.Sprintf("RPS %s (limit: %d)", t.client.ClientTitle, t.client.RPSLimit),
|
||||
})
|
||||
}
|
||||
if t.client.Limit4xx > 0 {
|
||||
rule := buildErrorRateAlertRule(t.client, 4, t.client.Limit4xx, tmpl, config)
|
||||
rule["folderUID"] = folderUID
|
||||
tasks = append(tasks, alertTask{
|
||||
rule: rule,
|
||||
label: fmt.Sprintf("4xx %s (limit: %d%%)", t.client.ClientTitle, t.client.Limit4xx),
|
||||
})
|
||||
}
|
||||
if t.client.Limit5xx > 0 {
|
||||
rule := buildErrorRateAlertRule(t.client, 5, t.client.Limit5xx, tmpl, config)
|
||||
rule["folderUID"] = folderUID
|
||||
tasks = append(tasks, alertTask{
|
||||
rule: rule,
|
||||
label: fmt.Sprintf("5xx %s (limit: %d%%)", t.client.ClientTitle, t.client.Limit5xx),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Alerts: sending %d dynamic rules (concurrency: 5)", len(tasks))
|
||||
sent, failed := parallelUpsert(tasks, config, dryRun, 5)
|
||||
log.Printf("Alerts: sent=%d, failed=%d", sent, failed)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d alert rules failed to upsert", failed)
|
||||
}
|
||||
|
||||
// Статические алерты — отправляются всегда вместе с динамическими
|
||||
if err := sendStaticAlerts(tmpl, config, dryRun); err != nil {
|
||||
log.Printf("Warning: static alerts error: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildErrorRateAlertRule строит алерт на процент 4xx или 5xx ответов для клиента.
|
||||
// Структура: два count-запроса (total + errors) → reduce → math (%) → threshold
|
||||
func buildErrorRateAlertRule(client ClientData, errCode, limitPct int, tmpl AlertRulesTemplate, config Config) map[string]interface{} {
|
||||
datasourceUID := config.AlertsDatasourceUID
|
||||
defaults := tmpl.Defaults
|
||||
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
||||
group := orDefault(config.AlertsGroup, defaults.Group)
|
||||
|
||||
// SID-запрос
|
||||
sids := make(map[string]bool)
|
||||
for _, d := range client.Domains {
|
||||
sids[d.SID] = true
|
||||
}
|
||||
unique := make([]string, 0, len(sids))
|
||||
for s := range sids {
|
||||
unique = append(unique, s)
|
||||
}
|
||||
sort.Strings(unique)
|
||||
parts := make([]string, len(unique))
|
||||
for i, s := range unique {
|
||||
parts[i] = "SID:" + s
|
||||
}
|
||||
sidQuery := strings.Join(parts, " OR ")
|
||||
|
||||
codeRange := "[400 TO 499]"
|
||||
codeName := "4xx"
|
||||
if errCode == 5 {
|
||||
codeRange = "[500 TO 599]"
|
||||
codeName = "5xx"
|
||||
}
|
||||
|
||||
uid := generateAlertUID(client.ClientTitle, codeName)
|
||||
totalRefID := "Total"
|
||||
errRefID := "Errors"
|
||||
totalReduceID := "TotalReduce"
|
||||
errReduceID := "ErrorsReduce"
|
||||
percentRefID := "ErrorPercent"
|
||||
thresholdRefID := "Превышение"
|
||||
|
||||
summary := fmt.Sprintf("🚨 %s — %s ошибок >%d%% трафика", client.ClientTitle, codeName, limitPct)
|
||||
|
||||
relFrom := int64(900)
|
||||
|
||||
return map[string]interface{}{
|
||||
"uid": uid,
|
||||
"title": fmt.Sprintf("%s %s", codeName, client.ClientTitle),
|
||||
"condition": thresholdRefID,
|
||||
"noDataState": defaults.NoDataState,
|
||||
"execErrState": defaults.ExecErrState,
|
||||
"isPaused": false,
|
||||
"folderUID": "",
|
||||
"ruleGroup": group,
|
||||
"for": "1m",
|
||||
"annotations": map[string]string{"summary": summary},
|
||||
"notification_settings": map[string]string{"receiver": receiver},
|
||||
"data": []interface{}{
|
||||
// Total запрос
|
||||
map[string]interface{}{
|
||||
"refId": totalRefID,
|
||||
"queryType": "lucene",
|
||||
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
||||
"datasourceUid": datasourceUID,
|
||||
"model": map[string]interface{}{
|
||||
"alias": "Total",
|
||||
"bucketAggs": []interface{}{map[string]interface{}{"field": "@timestamp", "id": "2", "settings": map[string]interface{}{"interval": "1m"}, "type": "date_histogram"}},
|
||||
"datasource": map[string]interface{}{"type": "grafana-opensearch-datasource", "uid": datasourceUID},
|
||||
"format": "table",
|
||||
"instant": false,
|
||||
"intervalMs": 1000,
|
||||
"luceneQueryType": "Metric",
|
||||
"maxDataPoints": 43200,
|
||||
"metrics": []interface{}{map[string]interface{}{"id": "1", "type": "count"}},
|
||||
"query": fmt.Sprintf("response_status_code:[0 TO 599] AND (%s)", sidQuery),
|
||||
"queryType": "lucene",
|
||||
"range": true,
|
||||
"refId": totalRefID,
|
||||
"timeField": "@timestamp",
|
||||
},
|
||||
},
|
||||
// Error запрос
|
||||
map[string]interface{}{
|
||||
"refId": errRefID,
|
||||
"queryType": "lucene",
|
||||
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
||||
"datasourceUid": datasourceUID,
|
||||
"model": map[string]interface{}{
|
||||
"alias": codeName,
|
||||
"bucketAggs": []interface{}{map[string]interface{}{"field": "@timestamp", "id": "2", "settings": map[string]interface{}{"interval": "1m"}, "type": "date_histogram"}},
|
||||
"datasource": map[string]interface{}{"type": "grafana-opensearch-datasource", "uid": datasourceUID},
|
||||
"format": "table",
|
||||
"instant": false,
|
||||
"intervalMs": 1000,
|
||||
"luceneQueryType": "Metric",
|
||||
"maxDataPoints": 43200,
|
||||
"metrics": []interface{}{map[string]interface{}{"id": "1", "type": "count"}},
|
||||
"query": fmt.Sprintf("response_status_code:%s AND (%s)", codeRange, sidQuery),
|
||||
"queryType": "lucene",
|
||||
"range": true,
|
||||
"refId": errRefID,
|
||||
"timeField": "@timestamp",
|
||||
},
|
||||
},
|
||||
// Reduce total
|
||||
map[string]interface{}{
|
||||
"refId": totalReduceID,
|
||||
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
||||
"expression": totalRefID,
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"reducer": "last",
|
||||
"refId": totalReduceID,
|
||||
"type": "reduce",
|
||||
},
|
||||
},
|
||||
// Reduce errors
|
||||
map[string]interface{}{
|
||||
"refId": errReduceID,
|
||||
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
||||
"expression": errRefID,
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"reducer": "last",
|
||||
"refId": errReduceID,
|
||||
"type": "reduce",
|
||||
},
|
||||
},
|
||||
// Math: процент ошибок
|
||||
map[string]interface{}{
|
||||
"refId": percentRefID,
|
||||
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
||||
"expression": fmt.Sprintf("($%s / $%s) * 100", errReduceID, totalReduceID),
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"refId": percentRefID,
|
||||
"type": "math",
|
||||
},
|
||||
},
|
||||
// Threshold
|
||||
map[string]interface{}{
|
||||
"refId": thresholdRefID,
|
||||
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
||||
"datasourceUid": "__expr__",
|
||||
"model": map[string]interface{}{
|
||||
"conditions": []interface{}{
|
||||
map[string]interface{}{
|
||||
"evaluator": map[string]interface{}{"params": []interface{}{limitPct, 0}, "type": "gt"},
|
||||
"operator": map[string]interface{}{"type": "and"},
|
||||
"query": map[string]interface{}{"params": []string{}},
|
||||
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
||||
"type": "query",
|
||||
},
|
||||
},
|
||||
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
||||
"expression": percentRefID,
|
||||
"intervalMs": 1000,
|
||||
"maxDataPoints": 43200,
|
||||
"refId": thresholdRefID,
|
||||
"type": "threshold",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// sendStaticAlerts отправляет статические алерты из шаблона (не зависят от БД)
|
||||
func sendStaticAlerts(tmpl AlertRulesTemplate, config Config, dryRun bool) error {
|
||||
if len(tmpl.StaticAlerts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
defaults := tmpl.Defaults
|
||||
folderUID := ""
|
||||
|
||||
if !dryRun {
|
||||
uid, err := ensureAlertFolder(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to ensure alerts folder for static alerts: %w", err)
|
||||
}
|
||||
folderUID = uid
|
||||
}
|
||||
|
||||
group := orDefault(config.AlertsGroup, defaults.Group)
|
||||
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
||||
|
||||
log.Printf("Alerts: sending %d static alert rules (concurrency: 5)", len(tmpl.StaticAlerts))
|
||||
|
||||
var tasks []alertTask
|
||||
for _, alertDef := range tmpl.StaticAlerts {
|
||||
rule := deepCopyMap(alertDef)
|
||||
|
||||
// Применяем defaults если не заданы в шаблоне
|
||||
if _, ok := rule["noDataState"]; !ok {
|
||||
rule["noDataState"] = defaults.NoDataState
|
||||
}
|
||||
if _, ok := rule["execErrState"]; !ok {
|
||||
rule["execErrState"] = defaults.ExecErrState
|
||||
}
|
||||
rule["folderUID"] = folderUID
|
||||
rule["ruleGroup"] = group
|
||||
if _, ok := rule["notification_settings"]; !ok {
|
||||
rule["notification_settings"] = map[string]string{"receiver": receiver}
|
||||
}
|
||||
|
||||
title, _ := rule["title"].(string)
|
||||
tasks = append(tasks, alertTask{rule: rule, label: title})
|
||||
}
|
||||
|
||||
sent, failed := parallelUpsert(tasks, config, dryRun, 5)
|
||||
log.Printf("Static alerts: sent=%d, failed=%d", sent, failed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// deepCopyMap делает глубокую копию map через JSON
|
||||
func deepCopyMap(src map[string]interface{}) map[string]interface{} {
|
||||
data, _ := json.Marshal(src)
|
||||
var dst map[string]interface{}
|
||||
json.Unmarshal(data, &dst)
|
||||
return dst
|
||||
}
|
||||
|
||||
// upsertAlertRule создаёт или обновляет правило алерта по UID
|
||||
func upsertAlertRule(rule map[string]interface{}, config Config) error {
|
||||
uid, _ := rule["uid"].(string)
|
||||
baseURL := strings.TrimRight(config.GrafanaURL, "/")
|
||||
|
||||
// Проверяем существует ли алерт с таким UID
|
||||
existing, err := getAlertRule(uid, config)
|
||||
if err == nil && existing != nil {
|
||||
// Обновляем — PUT /api/v1/provisioning/alert-rules/{uid}
|
||||
// Сохраняем version из существующего для оптимистичной блокировки
|
||||
if v, ok := existing["version"]; ok {
|
||||
rule["version"] = v
|
||||
}
|
||||
return putAlertRule(uid, rule, baseURL, config.GrafanaAPIKey)
|
||||
}
|
||||
|
||||
// Создаём — POST /api/v1/provisioning/alert-rules
|
||||
return postAlertRule(rule, baseURL, config.GrafanaAPIKey)
|
||||
}
|
||||
|
||||
func getAlertRule(uid string, config Config) (map[string]interface{}, error) {
|
||||
url := fmt.Sprintf("%s/api/v1/provisioning/alert-rules/%s",
|
||||
strings.TrimRight(config.GrafanaURL, "/"), uid)
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+config.GrafanaAPIKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == 404 {
|
||||
return nil, fmt.Errorf("not found")
|
||||
}
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func postAlertRule(rule map[string]interface{}, baseURL, apiKey string) error {
|
||||
return sendAlertRequest("POST",
|
||||
baseURL+"/api/v1/provisioning/alert-rules",
|
||||
rule, apiKey)
|
||||
}
|
||||
|
||||
func putAlertRule(uid string, rule map[string]interface{}, baseURL, apiKey string) error {
|
||||
return sendAlertRequest("PUT",
|
||||
fmt.Sprintf("%s/api/v1/provisioning/alert-rules/%s", baseURL, uid),
|
||||
rule, apiKey)
|
||||
}
|
||||
|
||||
func sendAlertRequest(method, url string, payload map[string]interface{}, apiKey string) error {
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal error: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(method, url, bytes.NewBuffer(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Disable-Provenance", "true") // разрешить редактирование в UI
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureAlertFolder возвращает UID папки для алертов, создаёт если нет
|
||||
func ensureAlertFolder(config Config) (string, error) {
|
||||
folder := orDefault(config.AlertsFolder, DefaultAlertsFolder)
|
||||
if folder == "" {
|
||||
return "", nil
|
||||
}
|
||||
baseURL := strings.TrimRight(config.GrafanaURL, "/")
|
||||
|
||||
// Ищем существующую папку
|
||||
req, _ := http.NewRequest("GET", baseURL+"/api/folders", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+config.GrafanaAPIKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var folders []map[string]interface{}
|
||||
if err := json.Unmarshal(body, &folders); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, f := range folders {
|
||||
if title, _ := f["title"].(string); title == folder {
|
||||
if uid, _ := f["uid"].(string); uid != "" {
|
||||
return uid, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Создаём папку
|
||||
payload := map[string]interface{}{"title": folder}
|
||||
data, _ := json.Marshal(payload)
|
||||
req, _ = http.NewRequest("POST", baseURL+"/api/folders", bytes.NewBuffer(data))
|
||||
req.Header.Set("Authorization", "Bearer "+config.GrafanaAPIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ = io.ReadAll(resp.Body)
|
||||
var created map[string]interface{}
|
||||
if err := json.Unmarshal(body, &created); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if uid, _ := created["uid"].(string); uid != "" {
|
||||
log.Printf("Created alerts folder: %s (uid: %s)", folder, uid)
|
||||
return uid, nil
|
||||
}
|
||||
return "", fmt.Errorf("failed to get folder UID")
|
||||
}
|
||||
|
||||
// orDefault возвращает val если не пустой, иначе def
|
||||
func orDefault(val, def string) string {
|
||||
if val != "" {
|
||||
return val
|
||||
}
|
||||
return def
|
||||
}
|
||||
64
config.go
64
config.go
|
|
@ -5,13 +5,13 @@ const (
|
|||
// Рекомендуется задавать через .env файл или переменные окружения
|
||||
DefaultDBUser = "" // Задать через DB_USER в .env
|
||||
DefaultDBPassword = "" // Задать через DB_PASSWORD в .env
|
||||
DefaultDBName = "test_info"
|
||||
DefaultDBName = "waf_info"
|
||||
DefaultDBPort = 5432
|
||||
DefaultPrimaryDBHost = "10.10.10.8"
|
||||
DefaultSecondaryDBHost = "10.10.10.5"
|
||||
DefaultPrimaryDBHost = "10.100.10.8"
|
||||
DefaultSecondaryDBHost = "10.100.13.5"
|
||||
|
||||
// Git repository settings
|
||||
DefaultTemplatesRepo = ""
|
||||
DefaultTemplatesRepo = "https://svc-git.cirex.ru/cloudstack/grafana_gen_template.git"
|
||||
DefaultTemplatesBranch = "master"
|
||||
DefaultTemplatesPath = "/etc/grafana_gen/templates" // Системный путь
|
||||
DefaultStateFile = "/var/lib/grafana_gen/state.json" // Системный путь для state
|
||||
|
|
@ -59,6 +59,12 @@ const (
|
|||
// Можно переопределить под ваши нужды:
|
||||
// DefaultGrafanaFolder = "WAF - PTAF"
|
||||
|
||||
// Alerts settings
|
||||
DefaultAlertsFolder = "WAF - PTAF"
|
||||
DefaultAlertsReceiver = "Telegram PTAF Grafana"
|
||||
DefaultAlertsGroup = "PTAF Grafana"
|
||||
DefaultAlertsDatasource = "af84zsvlp9blsa" // UID datasource OpenSearch
|
||||
|
||||
// DefaultDashboardTitle - название единого дашборда со всеми клиентами
|
||||
// Пример результата: Home -> Dashboards -> WAF - PTAF -> PT AF Nodes
|
||||
DefaultDashboardTitle = "PT AF Nodes"
|
||||
|
|
@ -70,29 +76,36 @@ const (
|
|||
)
|
||||
|
||||
type Config struct {
|
||||
DBHost string
|
||||
DBPort int
|
||||
DBUser string
|
||||
DBPassword string
|
||||
DBName string
|
||||
GrafanaURL string
|
||||
GrafanaAPIKey string
|
||||
GrafanaFolder string // Папка в Grafana для дашбордов
|
||||
DashboardTitle string // Название единого дашборда
|
||||
UseManualInfo bool
|
||||
DryRun bool
|
||||
TemplatesRepo string
|
||||
TemplatesBranch string
|
||||
TemplatesPath string
|
||||
SkipGitPull bool
|
||||
StateFile string
|
||||
ForceRun bool
|
||||
GitToken string // GitLab token для приватного репозитория
|
||||
DBHost string
|
||||
DBPort int
|
||||
DBUser string
|
||||
DBPassword string
|
||||
DBName string
|
||||
GrafanaURL string
|
||||
GrafanaAPIKey string
|
||||
GrafanaFolder string // Папка в Grafana для дашбордов
|
||||
DashboardTitle string // Название единого дашборда
|
||||
DryRun bool
|
||||
TemplatesRepo string
|
||||
TemplatesBranch string
|
||||
TemplatesPath string
|
||||
SkipGitPull bool
|
||||
StateFile string
|
||||
ForceRun bool
|
||||
GitToken string // GitLab token для приватного репозитория
|
||||
AlertsFolder string // Папка в Grafana для алертов
|
||||
AlertsReceiver string // Receiver (канал уведомлений) для алертов
|
||||
AlertsGroup string // Имя группы алертов в Grafana
|
||||
AlertsDatasourceUID string // UID datasource для алертов OpenSearch
|
||||
}
|
||||
|
||||
type ClientData struct {
|
||||
ClientTitle string
|
||||
Domains []DomainInfo
|
||||
ClientTitle string
|
||||
Domains []DomainInfo
|
||||
RPSLimit int // 0 = алерт не создавать (NULL или не задан в БД)
|
||||
RPSCommercialLimit int // коммерческий лимит, при NULL берётся 100
|
||||
Limit4xx int // порог 4xx ответов в % (0 = алерт не создавать)
|
||||
Limit5xx int // порог 5xx ответов в % (0 = алерт не создавать)
|
||||
}
|
||||
|
||||
type DomainInfo struct {
|
||||
|
|
@ -113,12 +126,15 @@ type DashboardTemplate struct {
|
|||
QueryTemplates map[string]interface{} `json:"query_templates"`
|
||||
Panels map[string]interface{} `json:"panels"`
|
||||
Queries map[string]interface{} `json:"queries"`
|
||||
Templating map[string]interface{} `json:"templating"` // Переменные дашборда Grafana
|
||||
}
|
||||
|
||||
type LayoutConfig struct {
|
||||
TenantPanels []PanelLayout `json:"tenant_panels"`
|
||||
DomainPanels []PanelLayout `json:"domain_panels"`
|
||||
UseCollapsedRows bool `json:"use_collapsed_rows"`
|
||||
OverviewPanels []PanelLayout `json:"overview_panels"` // Панели обзора перед клиентскими строками
|
||||
OverviewRowTitle string `json:"overview_row_title"` // Заголовок collapsed row обзора
|
||||
}
|
||||
|
||||
type PanelLayout struct {
|
||||
|
|
|
|||
49
database.go
49
database.go
|
|
@ -26,24 +26,29 @@ func connectDB(config Config) (*sql.DB, error) {
|
|||
return db, nil
|
||||
}
|
||||
|
||||
func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error) {
|
||||
tableName := "sp_info"
|
||||
if useManual {
|
||||
tableName = "manual_info"
|
||||
}
|
||||
|
||||
func fetchClientsData(db *sql.DB) (map[string]ClientData, error) {
|
||||
// aliases это JSON массив в БД, конвертируем в текст через ::text
|
||||
query := fmt.Sprintf(`
|
||||
SELECT DISTINCT
|
||||
// rps_limit берётся из client_info — NULL означает "алерт не нужен"
|
||||
// Данные всегда берутся из обеих таблиц: sp_info + manual_info (UNION)
|
||||
query := `
|
||||
SELECT DISTINCT
|
||||
COALESCE(a.client_title, 'Unknown') as client_title,
|
||||
s.sid,
|
||||
s.domain_name,
|
||||
COALESCE(s.aliases::text, 'null') as aliases
|
||||
FROM %s s
|
||||
COALESCE(s.aliases::text, 'null') as aliases,
|
||||
COALESCE(MAX(ci.rps_limit) OVER (PARTITION BY a.client_title), 0) as rps_limit,
|
||||
COALESCE(MAX(ci.rps_commercial_limit) OVER (PARTITION BY a.client_title), 100) as rps_commercial_limit,
|
||||
MAX(ci.four_hundred) OVER (PARTITION BY a.client_title) as limit_4xx,
|
||||
MAX(ci.five_hundred) OVER (PARTITION BY a.client_title) as limit_5xx
|
||||
FROM (
|
||||
SELECT sid, domain_name, aliases FROM sp_info WHERE sid IS NOT NULL
|
||||
UNION
|
||||
SELECT sid, domain_name, aliases FROM manual_info WHERE sid IS NOT NULL
|
||||
) s
|
||||
LEFT JOIN apps_settings a ON s.sid = a.l7resourceid
|
||||
WHERE s.sid IS NOT NULL
|
||||
LEFT JOIN client_info ci ON a.client_title = ci.client_title
|
||||
ORDER BY COALESCE(a.client_title, 'Unknown'), s.sid
|
||||
`, tableName)
|
||||
`
|
||||
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
|
|
@ -55,8 +60,10 @@ func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error)
|
|||
|
||||
for rows.Next() {
|
||||
var clientTitle, sid, domainName, aliasesJSON string
|
||||
var rpsLimit, rpsCommercialLimit int
|
||||
var limit4xx, limit5xx sql.NullInt64
|
||||
|
||||
if err := rows.Scan(&clientTitle, &sid, &domainName, &aliasesJSON); err != nil {
|
||||
if err := rows.Scan(&clientTitle, &sid, &domainName, &aliasesJSON, &rpsLimit, &rpsCommercialLimit, &limit4xx, &limit5xx); err != nil {
|
||||
return nil, fmt.Errorf("scan failed: %w", err)
|
||||
}
|
||||
|
||||
|
|
@ -65,8 +72,12 @@ func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error)
|
|||
|
||||
if _, exists := clients[clientTitle]; !exists {
|
||||
clients[clientTitle] = ClientData{
|
||||
ClientTitle: clientTitle,
|
||||
Domains: []DomainInfo{},
|
||||
ClientTitle: clientTitle,
|
||||
Domains: []DomainInfo{},
|
||||
RPSLimit: rpsLimit,
|
||||
RPSCommercialLimit: rpsCommercialLimit,
|
||||
Limit4xx: nullIntToInt(limit4xx),
|
||||
Limit5xx: nullIntToInt(limit5xx),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -82,6 +93,14 @@ func fetchClientsData(db *sql.DB, useManual bool) (map[string]ClientData, error)
|
|||
return clients, rows.Err()
|
||||
}
|
||||
|
||||
// nullIntToInt конвертирует sql.NullInt64 в int (0 если NULL)
|
||||
func nullIntToInt(n sql.NullInt64) int {
|
||||
if n.Valid {
|
||||
return int(n.Int64)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseAliasesJSON конвертирует JSON массив aliases в строку
|
||||
// Input: ["domain1.com", "domain2.com"] или null
|
||||
// Output: "domain1.com, domain2.com" или ""
|
||||
|
|
|
|||
47
main.go
47
main.go
|
|
@ -48,6 +48,13 @@ func main() {
|
|||
|
||||
log.Printf("Loaded template version: %s", template.Version)
|
||||
|
||||
// Load alert rules template
|
||||
alertTmpl, err := loadAlertTemplate(config.TemplatesPath)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to load alert template: %v", err)
|
||||
log.Printf("Alert rules will not be generated")
|
||||
}
|
||||
|
||||
// Connect to database
|
||||
db, err := connectDB(config)
|
||||
if err != nil {
|
||||
|
|
@ -56,7 +63,7 @@ func main() {
|
|||
defer db.Close()
|
||||
|
||||
// Fetch clients data
|
||||
clients, err := fetchClientsData(db, config.UseManualInfo)
|
||||
clients, err := fetchClientsData(db)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to fetch clients data: %v", err)
|
||||
}
|
||||
|
|
@ -87,10 +94,14 @@ func main() {
|
|||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Хэш шаблона алертов — считается независимо от дашборда
|
||||
alertTemplateHash := hashAlertTemplate(config.TemplatesPath)
|
||||
alertsChanged := stateManager.AlertsNeedUpdate(validClients) || stateManager.AlertTemplateChanged(alertTemplateHash)
|
||||
|
||||
// Check for changes
|
||||
hasChanges, changeReason := stateManager.CheckChanges(templateCommit, template.Version, clients)
|
||||
|
||||
if !hasChanges && !config.ForceRun {
|
||||
if !hasChanges && !alertsChanged && !config.ForceRun {
|
||||
log.Printf("=== No Changes Detected ===")
|
||||
log.Printf("No changes in templates or database since last run.")
|
||||
log.Printf("Skipping dashboard generation.")
|
||||
|
|
@ -103,6 +114,9 @@ func main() {
|
|||
log.Printf("=== Changes Detected ===")
|
||||
log.Printf("%s", changeReason)
|
||||
log.Printf("")
|
||||
} else if alertsChanged {
|
||||
log.Printf("=== Alert Rules Changed ===")
|
||||
log.Printf("")
|
||||
}
|
||||
|
||||
if config.ForceRun {
|
||||
|
|
@ -117,6 +131,10 @@ func main() {
|
|||
|
||||
if config.DryRun {
|
||||
logDryRun(config.DashboardTitle, dashboard)
|
||||
if alertTmpl != nil {
|
||||
log.Printf("DRY RUN: Generating alert rules preview...")
|
||||
_ = generateAndSendAlerts(validClients, *alertTmpl, config, true)
|
||||
}
|
||||
log.Printf("=== Summary ===")
|
||||
log.Printf("Clients: %d", len(validClients))
|
||||
log.Printf("Skipped: %d", skippedCount)
|
||||
|
|
@ -136,6 +154,20 @@ func main() {
|
|||
|
||||
log.Printf("Successfully created/updated dashboard: %s", config.DashboardTitle)
|
||||
|
||||
// Generate and send alert rules
|
||||
if alertTmpl != nil {
|
||||
if alertsChanged || config.ForceRun {
|
||||
if err := generateAndSendAlerts(validClients, *alertTmpl, config, false); err != nil {
|
||||
log.Printf("Warning: alerts generation failed: %v", err)
|
||||
} else {
|
||||
stateManager.UpdateAlertsHash(validClients)
|
||||
stateManager.UpdateAlertTemplateHash(alertTemplateHash)
|
||||
}
|
||||
} else {
|
||||
log.Printf("Alerts: no changes in rps_limit, skipping")
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
log.Printf("=== Summary ===")
|
||||
log.Printf("Dashboard: %s", config.DashboardTitle)
|
||||
|
|
@ -179,7 +211,6 @@ func parseFlags() Config {
|
|||
flag.StringVar(&config.DBUser, "db-user", dbUser, "PostgreSQL user")
|
||||
flag.StringVar(&config.DBPassword, "db-password", dbPassword, "PostgreSQL password")
|
||||
flag.StringVar(&config.DBName, "db-name", DefaultDBName, "PostgreSQL database name")
|
||||
flag.BoolVar(&config.UseManualInfo, "use-manual", false, "Use manual_info table instead of sp_info")
|
||||
|
||||
// Grafana flags
|
||||
var grafanaURLFlag, grafanaAPIKeyFlag string
|
||||
|
|
@ -196,6 +227,16 @@ func parseFlags() Config {
|
|||
flag.StringVar(&config.TemplatesPath, "templates-path", DefaultTemplatesPath, "Local path for templates")
|
||||
flag.BoolVar(&config.SkipGitPull, "skip-git-pull", false, "Skip updating templates from Git")
|
||||
|
||||
// Alerts flags
|
||||
alertsFolder := getEnvOrDefault("ALERTS_FOLDER", DefaultAlertsFolder)
|
||||
alertsReceiver := getEnvOrDefault("ALERTS_RECEIVER", DefaultAlertsReceiver)
|
||||
alertsGroup := getEnvOrDefault("ALERTS_GROUP", DefaultAlertsGroup)
|
||||
alertsDatasource := getEnvOrDefault("ALERTS_DATASOURCE_UID", DefaultAlertsDatasource)
|
||||
flag.StringVar(&config.AlertsFolder, "alerts-folder", alertsFolder, "Grafana folder for alert rules")
|
||||
flag.StringVar(&config.AlertsReceiver, "alerts-receiver", alertsReceiver, "Grafana notification receiver for alerts")
|
||||
flag.StringVar(&config.AlertsGroup, "alerts-group", alertsGroup, "Grafana alert rule group name")
|
||||
flag.StringVar(&config.AlertsDatasourceUID, "alerts-datasource-uid", alertsDatasource, "OpenSearch datasource UID for alerts")
|
||||
|
||||
// Other flags
|
||||
flag.BoolVar(&config.DryRun, "dry-run", false, "Generate JSON but don't send to Grafana")
|
||||
flag.StringVar(&config.StateFile, "state-file", DefaultStateFile, "Path to state file")
|
||||
|
|
|
|||
74
panels.go
74
panels.go
|
|
@ -19,6 +19,7 @@ var queryModeRegistry = map[string]targetBuilder{
|
|||
"multi_sum_expression": buildMultiSumExpressionTargets,
|
||||
"sum_expression": buildSumExpressionTargets,
|
||||
"bucket_logs": buildBucketLogsTargets,
|
||||
"static": buildStaticTargets,
|
||||
}
|
||||
|
||||
// conditionEvaluator — тип функции-вычислителя условия отображения панели.
|
||||
|
|
@ -73,6 +74,7 @@ func buildPanel(
|
|||
|
||||
queryConfigRaw, _ := panelMap["query_config"].(map[string]interface{})
|
||||
panel := copyPanelBase(panelMap, id, yPos, xPos, width)
|
||||
applyRPSThreshold(panel, panelKey, client.RPSLimit, client.RPSCommercialLimit)
|
||||
sidQuery := buildSIDQuery(domain.SID)
|
||||
|
||||
panel["targets"] = builder(queryConfigRaw, sidQuery, datasourceType, datasourceUID, template, domain, client)
|
||||
|
|
@ -85,6 +87,55 @@ func buildPanel(
|
|||
}
|
||||
|
||||
// evaluateCondition проверяет условие отображения панели через реестр
|
||||
// applyRPSThreshold подставляет rps_limit и rps_commercial_limit клиента в threshold панели rps.
|
||||
// Красная линия — rps_limit (если 0, берётся из шаблона).
|
||||
// Синяя линия — rps_commercial_limit (если 0, ставится 100).
|
||||
func applyRPSThreshold(panel map[string]interface{}, panelKey string, rpsLimit, rpsCommercialLimit int) {
|
||||
if panelKey != "rps" {
|
||||
return
|
||||
}
|
||||
// Ничего менять не нужно если оба лимита не заданы
|
||||
if rpsLimit == 0 && rpsCommercialLimit == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
fc, ok := panel["fieldConfig"].(map[string]interface{})
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defaults, ok := fc["defaults"].(map[string]interface{})
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
thresholds, ok := defaults["thresholds"].(map[string]interface{})
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Коммерческий лимит: если не задан — 100
|
||||
commercial := rpsCommercialLimit
|
||||
if commercial == 0 {
|
||||
commercial = 100
|
||||
}
|
||||
|
||||
steps := []interface{}{
|
||||
map[string]interface{}{"color": "green", "value": nil},
|
||||
map[string]interface{}{"color": "blue", "value": commercial},
|
||||
}
|
||||
if rpsLimit > 0 {
|
||||
steps = append(steps, map[string]interface{}{"color": "dark-red", "value": rpsLimit})
|
||||
}
|
||||
thresholds["steps"] = steps
|
||||
|
||||
// Включаем отображение линий если ещё не задано
|
||||
custom, ok := defaults["custom"].(map[string]interface{})
|
||||
if ok {
|
||||
if _, hasStyle := custom["thresholdsStyle"]; !hasStyle {
|
||||
custom["thresholdsStyle"] = map[string]interface{}{"mode": "line"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func evaluateCondition(condition *PanelCondition, client ClientData, domain *DomainInfo) bool {
|
||||
if condition == nil {
|
||||
return true
|
||||
|
|
@ -295,6 +346,29 @@ func buildSumExpressionTargets(cfg map[string]interface{}, sidQuery, datasourceT
|
|||
}
|
||||
}
|
||||
|
||||
// buildStaticTargets: панель без подстановки SID из БД.
|
||||
// targets берутся из query_config.targets как есть — для глобальных/обзорных панелей.
|
||||
// Поддерживает произвольные bucket_aggs из шаблона.
|
||||
func buildStaticTargets(cfg map[string]interface{}, _ string, datasourceType, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} {
|
||||
targetsRaw, ok := cfg["targets"].([]interface{})
|
||||
if !ok || len(targetsRaw) == 0 {
|
||||
return []interface{}{}
|
||||
}
|
||||
// Копируем targets как есть, только проставляем datasource
|
||||
targets := []interface{}{}
|
||||
for _, t := range targetsRaw {
|
||||
target := deepCopy(t).(map[string]interface{})
|
||||
if _, hasDatasource := target["datasource"]; !hasDatasource {
|
||||
target["datasource"] = map[string]interface{}{
|
||||
"type": datasourceType,
|
||||
"uid": datasourceUID,
|
||||
}
|
||||
}
|
||||
targets = append(targets, target)
|
||||
}
|
||||
return targets
|
||||
}
|
||||
|
||||
func buildBucketLogsTargets(cfg map[string]interface{}, sidQuery, datasourceType, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} {
|
||||
query := strings.ReplaceAll(getString(cfg, "base_query"), "{sid}", sidQuery)
|
||||
return []interface{}{
|
||||
|
|
|
|||
80
state.go
80
state.go
|
|
@ -11,14 +11,16 @@ import (
|
|||
)
|
||||
|
||||
type State struct {
|
||||
LastRun time.Time `json:"last_run"`
|
||||
TemplateCommit string `json:"template_commit"`
|
||||
TemplateVersion string `json:"template_version"`
|
||||
ClientsHash string `json:"clients_hash"`
|
||||
ClientCount int `json:"client_count"`
|
||||
DomainCount int `json:"domain_count"`
|
||||
SIDs []string `json:"sids"`
|
||||
LastChangeReason string `json:"last_change_reason,omitempty"`
|
||||
LastRun time.Time `json:"last_run"`
|
||||
TemplateCommit string `json:"template_commit"`
|
||||
TemplateVersion string `json:"template_version"`
|
||||
ClientsHash string `json:"clients_hash"`
|
||||
ClientCount int `json:"client_count"`
|
||||
DomainCount int `json:"domain_count"`
|
||||
SIDs []string `json:"sids"`
|
||||
LastChangeReason string `json:"last_change_reason,omitempty"`
|
||||
AlertsHash string `json:"alerts_hash,omitempty"` // Хэш лимитов алертов из БД
|
||||
AlertTemplateHash string `json:"alert_template_hash,omitempty"` // Хэш файла alert_rules_template.json
|
||||
}
|
||||
|
||||
type StateManager struct {
|
||||
|
|
@ -151,6 +153,7 @@ func calculateClientsHash(clients map[string]ClientData) string {
|
|||
// Create a deterministic representation
|
||||
type hashData struct {
|
||||
ClientTitle string
|
||||
RPSLimit int
|
||||
Domains []struct {
|
||||
SID string
|
||||
DomainName string
|
||||
|
|
@ -160,7 +163,7 @@ func calculateClientsHash(clients map[string]ClientData) string {
|
|||
|
||||
var data []hashData
|
||||
for _, client := range clients {
|
||||
hd := hashData{ClientTitle: client.ClientTitle}
|
||||
hd := hashData{ClientTitle: client.ClientTitle, RPSLimit: client.RPSLimit}
|
||||
for _, domain := range client.Domains {
|
||||
hd.Domains = append(hd.Domains, struct {
|
||||
SID string
|
||||
|
|
@ -190,6 +193,65 @@ func calculateClientsHash(clients map[string]ClientData) string {
|
|||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
// calculateAlertsHash считает хэш только от данных нужных для алертов (client_title + rps_limit)
|
||||
func calculateAlertsHash(clients map[string]ClientData) string {
|
||||
type alertData struct {
|
||||
ClientTitle string
|
||||
RPSLimit int
|
||||
Limit4xx int
|
||||
Limit5xx int
|
||||
}
|
||||
var data []alertData
|
||||
for _, client := range clients {
|
||||
if client.RPSLimit > 0 || client.Limit4xx > 0 || client.Limit5xx > 0 {
|
||||
data = append(data, alertData{
|
||||
ClientTitle: client.ClientTitle,
|
||||
RPSLimit: client.RPSLimit,
|
||||
Limit4xx: client.Limit4xx,
|
||||
Limit5xx: client.Limit5xx,
|
||||
})
|
||||
}
|
||||
}
|
||||
sort.Slice(data, func(i, j int) bool {
|
||||
return data[i].ClientTitle < data[j].ClientTitle
|
||||
})
|
||||
jsonData, _ := json.Marshal(data)
|
||||
hash := sha256.Sum256(jsonData)
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
// AlertTemplateChanged проверяет изменился ли файл alert_rules_template.json
|
||||
func (sm *StateManager) AlertTemplateChanged(templateHash string) bool {
|
||||
if sm.state == nil {
|
||||
return true
|
||||
}
|
||||
return sm.state.AlertTemplateHash != templateHash
|
||||
}
|
||||
|
||||
// UpdateAlertTemplateHash сохраняет хэш шаблона алертов
|
||||
func (sm *StateManager) UpdateAlertTemplateHash(templateHash string) {
|
||||
if sm.state == nil {
|
||||
sm.state = &State{}
|
||||
}
|
||||
sm.state.AlertTemplateHash = templateHash
|
||||
}
|
||||
|
||||
// AlertsNeedUpdate проверяет изменился ли хэш алертов
|
||||
func (sm *StateManager) AlertsNeedUpdate(clients map[string]ClientData) bool {
|
||||
if sm.state == nil {
|
||||
return true
|
||||
}
|
||||
return sm.state.AlertsHash != calculateAlertsHash(clients)
|
||||
}
|
||||
|
||||
// UpdateAlertsHash сохраняет хэш алертов в state
|
||||
func (sm *StateManager) UpdateAlertsHash(clients map[string]ClientData) {
|
||||
if sm.state == nil {
|
||||
sm.state = &State{}
|
||||
}
|
||||
sm.state.AlertsHash = calculateAlertsHash(clients)
|
||||
}
|
||||
|
||||
func extractSIDs(clients map[string]ClientData) []string {
|
||||
sidMap := make(map[string]bool)
|
||||
for _, client := range clients {
|
||||
|
|
|
|||
27
template.go
27
template.go
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
|
@ -23,6 +24,32 @@ func loadTemplate(templatesPath string) (*DashboardTemplate, error) {
|
|||
return &template, nil
|
||||
}
|
||||
|
||||
func loadAlertTemplate(templatesPath string) (*AlertRulesTemplate, error) {
|
||||
templateFile := filepath.Join(templatesPath, "alert_rules_template.json")
|
||||
|
||||
data, err := os.ReadFile(templateFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read alert template file: %w", err)
|
||||
}
|
||||
|
||||
var tmpl AlertRulesTemplate
|
||||
if err := json.Unmarshal(data, &tmpl); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse alert template JSON: %w", err)
|
||||
}
|
||||
|
||||
return &tmpl, nil
|
||||
}
|
||||
|
||||
// hashAlertTemplate считает SHA256 от содержимого alert_rules_template.json
|
||||
func hashAlertTemplate(templatesPath string) string {
|
||||
data, err := os.ReadFile(filepath.Join(templatesPath, "alert_rules_template.json"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
hash := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
func deepCopy(src interface{}) interface{} {
|
||||
data, _ := json.Marshal(src)
|
||||
var dst interface{}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue