557 lines
20 KiB
Go
557 lines
20 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// buildRPSAlertRuleVL строит правило алерта на RPS для VictoriaLogs datasource.
|
|
func buildRPSAlertRuleVL(client ClientData, tmpl AlertRulesTemplate, config Config) map[string]interface{} {
|
|
datasourceUID := config.VLDatasourceUID
|
|
datasourceType := DefaultVLDatasourceType
|
|
|
|
// SID-запрос в формате LogsQL
|
|
sidMap := make(map[string]bool)
|
|
for _, d := range client.Domains {
|
|
sidMap[d.SID] = true
|
|
}
|
|
uniqueSIDs := make([]string, 0, len(sidMap))
|
|
for s := range sidMap {
|
|
uniqueSIDs = append(uniqueSIDs, s)
|
|
}
|
|
sort.Strings(uniqueSIDs)
|
|
sidQuery := buildTenantSIDQueryVL(client.Domains)
|
|
|
|
defaults := tmpl.Defaults
|
|
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
|
group := client.ClientTitle
|
|
|
|
timeRangeFrom := int64(1800)
|
|
if v, ok := tmpl.RPSAlert["relative_time_range_from"].(float64); ok {
|
|
timeRangeFrom = int64(v)
|
|
}
|
|
|
|
summaryTemplate := "Превышен порог в {rps_limit} RPS в {client_title}"
|
|
if v, ok := tmpl.RPSAlert["annotation_summary"].(string); ok {
|
|
summaryTemplate = v
|
|
}
|
|
summary := strings.NewReplacer(
|
|
"{rps_limit}", fmt.Sprintf("%d", client.RPSLimit),
|
|
"{client_title}", client.ClientTitle,
|
|
).Replace(summaryTemplate)
|
|
|
|
_ = uniqueSIDs // используется через sidQuery
|
|
|
|
uid := generateAlertUID(client.ClientTitle+"_vl", "rps")
|
|
dataRefID := "A"
|
|
rpsRefID := "RPS"
|
|
reduceRefID := "_Reduce"
|
|
thresholdRefID := "Превышение"
|
|
|
|
// LogsQL запрос: count за минуту
|
|
expr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() rps", sidQuery)
|
|
|
|
return map[string]interface{}{
|
|
"uid": uid,
|
|
"title": fmt.Sprintf("VL RPS %s", client.ClientTitle),
|
|
"condition": thresholdRefID,
|
|
"noDataState": "NoData", // VL: нет данных = нет алерта
|
|
"execErrState": "Error", // VL: ошибка datasource = Error, не Alerting
|
|
"isPaused": false,
|
|
"folderUID": "",
|
|
"ruleGroup": group,
|
|
"annotations": map[string]string{"summary": summary},
|
|
"notification_settings": map[string]string{
|
|
"receiver": receiver,
|
|
},
|
|
"data": []interface{}{
|
|
// Шаг 1: VL statsRange запрос
|
|
map[string]interface{}{
|
|
"refId": dataRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": timeRangeFrom,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": datasourceUID,
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{
|
|
"type": datasourceType,
|
|
"uid": datasourceUID,
|
|
},
|
|
"editorMode": "code",
|
|
"expr": expr,
|
|
"legendFormat": fmt.Sprintf("RPS %s", client.ClientTitle),
|
|
"maxLines": 1000,
|
|
"queryType": "statsRange",
|
|
"refId": dataRefID,
|
|
"intervalMs": 60000,
|
|
"maxDataPoints": 43200,
|
|
},
|
|
},
|
|
// Шаг 2: math $A / 60 → RPS (интервал 1m = 60 секунд)
|
|
map[string]interface{}{
|
|
"refId": rpsRefID,
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": timeRangeFrom,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{
|
|
"name": "Expression",
|
|
"type": "__expr__",
|
|
"uid": "__expr__",
|
|
},
|
|
"expression": fmt.Sprintf("$%s / 60", dataRefID),
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": rpsRefID,
|
|
"type": "math",
|
|
"window": "",
|
|
},
|
|
},
|
|
// Шаг 3: reduce median
|
|
map[string]interface{}{
|
|
"refId": reduceRefID,
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": 0,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{"params": []int{0, 0}, "type": "gt"},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{
|
|
"name": "Expression",
|
|
"type": "__expr__",
|
|
"uid": "__expr__",
|
|
},
|
|
"expression": rpsRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"reducer": "median",
|
|
"refId": reduceRefID,
|
|
"settings": map[string]interface{}{"mode": ""},
|
|
"type": "reduce",
|
|
},
|
|
},
|
|
// Шаг 4: threshold > rps_limit
|
|
map[string]interface{}{
|
|
"refId": thresholdRefID,
|
|
"relativeTimeRange": map[string]interface{}{
|
|
"from": 0,
|
|
"to": 0,
|
|
},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{
|
|
"params": []interface{}{client.RPSLimit, 0},
|
|
"type": "gt",
|
|
},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{
|
|
"name": "Expression",
|
|
"type": "__expr__",
|
|
"uid": "__expr__",
|
|
},
|
|
"expression": reduceRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": thresholdRefID,
|
|
"type": "threshold",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// buildErrorRateAlertRulesVL строит алерт на процент ошибок для конкретного домена (VictoriaLogs).
|
|
// statusField: "response_status_code" или "upstream_status"
|
|
func buildErrorRateAlertRulesVL(client ClientData, domain DomainInfo, errCode, limitPct int, statusField string, tmpl AlertRulesTemplate, config Config) []map[string]interface{} {
|
|
datasourceUID := config.VLDatasourceUID
|
|
datasourceType := DefaultVLDatasourceType
|
|
|
|
defaults := tmpl.Defaults
|
|
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
|
group := client.ClientTitle
|
|
|
|
codeFrom, codeTo := 400, 499
|
|
codeName := "4xx"
|
|
if errCode == 5 {
|
|
codeFrom, codeTo = 500, 599
|
|
codeName = "5xx"
|
|
}
|
|
|
|
forDuration := "1m"
|
|
if errCode == 4 {
|
|
forDuration = "3m"
|
|
}
|
|
|
|
relFrom := int64(900)
|
|
|
|
var rules []map[string]interface{}
|
|
|
|
{
|
|
sidQuery := fmt.Sprintf("SID:%s", domain.SID)
|
|
// Короткое имя поля для использования в названии алерта
|
|
fieldShort := "response"
|
|
if statusField == "upstream_status" {
|
|
fieldShort = "upstream"
|
|
}
|
|
uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID+"_"+fieldShort, codeName)
|
|
title := fmt.Sprintf("VL %s %s %s / %s SID:%s", codeName, fieldShort, client.ClientTitle, domain.DomainName, domain.SID)
|
|
fieldDesc := "ответ от PTAF"
|
|
if statusField == "upstream_status" {
|
|
fieldDesc = "ответ от origin"
|
|
}
|
|
circle := "🟡"
|
|
summary := fmt.Sprintf(
|
|
"%s %s / %s (SID:%s) — %s ошибок > %d%% держится > %s. [%s]",
|
|
circle, client.ClientTitle, domain.DomainName, domain.SID, codeName, limitPct, forDuration, fieldShort,
|
|
)
|
|
thresholdAnnotation := fmt.Sprintf(">%d%% для %s (%s)", limitPct, statusField, fieldDesc)
|
|
fieldInfoAnnotation := fmt.Sprintf("для %s (%s)", statusField, fieldDesc)
|
|
thresholdValueAnnotation := fmt.Sprintf("%d", limitPct)
|
|
|
|
totalRefID := "Total"
|
|
errRefID := "Errors"
|
|
totalReduceID := "_TotalReduce"
|
|
errReduceID := "_ErrorsReduce"
|
|
percentRefID := "Значение"
|
|
thresholdRefID := "Превышение"
|
|
|
|
totalExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF | stats by (_time:1m) count() hits", sidQuery)
|
|
errExpr := fmt.Sprintf("%s log_type:nginx-access-PTAF %s:>=%d %s:<=%d | stats by (_time:1m) count() hits", sidQuery, statusField, codeFrom, statusField, codeTo)
|
|
|
|
makeVLModel := func(refID, expr, legend string) map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"datasource": map[string]interface{}{
|
|
"type": datasourceType,
|
|
"uid": datasourceUID,
|
|
},
|
|
"editorMode": "code",
|
|
"expr": expr,
|
|
"legendFormat": legend,
|
|
"maxLines": 1000,
|
|
"queryType": "statsRange",
|
|
"refId": refID,
|
|
"intervalMs": 60000,
|
|
"maxDataPoints": 43200,
|
|
}
|
|
}
|
|
|
|
makeReduce := func(refID, expression, reducer string) map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": expression,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"reducer": reducer,
|
|
"refId": refID,
|
|
"type": "reduce",
|
|
}
|
|
}
|
|
|
|
rule := map[string]interface{}{
|
|
"uid": uid,
|
|
"title": title,
|
|
"condition": thresholdRefID,
|
|
"noDataState": "NoData", // VL: нет данных = нет алерта
|
|
"execErrState": "Error", // VL: ошибка datasource = Error, не Alerting
|
|
"isPaused": false,
|
|
"folderUID": "",
|
|
"ruleGroup": group,
|
|
"for": forDuration,
|
|
"annotations": map[string]string{
|
|
"summary": summary,
|
|
"client_domain": fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID),
|
|
"code_field": codeName,
|
|
"for_duration": forDuration,
|
|
"threshold": thresholdAnnotation,
|
|
"threshold_value": thresholdValueAnnotation,
|
|
"field_info": fieldInfoAnnotation,
|
|
"severity": "warning",
|
|
},
|
|
"notification_settings": map[string]string{"receiver": receiver},
|
|
"data": []interface{}{
|
|
map[string]interface{}{
|
|
"refId": totalRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
|
"datasourceUid": datasourceUID,
|
|
"model": makeVLModel(totalRefID, totalExpr, "Total"),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": errRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
|
"datasourceUid": datasourceUID,
|
|
"model": makeVLModel(errRefID, errExpr, codeName),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": totalReduceID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": makeReduce(totalReduceID, totalRefID, "sum"),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": errReduceID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": makeReduce(errReduceID, errRefID, "sum"),
|
|
},
|
|
map[string]interface{}{
|
|
"refId": percentRefID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": fmt.Sprintf("($%s / $%s) * 100", errReduceID, totalReduceID),
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": percentRefID,
|
|
"type": "math",
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"refId": thresholdRefID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{"params": []interface{}{limitPct, 0}, "type": "gt"},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": percentRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": thresholdRefID,
|
|
"type": "threshold",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rules = append(rules, rule)
|
|
}
|
|
|
|
return rules
|
|
}
|
|
|
|
// generateAndSendAlertsVL генерирует и отправляет алерты для VictoriaLogs.
|
|
func generateAndSendAlertsVL(clients map[string]ClientData, tmpl AlertRulesTemplate, config Config, dryRun bool) ([]string, error) {
|
|
if config.VLDatasourceUID == "" {
|
|
log.Printf("Skipping VL alerts: VLDatasourceUID not configured")
|
|
return nil, nil
|
|
}
|
|
|
|
folderUID, err := ensureAlertFolder(config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to ensure alert folder: %w", err)
|
|
}
|
|
|
|
var tasks []alertTask
|
|
|
|
for _, client := range clients {
|
|
if len(client.Domains) == 0 {
|
|
continue
|
|
}
|
|
|
|
// RPS алерт
|
|
if client.RPSLimit > 0 {
|
|
rule := buildRPSAlertRuleVL(client, tmpl, config)
|
|
rule["folderUID"] = folderUID
|
|
tasks = append(tasks, alertTask{
|
|
rule: rule,
|
|
label: fmt.Sprintf("VL RPS %s", client.ClientTitle),
|
|
})
|
|
}
|
|
|
|
// WAF block алерт — по одному на каждый SID (пропускаем если ptaf_fallback_code = "pass" или пустой)
|
|
if client.PtafFallbackCode != "" && strings.ToLower(client.PtafFallbackCode) != "pass" {
|
|
for _, domain := range client.Domains {
|
|
rule := buildWAFBlockAlertRuleVL(client, domain, tmpl, config)
|
|
rule["folderUID"] = folderUID
|
|
tasks = append(tasks, alertTask{
|
|
rule: rule,
|
|
label: fmt.Sprintf("VL %s WAF %s / %s SID:%s", client.PtafFallbackCode, client.ClientTitle, domain.DomainName, domain.SID),
|
|
})
|
|
}
|
|
}
|
|
|
|
// Error rate алерты — пороги берутся из каждого домена (apps_settings)
|
|
// Генерируем для response_status_code и upstream_status_code
|
|
for _, domain := range client.Domains {
|
|
for _, statusField := range []string{"response_status_code", "upstream_status"} {
|
|
if domain.Limit4xx > 0 {
|
|
for _, rule := range buildErrorRateAlertRulesVL(client, domain, 4, domain.Limit4xx, statusField, tmpl, config) {
|
|
rule["folderUID"] = folderUID
|
|
tasks = append(tasks, alertTask{
|
|
rule: rule,
|
|
label: fmt.Sprintf("VL 4xx %s %s / %s SID:%s", statusField, client.ClientTitle, domain.DomainName, domain.SID),
|
|
})
|
|
}
|
|
}
|
|
if domain.Limit5xx > 0 {
|
|
for _, rule := range buildErrorRateAlertRulesVL(client, domain, 5, domain.Limit5xx, statusField, tmpl, config) {
|
|
rule["folderUID"] = folderUID
|
|
tasks = append(tasks, alertTask{
|
|
rule: rule,
|
|
label: fmt.Sprintf("VL 5xx %s %s / %s SID:%s", statusField, client.ClientTitle, domain.DomainName, domain.SID),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Обновляем шаблон contact point (если OS алерты отключены — делаем здесь)
|
|
if !config.AlertsOSEnabled {
|
|
if err := upsertContactPoint(tmpl, config, dryRun); err != nil {
|
|
log.Printf("Warning: failed to update contact point: %v", err)
|
|
}
|
|
}
|
|
|
|
sent, failed := parallelUpsert(tasks, config, dryRun, 20)
|
|
log.Printf("VL Alerts: sent=%d failed=%d", sent, failed)
|
|
|
|
// Статические алерты отправляются отдельно из main.go чтобы избежать
|
|
// дублирования при вызове функции для разных провайдеров (spik/sp)
|
|
|
|
// Собираем активные UID для возврата
|
|
activeUIDs := []string{}
|
|
for _, task := range tasks {
|
|
if uid, ok := task.rule["uid"].(string); ok {
|
|
activeUIDs = append(activeUIDs, uid)
|
|
}
|
|
}
|
|
|
|
return activeUIDs, nil
|
|
}
|
|
|
|
// buildWAFBlockAlertRuleVL строит алерт на количество 418 ответов за минуту (VictoriaLogs).
|
|
func buildWAFBlockAlertRuleVL(client ClientData, domain DomainInfo, tmpl AlertRulesTemplate, config Config) map[string]interface{} {
|
|
datasourceUID := config.VLDatasourceUID
|
|
datasourceType := DefaultVLDatasourceType
|
|
|
|
defaults := tmpl.Defaults
|
|
receiver := orDefault(config.AlertsReceiver, defaults.Receiver)
|
|
group := client.ClientTitle
|
|
|
|
uid := generateAlertUID(client.ClientTitle+"_vl_"+domain.SID, "waf_block")
|
|
title := fmt.Sprintf("VL %s WAF %s / %s SID:%s", client.PtafFallbackCode, client.ClientTitle, domain.DomainName, domain.SID)
|
|
|
|
clientDomain := fmt.Sprintf("%s / %s (SID:%s)", client.ClientTitle, domain.DomainName, domain.SID)
|
|
summary := fmt.Sprintf("🟡 %s — %s ошибка WAF - кол-во штук за 1 минуту.", clientDomain, client.PtafFallbackCode)
|
|
thresholdAnnotation := fmt.Sprintf(">%d штук(и) для response_status_code (ответ от PTAF)", domain.PtafFallbackCodeAlertCount)
|
|
thresholdCountAnnotation := fmt.Sprintf("%d шт.", domain.PtafFallbackCodeAlertCount)
|
|
|
|
expr := fmt.Sprintf("SID:%s log_type:nginx-access-PTAF response_status_code:=%s | stats by (_time:1m) count() hits", domain.SID, client.PtafFallbackCode)
|
|
|
|
dataRefID := "A"
|
|
reduceRefID := "_Reduce"
|
|
thresholdRefID := "Превышение"
|
|
relFrom := int64(120)
|
|
|
|
return map[string]interface{}{
|
|
"uid": uid,
|
|
"title": title,
|
|
"condition": thresholdRefID,
|
|
"noDataState": "NoData",
|
|
"execErrState": "Error",
|
|
"isPaused": false,
|
|
"folderUID": "",
|
|
"ruleGroup": group,
|
|
"for": "1m",
|
|
"annotations": map[string]string{
|
|
"summary": summary,
|
|
"client_domain": clientDomain,
|
|
"code_field": fmt.Sprintf("%s WAF", client.PtafFallbackCode),
|
|
"for_duration": "1m",
|
|
"threshold": thresholdAnnotation,
|
|
"field_info": "для response_status_code (ответ от PTAF)",
|
|
"runbook": "проверить логи docker и эскалировать на аналитика.",
|
|
"severity": "warning",
|
|
"value_unit": "шт.",
|
|
"threshold_count": thresholdCountAnnotation,
|
|
},
|
|
"notification_settings": map[string]string{"receiver": receiver},
|
|
"data": []interface{}{
|
|
map[string]interface{}{
|
|
"refId": dataRefID,
|
|
"queryType": "statsRange",
|
|
"relativeTimeRange": map[string]interface{}{"from": relFrom, "to": 0},
|
|
"datasourceUid": datasourceUID,
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{"type": datasourceType, "uid": datasourceUID},
|
|
"editorMode": "code",
|
|
"expr": expr,
|
|
"legendFormat": fmt.Sprintf("418 %s", domain.DomainName),
|
|
"maxLines": 1000,
|
|
"queryType": "statsRange",
|
|
"refId": dataRefID,
|
|
"intervalMs": 60000,
|
|
"maxDataPoints": 43200,
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"refId": reduceRefID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": dataRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"reducer": "last",
|
|
"refId": reduceRefID,
|
|
"type": "reduce",
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"refId": thresholdRefID,
|
|
"relativeTimeRange": map[string]interface{}{"from": 0, "to": 0},
|
|
"datasourceUid": "__expr__",
|
|
"model": map[string]interface{}{
|
|
"conditions": []interface{}{
|
|
map[string]interface{}{
|
|
"evaluator": map[string]interface{}{"params": []interface{}{domain.PtafFallbackCodeAlertCount, 0}, "type": "gt"},
|
|
"operator": map[string]interface{}{"type": "and"},
|
|
"query": map[string]interface{}{"params": []string{}},
|
|
"reducer": map[string]interface{}{"params": []string{}, "type": "avg"},
|
|
"type": "query",
|
|
},
|
|
},
|
|
"datasource": map[string]interface{}{"name": "Expression", "type": "__expr__", "uid": "__expr__"},
|
|
"expression": reduceRefID,
|
|
"intervalMs": 1000,
|
|
"maxDataPoints": 43200,
|
|
"refId": thresholdRefID,
|
|
"type": "threshold",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|