From 118c511e77646f463133d5ff8a6a51fe2f5f328d Mon Sep 17 00:00:00 2001 From: Magnus Root Date: Fri, 6 Mar 2026 12:00:26 +0300 Subject: [PATCH] Added VL --- config.go | 15 +- dashboard.go | 15 ++ dashboard_vl.go | 396 ++++++++++++++++++++++++++++++++++++++++++ main.go | 25 +++ panels.go | 58 ++++++- panels_vl.go | 451 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 958 insertions(+), 2 deletions(-) create mode 100644 dashboard_vl.go create mode 100644 panels_vl.go diff --git a/config.go b/config.go index 45a3d7d..03d5723 100644 --- a/config.go +++ b/config.go @@ -65,9 +65,16 @@ const ( DefaultAlertsGroup = "PTAF Grafana" DefaultAlertsDatasource = "af84zsvlp9blsa" // UID datasource OpenSearch + // VictoriaLogs second dashboard + DefaultVLDatasourceUID = "efewdonokxybkf" + DefaultVLDashboardTitle = "PT AF Requests (VictoriaLogs)" // Название второго дашборда + DefaultVLGrafanaFolder = "" // Пустая = та же папка что и основная + DefaultVLDatasourceType = "victoriametrics-logs-datasource-ptaf-sw" // Тип VL datasource плагина + DefaultVLDashboardUID = "pt-af-requests-auto-vl" // UID второго дашборда + // DefaultDashboardTitle - название единого дашборда со всеми клиентами // Пример результата: Home -> Dashboards -> WAF - PTAF -> PT AF Nodes - DefaultDashboardTitle = "PT AF Nodes" + DefaultDashboardTitle = "PT AF Requests" // Другие примеры названий: // DefaultDashboardTitle = "WAF Monitoring" @@ -97,6 +104,12 @@ type Config struct { AlertsReceiver string // Receiver (канал уведомлений) для алертов AlertsGroup string // Имя группы алертов в Grafana AlertsDatasourceUID string // UID datasource для алертов OpenSearch + + // VictoriaLogs второй дашборд + VLDatasourceUID string // UID VictoriaLogs datasource + VLDashboardTitle string // Название второго дашборда (пустая строка = не генерировать) + VLGrafanaFolder string // Папка для VL дашборда (пустая = та же что и основная) + VLDashboardUID string // UID второго дашборда в Grafana } type ClientData struct { diff --git a/dashboard.go b/dashboard.go index f27483f..b17c1b6 100644 --- a/dashboard.go +++ b/dashboard.go @@ -83,6 +83,21 @@ func generateSingleDashboard(clients map[string]ClientData, template *DashboardT applyDashboardMeta(dashboardBase, template, config.DashboardTitle) dashboardBase["panels"] = panels + return buildDashboardJSON(panels, config.DashboardTitle, template, "") +} + +// buildDashboardJSON собирает финальный JSON дашборда из панелей и шаблона. +// uid — переопределение UID (пустая строка = берётся из шаблона). +func buildDashboardJSON(panels []interface{}, title string, template *DashboardTemplate, uid string) map[string]interface{} { + dashboardBase := deepCopy(template.Dashboard).(map[string]interface{}) + applyDashboardMeta(dashboardBase, template, title) + dashboardBase["panels"] = panels + + // Если передан явный uid — переопределяем (для VL дашборда) + if uid != "" { + dashboardBase["uid"] = uid + } + return map[string]interface{}{ "dashboard": dashboardBase, "overwrite": true, diff --git a/dashboard_vl.go b/dashboard_vl.go new file mode 100644 index 0000000..f185b03 --- /dev/null +++ b/dashboard_vl.go @@ -0,0 +1,396 @@ +package main + +import ( + "fmt" + "log" + "strings" +) + +// generateVLDashboard создаёт дашборд для VictoriaLogs datasource. +// Использует те же панели и layout что и основной дашборд, +// но targets строятся через queryModeRegistryVL с LogsQL запросами. +func generateVLDashboard(clients map[string]ClientData, template *DashboardTemplate, config Config) map[string]interface{} { + panels := []interface{}{} + yPos := 0 + panelID := 100 + + datasourceUID := config.VLDatasourceUID + datasourceType := DefaultVLDatasourceType + useCollapsedRows := template.Layout.UseCollapsedRows + + if len(template.Layout.OverviewPanels) > 0 { + overview := createOverviewRowVL(template, datasourceType, datasourceUID, panelID, yPos) + panels = append(panels, overview) + panelID += 50 + yPos++ + } + + for _, client := range clients { + if len(client.Domains) == 0 { + continue + } + + if useCollapsedRows { + row := createCollapsedRowVL(client, template, datasourceType, datasourceUID, panelID, yPos) + panels = append(panels, row) + panelID += 100 + yPos++ + } else { + panels = append(panels, createRowPanel(client.ClientTitle, template, panelID, yPos)) + panelID++ + yPos++ + + tenantPanels, nextID, nextY := generateLayoutPanelsVL( + template.Layout.TenantPanels, client, nil, + template, datasourceType, datasourceUID, panelID, yPos, + ) + panels = append(panels, tenantPanels...) + panelID = nextID + yPos = nextY + + for _, domain := range client.Domains { + domainPanels, nextID, nextY := generateLayoutPanelsVL( + template.Layout.DomainPanels, client, &domain, + template, datasourceType, datasourceUID, panelID, yPos, + ) + panels = append(panels, domainPanels...) + panelID = nextID + yPos = nextY + } + + interPanels, nextID, nextY := generateLayoutPanelsVL( + template.Layout.InterDomainPanels, client, nil, + template, datasourceType, datasourceUID, panelID, yPos, + ) + panels = append(panels, interPanels...) + panelID = nextID + yPos = nextY + + for _, domain := range client.Domains { + trafficPanels, nextID, nextY := generateLayoutPanelsVL( + template.Layout.TrafficDomainPanels, client, &domain, + template, datasourceType, datasourceUID, panelID, yPos, + ) + panels = append(panels, trafficPanels...) + panelID = nextID + yPos = nextY + } + } + } + + title := config.VLDashboardTitle + dashboard := buildDashboardJSON(panels, title, template, config.VLDashboardUID) + return dashboard +} + +// generateLayoutPanelsVL — аналог generateLayoutPanels но использует VL builders. +func generateLayoutPanelsVL( + layouts []PanelLayout, + client ClientData, + domain *DomainInfo, + template *DashboardTemplate, + datasourceType, datasourceUID string, + panelID, yPos int, +) ([]interface{}, int, int) { + panels := []interface{}{} + for _, layout := range layouts { + if layout.Condition != nil { + evaluator, ok := conditionRegistry[layout.Condition.Type] + if !ok || !evaluator(layout.Condition.Value, client, domain) { + continue + } + } + domainInfo := DomainInfo{} + if domain != nil { + domainInfo = *domain + } + width := layout.Width + if width == 0 { + // Берём ширину из gridPos шаблона панели + if panelDef, ok := template.Panels[layout.PanelKey]; ok { + if pm, ok := panelDef.(map[string]interface{}); ok { + if gp, ok := pm["gridPos"].(map[string]interface{}); ok { + if w, ok := gp["w"].(float64); ok && w > 0 { + width = int(w) + } + } + } + } + } + if width == 0 { + width = 12 + } + xPos := layout.XOffset + panel := buildPanelVL(layout.PanelKey, template, domainInfo, client, datasourceType, datasourceUID, panelID, yPos, xPos, width) + if panel == nil { + continue + } + // Применяем title format или title suffix + if layout.TitleFormat != "" { + panel["title"] = formatPanelTitle(layout.TitleFormat, client, domainInfo) + } else if layout.TitleSuffix != "" && domainInfo.DomainName != "" { + panel["title"] = fmt.Sprintf("%s %s", domainInfo.DomainName, formatPanelTitle(layout.TitleSuffix, client, domainInfo)) + } else if layout.TitleSuffix != "" { + panel["title"] = formatPanelTitle(layout.TitleSuffix, client, domainInfo) + } + panels = append(panels, panel) + panelID++ + // Увеличиваем yPos только если следующая панель не рядом (другой x) + if xPos == 0 || xPos+width >= 24 { + yPos += getGridH(panel) + } + } + return panels, panelID, yPos +} + +// buildPanelVL — аналог buildPanel но использует queryModeRegistryVL. +func buildPanelVL( + panelKey string, + template *DashboardTemplate, + domain DomainInfo, + client ClientData, + datasourceType, datasourceUID string, + id, yPos, xPos, width int, +) map[string]interface{} { + panelDef, ok := template.Panels[panelKey] + if !ok { + log.Printf("Warning: panel '%s' not found in template", panelKey) + return nil + } + + panelMap, ok := panelDef.(map[string]interface{}) + if !ok { + return nil + } + + panel := deepCopyMap(panelMap) + + // gridPos + h := 8 + if gridPos, ok := panel["gridPos"].(map[string]interface{}); ok { + if hVal, ok := gridPos["h"].(float64); ok { + h = int(hVal) + } + } + panel["gridPos"] = map[string]interface{}{ + "h": h, "w": width, "x": xPos, "y": yPos, + } + panel["id"] = id + + // Подменяем datasource + panel["datasource"] = map[string]interface{}{ + "type": datasourceType, + "uid": datasourceUID, + } + + // Строим targets через VL registry + queryMode, _ := panel["query_mode"].(string) + if queryMode == "" || queryMode == "row" { + return panel + } + + builder, ok := queryModeRegistryVL[queryMode] + if !ok { + log.Printf("Warning VL: unknown query_mode '%s' for panel '%s'", queryMode, panelKey) + return panel + } + + queryConfigRaw, _ := panel["query_config"].(map[string]interface{}) + if queryConfigRaw == nil { + queryConfigRaw = map[string]interface{}{} + } + + // Строим sidQuery в LogsQL формате + var sidQuery string + if domain.SID != "" { + sidQuery = fmt.Sprintf("SID:%s", domain.SID) + } else { + sidQuery = buildTenantSIDQueryVL(client.Domains) + } + + panel["targets"] = builder(queryConfigRaw, sidQuery, datasourceType, datasourceUID, template, domain, client) + + // Минимальный интервал агрегации для панелей где нужна агрегация по минутам + // Читаем из шаблона панели поле vl_interval, иначе не устанавливаем (auto) + if vlInterval, ok := panel["vl_interval"].(string); ok && vlInterval != "" { + panel["interval"] = vlInterval + } + delete(panel, "vl_interval") + + // Сдвиг на 1 минуту назад чтобы скрыть неполную последнюю минуту + panel["timeShift"] = "1m" + + // Подставляем плейсхолдеры в title панели из шаблона + if title, ok := panel["title"].(string); ok { + panel["title"] = formatPanelTitle(title, client, domain) + } + + // Для range_percent добавляем transformation которая убирает " count(*)" из названий серий + if queryMode == "range_percent" { + panel["transformations"] = []interface{}{ + map[string]interface{}{ + "id": "renameByRegex", + "options": map[string]interface{}{ + "regex": "^(.+) count[(][*][)]$", + "renamePattern": "$1", + }, + }, + } + } + + // Применяем RPS threshold (те же данные) + applyRPSThreshold(panel, panelKey, client.RPSLimit, client.RPSCommercialLimit) + + return panel +} + +// createOverviewRowVL — аналог createOverviewRow для VL. +func createOverviewRowVL(template *DashboardTemplate, datasourceType, datasourceUID string, panelID, yPos int) map[string]interface{} { + overviewPanels := []interface{}{} + dummyClient := ClientData{} + + for _, layout := range template.Layout.OverviewPanels { + width := layout.Width + if width == 0 { + width = 24 + } + panel := buildPanelVL(layout.PanelKey, template, DomainInfo{}, dummyClient, datasourceType, datasourceUID, panelID, 1, layout.XOffset, width) + if panel == nil { + continue + } + if layout.TitleFormat != "" { + panel["title"] = layout.TitleFormat + } + overviewPanels = append(overviewPanels, panel) + panelID++ + } + + rowTitle := template.Layout.OverviewRowTitle + if rowTitle == "" { + rowTitle = "Обзор" + } + + return map[string]interface{}{ + "type": "row", + "title": rowTitle, + "id": panelID, + "collapsed": true, + "gridPos": map[string]interface{}{"h": 1, "w": 24, "x": 0, "y": yPos}, + "panels": overviewPanels, + } +} + +// createCollapsedRowVL — аналог createCollapsedRow для VL. +func createCollapsedRowVL(client ClientData, template *DashboardTemplate, datasourceType, datasourceUID string, panelID, yPos int) map[string]interface{} { + innerPanels := []interface{}{} + currentY := 1 + + addPanelVL := func(layouts []PanelLayout, domain *DomainInfo, rowStartY int) { + for _, layout := range layouts { + if layout.Condition != nil { + evaluator, ok := conditionRegistry[layout.Condition.Type] + if !ok || !evaluator(layout.Condition.Value, client, domain) { + continue + } + } + domainInfo := DomainInfo{} + if domain != nil { + domainInfo = *domain + } + width := layout.Width + if width == 0 { + if panelDef, ok := template.Panels[layout.PanelKey]; ok { + if pm, ok := panelDef.(map[string]interface{}); ok { + if gp, ok := pm["gridPos"].(map[string]interface{}); ok { + if w, ok := gp["w"].(float64); ok && w > 0 { + width = int(w) + } + } + } + } + } + if width == 0 { + width = 12 + } + xPos := layout.XOffset + panel := buildPanelVL(layout.PanelKey, template, domainInfo, client, datasourceType, datasourceUID, panelID, rowStartY, xPos, width) + if panel == nil { + continue + } + if layout.TitleFormat != "" { + panel["title"] = formatPanelTitle(layout.TitleFormat, client, domainInfo) + } else if layout.TitleSuffix != "" && domainInfo.DomainName != "" { + panel["title"] = fmt.Sprintf("%s %s", domainInfo.DomainName, formatPanelTitle(layout.TitleSuffix, client, domainInfo)) + } else if layout.TitleSuffix != "" { + panel["title"] = formatPanelTitle(layout.TitleSuffix, client, domainInfo) + } + innerPanels = append(innerPanels, panel) + panelID++ + } + } + + // tenant_panels + addPanelVL(template.Layout.TenantPanels, nil, currentY) + currentY += 8 + + // domain_panels (side by side) + for _, domain := range client.Domains { + d := domain + addPanelVL(template.Layout.DomainPanels, &d, currentY) + currentY += 8 + } + + // inter_domain_panels + if len(template.Layout.InterDomainPanels) > 0 { + addPanelVL(template.Layout.InterDomainPanels, nil, currentY) + currentY += 8 + } + + // traffic_domain_panels + for _, domain := range client.Domains { + d := domain + addPanelVL(template.Layout.TrafficDomainPanels, &d, currentY) + currentY += 8 + } + + return map[string]interface{}{ + "type": "row", + "title": client.ClientTitle, + "id": panelID, + "collapsed": true, + "gridPos": map[string]interface{}{"h": 1, "w": 24, "x": 0, "y": yPos}, + "panels": innerPanels, + } +} + +// getGridH возвращает высоту панели из gridPos. +func getGridH(panel map[string]interface{}) int { + if gp, ok := panel["gridPos"].(map[string]interface{}); ok { + if h, ok := gp["h"].(int); ok { + return h + } + if h, ok := gp["h"].(float64); ok { + return int(h) + } + } + return 8 +} + +// formatPanelTitle форматирует заголовок панели подстановкой переменных. +func formatPanelTitle(format string, client ClientData, domain DomainInfo) string { + title := format + title = strings.ReplaceAll(title, "{client}", client.ClientTitle) + title = strings.ReplaceAll(title, "{domain}", domain.DomainName) + title = strings.ReplaceAll(title, "{sid}", domain.SID) + return title +} + +// sendVLDashboardToGrafana отправляет VL дашборд в Grafana. +func sendVLDashboardToGrafana(dashboard map[string]interface{}, config Config) error { + vlConfig := config + if config.VLGrafanaFolder != "" { + vlConfig.GrafanaFolder = config.VLGrafanaFolder + } + // Используем тот же sendToGrafana + return sendToGrafana(dashboard, vlConfig) +} diff --git a/main.go b/main.go index 43df148..1432339 100644 --- a/main.go +++ b/main.go @@ -139,6 +139,10 @@ func main() { log.Printf("Clients: %d", len(validClients)) log.Printf("Skipped: %d", skippedCount) log.Printf("DRY RUN: Dashboard generated but not sent to Grafana") + if config.VLDatasourceUID != "" { + vlDashboard := generateVLDashboard(validClients, template, config) + logDryRun(config.VLDashboardTitle, vlDashboard) + } os.Exit(0) } @@ -154,6 +158,17 @@ func main() { log.Printf("Successfully created/updated dashboard: %s", config.DashboardTitle) + // Generate VictoriaLogs dashboard if configured + if config.VLDatasourceUID != "" { + log.Printf("Generating VictoriaLogs dashboard: %s", config.VLDashboardTitle) + vlDashboard := generateVLDashboard(validClients, template, config) + if err := sendVLDashboardToGrafana(vlDashboard, config); err != nil { + log.Printf("Warning: VictoriaLogs dashboard failed: %v", err) + } else { + log.Printf("Successfully created/updated VL dashboard: %s", config.VLDashboardTitle) + } + } + // Generate and send alert rules if alertTmpl != nil { if alertsChanged || config.ForceRun { @@ -237,6 +252,16 @@ func parseFlags() Config { flag.StringVar(&config.AlertsGroup, "alerts-group", alertsGroup, "Grafana alert rule group name") flag.StringVar(&config.AlertsDatasourceUID, "alerts-datasource-uid", alertsDatasource, "OpenSearch datasource UID for alerts") + // VictoriaLogs second dashboard flags + vlDatasourceUID := getEnvOrDefault("VL_DATASOURCE_UID", DefaultVLDatasourceUID) + vlDashboardTitle := getEnvOrDefault("VL_DASHBOARD_TITLE", DefaultVLDashboardTitle) + vlGrafanaFolder := getEnvOrDefault("VL_GRAFANA_FOLDER", DefaultVLGrafanaFolder) + vlDashboardUID := getEnvOrDefault("VL_DASHBOARD_UID", DefaultVLDashboardUID) + flag.StringVar(&config.VLDatasourceUID, "vl-datasource-uid", vlDatasourceUID, "VictoriaLogs datasource UID (empty = skip VL dashboard)") + flag.StringVar(&config.VLDashboardTitle, "vl-dashboard-title", vlDashboardTitle, "VictoriaLogs dashboard title") + flag.StringVar(&config.VLGrafanaFolder, "vl-grafana-folder", vlGrafanaFolder, "Grafana folder for VictoriaLogs dashboard (empty = same as main)") + flag.StringVar(&config.VLDashboardUID, "vl-dashboard-uid", vlDashboardUID, "VictoriaLogs dashboard UID in Grafana") + // Other flags flag.BoolVar(&config.DryRun, "dry-run", false, "Generate JSON but don't send to Grafana") flag.StringVar(&config.StateFile, "state-file", DefaultStateFile, "Path to state file") diff --git a/panels.go b/panels.go index 43a6e0c..909f8dc 100644 --- a/panels.go +++ b/panels.go @@ -15,6 +15,8 @@ type targetBuilder func(cfg map[string]interface{}, sidQuery string, datasourceT // Всё остальное — только в JSON. var queryModeRegistry = map[string]targetBuilder{ "rps": buildRPSTargets, + "rps_var": buildRPSVarTargets, + "rps_multi": buildRPSMultiTargets, "multi_target": buildMultiTargets, "range_percent": buildRangePercentTargets, "multi_sum_expression": buildMultiSumExpressionTargets, @@ -92,7 +94,7 @@ func buildPanel( // Красная линия — rps_limit (если 0, берётся из шаблона). // Синяя линия — rps_commercial_limit (если 0, ставится 100). func applyRPSThreshold(panel map[string]interface{}, panelKey string, rpsLimit, rpsCommercialLimit int) { - if panelKey != "rps" { + if panelKey != "rps" && panelKey != "rps_filtered" && panelKey != "rps_multi" { return } // Ничего менять не нужно если оба лимита не заданы @@ -269,6 +271,20 @@ func buildRPSTargets(cfg map[string]interface{}, sidQuery, datasourceType, datas } } +// buildRPSVarTargets — RPS с фильтрацией через Grafana-переменную $rps_sid. +// base_query может содержать ${rps_sid:lucene} напрямую — Go не трогает эти плейсхолдеры. +func buildRPSVarTargets(cfg map[string]interface{}, sidQuery, datasourceType, datasourceUID string, template *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} { + query := replacePlaceholders(getString(cfg, "base_query"), sidQuery, client.ClientTitle) + alias := replacePlaceholders(getString(cfg, "alias"), sidQuery, client.ClientTitle) + bucketAgg := getBucketAgg(template) + countMetric := map[string]interface{}{"id": "1", "type": "count"} + + return []interface{}{ + makeDataTarget(alias, query, "A", countMetric, true, datasourceType, datasourceUID, bucketAgg), + makeMathTarget(getString(cfg, "expression"), getString(cfg, "expression_ref_id")), + } +} + func buildMultiTargets(cfg map[string]interface{}, sidQuery, datasourceType, datasourceUID string, template *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { targetsRaw, _ := cfg["targets"].([]interface{}) bucketAgg := getBucketAgg(template) @@ -403,3 +419,43 @@ func buildBucketLogsTargets(cfg map[string]interface{}, sidQuery, datasourceType }, } } + +// buildRPSMultiTargets — RPS с одной суммарной линией + по линии на каждый SID клиента. +func buildRPSMultiTargets(cfg map[string]interface{}, _ string, datasourceType, datasourceUID string, template *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} { + bucketAgg := getBucketAgg(template) + countMetric := map[string]interface{}{"id": "1", "type": "count"} + expr := getString(cfg, "expression") + targets := []interface{}{} + + // Суммарная линия по всем SID + allSIDs := buildTenantSIDQuery(client.Domains) + allQuery := fmt.Sprintf("SID:(%s)", allSIDs) + totalLabel := client.ClientTitle + " Total" + targets = append(targets, + makeDataTarget(totalLabel, allQuery, "A", countMetric, true, datasourceType, datasourceUID, bucketAgg), + makeMathTargetWithAlias(strings.ReplaceAll(expr, "$A", "$A"), "RPS_Total", totalLabel), + ) + + // Линия на каждый домен — refId используется как имя серии в Grafana + for i, domain := range client.Domains { + dataRefID := fmt.Sprintf("D%d", i) + // exprRefID = читаемое имя: "domain.name_SID" — Grafana показывает его в легенде + name := domain.SID + if domain.DomainName != "" { + // Заменяем пробелы и точки на _ для валидного refId + safeName := strings.NewReplacer(" ", "_", ".", "_").Replace(domain.DomainName) + name = fmt.Sprintf("%s_%s", safeName, domain.SID) + } + domainQuery := fmt.Sprintf("SID:%s", domain.SID) + domainExpr := strings.ReplaceAll(expr, "$A", "$"+dataRefID) + label := domain.SID + if domain.DomainName != "" { + label = fmt.Sprintf("%s %s", domain.DomainName, domain.SID) + } + targets = append(targets, + makeDataTarget(label, domainQuery, dataRefID, countMetric, true, datasourceType, datasourceUID, bucketAgg), + makeMathTargetWithAlias(domainExpr, name, label), + ) + } + return targets +} diff --git a/panels_vl.go b/panels_vl.go new file mode 100644 index 0000000..a40e4d0 --- /dev/null +++ b/panels_vl.go @@ -0,0 +1,451 @@ +package main + +import ( + "crypto/md5" + "fmt" + "strings" +) + +// queryModeRegistryVL — реестр builders для VictoriaLogs datasource. +// Каждый builder возвращает []interface{} targets в формате VL datasource. +var queryModeRegistryVL = map[string]targetBuilder{ + "rps": buildVLRPSTargets, + "rps_var": buildVLRPSVarTargets, + "rps_multi": buildVLRPSMultiTargets, + "multi_target": buildVLMultiTargets, + "range_percent": buildVLRangePercentTargets, + "multi_sum_expression": buildVLMultiSumExpressionTargets, + "sum_expression": buildVLSumExpressionTargets, + "bucket_logs": buildVLBucketLogsTargets, + "static": buildStaticTargets, // переиспользуем — не зависит от datasource +} + +// makeVLTarget создаёт target для VictoriaLogs datasource. +// queryType: "range" — для запросов с группировкой by (_time:$__interval, field), +// "statsRange" — для stats-запросов без группировки по времени (возвращает числа напрямую). +func makeVLTarget(refID, expr, datasourceUID, legendFormat string) map[string]interface{} { + return makeVLTargetWithType(refID, expr, datasourceUID, legendFormat, "range") +} + +// makeVLStatsTarget создаёт target с queryType="statsRange" для stats-запросов +// без группировки по времени. Плагин VictoriaLogs возвращает числовые данные напрямую. +func makeVLStatsTarget(refID, expr, datasourceUID, legendFormat string) map[string]interface{} { + return makeVLTargetWithType(refID, expr, datasourceUID, legendFormat, "statsRange") +} + +// makeVLHiddenStatsTarget создаёт скрытый statsRange target — для использования только в math expressions. +func makeVLHiddenStatsTarget(refID, expr, datasourceUID string) map[string]interface{} { + t := makeVLTargetWithType(refID, expr, datasourceUID, "", "statsRange") + t["hide"] = true + return t +} + +func makeVLTargetWithType(refID, expr, datasourceUID, legendFormat, queryType string) map[string]interface{} { + return map[string]interface{}{ + "refId": refID, + "expr": expr, + "queryType": queryType, + "editorMode": "code", + "legendFormat": legendFormat, + "maxLines": 1000, + "datasource": map[string]interface{}{ + "type": DefaultVLDatasourceType, + "uid": datasourceUID, + }, + } +} + +// luceneToLogsQL конвертирует базовый Lucene-запрос в LogsQL. +// Поддерживает: AND→пробел, SID:(a OR b)→SID:in(a,b), field:[X TO Y]→field:>=X AND field:<=Y +func luceneToLogsQL(q string) string { + // AND → пробел (LogsQL использует пробел как AND) + q = strings.ReplaceAll(q, " AND ", " ") + + // SID:(a OR b OR c) → SID:in(a, b, c) + q = convertORGroupToIn(q, "SID") + + // SID:({sids}) останется как SID:({sids}) — заменим при построении + // field:[X TO Y] → field:>=X AND field:<=Y + q = convertRangeFilters(q) + + return q +} + +// convertORGroupToIn конвертирует field:(a OR b OR c) → field:in(a, b, c) +func convertORGroupToIn(q, field string) string { + prefix := field + ":(" + start := strings.Index(q, prefix) + if start == -1 { + return q + } + end := strings.Index(q[start:], ")") + if end == -1 { + return q + } + end += start + inner := q[start+len(prefix) : end] + // Заменяем OR на запятые + parts := strings.Split(inner, " OR ") + for i, p := range parts { + p = strings.TrimSpace(p) + // Убираем лишние скобки и кавычки которые Grafana добавляет для single-value переменных + p = strings.Trim(p, "()") + p = strings.Trim(p, `"'`) + parts[i] = strings.TrimSpace(p) + } + newFilter := fmt.Sprintf("%s:in(%s)", field, strings.Join(parts, ", ")) + return q[:start] + newFilter + q[end+1:] +} + +// convertRangeFilters конвертирует field:[X TO Y] → field:>=X AND field:<=Y +// field:[X TO *] → field:>=X +func convertRangeFilters(q string) string { + // Простая обработка Lucene range syntax + for { + startIdx := strings.Index(q, ":[") + if startIdx == -1 { + break + } + endIdx := strings.Index(q[startIdx:], "]") + if endIdx == -1 { + break + } + endIdx += startIdx + + fieldStart := strings.LastIndex(q[:startIdx], " ") + if fieldStart == -1 { + fieldStart = 0 + } else { + fieldStart++ + } + field := q[fieldStart:startIdx] + inner := q[startIdx+2 : endIdx] + parts := strings.Split(inner, " TO ") + if len(parts) != 2 { + break + } + from := strings.TrimSpace(parts[0]) + to := strings.TrimSpace(parts[1]) + + var newFilter string + if to == "*" { + newFilter = fmt.Sprintf("%s:>=%s", field, from) + } else if from == "*" { + newFilter = fmt.Sprintf("%s:<=%s", field, to) + } else { + newFilter = fmt.Sprintf("%s:>=%s %s:<=%s", field, from, field, to) + } + q = q[:fieldStart] + newFilter + q[endIdx+1:] + } + return q +} + +// buildSIDQueryVL строит LogsQL фильтр по SID. +// SID:12345 (один) или SID:in(12345, 67890) (несколько) +func buildSIDQueryVL(sids []string) string { + if len(sids) == 0 { + return "*" + } + if len(sids) == 1 { + return fmt.Sprintf("SID:%s", sids[0]) + } + return fmt.Sprintf("SID:in(%s)", strings.Join(sids, ", ")) +} + +// buildTenantSIDQueryVL строит LogsQL фильтр по всем SID тенанта. +func buildTenantSIDQueryVL(domains []DomainInfo) string { + sids := make([]string, len(domains)) + for i, d := range domains { + sids[i] = d.SID + } + return buildSIDQueryVL(sids) +} + +// replaceVLPlaceholders заменяет {sids}, {sid}, {client_title} в LogsQL запросе. +// sidFilter может быть как "SID:in(...)" так и голыми SID-значениями. +// Если base_query содержит "SID:{sids}", передаём только значения без префикса SID:. +func replaceVLPlaceholders(q, sidFilter, clientTitle string) string { + // Убираем префикс SID: если он есть — base_query сам содержит SID: + sidValue := sidFilter + if strings.HasPrefix(sidValue, "SID:") { + sidValue = sidValue[4:] + } + // Если base_query содержит SID:{sids} — подставляем без префикса + if strings.Contains(q, "SID:{sids}") || strings.Contains(q, "SID:{sid}") { + q = strings.ReplaceAll(q, "{sids}", sidValue) + q = strings.ReplaceAll(q, "{sid}", sidValue) + } else { + // base_query не содержит SID: — подставляем полный фильтр + q = strings.ReplaceAll(q, "{sids}", sidFilter) + q = strings.ReplaceAll(q, "{sid}", sidFilter) + } + q = strings.ReplaceAll(q, "{client_title}", clientTitle) + return q +} + +// ---------- RPS builders ---------- + +func buildVLRPSTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} { + vlSID := buildTenantSIDQueryVL(client.Domains) + alias := getString(cfg, "alias") + alias = strings.ReplaceAll(alias, "{client_title}", client.ClientTitle) + if alias == "" { + alias = client.ClientTitle + } + expr := fmt.Sprintf("%s log_type:access | stats by (_time:$__interval) count() rps", vlSID) + + return []interface{}{ + makeVLHiddenStatsTarget("A", expr, datasourceUID), + makeMathTargetWithAlias("$A / $__interval_ms * 1000", alias, alias), + } +} + +func buildVLRPSVarTargets(cfg map[string]interface{}, _ string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} { + alias := getString(cfg, "alias") + alias = strings.ReplaceAll(alias, "{client_title}", client.ClientTitle) + if alias == "" { + alias = "rps" + } + expr := "SID:${rps_sid} log_type:access | stats by (_time:$__interval) count() rps" + + return []interface{}{ + makeVLHiddenStatsTarget("A", expr, datasourceUID), + makeMathTargetWithAlias("$A / $__interval_ms * 1000", alias, alias), + } +} + +func buildVLRPSMultiTargets(cfg map[string]interface{}, _ string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} { + targets := []interface{}{} + + // Суммарная линия + allSID := buildTenantSIDQueryVL(client.Domains) + totalExpr := fmt.Sprintf("%s log_type:access | stats by (_time:$__interval) count() rps", allSID) + targets = append(targets, makeVLHiddenStatsTarget("A", totalExpr, datasourceUID)) + targets = append(targets, makeMathTargetWithAlias("$A / $__interval_ms * 1000", "Total", "Total")) + + // Линия на каждый домен + for i, domain := range client.Domains { + rawRefID := fmt.Sprintf("D%d", i) + domainExpr := fmt.Sprintf("SID:%s log_type:access | stats by (_time:$__interval) count() rps", domain.SID) + legend := domain.SID + "_" + domain.DomainName + if domain.DomainName == "" { + legend = domain.SID + } + mathRefID := strings.ReplaceAll(legend, "-", "_") + mathRefID = strings.ReplaceAll(mathRefID, ".", "_") + targets = append(targets, makeVLHiddenStatsTarget(rawRefID, domainExpr, datasourceUID)) + targets = append(targets, makeMathTargetWithAlias(fmt.Sprintf("$%s / $__interval_ms * 1000", rawRefID), mathRefID, legend)) + } + return targets +} + +// ---------- Status codes (multi_target) ---------- + +func buildVLMultiTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { + targetsRaw, _ := cfg["targets"].([]interface{}) + + // Уникальный суффикс на основе sidQuery чтобы избежать конфликта refId между панелями + h := md5.Sum([]byte(sidQuery)) + uidSuffix := fmt.Sprintf("%x", h[:3]) + + targets := []interface{}{} + for _, t := range targetsRaw { + tMap, ok := t.(map[string]interface{}) + if !ok { + continue + } + refID := getString(tMap, "ref_id") + "_" + uidSuffix + alias := getString(tMap, "alias") + suffix := getString(tMap, "query_suffix") + + // Конвертируем suffix из Lucene в LogsQL и добавляем к sidQuery + vlSuffix := luceneToLogsQL(strings.TrimPrefix(suffix, " ")) + + var expr string + if vlSuffix != "" { + expr = fmt.Sprintf("%s %s | stats by (_time:$__interval) count() hits", sidQuery, vlSuffix) + } else { + expr = fmt.Sprintf("%s | stats by (_time:$__interval) count() hits", sidQuery) + } + + legend := alias + if legend == "" { + legend = refID + } + targets = append(targets, makeVLStatsTarget(refID, expr, datasourceUID, legend)) + } + return targets +} + +// ---------- Response time (range_percent) ---------- + +func buildVLRangePercentTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { + baseQuery := luceneToLogsQL(getString(cfg, "base_query")) + baseQuery = replaceVLPlaceholders(baseQuery, sidQuery, "") + rangeField := getString(cfg, "range_field") + rangesRaw, _ := cfg["ranges"].([]interface{}) + totalRefID := getString(cfg, "total_ref_id") + if totalRefID == "" { + totalRefID = "F" + } + + // Уникальный суффикс чтобы избежать конфликта refId между панелями + h := md5.Sum([]byte(baseQuery)) + uidSuffix := fmt.Sprintf("%x", h[:3]) + totalRefIDUniq := totalRefID + "_" + uidSuffix + + targets := []interface{}{} + + // Total target — скрытый, используется только для math + totalExpr := fmt.Sprintf("%s %s:>=0 | stats by (_time:$__interval) count()", baseQuery, rangeField) + targets = append(targets, makeVLHiddenStatsTarget(totalRefIDUniq, totalExpr, datasourceUID)) + + // Targets для каждого диапазона — скрытые, используются только для math + for _, r := range rangesRaw { + rMap, ok := r.(map[string]interface{}) + if !ok { + continue + } + refID := getString(rMap, "ref_id") + "_" + uidSuffix + from := getString(rMap, "from") + to := getString(rMap, "to") + + var rangeFilter string + if to == "*" { + rangeFilter = fmt.Sprintf("%s:>=%s", rangeField, from) + } else { + rangeFilter = fmt.Sprintf("%s:>=%s %s:<%s", rangeField, from, rangeField, to) + } + expr := fmt.Sprintf("%s %s | stats by (_time:$__interval) count()", baseQuery, rangeFilter) + targets = append(targets, makeVLHiddenStatsTarget(refID, expr, datasourceUID)) + } + + // Math targets для процентов — видимые, refId = label (Grafana показывает refId как имя серии) + for _, r := range rangesRaw { + rMap, ok := r.(map[string]interface{}) + if !ok { + continue + } + refID := getString(rMap, "ref_id") + "_" + uidSuffix + label := getString(rMap, "label") + if label == "" { + label = refID + } + // Используем label как refId — Grafana показывает refId как имя серии для math targets + safeLabel := strings.ReplaceAll(label, ".", "_") + mathExpr := fmt.Sprintf("$%s / $%s * 100", refID, totalRefIDUniq) + targets = append(targets, makeMathTargetWithAlias(mathExpr, safeLabel, label)) + } + + return targets +} + +// ---------- Traffic (sum_expression) ---------- + +func buildVLSumExpressionTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { + baseQuery := luceneToLogsQL(getString(cfg, "base_query")) + baseQuery = replaceVLPlaceholders(baseQuery, sidQuery, "") + field := getString(cfg, "field") + refID := getString(cfg, "ref_id") + if refID == "" { + refID = "A" + } + expression := getString(cfg, "expression") + exprRefID := getString(cfg, "expression_ref_id") + + // VictoriaLogs: sum по числовому полю, без group by time + expr := fmt.Sprintf("%s | stats by (_time:$__interval) sum(%s) traffic", baseQuery, field) + + // Raw target скрытый — используется только для math expression + rawTarget := makeVLHiddenStatsTarget(refID, expr, datasourceUID) + return []interface{}{ + rawTarget, + makeMathTargetWithAlias(expression, exprRefID, exprRefID), + } +} + +// ---------- Traffic combined (multi_sum_expression) ---------- + +func buildVLMultiSumExpressionTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} { + targetsRaw, _ := cfg["targets"].([]interface{}) + baseQuery := luceneToLogsQL(getString(cfg, "base_query")) + baseQuery = replaceVLPlaceholders(baseQuery, sidQuery, client.ClientTitle) + + targets := []interface{}{} + for _, t := range targetsRaw { + tMap, ok := t.(map[string]interface{}) + if !ok { + continue + } + refID := getString(tMap, "ref_id") + expression := getString(tMap, "expression") + alias := getString(tMap, "alias") + if alias == "" { + alias = refID + } + + field := getString(tMap, "field") + expr := fmt.Sprintf("%s | stats by (_time:$__interval) sum(%s) traffic", baseQuery, field) + // Raw target скрытый — используется только для math expression + rawTarget := makeVLHiddenStatsTarget(refID, expr, datasourceUID) + // Math target с alias как refId (Grafana показывает refId как имя серии) + targets = append(targets, + rawTarget, + makeMathTargetWithAlias(expression, alias, alias), + ) + } + return targets +} + +// ---------- Bucket logs (overview panels) ---------- + +func buildVLBucketLogsTargets(cfg map[string]interface{}, _ string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} { + raw := getString(cfg, "base_query") + // Заменяем Grafana переменные в Lucene формате на LogsQL формат + // SID:({${SID:lucene}}) → SID:in(${SID:csv}) чтобы избежать двойных скобок + raw = strings.ReplaceAll(raw, "SID:(${SID:lucene})", "SID:in(${SID:csv})") + raw = strings.ReplaceAll(raw, "app_name:${app_name:lucene}", "app_name:${app_name}") + raw = strings.ReplaceAll(raw, "response_status_code:${response_status_code:lucene}", "response_status_code:${response_status_code}") + baseQuery := luceneToLogsQL(raw) + bucketAggsRaw, _ := cfg["bucket_aggs"].([]interface{}) + + // Разбираем bucket_aggs чтобы понять по какому полю группировать и top N + var termsField string + var termsSize int + + for _, a := range bucketAggsRaw { + aMap, ok := a.(map[string]interface{}) + if !ok { + continue + } + if aMap["type"] == "terms" { + termsField = getString(aMap, "field") + // Убираем .keyword суффикс для LogsQL + termsField = strings.TrimSuffix(termsField, ".keyword") + settings, _ := aMap["settings"].(map[string]interface{}) + if settings != nil { + if sizeStr, ok := settings["size"].(string); ok && sizeStr != "0" && sizeStr != "" { + fmt.Sscanf(sizeStr, "%d", &termsSize) + } + } + } + } + + if termsField == "" { + termsField = "_msg" + } + + // LogsQL: группировка по полю + top N + var expr string + if termsSize > 0 { + expr = fmt.Sprintf("%s | stats by (%s) count() hits | sort by (hits desc) | limit %d", baseQuery, termsField, termsSize) + } else { + expr = fmt.Sprintf("%s | stats by (%s) count() hits", baseQuery, termsField) + } + + // legendFormat с именем поля — плагин подставит значение поля группировки + legendFormat := fmt.Sprintf("{{%s}}", termsField) + return []interface{}{ + makeVLStatsTarget("A", expr, datasourceUID, legendFormat), + } +}