470 lines
18 KiB
Go
470 lines
18 KiB
Go
package main
|
||
|
||
import (
|
||
"crypto/md5"
|
||
"fmt"
|
||
"strings"
|
||
)
|
||
|
||
// queryModeRegistryVL — реестр builders для VictoriaLogs datasource.
|
||
// Каждый builder возвращает []interface{} targets в формате VL datasource.
|
||
var queryModeRegistryVL = map[string]targetBuilder{
|
||
"rps": buildVLRPSTargets,
|
||
"rps_var": buildVLRPSVarTargets,
|
||
"rps_multi": buildVLRPSMultiTargets,
|
||
"multi_target": buildVLMultiTargets,
|
||
"range_percent": buildVLRangePercentTargets,
|
||
"multi_sum_expression": buildVLMultiSumExpressionTargets,
|
||
"sum_expression": buildVLSumExpressionTargets,
|
||
"bucket_logs": buildVLBucketLogsTargets,
|
||
"static": buildStaticTargets, // переиспользуем — не зависит от datasource
|
||
}
|
||
|
||
// makeVLTarget создаёт target для VictoriaLogs datasource.
|
||
// queryType: "range" — для запросов с группировкой by (_time:$__interval, field),
|
||
// "statsRange" — для stats-запросов без группировки по времени (возвращает числа напрямую).
|
||
func makeVLTarget(refID, expr, datasourceUID, legendFormat string) map[string]interface{} {
|
||
return makeVLTargetWithType(refID, expr, datasourceUID, legendFormat, "range")
|
||
}
|
||
|
||
// makeVLStatsTarget создаёт target с queryType="statsRange" для stats-запросов
|
||
// без группировки по времени. Плагин VictoriaLogs возвращает числовые данные напрямую.
|
||
func makeVLStatsTarget(refID, expr, datasourceUID, legendFormat string) map[string]interface{} {
|
||
return makeVLTargetWithType(refID, expr, datasourceUID, legendFormat, "statsRange")
|
||
}
|
||
|
||
// makeVLHiddenStatsTarget создаёт скрытый statsRange target — для использования только в math expressions.
|
||
func makeVLHiddenStatsTarget(refID, expr, datasourceUID string) map[string]interface{} {
|
||
t := makeVLTargetWithType(refID, expr, datasourceUID, "", "statsRange")
|
||
t["hide"] = true
|
||
return t
|
||
}
|
||
|
||
func makeVLTargetWithType(refID, expr, datasourceUID, legendFormat, queryType string) map[string]interface{} {
|
||
return map[string]interface{}{
|
||
"refId": refID,
|
||
"expr": expr,
|
||
"queryType": queryType,
|
||
"editorMode": "code",
|
||
"legendFormat": legendFormat,
|
||
"maxLines": 1000,
|
||
"datasource": map[string]interface{}{
|
||
"type": DefaultVLDatasourceType,
|
||
"uid": datasourceUID,
|
||
},
|
||
}
|
||
}
|
||
|
||
// luceneToLogsQL конвертирует базовый Lucene-запрос в LogsQL.
|
||
// Поддерживает: AND→пробел, SID:(a OR b)→SID:in(a,b), field:[X TO Y]→field:>=X AND field:<=Y
|
||
func luceneToLogsQL(q string) string {
|
||
// Убираем Lucene-экранирование — заменяем все \X на X
|
||
q = unescapeLucene(q)
|
||
// AND → пробел (LogsQL использует пробел как AND)
|
||
q = strings.ReplaceAll(q, " AND ", " ")
|
||
|
||
// SID:(a OR b OR c) → SID:in(a, b, c)
|
||
q = convertORGroupToIn(q, "SID")
|
||
|
||
// SID:({sids}) останется как SID:({sids}) — заменим при построении
|
||
// field:[X TO Y] → field:>=X AND field:<=Y
|
||
q = convertRangeFilters(q)
|
||
|
||
return q
|
||
}
|
||
|
||
// convertORGroupToIn конвертирует field:(a OR b OR c) → field:in(a, b, c)
|
||
func convertORGroupToIn(q, field string) string {
|
||
prefix := field + ":("
|
||
start := strings.Index(q, prefix)
|
||
if start == -1 {
|
||
return q
|
||
}
|
||
end := strings.Index(q[start:], ")")
|
||
if end == -1 {
|
||
return q
|
||
}
|
||
end += start
|
||
inner := q[start+len(prefix) : end]
|
||
// Заменяем OR на запятые
|
||
parts := strings.Split(inner, " OR ")
|
||
for i, p := range parts {
|
||
p = strings.TrimSpace(p)
|
||
// Убираем лишние скобки и кавычки которые Grafana добавляет для single-value переменных
|
||
p = strings.Trim(p, "()")
|
||
p = strings.Trim(p, `"'`)
|
||
parts[i] = strings.TrimSpace(p)
|
||
}
|
||
newFilter := fmt.Sprintf("%s:in(%s)", field, strings.Join(parts, ", "))
|
||
return q[:start] + newFilter + q[end+1:]
|
||
}
|
||
|
||
// convertRangeFilters конвертирует field:[X TO Y] → field:>=X AND field:<=Y
|
||
// field:[X TO *] → field:>=X
|
||
func convertRangeFilters(q string) string {
|
||
// Простая обработка Lucene range syntax
|
||
for {
|
||
startIdx := strings.Index(q, ":[")
|
||
if startIdx == -1 {
|
||
break
|
||
}
|
||
endIdx := strings.Index(q[startIdx:], "]")
|
||
if endIdx == -1 {
|
||
break
|
||
}
|
||
endIdx += startIdx
|
||
|
||
fieldStart := strings.LastIndex(q[:startIdx], " ")
|
||
if fieldStart == -1 {
|
||
fieldStart = 0
|
||
} else {
|
||
fieldStart++
|
||
}
|
||
field := q[fieldStart:startIdx]
|
||
inner := q[startIdx+2 : endIdx]
|
||
parts := strings.Split(inner, " TO ")
|
||
if len(parts) != 2 {
|
||
break
|
||
}
|
||
from := strings.TrimSpace(parts[0])
|
||
to := strings.TrimSpace(parts[1])
|
||
|
||
var newFilter string
|
||
if to == "*" {
|
||
newFilter = fmt.Sprintf("%s:>=%s", field, from)
|
||
} else if from == "*" {
|
||
newFilter = fmt.Sprintf("%s:<=%s", field, to)
|
||
} else {
|
||
newFilter = fmt.Sprintf("%s:>=%s %s:<=%s", field, from, field, to)
|
||
}
|
||
q = q[:fieldStart] + newFilter + q[endIdx+1:]
|
||
}
|
||
return q
|
||
}
|
||
|
||
// buildSIDQueryVL строит LogsQL фильтр по SID.
|
||
// SID:12345 (один) или SID:in(12345, 67890) (несколько)
|
||
func buildSIDQueryVL(sids []string) string {
|
||
if len(sids) == 0 {
|
||
return "*"
|
||
}
|
||
if len(sids) == 1 {
|
||
return fmt.Sprintf("SID:%s", sids[0])
|
||
}
|
||
return fmt.Sprintf("SID:in(%s)", strings.Join(sids, ", "))
|
||
}
|
||
|
||
// buildTenantSIDQueryVL строит LogsQL фильтр по всем SID тенанта.
|
||
func buildTenantSIDQueryVL(domains []DomainInfo) string {
|
||
sids := make([]string, len(domains))
|
||
for i, d := range domains {
|
||
sids[i] = d.SID
|
||
}
|
||
return buildSIDQueryVL(sids)
|
||
}
|
||
|
||
// replaceVLPlaceholders заменяет {sids}, {sid}, {client_title} в LogsQL запросе.
|
||
// sidFilter может быть как "SID:in(...)" так и голыми SID-значениями.
|
||
// Если base_query содержит "SID:{sids}", передаём только значения без префикса SID:.
|
||
func replaceVLPlaceholders(q, sidFilter, clientTitle string) string {
|
||
// Убираем префикс SID: если он есть — base_query сам содержит SID:
|
||
sidValue := sidFilter
|
||
if strings.HasPrefix(sidValue, "SID:") {
|
||
sidValue = sidValue[4:]
|
||
}
|
||
// Если base_query содержит SID:{sids} — подставляем без префикса
|
||
if strings.Contains(q, "SID:{sids}") || strings.Contains(q, "SID:{sid}") {
|
||
q = strings.ReplaceAll(q, "{sids}", sidValue)
|
||
q = strings.ReplaceAll(q, "{sid}", sidValue)
|
||
} else {
|
||
// base_query не содержит SID: — подставляем полный фильтр
|
||
q = strings.ReplaceAll(q, "{sids}", sidFilter)
|
||
q = strings.ReplaceAll(q, "{sid}", sidFilter)
|
||
}
|
||
q = strings.ReplaceAll(q, "{client_title}", clientTitle)
|
||
return q
|
||
}
|
||
|
||
// ---------- RPS builders ----------
|
||
|
||
func buildVLRPSTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} {
|
||
vlSID := buildTenantSIDQueryVL(client.Domains)
|
||
alias := getString(cfg, "alias")
|
||
alias = strings.ReplaceAll(alias, "{client_title}", client.ClientTitle)
|
||
if alias == "" {
|
||
alias = client.ClientTitle
|
||
}
|
||
expr := fmt.Sprintf("%s log_type:access | stats by (_time:$__interval) count() rps", vlSID)
|
||
|
||
return []interface{}{
|
||
makeVLHiddenStatsTarget("A", expr, datasourceUID),
|
||
makeMathTargetWithAlias("$A / $__interval_ms * 1000", alias, alias),
|
||
}
|
||
}
|
||
|
||
func buildVLRPSVarTargets(cfg map[string]interface{}, _ string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} {
|
||
alias := getString(cfg, "alias")
|
||
alias = strings.ReplaceAll(alias, "{client_title}", client.ClientTitle)
|
||
if alias == "" {
|
||
alias = "rps"
|
||
}
|
||
expr := "SID:${rps_sid} log_type:access | stats by (_time:$__interval) count() rps"
|
||
|
||
return []interface{}{
|
||
makeVLHiddenStatsTarget("A", expr, datasourceUID),
|
||
makeMathTargetWithAlias("$A / $__interval_ms * 1000", alias, alias),
|
||
}
|
||
}
|
||
|
||
func buildVLRPSMultiTargets(cfg map[string]interface{}, _ string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} {
|
||
targets := []interface{}{}
|
||
|
||
// Суммарная линия
|
||
allSID := buildTenantSIDQueryVL(client.Domains)
|
||
totalExpr := fmt.Sprintf("%s log_type:access | stats by (_time:$__interval) count() rps", allSID)
|
||
targets = append(targets, makeVLHiddenStatsTarget("A", totalExpr, datasourceUID))
|
||
targets = append(targets, makeMathTargetWithAlias("$A / $__interval_ms * 1000", "Total", "Total"))
|
||
|
||
// Линия на каждый домен
|
||
for i, domain := range client.Domains {
|
||
rawRefID := fmt.Sprintf("D%d", i)
|
||
domainExpr := fmt.Sprintf("SID:%s log_type:access | stats by (_time:$__interval) count() rps", domain.SID)
|
||
legend := domain.SID + "_" + domain.DomainName
|
||
if domain.DomainName == "" {
|
||
legend = domain.SID
|
||
}
|
||
mathRefID := strings.ReplaceAll(legend, "-", "_")
|
||
mathRefID = strings.ReplaceAll(mathRefID, ".", "_")
|
||
targets = append(targets, makeVLHiddenStatsTarget(rawRefID, domainExpr, datasourceUID))
|
||
targets = append(targets, makeMathTargetWithAlias(fmt.Sprintf("$%s / $__interval_ms * 1000", rawRefID), mathRefID, legend))
|
||
}
|
||
return targets
|
||
}
|
||
|
||
// ---------- Status codes (multi_target) ----------
|
||
|
||
func buildVLMultiTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} {
|
||
targetsRaw, _ := cfg["targets"].([]interface{})
|
||
|
||
// Уникальный суффикс на основе sidQuery чтобы избежать конфликта refId между панелями
|
||
h := md5.Sum([]byte(sidQuery))
|
||
uidSuffix := fmt.Sprintf("%x", h[:3])
|
||
|
||
targets := []interface{}{}
|
||
for _, t := range targetsRaw {
|
||
tMap, ok := t.(map[string]interface{})
|
||
if !ok {
|
||
continue
|
||
}
|
||
refID := getString(tMap, "ref_id") + "_" + uidSuffix
|
||
alias := getString(tMap, "alias")
|
||
suffix := getString(tMap, "query_suffix")
|
||
|
||
// Конвертируем suffix из Lucene в LogsQL и добавляем к sidQuery
|
||
vlSuffix := luceneToLogsQL(strings.TrimPrefix(suffix, " "))
|
||
|
||
var expr string
|
||
if vlSuffix != "" {
|
||
expr = fmt.Sprintf("%s %s | stats by (_time:$__interval) count() hits", sidQuery, vlSuffix)
|
||
} else {
|
||
expr = fmt.Sprintf("%s | stats by (_time:$__interval) count() hits", sidQuery)
|
||
}
|
||
|
||
legend := alias
|
||
if legend == "" {
|
||
legend = refID
|
||
}
|
||
targets = append(targets, makeVLStatsTarget(refID, expr, datasourceUID, legend))
|
||
}
|
||
return targets
|
||
}
|
||
|
||
// ---------- Response time (range_percent) ----------
|
||
|
||
func buildVLRangePercentTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} {
|
||
baseQuery := luceneToLogsQL(getString(cfg, "base_query"))
|
||
baseQuery = replaceVLPlaceholders(baseQuery, sidQuery, "")
|
||
rangeField := getString(cfg, "range_field")
|
||
rangesRaw, _ := cfg["ranges"].([]interface{})
|
||
totalRefID := getString(cfg, "total_ref_id")
|
||
if totalRefID == "" {
|
||
totalRefID = "F"
|
||
}
|
||
|
||
// Уникальный суффикс чтобы избежать конфликта refId между панелями
|
||
h := md5.Sum([]byte(baseQuery))
|
||
uidSuffix := fmt.Sprintf("%x", h[:3])
|
||
totalRefIDUniq := totalRefID + "_" + uidSuffix
|
||
|
||
targets := []interface{}{}
|
||
|
||
// Total target — скрытый, используется только для math
|
||
totalExpr := fmt.Sprintf("%s %s:>=0 | stats by (_time:$__interval) count()", baseQuery, rangeField)
|
||
targets = append(targets, makeVLHiddenStatsTarget(totalRefIDUniq, totalExpr, datasourceUID))
|
||
|
||
// Targets для каждого диапазона — скрытые, используются только для math
|
||
for _, r := range rangesRaw {
|
||
rMap, ok := r.(map[string]interface{})
|
||
if !ok {
|
||
continue
|
||
}
|
||
refID := getString(rMap, "ref_id") + "_" + uidSuffix
|
||
from := getString(rMap, "from")
|
||
to := getString(rMap, "to")
|
||
|
||
var rangeFilter string
|
||
if to == "*" {
|
||
rangeFilter = fmt.Sprintf("%s:>=%s", rangeField, from)
|
||
} else {
|
||
rangeFilter = fmt.Sprintf("%s:>=%s %s:<%s", rangeField, from, rangeField, to)
|
||
}
|
||
expr := fmt.Sprintf("%s %s | stats by (_time:$__interval) count()", baseQuery, rangeFilter)
|
||
targets = append(targets, makeVLHiddenStatsTarget(refID, expr, datasourceUID))
|
||
}
|
||
|
||
// Math targets для процентов — видимые, refId = label (Grafana показывает refId как имя серии)
|
||
for _, r := range rangesRaw {
|
||
rMap, ok := r.(map[string]interface{})
|
||
if !ok {
|
||
continue
|
||
}
|
||
refID := getString(rMap, "ref_id") + "_" + uidSuffix
|
||
label := getString(rMap, "label")
|
||
if label == "" {
|
||
label = refID
|
||
}
|
||
// Используем label как refId — Grafana показывает refId как имя серии для math targets
|
||
safeLabel := strings.ReplaceAll(label, ".", "_")
|
||
mathExpr := fmt.Sprintf("$%s / $%s * 100", refID, totalRefIDUniq)
|
||
targets = append(targets, makeMathTargetWithAlias(mathExpr, safeLabel, label))
|
||
}
|
||
|
||
return targets
|
||
}
|
||
|
||
// ---------- Traffic (sum_expression) ----------
|
||
|
||
func buildVLSumExpressionTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} {
|
||
baseQuery := luceneToLogsQL(getString(cfg, "base_query"))
|
||
baseQuery = replaceVLPlaceholders(baseQuery, sidQuery, "")
|
||
field := getString(cfg, "field")
|
||
refID := getString(cfg, "ref_id")
|
||
if refID == "" {
|
||
refID = "A"
|
||
}
|
||
expression := getString(cfg, "expression")
|
||
exprRefID := getString(cfg, "expression_ref_id")
|
||
|
||
// VictoriaLogs: sum по числовому полю, без group by time
|
||
expr := fmt.Sprintf("%s | stats by (_time:$__interval) sum(%s) traffic", baseQuery, field)
|
||
|
||
// Raw target скрытый — используется только для math expression
|
||
rawTarget := makeVLHiddenStatsTarget(refID, expr, datasourceUID)
|
||
return []interface{}{
|
||
rawTarget,
|
||
makeMathTargetWithAlias(expression, exprRefID, exprRefID),
|
||
}
|
||
}
|
||
|
||
// ---------- Traffic combined (multi_sum_expression) ----------
|
||
|
||
func buildVLMultiSumExpressionTargets(cfg map[string]interface{}, sidQuery string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, client ClientData) []interface{} {
|
||
targetsRaw, _ := cfg["targets"].([]interface{})
|
||
baseQuery := luceneToLogsQL(getString(cfg, "base_query"))
|
||
baseQuery = replaceVLPlaceholders(baseQuery, sidQuery, client.ClientTitle)
|
||
|
||
targets := []interface{}{}
|
||
for _, t := range targetsRaw {
|
||
tMap, ok := t.(map[string]interface{})
|
||
if !ok {
|
||
continue
|
||
}
|
||
refID := getString(tMap, "ref_id")
|
||
expression := getString(tMap, "expression")
|
||
alias := getString(tMap, "alias")
|
||
if alias == "" {
|
||
alias = refID
|
||
}
|
||
|
||
field := getString(tMap, "field")
|
||
expr := fmt.Sprintf("%s | stats by (_time:$__interval) sum(%s) traffic", baseQuery, field)
|
||
// Raw target скрытый — используется только для math expression
|
||
rawTarget := makeVLHiddenStatsTarget(refID, expr, datasourceUID)
|
||
// Math target с alias как refId (Grafana показывает refId как имя серии)
|
||
targets = append(targets,
|
||
rawTarget,
|
||
makeMathTargetWithAlias(expression, alias, alias),
|
||
)
|
||
}
|
||
return targets
|
||
}
|
||
|
||
// ---------- Bucket logs (overview panels) ----------
|
||
|
||
func buildVLBucketLogsTargets(cfg map[string]interface{}, _ string, _ string, datasourceUID string, _ *DashboardTemplate, _ DomainInfo, _ ClientData) []interface{} {
|
||
raw := getString(cfg, "base_query")
|
||
// Заменяем Grafana переменные в Lucene формате на LogsQL формат
|
||
// SID:({${SID:lucene}}) → SID:in(${SID:csv}) чтобы избежать двойных скобок
|
||
raw = strings.ReplaceAll(raw, "SID:(${SID:lucene})", "SID:in(${SID:csv})")
|
||
raw = strings.ReplaceAll(raw, "app_name:${app_name:lucene}", "app_name:${app_name}")
|
||
raw = strings.ReplaceAll(raw, "response_status_code:${response_status_code:lucene}", "response_status_code:${response_status_code}")
|
||
baseQuery := luceneToLogsQL(raw)
|
||
bucketAggsRaw, _ := cfg["bucket_aggs"].([]interface{})
|
||
|
||
// Разбираем bucket_aggs чтобы понять по какому полю группировать и top N
|
||
var termsField string
|
||
var termsSize int
|
||
|
||
for _, a := range bucketAggsRaw {
|
||
aMap, ok := a.(map[string]interface{})
|
||
if !ok {
|
||
continue
|
||
}
|
||
if aMap["type"] == "terms" {
|
||
termsField = getString(aMap, "field")
|
||
// Убираем .keyword суффикс для LogsQL
|
||
termsField = strings.TrimSuffix(termsField, ".keyword")
|
||
settings, _ := aMap["settings"].(map[string]interface{})
|
||
if settings != nil {
|
||
if sizeStr, ok := settings["size"].(string); ok && sizeStr != "0" && sizeStr != "" {
|
||
fmt.Sscanf(sizeStr, "%d", &termsSize)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if termsField == "" {
|
||
termsField = "_msg"
|
||
}
|
||
|
||
// LogsQL: группировка по полю + top N
|
||
var expr string
|
||
if termsSize > 0 {
|
||
expr = fmt.Sprintf("%s | stats by (%s) count() hits | sort by (hits desc) | limit %d", baseQuery, termsField, termsSize)
|
||
} else {
|
||
expr = fmt.Sprintf("%s | stats by (%s) count() hits", baseQuery, termsField)
|
||
}
|
||
|
||
// legendFormat с именем поля — плагин подставит значение поля группировки
|
||
legendFormat := fmt.Sprintf("{{%s}}", termsField)
|
||
return []interface{}{
|
||
makeVLStatsTarget("A", expr, datasourceUID, legendFormat),
|
||
}
|
||
}
|
||
|
||
// unescapeLucene убирает Lucene-экранирование символов вида \X → X
|
||
func unescapeLucene(q string) string {
|
||
var result strings.Builder
|
||
i := 0
|
||
for i < len(q) {
|
||
if q[i] == '\\' && i+1 < len(q) {
|
||
// Пропускаем обратный слеш, берём следующий символ как есть
|
||
result.WriteByte(q[i+1])
|
||
i += 2
|
||
} else {
|
||
result.WriteByte(q[i])
|
||
i++
|
||
}
|
||
}
|
||
return result.String()
|
||
}
|