From bdb54eb7ba21b34b56c796246ed242f59921cb68 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Tue, 2 Jun 2026 23:08:03 +0100 Subject: [PATCH 01/18] gateway: Add initial gateway scafolding --- .dockerignore | 11 + Makefile | 14 +- configs/datasource-gateway/config.yaml | 25 +++ datasource-gateway/README.md | 61 ++++++ datasource-gateway/go.mod | 5 + datasource-gateway/go.sum | 4 + datasource-gateway/internal/api/handlers.go | 33 +++ datasource-gateway/internal/config/config.go | 195 ++++++++++++++++++ datasource-gateway/internal/logger/logger.go | 69 +++++++ datasource-gateway/main.go | 94 +++++++++ .../docker/Dockerfile.datasource-gateway | 32 +++ .../docker/compose.datasource-gateway.yml | 14 ++ 12 files changed, 556 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 configs/datasource-gateway/config.yaml create mode 100644 datasource-gateway/README.md create mode 100644 datasource-gateway/go.mod create mode 100644 datasource-gateway/go.sum create mode 100644 datasource-gateway/internal/api/handlers.go create mode 100644 datasource-gateway/internal/config/config.go create mode 100644 datasource-gateway/internal/logger/logger.go create mode 100644 datasource-gateway/main.go create mode 100644 deployments/docker/Dockerfile.datasource-gateway create mode 100644 deployments/docker/compose.datasource-gateway.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b4e018b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# Keep the Docker build context small for the Go-service images +# (config-manager, datasource-gateway). These builds only need the +# Makefile, Go sources, and configs — not JS deps, build artifacts, or VCS. +.git +.gitignore +**/node_modules +**/dist +bin +data +temp_path +*.log diff --git a/Makefile b/Makefile index d19580a..45585bc 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,23 @@ .PHONY: build clean test lint help # Default target: Build all services -build: build-cm +build: build-cm build-gateway # Build the config-manager service build-cm: @echo "Building config-manager service..." @cd config-manager && go build -o ../bin/config-manager . +# Build the datasource-gateway service +build-gateway: + @echo "Building datasource-gateway service..." + @cd datasource-gateway && go build -o ../bin/datasource-gateway . + +# Build the datasource-gateway service docker image (standalone compose project) +build-gateway-docker: build-gateway + @echo "Building datasource-gateway service docker image..." + @docker compose -f deployments/docker/compose.datasource-gateway.yml up --build -d + # Build the config-manager service docker image build-cm-docker: build-cm @echo "Building config-manager service docker image..." @@ -71,6 +81,8 @@ help: @echo " build - Build all services" @echo " build-cm - Build config-manager service" @echo " build-cm-docker - Build config-manager service docker image" + @echo " build-gateway - Build datasource-gateway service" + @echo " build-gateway-docker - Build datasource-gateway service docker image" @echo " build-plugin - Build cbmonitor grafana-app plugin" @echo " build-plugin-docker - Build cbmonitor grafana-app plugin docker image" @echo " reload-plugin - Rebuild plugin and restart container" diff --git a/configs/datasource-gateway/config.yaml b/configs/datasource-gateway/config.yaml new file mode 100644 index 0000000..80a65f3 --- /dev/null +++ b/configs/datasource-gateway/config.yaml @@ -0,0 +1,25 @@ +# datasource-gateway Service Configuration +server: + port: 8090 + host: "0.0.0.0" + +logging: + level: "info" + +# Upstream Prometheus-compatible store, used for the passthrough path +# (Prometheus-backed snapshots). +prometheus: + url: "http://localhost:9009/prometheus" + +# Couchbase access for snapshot metadata (routing + time windows) and +# metrics (the PromQL->SQL++ translation path). Clients are wired in a +# later task; these values are loaded now. +couchbase: + enabled: true + host: "localhost" + username: "Administrator" + password: "password" + metadata_bucket: "metadata" + metrics_bucket: "cbmonitor" + metrics_scope: "_default" + metrics_collection: "_default" diff --git a/datasource-gateway/README.md b/datasource-gateway/README.md new file mode 100644 index 0000000..60cbf39 --- /dev/null +++ b/datasource-gateway/README.md @@ -0,0 +1,61 @@ +# datasource-gateway + +A standalone, Prometheus-compatible query gateway for cbmonitor2. It is the +single datasource the Grafana frontend talks to for panel time-series: the +frontend always speaks PromQL, and the gateway hides where the data actually +lives. + +Per query it routes by snapshot: + +- **Prometheus-backed snapshots** — pass the PromQL straight through to the + upstream Prometheus/Mimir (zero-copy streaming). +- **Couchbase-backed snapshots** — translate the PromQL to SQL++ and execute it + against Couchbase (rate/irate/increase computed in Go; no Couchbase UDFs). +- **Overlap comparison** (`job=~"a|b"`) — fetch each snapshot over its own + window and shift timestamps to a shared `t=0` axis so they can be overlaid. + +Because the interface is a strict superset of the Prometheus HTTP API, a +deployment that doesn't need Couchbase or overlap can simply point its Grafana +Prometheus datasource straight at Prometheus and skip the gateway entirely. + +## Origin + +This service originates from the **[SyncedApp](https://github.com/m-tarhon/SyncedApp)** +repo — the original `proxyprometheus` reverse proxy that time-pads snapshots for +overlap comparison. `datasource-gateway` generalises that proxy into the unified +gateway described above. + +## Why a separate sidecar + +The gateway has no dependency on the Grafana plugin backend and keeps running across Grafana restarts. + +## Build & run + +```sh +# Binary (from the repo root) +make build-gateway +./bin/datasource-gateway --config configs/datasource-gateway/config.yaml + +# Docker (standalone compose project) +docker compose -f deployments/docker/compose.datasource-gateway.yml up --build -d +``` + +`/healthz` returns `200 {"status":"ok"}` once it's up. + +## Configuration + +Defaults live in [`configs/datasource-gateway/config.yaml`](../configs/datasource-gateway/config.yaml) +and can be overridden with `section.field=value` arguments (the container passes +them from `DSG_*` environment variables): + +```sh +./bin/datasource-gateway --config config.yaml server.port=8090 logging.level=debug +``` + +| Setting | Default | Notes | +|---|---|---| +| `server.port` | `8090` | Deliberately off the Prometheus (9090) / Mimir (9009) defaults so it can share a host. | +| `server.host` | `0.0.0.0` | | +| `logging.level` | `info` | `debug` / `info` / `warn` / `error` | +| `prometheus.url` | `http://localhost:9009/prometheus` | Upstream Prometheus-compatible store for the passthrough path. | +| `couchbase.*` | — | Connection + metadata/metrics buckets for routing and the SQL++ path. | diff --git a/datasource-gateway/go.mod b/datasource-gateway/go.mod new file mode 100644 index 0000000..cc801c2 --- /dev/null +++ b/datasource-gateway/go.mod @@ -0,0 +1,5 @@ +module github.com/couchbase/datasource-gateway + +go 1.24 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/datasource-gateway/go.sum b/datasource-gateway/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/datasource-gateway/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/datasource-gateway/internal/api/handlers.go b/datasource-gateway/internal/api/handlers.go new file mode 100644 index 0000000..99e0870 --- /dev/null +++ b/datasource-gateway/internal/api/handlers.go @@ -0,0 +1,33 @@ +package api + +import ( + "encoding/json" + "net/http" +) + +// Handler holds the gateway HTTP handlers. It is intentionally minimal for +// now; the Prometheus-compatible query routes (/api/v1/*) and the routing / +// translation dependencies are added in subsequent tasks. +type Handler struct{} + +// NewHandler constructs the gateway HTTP handler. +func NewHandler() *Handler { + return &Handler{} +} + +// Register wires the handler's routes onto the given mux. This is the seam +// the Prometheus-API routes attach to in a later task. +func (h *Handler) Register(mux *http.ServeMux) { + mux.HandleFunc("/healthz", h.Health) +} + +// Health reports service liveness. +func (h *Handler) Health(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) +} diff --git a/datasource-gateway/internal/config/config.go b/datasource-gateway/internal/config/config.go new file mode 100644 index 0000000..80ab977 --- /dev/null +++ b/datasource-gateway/internal/config/config.go @@ -0,0 +1,195 @@ +package config + +import ( + "fmt" + "os" + "reflect" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// Config holds the datasource-gateway service configuration. +// +// The gateway is a Prometheus-compatible sidecar: it serves a single +// Grafana Prometheus datasource, passing PromQL through to Prometheus for +// Prometheus-backed snapshots and (in later tasks) translating PromQL to +// SQL++ for Couchbase-backed snapshots. The Prometheus and Couchbase sections +// below are loaded now; their clients are wired in subsequent tasks. +type Config struct { + Server struct { + Port int `yaml:"port"` + Host string `yaml:"host"` + } `yaml:"server"` + Logging struct { + Level string `yaml:"level"` + } `yaml:"logging"` + // Prometheus is the upstream Prometheus-compatible store used for the + // passthrough path (Prometheus-backed snapshots). + Prometheus struct { + URL string `yaml:"url"` + } `yaml:"prometheus"` + // Couchbase access for snapshot metadata (routing/time-windows) and + // metrics (the PromQL->SQL++ translation path). + Couchbase struct { + Enabled bool `yaml:"enabled"` + Host string `yaml:"host"` + Username string `yaml:"username"` + Password string `yaml:"password"` + MetadataBucket string `yaml:"metadata_bucket"` + MetricsBucket string `yaml:"metrics_bucket"` + MetricsScope string `yaml:"metrics_scope"` + MetricsCollection string `yaml:"metrics_collection"` + } `yaml:"couchbase"` +} + +// LoadConfig loads configuration from file (if provided) over the built-in +// defaults, then applies any dot-notation flag overrides. +func LoadConfig(configPath string, flagOverrides map[string]string) (*Config, error) { + var config Config + setDefaults(&config) + + if len(configPath) > 0 { + if err := LoadConfigFromFile(&config, configPath); err != nil { + return nil, fmt.Errorf("failed to load config file: %w", err) + } + } + + if len(flagOverrides) > 0 { + if err := ApplyFlagOverrides(&config, flagOverrides); err != nil { + return nil, fmt.Errorf("failed to apply flag overrides: %w", err) + } + } + + return &config, nil +} + +// LoadConfigFromFile loads configuration from a YAML file. +func LoadConfigFromFile(config *Config, configPath string) error { + data, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } + + if err := yaml.Unmarshal(data, config); err != nil { + return fmt.Errorf("failed to parse config file: %w", err) + } + + return nil +} + +// ApplyFlagOverrides applies dot-notation overrides (e.g. "server.port=8090"). +func ApplyFlagOverrides(config *Config, overrides map[string]string) error { + for path, value := range overrides { + if err := setConfigValue(config, path, value); err != nil { + return fmt.Errorf("failed to set %s: %w", path, err) + } + } + return nil +} + +// setConfigValue sets a config value using a "section.field" path. +func setConfigValue(config *Config, path, value string) error { + parts := strings.Split(path, ".") + if len(parts) < 2 { + return fmt.Errorf("invalid path format: %s (expected format: section.field)", path) + } + + section := parts[0] + field := parts[1] + + configValue := reflect.ValueOf(config).Elem() + sectionField := configValue.FieldByName(strings.Title(section)) + if !sectionField.IsValid() { + return fmt.Errorf("unknown section: %s", section) + } + + if sectionField.Kind() != reflect.Struct { + return fmt.Errorf("section %s is not a struct", section) + } + + fieldValue := sectionField.FieldByName(strings.Title(field)) + if !fieldValue.IsValid() { + return fmt.Errorf("unknown field: %s in section: %s", field, section) + } + + if !fieldValue.CanSet() { + return fmt.Errorf("field %s in section %s cannot be set", field, section) + } + + if err := setFieldValue(fieldValue, value); err != nil { + return fmt.Errorf("failed to set %s.%s: %w", section, field, err) + } + + return nil +} + +// setFieldValue sets a field value with proper type conversion. +func setFieldValue(field reflect.Value, value string) error { + if field.Type() == reflect.TypeOf(time.Duration(0)) { + dur, err := time.ParseDuration(value) + if err != nil { + return fmt.Errorf("invalid duration value: %s", value) + } + field.Set(reflect.ValueOf(dur)) + return nil + } + switch field.Kind() { + case reflect.String: + field.SetString(value) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + intVal, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("invalid integer value: %s", value) + } + field.SetInt(intVal) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + uintVal, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return fmt.Errorf("invalid unsigned integer value: %s", value) + } + field.SetUint(uintVal) + case reflect.Bool: + boolVal, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid boolean value: %s", value) + } + field.SetBool(boolVal) + case reflect.Float32, reflect.Float64: + floatVal, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid float value: %s", value) + } + field.SetFloat(floatVal) + default: + return fmt.Errorf("unsupported field type: %s", field.Kind()) + } + return nil +} + +// setDefaults sets default values for the configuration. +func setDefaults(config *Config) { + // Server defaults. Port 8090 deliberately avoids the default + // Prometheus (9090) and Mimir (9009) ports so the gateway can run on + // the same host as either without colliding. + config.Server.Port = 8090 + config.Server.Host = "0.0.0.0" + + // Logging defaults + config.Logging.Level = "info" + + // Prometheus (upstream) defaults + config.Prometheus.URL = "http://localhost:9009/prometheus" + + // Couchbase defaults + config.Couchbase.Enabled = true + config.Couchbase.Host = "localhost" + config.Couchbase.Username = "Administrator" + config.Couchbase.Password = "password" + config.Couchbase.MetadataBucket = "metadata" + config.Couchbase.MetricsBucket = "cbmonitor" + config.Couchbase.MetricsScope = "_default" + config.Couchbase.MetricsCollection = "_default" +} diff --git a/datasource-gateway/internal/logger/logger.go b/datasource-gateway/internal/logger/logger.go new file mode 100644 index 0000000..a29de11 --- /dev/null +++ b/datasource-gateway/internal/logger/logger.go @@ -0,0 +1,69 @@ +package logger + +import ( + "log/slog" + "os" + "strings" +) + +var globalLogger *slog.Logger + +// InitLogger initializes the global logger with the specified log level. +func InitLogger(level string) { + var logLevel slog.Level + + switch strings.ToLower(level) { + case "debug": + logLevel = slog.LevelDebug + case "info": + logLevel = slog.LevelInfo + case "warn", "warning": + logLevel = slog.LevelWarn + case "error": + logLevel = slog.LevelError + default: + logLevel = slog.LevelInfo + } + + handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: logLevel, + }) + globalLogger = slog.New(handler) + + // Set as default logger + slog.SetDefault(globalLogger) +} + +// GetLogger returns the global logger instance. +func GetLogger() *slog.Logger { + if globalLogger == nil { + // Fallback to default if not initialized + InitLogger("info") + } + return globalLogger +} + +// Debug logs a debug message. +func Debug(msg string, args ...any) { + GetLogger().Debug(msg, args...) +} + +// Info logs an info message. +func Info(msg string, args ...any) { + GetLogger().Info(msg, args...) +} + +// Warn logs a warning message. +func Warn(msg string, args ...any) { + GetLogger().Warn(msg, args...) +} + +// Error logs an error message. +func Error(msg string, args ...any) { + GetLogger().Error(msg, args...) +} + +// With returns a logger with the given attributes. +func With(args ...any) *slog.Logger { + return GetLogger().With(args...) +} diff --git a/datasource-gateway/main.go b/datasource-gateway/main.go new file mode 100644 index 0000000..dc5faeb --- /dev/null +++ b/datasource-gateway/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/couchbase/datasource-gateway/internal/api" + "github.com/couchbase/datasource-gateway/internal/config" + "github.com/couchbase/datasource-gateway/internal/logger" +) + +const shutdownTimeout = 10 * time.Second + +func main() { + var configPath string + flag.StringVar(&configPath, "config", "", "Path to the configuration file") + flag.Parse() + + // Collect any remaining "section.field=value" args as dot-notation overrides. + flagOverrides := make(map[string]string) + for _, arg := range flag.Args() { + if strings.Contains(arg, "=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + flagName := strings.TrimLeft(parts[0], "-") + flagOverrides[flagName] = parts[1] + } + } + } + + cfg, err := config.LoadConfig(configPath, flagOverrides) + if err != nil { + logger.GetLogger().Error("Failed to load configuration", "error", err) + os.Exit(1) + } + + logger.InitLogger(cfg.Logging.Level) + + logger.Info("datasource-gateway starting...") + logger.Info("Configuration loaded", + "server_host", cfg.Server.Host, + "server_port", cfg.Server.Port, + "logging_level", cfg.Logging.Level, + "prometheus_url", cfg.Prometheus.URL, + "couchbase_enabled", cfg.Couchbase.Enabled, + "couchbase_host", cfg.Couchbase.Host, + "couchbase_metadata_bucket", cfg.Couchbase.MetadataBucket, + "couchbase_metrics_bucket", cfg.Couchbase.MetricsBucket, + ) + + handler := api.NewHandler() + + mux := http.NewServeMux() + handler.Register(mux) + + addr := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) + server := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } + + go func() { + logger.Info("Starting HTTP server", "address", addr) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("HTTP server failed", "error", err) + os.Exit(1) + } + }() + + logger.Info("datasource-gateway started") + + // Wait for an interrupt signal to gracefully shut down. + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + + logger.Info("Shutting down server...") + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := server.Shutdown(ctx); err != nil { + logger.Error("Server forced to shutdown", "error", err) + os.Exit(1) + } + + logger.Info("Server exited") +} diff --git a/deployments/docker/Dockerfile.datasource-gateway b/deployments/docker/Dockerfile.datasource-gateway new file mode 100644 index 0000000..b3fce72 --- /dev/null +++ b/deployments/docker/Dockerfile.datasource-gateway @@ -0,0 +1,32 @@ +FROM golang:1.24-alpine AS builder + +# Install make for building +RUN apk add --no-cache make + +# Set working directory to the project root +WORKDIR /app + +# Copy the entire project +COPY . . + +# Build the datasource-gateway service using the Makefile +RUN make build-gateway + +FROM alpine:latest + +ARG DSG_SERVER_PORT=8090 +ARG DSG_LOG_LEVEL=info + +RUN apk --no-cache add ca-certificates +WORKDIR /root/ + +# Copy the built binary and default config +COPY --from=builder /app/bin/datasource-gateway . +COPY --from=builder /app/configs/datasource-gateway/config.yaml ./config.yaml + +EXPOSE ${DSG_SERVER_PORT} + +ENV DSG_SERVER_PORT=${DSG_SERVER_PORT} +ENV DSG_LOG_LEVEL=${DSG_LOG_LEVEL} + +CMD ["sh", "-c", "./datasource-gateway --config /root/config.yaml server.port=$DSG_SERVER_PORT logging.level=$DSG_LOG_LEVEL"] diff --git a/deployments/docker/compose.datasource-gateway.yml b/deployments/docker/compose.datasource-gateway.yml new file mode 100644 index 0000000..05b220f --- /dev/null +++ b/deployments/docker/compose.datasource-gateway.yml @@ -0,0 +1,14 @@ +# Usage: docker compose -f deployments/docker/compose.datasource-gateway.yml up --build -d +name: datasource-gateway + +services: + datasource-gateway: + build: + context: ../.. + dockerfile: deployments/docker/Dockerfile.datasource-gateway + args: + - DSG_SERVER_PORT=${DSG_SERVER_PORT:-8090} + ports: + - "${DSG_SERVER_PORT:-8090}:${DSG_SERVER_PORT:-8090}" + environment: + - DSG_LOG_LEVEL=${DSG_LOG_LEVEL:-info} From 42ca53f434a754274ab6901b285e938a15d4380d Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Tue, 2 Jun 2026 23:20:36 +0100 Subject: [PATCH 02/18] gateway: Migrate promql builder from grafana backend --- datasource-gateway/go.mod | 21 +- datasource-gateway/go.sum | 166 +++++- datasource-gateway/internal/promql/parser.go | 113 ++++ datasource-gateway/internal/promql/planner.go | 210 ++++++++ .../internal/promql/promql_test.go | 86 +++ .../internal/promql/sqlbuilder.go | 319 +++++++++++ .../internal/promql/transformer.go | 510 ++++++++++++++++++ .../internal/querybuilder/querybuilder.go | 248 +++++++++ 8 files changed, 1671 insertions(+), 2 deletions(-) create mode 100644 datasource-gateway/internal/promql/parser.go create mode 100644 datasource-gateway/internal/promql/planner.go create mode 100644 datasource-gateway/internal/promql/promql_test.go create mode 100644 datasource-gateway/internal/promql/sqlbuilder.go create mode 100644 datasource-gateway/internal/promql/transformer.go create mode 100644 datasource-gateway/internal/querybuilder/querybuilder.go diff --git a/datasource-gateway/go.mod b/datasource-gateway/go.mod index cc801c2..e0fe5ce 100644 --- a/datasource-gateway/go.mod +++ b/datasource-gateway/go.mod @@ -1,5 +1,24 @@ module github.com/couchbase/datasource-gateway -go 1.24 +go 1.24.0 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dennwc/varint v1.0.0 // indirect + github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/prometheus v0.307.3 + go.uber.org/atomic v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect +) diff --git a/datasource-gateway/go.sum b/datasource-gateway/go.sum index a62c313..82de7c9 100644 --- a/datasource-gateway/go.sum +++ b/datasource-gateway/go.sum @@ -1,4 +1,168 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= +cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.8.4 h1:oXMa1VMQBVCyewMIOm3WQsnVd9FbKBtm8reqWRaXnHQ= +cloud.google.com/go/compute/metadata v0.8.4/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.12.0 h1:wL5IEG5zb7BVv1Kv0Xm92orq+5hB5Nipn3B5tn4Rqfk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.12.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= +github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I= +github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2/config v1.31.12 h1:pYM1Qgy0dKZLHX2cXslNacbcEFMkDMl+Bcj5ROuS6p8= +github.com/aws/aws-sdk-go-v2/config v1.31.12/go.mod h1:/MM0dyD7KSDPR+39p9ZNVKaHDLb9qnfDurvVS2KAhN8= +github.com/aws/aws-sdk-go-v2/credentials v1.18.16 h1:4JHirI4zp958zC026Sm+V4pSDwW4pwLefKrc0bF2lwI= +github.com/aws/aws-sdk-go-v2/credentials v1.18.16/go.mod h1:qQMtGx9OSw7ty1yLclzLxXCRbrkjWAM7JnObZjmCB7I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.9 h1:Mv4Bc0mWmv6oDuSWTKnk+wgeqPL5DRFu5bQL9BGPQ8Y= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.9/go.mod h1:IKlKfRppK2a1y0gy1yH6zD+yX5uplJ6UuPlgd48dJiQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 h1:se2vOWGD3dWQUtfn4wEjRQJb1HK1XsNIt825gskZ970= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9/go.mod h1:hijCGH2VfbZQxqCDN7bwz/4dzxV+hkyhjawAtdPWKZA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 h1:6RBnKZLkJM4hQ+kN6E7yWFveOTg8NLPHAkqrs4ZPlTU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9/go.mod h1:V9rQKRmK7AWuEsOMnHzKj8WyrIir1yUJbZxDuZLFvXI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD4WZudeEKZ9/iKpiT6cM1JyEROpXjOcdWv8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.6 h1:A1oRkiSQOWstGh61y4Wc/yQ04sqrQZr1Si/oAXj20/s= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.6/go.mod h1:5PfYspyCU5Vw1wNPsxi15LZovOnULudOQuVxphSflQA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.1 h1:5fm5RTONng73/QA73LhCNR7UT9RpFH3hR6HWL6bIgVY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.1/go.mod h1:xBEjWD13h+6nq+z4AkqSfSvqRKFgDIQeaMguAJndOWo= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.6 h1:p3jIvqYwUZgu/XYeI48bJxOhvm47hZb5HUQ0tn6Q9kA= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.6/go.mod h1:WtKK+ppze5yKPkZ0XwqIVWD4beCwv056ZbPQNoeHqM8= +github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= +github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= +github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= +github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI= +github.com/prometheus/common v0.67.1/go.mod h1:RpmT9v35q2Y+lsieQsdOh5sXZ6ajUGC8NjZAmr8vb0Q= +github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= +github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/prometheus v0.307.3 h1:zGIN3EpiKacbMatcUL2i6wC26eRWXdoXfNPjoBc2l34= +github.com/prometheus/prometheus v0.307.3/go.mod h1:sPbNW+KTS7WmzFIafC3Inzb6oZVaGLnSvwqTdz2jxRQ= +github.com/prometheus/sigv4 v0.2.1 h1:hl8D3+QEzU9rRmbKIRwMKRwaFGyLkbPdH5ZerglRHY0= +github.com/prometheus/sigv4 v0.2.1/go.mod h1:ySk6TahIlsR2sxADuHy4IBFhwEjRGGsfbbLGhFYFj6Q= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20250808145144-a408d31f581a h1:Y+7uR/b1Mw2iSXZ3G//1haIiSElDQZ8KWh0h+sZPG90= +golang.org/x/exp v0.0.0-20250808145144-a408d31f581a/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= +golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= +golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/api v0.250.0 h1:qvkwrf/raASj82UegU2RSDGWi/89WkLckn4LuO4lVXM= +google.golang.org/api v0.250.0/go.mod h1:Y9Uup8bDLJJtMzJyQnu+rLRJLA0wn+wTtc6vTlOvfXo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 h1:V1jCN2HBa8sySkR5vLcCSqJSTMv093Rw9EJefhQGP7M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/datasource-gateway/internal/promql/parser.go b/datasource-gateway/internal/promql/parser.go new file mode 100644 index 0000000..15c769b --- /dev/null +++ b/datasource-gateway/internal/promql/parser.go @@ -0,0 +1,113 @@ +package promql + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/prometheus/prometheus/promql/parser" +) + +// ParseQuery parses a PromQL query string into an AST +func ParseQuery(query string) (parser.Expr, error) { + expr, err := parser.ParseExpr(query) + if err != nil { + return nil, fmt.Errorf("failed to parse PromQL query: %w", err) + } + return expr, nil +} + +// QueryContext holds context for query execution +type QueryContext struct { + Context context.Context + Query string + Time time.Time + StartTime time.Time + EndTime time.Time + Step time.Duration + IsRange bool + SnapshotID string // Optional snapshot ID from context +} + +// ParseQueryContext parses query parameters into a QueryContext +func ParseQueryContext(query string, timeStr string, startStr, endStr string, stepStr string, snapshotID string) (*QueryContext, error) { + ctx := &QueryContext{ + Query: query, + SnapshotID: snapshotID, + } + + // Parse time (for instant queries) + if timeStr != "" { + t, err := parseTime(timeStr) + if err != nil { + return nil, fmt.Errorf("invalid time parameter: %w", err) + } + ctx.Time = t + ctx.IsRange = false + } + + // Parse time range (for range queries) + if startStr != "" && endStr != "" { + start, err := parseTime(startStr) + if err != nil { + return nil, fmt.Errorf("invalid start time: %w", err) + } + end, err := parseTime(endStr) + if err != nil { + return nil, fmt.Errorf("invalid end time: %w", err) + } + ctx.StartTime = start + ctx.EndTime = end + ctx.IsRange = true + ctx.Time = end // Use end time as the evaluation time + } + + // Parse step (for range queries) + if stepStr != "" { + step, err := parseDuration(stepStr) + if err != nil { + return nil, fmt.Errorf("invalid step parameter: %w", err) + } + ctx.Step = step + } + + // Default to current time if no time specified + if ctx.Time.IsZero() { + ctx.Time = time.Now() + } + + return ctx, nil +} + +// parseTime parses a time string (Unix timestamp or RFC3339) +func parseTime(timeStr string) (time.Time, error) { + // Try RFC3339 first (most common format from Prometheus API) + // This includes formats like: 2025-11-26T14:54:31.661792773Z + if t, err := time.Parse(time.RFC3339, timeStr); err == nil { + return t, nil + } + + // Try RFC3339Nano for nanosecond precision + if t, err := time.Parse(time.RFC3339Nano, timeStr); err == nil { + return t, nil + } + + // Try Unix timestamp (only if the entire string is numeric) + // Use strconv.ParseFloat which requires the entire string to be a number + if timestamp, err := strconv.ParseFloat(timeStr, 64); err == nil { + // Check if it's in seconds or milliseconds + if timestamp > 1e10 { + // Likely milliseconds + return time.Unix(0, int64(timestamp*1e6)), nil + } + return time.Unix(int64(timestamp), 0), nil + } + + return time.Time{}, fmt.Errorf("unable to parse time: %s (expected RFC3339 format or Unix timestamp)", timeStr) +} + +// parseDuration parses a duration string (e.g., "5m", "1h", "30s") +func parseDuration(durationStr string) (time.Duration, error) { + return time.ParseDuration(durationStr) +} diff --git a/datasource-gateway/internal/promql/planner.go b/datasource-gateway/internal/promql/planner.go new file mode 100644 index 0000000..7233ddd --- /dev/null +++ b/datasource-gateway/internal/promql/planner.go @@ -0,0 +1,210 @@ +package promql + +import ( + "fmt" + "strings" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql/parser" +) + +// QueryPlan represents an optimized query execution plan +type QueryPlan struct { + BaseExpr parser.Expr + SeriesQueries []SeriesQuery + Aggregation *AggregationPlan + Function *FunctionPlan +} + +// SeriesQuery represents a single series query to Couchbase +type SeriesQuery struct { + MetricName string + Snapshot string // Extracted from 'job' label in PromQL + Labels map[string]string // Additional labels to filter (including node, instance, etc.) +} + +// AggregationPlan represents aggregation operations +type AggregationPlan struct { + Operation string // sum, avg, max, min, count + By []string + Without []string +} + +// FunctionPlan represents function operations +type FunctionPlan struct { + Name string // rate, increase, irate, etc. + Range string // For range functions like rate[5m] + Parameter string // Optional parameter +} + +// PlanQuery creates an optimized query plan from a PromQL expression +func PlanQuery(expr parser.Expr, snapshotID string) (*QueryPlan, error) { + plan := &QueryPlan{ + BaseExpr: expr, + } + + // Walk the AST to extract series selectors and operations + err := parser.Walk(&queryPlanner{ + plan: plan, + snapshotID: snapshotID, + }, expr, nil) + + if err != nil { + return nil, fmt.Errorf("failed to plan query: %w", err) + } + + return plan, nil +} + +// queryPlanner implements parser.Visitor to walk the AST +type queryPlanner struct { + plan *QueryPlan + snapshotID string +} + +func (v *queryPlanner) Visit(node parser.Node, path []parser.Node) (parser.Visitor, error) { + switch n := node.(type) { + case *parser.VectorSelector: + // Extract series selector + seriesQuery, err := v.extractSeriesSelector(n) + if err != nil { + return nil, err + } + v.plan.SeriesQueries = append(v.plan.SeriesQueries, seriesQuery) + + case *parser.AggregateExpr: + // Extract aggregation + v.plan.Aggregation = &AggregationPlan{ + Operation: n.Op.String(), + By: v.extractLabels(n.Grouping), + Without: v.extractLabels(n.Grouping), // Simplified - would need to check Without flag + } + + case *parser.Call: + // Extract function call + v.plan.Function = &FunctionPlan{ + Name: n.Func.Name, + } + // Extract range parameter if it's a range function + if len(n.Args) > 0 { + if matrixSelector, ok := n.Args[0].(*parser.MatrixSelector); ok { + v.plan.Function.Range = matrixSelector.Range.String() + } + } + } + + return v, nil +} + +// extractSeriesSelector extracts series information from a vector selector +func (v *queryPlanner) extractSeriesSelector(selector *parser.VectorSelector) (SeriesQuery, error) { + query := SeriesQuery{ + MetricName: selector.Name, + Snapshot: v.snapshotID, // Will be overridden by 'job' label if present + Labels: make(map[string]string), + } + + // Extract labels from matchers + for _, matcher := range selector.LabelMatchers { + labelName := matcher.Name + labelValue := matcher.Value + + switch matcher.Type { + case labels.MatchEqual: + // Map known labels to query parameters + switch labelName { + case "job", "snapshot", "snapshot_id": + // 'job' label maps to snapshot in get_metric_for function + query.Snapshot = labelValue + default: + // Store as additional label filter (node, instance, bucket, etc.) + query.Labels[labelName] = labelValue + } + + case labels.MatchNotEqual: + // For !=, we'll need to filter in WHERE clause + query.Labels["!"+labelName] = labelValue + + case labels.MatchRegexp: + // For =~, we'll need regex matching in WHERE clause + query.Labels["=~"+labelName] = labelValue + + case labels.MatchNotRegexp: + // For !~, we'll need negative regex matching + query.Labels["!~"+labelName] = labelValue + } + } + + // Default snapshot if not provided (from job label or context) + if query.Snapshot == "" { + query.Snapshot = v.snapshotID + } + + return query, nil +} + +// extractLabels extracts label names from grouping +func (v *queryPlanner) extractLabels(grouping []string) []string { + return grouping +} + +// GetMetricName extracts the metric name from the expression +func GetMetricName(expr parser.Expr) string { + switch e := expr.(type) { + case *parser.VectorSelector: + return e.Name + case *parser.MatrixSelector: + return GetMetricName(e.VectorSelector) + case *parser.Call: + if len(e.Args) > 0 { + return GetMetricName(e.Args[0]) + } + } + return "" +} + +// HasMultipleSeries checks if the query involves multiple series +func (p *QueryPlan) HasMultipleSeries() bool { + return len(p.SeriesQueries) > 1 +} + +// ShouldBatch determines if queries should be batched +func (p *QueryPlan) ShouldBatch() bool { + // Batch if we have multiple series with the same metric name + if len(p.SeriesQueries) <= 1 { + return false + } + + // Check if all series have the same metric name + firstMetric := p.SeriesQueries[0].MetricName + for _, sq := range p.SeriesQueries[1:] { + if sq.MetricName != firstMetric { + return false + } + } + + // Batch if we have 5-20 series (adaptive threshold) + return len(p.SeriesQueries) >= 5 && len(p.SeriesQueries) <= 20 +} + +// GetBatchedQueries groups queries by metric name for batching +func (p *QueryPlan) GetBatchedQueries() map[string][]SeriesQuery { + batched := make(map[string][]SeriesQuery) + for _, sq := range p.SeriesQueries { + batched[sq.MetricName] = append(batched[sq.MetricName], sq) + } + return batched +} + +// String returns a human-readable representation of the plan +func (p *QueryPlan) String() string { + var parts []string + parts = append(parts, fmt.Sprintf("SeriesQueries: %d", len(p.SeriesQueries))) + if p.Aggregation != nil { + parts = append(parts, fmt.Sprintf("Aggregation: %s", p.Aggregation.Operation)) + } + if p.Function != nil { + parts = append(parts, fmt.Sprintf("Function: %s", p.Function.Name)) + } + return strings.Join(parts, ", ") +} diff --git a/datasource-gateway/internal/promql/promql_test.go b/datasource-gateway/internal/promql/promql_test.go new file mode 100644 index 0000000..b214a62 --- /dev/null +++ b/datasource-gateway/internal/promql/promql_test.go @@ -0,0 +1,86 @@ +package promql + +import ( + "strings" + "testing" +) + +// TestTranslateSmoke proves the relocated PromQL->SQL++ translator compiles +// and runs end-to-end (parse -> plan -> SQL) inside the gateway module. Full +// subset coverage and correctness are hardened in a later task (G7/G8). +func TestTranslateSmoke(t *testing.T) { + cases := []struct { + name string + query string + wantInSQL []string + }{ + { + name: "vector selector with job", + query: `kv_curr_items{job="snap-1"}`, + wantInSQL: []string{"kv_curr_items", "snap-1"}, + }, + { + name: "aggregation over rate", + query: `sum by (instance) (rate(kv_ops{job="snap-1",bucket="default"}[5m]))`, + wantInSQL: []string{"kv_ops"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx, err := ParseQueryContext(tc.query, "", "1700000000", "1700003600", "15s", "") + if err != nil { + t.Fatalf("ParseQueryContext: %v", err) + } + + expr, err := ParseQuery(tc.query) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + + plan, err := PlanQuery(expr, "") + if err != nil { + t.Fatalf("PlanQuery: %v", err) + } + if len(plan.SeriesQueries) == 0 { + t.Fatalf("expected at least one series query, got plan: %s", plan.String()) + } + + queries, err := NewSQLBuilder(plan, ctx).Build() + if err != nil { + t.Fatalf("Build: %v", err) + } + if len(queries) == 0 { + t.Fatal("expected at least one SQL++ query") + } + sql := strings.Join(queries, " ") + for _, want := range tc.wantInSQL { + if !strings.Contains(sql, want) { + t.Errorf("generated SQL missing %q\nSQL: %s", want, sql) + } + } + }) + } +} + +// TestExtractJobAsSnapshot verifies the `job` label is mapped to the snapshot +// the generated SQL filters on. +func TestExtractJobAsSnapshot(t *testing.T) { + expr, err := ParseQuery(`kv_ops{job="snap-42"}`) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + plan, err := PlanQuery(expr, "") + if err != nil { + t.Fatalf("PlanQuery: %v", err) + } + if len(plan.SeriesQueries) != 1 { + t.Fatalf("expected 1 series query, got %d", len(plan.SeriesQueries)) + } + if got := plan.SeriesQueries[0].Snapshot; got != "snap-42" { + t.Errorf("snapshot = %q, want %q", got, "snap-42") + } + if got := plan.SeriesQueries[0].MetricName; got != "kv_ops" { + t.Errorf("metric = %q, want %q", got, "kv_ops") + } +} diff --git a/datasource-gateway/internal/promql/sqlbuilder.go b/datasource-gateway/internal/promql/sqlbuilder.go new file mode 100644 index 0000000..da7c749 --- /dev/null +++ b/datasource-gateway/internal/promql/sqlbuilder.go @@ -0,0 +1,319 @@ +package promql + +import ( + "fmt" + "strings" + "time" + + "github.com/couchbase/datasource-gateway/internal/querybuilder" +) + +// SQLBuilder builds SQL++ queries from query plans +type SQLBuilder struct { + plan *QueryPlan + queryCtx *QueryContext + useBatching bool +} + +// NewSQLBuilder creates a new SQL++ query builder +func NewSQLBuilder(plan *QueryPlan, queryCtx *QueryContext) *SQLBuilder { + return &SQLBuilder{ + plan: plan, + queryCtx: queryCtx, + useBatching: plan.ShouldBatch(), + } +} + +// Build generates SQL++ query string(s) from the plan +func (b *SQLBuilder) Build() ([]string, error) { + if b.useBatching { + return b.buildBatchedQueries() + } + return b.buildIndividualQueries() +} + +// buildIndividualQueries builds separate queries for each series +func (b *SQLBuilder) buildIndividualQueries() ([]string, error) { + var queries []string + + for _, seriesQuery := range b.plan.SeriesQueries { + query, err := b.buildSeriesQuery(seriesQuery) + if err != nil { + return nil, err + } + queries = append(queries, query) + } + + return queries, nil +} + +// buildBatchedQueries builds batched queries using UNION ALL +func (b *SQLBuilder) buildBatchedQueries() ([]string, error) { + batched := b.plan.GetBatchedQueries() + var queries []string + + for _, seriesQueries := range batched { + if len(seriesQueries) == 1 { + // Single series, no need to batch + query, err := b.buildSeriesQuery(seriesQueries[0]) + if err != nil { + return nil, err + } + queries = append(queries, query) + continue + } + + // Build UNION ALL query + var unionParts []string + for _, sq := range seriesQueries { + part := b.buildSeriesQueryPart(sq) + unionParts = append(unionParts, part) + } + + // Combine with UNION ALL and add aggregation if needed + unionQuery := strings.Join(unionParts, " UNION ALL ") + + // If aggregation is needed, wrap in aggregation query + if b.plan.Aggregation != nil { + unionQuery = b.wrapWithAggregation(unionQuery) + } + + queries = append(queries, unionQuery) + } + + return queries, nil +} + +// buildSeriesQuery builds a SQL++ query for a single series +func (b *SQLBuilder) buildSeriesQuery(seriesQuery SeriesQuery) (string, error) { + baseQuery := b.buildSeriesQueryPart(seriesQuery) + + // Apply function transformation if needed + if b.plan.Function != nil { + baseQuery = b.applyFunction(baseQuery) + } + + // Apply aggregation if needed + if b.plan.Aggregation != nil { + baseQuery = b.wrapWithAggregation(baseQuery) + } + + return baseQuery, nil +} + +// buildSeriesQueryPart builds the base SQL++ query part for a series +func (b *SQLBuilder) buildSeriesQueryPart(seriesQuery SeriesQuery) string { + // Build metrics_filter function call + metricFilter := b.buildMetricFilter(seriesQuery) + + // Build time range + fromMillis := b.queryCtx.StartTime.UnixMilli() + toMillis := b.queryCtx.EndTime.UnixMilli() + if !b.queryCtx.IsRange { + // For instant queries, use a small range around the time (~30s) + fromMillis = b.queryCtx.Time.UnixMilli() - 30000 + toMillis = b.queryCtx.Time.UnixMilli() + 1000 + } + + // Build SELECT clause + selectClause := b.buildSelectClause(seriesQuery) + + // Build time range condition + // Note: label filters are already embedded in metricFilter (before UNNEST) + // Time filtering happens after UNNEST but before final output + timeCondition := fmt.Sprintf("d.time_millis >= %d AND d.time_millis <= %d", fromMillis, toMillis) + + query := fmt.Sprintf( + "SELECT %s FROM %s AS d WHERE %s", + selectClause, + metricFilter, + timeCondition, + ) + + return query +} + +// buildMetricFilter builds the query with all label conditions embedded BEFORE UNNEST +// This ensures optimal performance by filtering at the earliest possible point +func (b *SQLBuilder) buildMetricFilter(seriesQuery SeriesQuery) string { + // Build all label conditions including job/snapshot + labelConditions := b.buildLabelWhereClause(seriesQuery) + + // Construct query inline with conditions embedded - filters BEFORE UNNEST + baseQuery := fmt.Sprintf( + "SELECT t._t AS time_millis, MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels FROM cbmonitor._default._default AS d UNNEST _timeseries(d, {'ts_ranges':[0, 9223372036854775807]}) AS t WHERE d.metric_name = '%s'", + seriesQuery.MetricName, + ) + + if labelConditions != "" { + baseQuery = fmt.Sprintf("%s AND %s", baseQuery, labelConditions) + } + + return fmt.Sprintf("(%s)", baseQuery) +} + +// buildSelectClause builds the SELECT clause +func (b *SQLBuilder) buildSelectClause(seriesQuery SeriesQuery) string { + parts := []string{ + "d.time", + "d.`value`", + } + + // Add label fields if needed + if len(seriesQuery.Labels) > 0 || b.plan.Aggregation != nil { + // Include labels for grouping/aggregation + parts = append(parts, "d.labels") + } + + return strings.Join(parts, ", ") +} + +// buildLabelWhereClause builds the WHERE clause for label filters including job/snapshot +func (b *SQLBuilder) buildLabelWhereClause(seriesQuery SeriesQuery) string { + // Convert seriesQuery.Labels to LabelFilter slice for shared utility + var filters []querybuilder.LabelFilter + + // Check if job label is already in Labels (could be with operators like !=, =~, !~) + hasJobInLabels := false + for labelName := range seriesQuery.Labels { + // Remove operator prefixes to get actual label name + actualLabel := labelName + if strings.HasPrefix(labelName, "!") { + actualLabel = strings.TrimPrefix(labelName, "!") + } else if strings.HasPrefix(labelName, "=~") { + actualLabel = strings.TrimPrefix(labelName, "=~") + } else if strings.HasPrefix(labelName, "!~") { + actualLabel = strings.TrimPrefix(labelName, "!~") + } + if actualLabel == "job" || actualLabel == "snapshot" || actualLabel == "snapshot_id" { + hasJobInLabels = true + break + } + } + + // Add job/snapshot filter if not already present in Labels + if !hasJobInLabels { + snapshot := seriesQuery.Snapshot + if snapshot == "" { + snapshot = b.queryCtx.SnapshotID + } + if snapshot != "" { + // Add job label filter (snapshot is stored as job label) + filters = append(filters, querybuilder.LabelFilter{ + Name: "job", + Value: snapshot, + Op: "=", + }) + } + } + + // Process all label filters from Labels + for labelName, labelValue := range seriesQuery.Labels { + if strings.HasPrefix(labelName, "!") { + // NOT EQUAL + actualLabel := strings.TrimPrefix(labelName, "!") + filters = append(filters, querybuilder.LabelFilter{ + Name: actualLabel, + Value: labelValue, + Op: "!=", + }) + } else if strings.HasPrefix(labelName, "=~") { + // REGEX MATCH + actualLabel := strings.TrimPrefix(labelName, "=~") + filters = append(filters, querybuilder.LabelFilter{ + Name: actualLabel, + Value: labelValue, + Op: "=~", + }) + } else if strings.HasPrefix(labelName, "!~") { + // NOT REGEX MATCH + actualLabel := strings.TrimPrefix(labelName, "!~") + filters = append(filters, querybuilder.LabelFilter{ + Name: actualLabel, + Value: labelValue, + Op: "!~", + }) + } else { + // EQUAL - process all labels (job will be handled above if not in Labels) + filters = append(filters, querybuilder.LabelFilter{ + Name: labelName, + Value: labelValue, + Op: "=", + }) + } + } + + return querybuilder.BuildLabelWhereClauseFromFilters(filters) +} + +// applyFunction applies PromQL function transformations +func (b *SQLBuilder) applyFunction(query string) string { + if b.plan.Function == nil { + return query + } + + switch b.plan.Function.Name { + case "rate", "irate", "increase": + // These need to be calculated post-query for now + // We'll mark the query for post-processing + return query + default: + return query + } +} + +// wrapWithAggregation wraps query with aggregation +func (b *SQLBuilder) wrapWithAggregation(query string) string { + if b.plan.Aggregation == nil { + return query + } + + // Build GROUP BY clause + var groupBy []string + if len(b.plan.Aggregation.By) > 0 { + for _, label := range b.plan.Aggregation.By { + groupBy = append(groupBy, fmt.Sprintf("d.labels.%s", querybuilder.EscapeLabel(label))) + } + } + groupBy = append(groupBy, "d.time") // Always group by time + + // Build aggregation function + aggFunc := strings.ToUpper(b.plan.Aggregation.Operation) + if aggFunc == "" { + aggFunc = "SUM" + } + + // Wrap query + wrapped := fmt.Sprintf( + "SELECT time, %s(value) AS value %s FROM (%s) AS subq GROUP BY %s ORDER BY time", + aggFunc, + b.buildLabelSelect(groupBy), + query, + strings.Join(groupBy, ", "), + ) + + return wrapped +} + +// buildLabelSelect builds label selection for GROUP BY +func (b *SQLBuilder) buildLabelSelect(groupBy []string) string { + var labels []string + for _, gb := range groupBy { + if strings.HasPrefix(gb, "d.labels.") { + labels = append(labels, gb) + } + } + if len(labels) > 0 { + return ", " + strings.Join(labels, ", ") + } + return "" +} + +// GetTimeRange returns the time range for the query +func (b *SQLBuilder) GetTimeRange() (time.Time, time.Time) { + if b.queryCtx.IsRange { + return b.queryCtx.StartTime, b.queryCtx.EndTime + } + // For instant queries, return a small range + return b.queryCtx.Time.Add(-time.Second), b.queryCtx.Time.Add(time.Second) +} diff --git a/datasource-gateway/internal/promql/transformer.go b/datasource-gateway/internal/promql/transformer.go new file mode 100644 index 0000000..3f10c53 --- /dev/null +++ b/datasource-gateway/internal/promql/transformer.go @@ -0,0 +1,510 @@ +package promql + +import ( + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" +) + +// PrometheusResult represents a Prometheus API result +type PrometheusResult struct { + Status string `json:"status"` + Data ResultData `json:"data"` + Error string `json:"error,omitempty"` + ErrorType string `json:"errorType,omitempty"` + // Warnings carries non-fatal issues encountered while serving the query + // (e.g. some sub-queries failed but enough succeeded to return data). + // Matches the Prometheus HTTP API spec. + Warnings []string `json:"warnings,omitempty"` +} + +// ResultData represents the data portion of Prometheus response +type ResultData struct { + ResultType string `json:"resultType"` + Result []interface{} `json:"result"` +} + +// Sample represents a single time-value pair +type Sample struct { + Timestamp float64 `json:"timestamp"` // Unix timestamp in seconds + Value string `json:"value"` // String representation of float +} + +// Series represents a time series with labels and samples +type Series struct { + Metric map[string]string `json:"metric"` + Values []Sample `json:"values,omitempty"` // For range queries + Value Sample `json:"value,omitempty"` // For instant queries +} + +// QueryResult represents raw query results from Couchbase +type QueryResult struct { + Time string `json:"time"` + Value interface{} `json:"value"` + Labels map[string]interface{} `json:"labels,omitempty"` +} + +// TransformResults transforms Couchbase query results to Prometheus format +func TransformResults(results []QueryResult, plan *QueryPlan, queryCtx *QueryContext) (*PrometheusResult, error) { + if len(results) == 0 { + return &PrometheusResult{ + Status: "success", + Data: ResultData{ + ResultType: getResultType(plan, queryCtx), + Result: []interface{}{}, + }, + }, nil + } + + // Group results by labels to form series + seriesMap := make(map[string]*Series) + + for _, result := range results { + // Extract labels + labels := extractLabels(result, plan) + labelKey := buildLabelKey(labels) + + // Get or create series + series, exists := seriesMap[labelKey] + if !exists { + series = &Series{ + Metric: labels, + } + seriesMap[labelKey] = series + } + + // Parse time and value + timestamp, err := parseTimeFromResult(result.Time) + if err != nil { + return nil, fmt.Errorf("failed to parse time: %w", err) + } + + value, err := parseValue(result.Value) + if err != nil { + return nil, fmt.Errorf("failed to parse value: %w", err) + } + + // Create sample + sample := Sample{ + Timestamp: float64(timestamp.Unix()), + Value: formatValue(value), + } + + // Add to series + if queryCtx.IsRange { + series.Values = append(series.Values, sample) + } else { + series.Value = sample + } + } + + // Convert map to slice and sort + var seriesList []interface{} + for _, series := range seriesMap { + // Sort values by timestamp + if queryCtx.IsRange { + sort.Slice(series.Values, func(i, j int) bool { + return series.Values[i].Timestamp < series.Values[j].Timestamp + }) + } + seriesList = append(seriesList, series) + } + + // Apply function transformations + if plan.Function != nil { + seriesList, err := applyFunction(seriesList, plan.Function, queryCtx) + if err != nil { + return nil, fmt.Errorf("failed to apply function: %w", err) + } + // Update seriesList with transformed results + _ = seriesList // Will be used after implementing function application + } + + // Apply aggregation if needed + if plan.Aggregation != nil { + seriesList = applyAggregation(seriesList, plan.Aggregation) + } + + return &PrometheusResult{ + Status: "success", + Data: ResultData{ + ResultType: getResultType(plan, queryCtx), + Result: seriesList, + }, + }, nil +} + +// extractLabels extracts labels from query result +func extractLabels(result QueryResult, plan *QueryPlan) map[string]string { + labels := make(map[string]string) + + // Add metric name from plan + if len(plan.SeriesQueries) > 0 { + metricName := plan.SeriesQueries[0].MetricName + if metricName != "" { + labels["__name__"] = metricName + } + } + + // Extract labels from result + if result.Labels != nil { + for key, value := range result.Labels { + if strValue, ok := value.(string); ok { + labels[key] = strValue + } else if strValue := fmt.Sprintf("%v", value); strValue != "" { + labels[key] = strValue + } + } + } + + return labels +} + +// buildLabelKey creates a unique key for a label set +func buildLabelKey(labels map[string]string) string { + // Sort keys for consistent ordering + var keys []string + for k := range labels { + keys = append(keys, k) + } + sort.Strings(keys) + + var parts []string + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%s", k, labels[k])) + } + return fmt.Sprintf("{%s}", strings.Join(parts, ",")) +} + +// parseTimeFromResult parses time from result string +func parseTimeFromResult(timeStr string) (time.Time, error) { + // Try RFC3339 first + if t, err := time.Parse(time.RFC3339, timeStr); err == nil { + return t, nil + } + + // Try Unix timestamp + if timestamp, err := strconv.ParseFloat(timeStr, 64); err == nil { + if timestamp > 1e10 { + // Milliseconds + return time.Unix(0, int64(timestamp*1e6)), nil + } + return time.Unix(int64(timestamp), 0), nil + } + + return time.Time{}, fmt.Errorf("unable to parse time: %s", timeStr) +} + +// parseValue parses value from result +func parseValue(value interface{}) (float64, error) { + switch v := value.(type) { + case float64: + return v, nil + case int: + return float64(v), nil + case int64: + return float64(v), nil + case string: + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, fmt.Errorf("invalid value: %s", v) + } + return f, nil + default: + return 0, fmt.Errorf("unsupported value type: %T", value) + } +} + +// formatValue formats a float value as string (Prometheus format) +func formatValue(value float64) string { + // Handle special values + if math.IsNaN(value) { + return "NaN" + } + if math.IsInf(value, 1) { + return "+Inf" + } + if math.IsInf(value, -1) { + return "-Inf" + } + + // Format with appropriate precision + return strconv.FormatFloat(value, 'f', -1, 64) +} + +// getResultType determines the result type based on query +func getResultType(plan *QueryPlan, queryCtx *QueryContext) string { + if queryCtx.IsRange { + return "matrix" + } + return "vector" +} + +// applyFunction applies PromQL function transformations +func applyFunction(seriesList []interface{}, function *FunctionPlan, queryCtx *QueryContext) ([]interface{}, error) { + switch function.Name { + case "rate": + return applyRate(seriesList, function.Range) + case "irate": + return applyIRate(seriesList) + case "increase": + return applyIncrease(seriesList, function.Range) + default: + // Unknown function, return as-is + return seriesList, nil + } +} + +// applyRate calculates rate (per-second average rate) +func applyRate(seriesList []interface{}, rangeStr string) ([]interface{}, error) { + // Parse range duration (for future use in range-based rate calculation) + _, err := time.ParseDuration(rangeStr) + if err != nil && rangeStr != "" { + return nil, fmt.Errorf("invalid range: %w", err) + } + + // Apply rate to each series + var result []interface{} + for _, item := range seriesList { + series, ok := item.(*Series) + if !ok { + continue + } + + if len(series.Values) < 2 { + // Need at least 2 points for rate + continue + } + + // Calculate rate for each interval + newValues := make([]Sample, 0, len(series.Values)-1) + for i := 1; i < len(series.Values); i++ { + prev := series.Values[i-1] + curr := series.Values[i] + + prevVal, _ := strconv.ParseFloat(prev.Value, 64) + currVal, _ := strconv.ParseFloat(curr.Value, 64) + + timeDiff := curr.Timestamp - prev.Timestamp + if timeDiff <= 0 { + continue + } + + // Rate = (value change) / (time change in seconds) + rate := (currVal - prevVal) / timeDiff + + newValues = append(newValues, Sample{ + Timestamp: curr.Timestamp, + Value: formatValue(rate), + }) + } + + // Update series + newSeries := *series + newSeries.Values = newValues + result = append(result, &newSeries) + } + + return result, nil +} + +// applyIRate calculates instant rate (per-second rate from last two points) +func applyIRate(seriesList []interface{}) ([]interface{}, error) { + // Similar to rate but only uses last two points + return applyRate(seriesList, "") +} + +// applyIncrease calculates increase over time range +func applyIncrease(seriesList []interface{}, rangeStr string) ([]interface{}, error) { + // Similar to rate but returns absolute increase + // For now, use rate implementation + return applyRate(seriesList, rangeStr) +} + +// applyAggregation applies aggregation operations +func applyAggregation(seriesList []interface{}, agg *AggregationPlan) []interface{} { + // Group series by labels (excluding aggregation labels) + grouped := make(map[string][]*Series) + + for _, item := range seriesList { + series, ok := item.(*Series) + if !ok { + continue + } + + // Build group key + groupKey := buildAggregationKey(series.Metric, agg) + grouped[groupKey] = append(grouped[groupKey], series) + } + + // Apply aggregation to each group + var result []interface{} + for _, group := range grouped { + aggSeries := aggregateSeries(group, agg) + if aggSeries != nil { + result = append(result, aggSeries) + } + } + + return result +} + +// buildAggregationKey builds key for grouping +func buildAggregationKey(labels map[string]string, agg *AggregationPlan) string { + var keys []string + for k, v := range labels { + // Skip __name__ and aggregation-excluded labels + if k == "__name__" { + continue + } + // Include only labels in "by" or exclude those in "without" + keys = append(keys, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +// aggregateSeries aggregates multiple series into one +func aggregateSeries(seriesList []*Series, agg *AggregationPlan) *Series { + if len(seriesList) == 0 { + return nil + } + + // Use first series as base + result := &Series{ + Metric: make(map[string]string), + } + + // Copy labels from first series (excluding aggregation-excluded labels) + for k, v := range seriesList[0].Metric { + if k != "__name__" { + result.Metric[k] = v + } + } + + // Aggregate values + if len(seriesList[0].Values) > 0 { + // Range query - aggregate across series for each timestamp + result.Values = aggregateValues(seriesList, agg) + } else { + // Instant query - aggregate single values + result.Value = aggregateValue(seriesList, agg) + } + + return result +} + +// aggregateValues aggregates values across series for range queries +func aggregateValues(seriesList []*Series, agg *AggregationPlan) []Sample { + // Find all unique timestamps + timestampMap := make(map[float64]bool) + for _, series := range seriesList { + for _, sample := range series.Values { + timestampMap[sample.Timestamp] = true + } + } + + // Sort timestamps + var timestamps []float64 + for ts := range timestampMap { + timestamps = append(timestamps, ts) + } + sort.Float64s(timestamps) + + // Aggregate for each timestamp + var result []Sample + for _, ts := range timestamps { + var values []float64 + for _, series := range seriesList { + for _, sample := range series.Values { + if sample.Timestamp == ts { + if val, err := strconv.ParseFloat(sample.Value, 64); err == nil { + values = append(values, val) + } + break + } + } + } + + if len(values) > 0 { + aggValue := aggregateFloat(values, agg.Operation) + result = append(result, Sample{ + Timestamp: ts, + Value: formatValue(aggValue), + }) + } + } + + return result +} + +// aggregateValue aggregates single values for instant queries +func aggregateValue(seriesList []*Series, agg *AggregationPlan) Sample { + var values []float64 + var timestamp float64 + + for _, series := range seriesList { + if val, err := strconv.ParseFloat(series.Value.Value, 64); err == nil { + values = append(values, val) + if timestamp == 0 { + timestamp = series.Value.Timestamp + } + } + } + + aggValue := aggregateFloat(values, agg.Operation) + return Sample{ + Timestamp: timestamp, + Value: formatValue(aggValue), + } +} + +// aggregateFloat aggregates float values based on operation +func aggregateFloat(values []float64, operation string) float64 { + if len(values) == 0 { + return 0 + } + + switch strings.ToUpper(operation) { + case "SUM": + sum := 0.0 + for _, v := range values { + sum += v + } + return sum + case "AVG", "AVERAGE": + sum := 0.0 + for _, v := range values { + sum += v + } + return sum / float64(len(values)) + case "MAX": + max := values[0] + for _, v := range values { + if v > max { + max = v + } + } + return max + case "MIN": + min := values[0] + for _, v := range values { + if v < min { + min = v + } + } + return min + case "COUNT": + return float64(len(values)) + default: + // Default to sum + sum := 0.0 + for _, v := range values { + sum += v + } + return sum + } +} diff --git a/datasource-gateway/internal/querybuilder/querybuilder.go b/datasource-gateway/internal/querybuilder/querybuilder.go new file mode 100644 index 0000000..3315d15 --- /dev/null +++ b/datasource-gateway/internal/querybuilder/querybuilder.go @@ -0,0 +1,248 @@ +package querybuilder + +import ( + "fmt" + "strings" +) + +// LabelFilter represents a label filter condition +type LabelFilter struct { + Name string + Value string + Op string // "=", "!=", "=~", "!~" +} + +// BuildLabelWhereClause builds a WHERE clause for label filters +// Returns empty string if no filters, otherwise returns conditions joined with AND +func BuildLabelWhereClause(labelFilters map[string]string) string { + if len(labelFilters) == 0 { + return "" + } + + conditions := []string{} + for labelName, labelValue := range labelFilters { + // Basic SQL injection prevention - escape single quotes in value + escapedValue := strings.ReplaceAll(labelValue, "'", "''") + + // Special case: "cluster" matches against both cluster_uuid and cluster_name + if labelName == "cluster" { + condition := fmt.Sprintf(`(d.labels.%s = '%s' OR d.labels.%s = '%s')`, + EscapeLabel("cluster_uuid"), escapedValue, + EscapeLabel("cluster_name"), escapedValue) + conditions = append(conditions, condition) + continue + } + + // Escape label name and value for SQL injection prevention + escapedLabel := EscapeLabel(labelName) + conditions = append(conditions, fmt.Sprintf(`d.labels.%s = '%s'`, escapedLabel, escapedValue)) + } + + return strings.Join(conditions, " AND ") +} + +// BuildLabelWhereClauseFromFilters builds a WHERE clause from LabelFilter slice +// Supports different operators: =, !=, =~, !~ +func BuildLabelWhereClauseFromFilters(filters []LabelFilter) string { + if len(filters) == 0 { + return "" + } + + conditions := []string{} + for _, filter := range filters { + escapedLabel := EscapeLabel(filter.Name) + escapedValue := strings.ReplaceAll(filter.Value, "'", "''") + + var condition string + switch filter.Op { + case "!=": + condition = fmt.Sprintf(`d.labels.%s != '%s'`, escapedLabel, escapedValue) + case "=~": + condition = fmt.Sprintf(`d.labels.%s LIKE '%s'`, escapedLabel, escapedValue) + case "!~": + condition = fmt.Sprintf(`d.labels.%s NOT LIKE '%s'`, escapedLabel, escapedValue) + default: // "=" + condition = fmt.Sprintf(`d.labels.%s = '%s'`, escapedLabel, escapedValue) + } + conditions = append(conditions, condition) + } + + return strings.Join(conditions, " AND ") +} + +// EscapeLabel escapes label names for SQL++ by wrapping them in backticks +func EscapeLabel(label string) string { + return fmt.Sprintf("`%s`", label) +} + +// StripPortsFromTargets removes port numbers from target addresses +// "172.23.96.97:18091" -> "172.23.96.97" +// "172.23.96.97:8091" -> "172.23.96.97" +func StripPortsFromTargets(targets []string) []string { + stripped := make([]string, 0, len(targets)) + seen := make(map[string]struct{}) + + for _, target := range targets { + // Find the last colon (handles IPv6 addresses with brackets) + host := target + if idx := strings.LastIndex(target, ":"); idx != -1 { + host = target[:idx] + } + // Remove brackets from IPv6 addresses if present + host = strings.TrimPrefix(strings.TrimSuffix(host, "]"), "[") + + if host == "" { + continue + } + // Deduplicate + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + stripped = append(stripped, host) + } + } + return stripped +} + +// BuildInstanceInClause builds a WHERE clause condition for instance filtering +// Returns empty string if no instances, otherwise returns: d.labels.`instance` IN ['host1', 'host2'] +func BuildInstanceInClause(instances []string) string { + if len(instances) == 0 { + return "" + } + + quotedInstances := make([]string, len(instances)) + for i, inst := range instances { + // Escape single quotes + escaped := strings.ReplaceAll(inst, "'", "''") + quotedInstances[i] = fmt.Sprintf("'%s'", escaped) + } + + return fmt.Sprintf("d.labels.`instance` IN [%s]", strings.Join(quotedInstances, ", ")) +} + +// buildSelectClause builds a SELECT clause from field names, prefixing each with "d." +// Handles fields that may already have aliases (e.g., "time AS timestamp" -> "d.time AS timestamp") +func buildSelectClause(fields []string) string { + if len(fields) == 0 { + return "*" + } + + selectParts := []string{} + for _, field := range fields { + field = strings.TrimSpace(field) + if field == "" { + continue + } + + // Check if field already has "d." prefix + if strings.HasPrefix(field, "d.") { + selectParts = append(selectParts, field) + continue + } + + // Check if field has an alias (e.g., "time AS timestamp") + if strings.Contains(strings.ToUpper(field), " AS ") { + parts := strings.SplitN(field, " AS ", 2) + if len(parts) == 2 { + // Prefix the field name with d., keep the alias + selectParts = append(selectParts, fmt.Sprintf("d.`%s` AS `%s`", strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]))) + continue + } + } + + // Simple field name - just prefix with d. + selectParts = append(selectParts, fmt.Sprintf("d.`%s`", field)) + } + + return strings.Join(selectParts, ", ") +} + +// BuildMetricQuery builds a query with label conditions embedded BEFORE UNNEST for optimal performance +// metricName: name of the metric +// labelConditions: WHERE clause conditions for label filtering (e.g., "d.labels.job = 'snapshot-1' AND d.labels.node = 'node1'") +// selectFields: list of fields to select (e.g., []string{"time", "value"} or []string{"value"}) +// All label filters are embedded in the query to be applied BEFORE UNNEST for optimal performance +// Since SQL++ UDFs can't execute dynamic SQL from string parameters, we construct the query inline +func BuildMetricQuery(metricName, labelConditions string, selectFields []string) string { + // Construct query with conditions embedded - this ensures filtering happens BEFORE UNNEST + // This is more efficient than using a UDF with post-UNNEST filtering + baseQuery := fmt.Sprintf( + "SELECT t._t AS time_millis, MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels FROM cbmonitor._default._default AS d UNNEST _timeseries(d, {'ts_ranges':[0, 9223372036854775807]}) AS t WHERE d.metric_name = '%s'", + metricName, + ) + + if labelConditions != "" { + // Add label conditions - these filter BEFORE UNNEST for optimal performance + baseQuery = fmt.Sprintf("%s AND %s", baseQuery, labelConditions) + } + + // Build SELECT clause with d. prefix for each field + selectClause := buildSelectClause(selectFields) + + // Wrap in outer SELECT to get only the requested fields + return fmt.Sprintf("SELECT %s FROM (%s) AS d", selectClause, baseQuery) +} + +// BuildSnapshotQuery builds a query with all label conditions embedded BEFORE UNNEST +// metricName: name of the metric +// snapshotID: snapshot/job identifier (always included as d.labels.job filter) +// selectFields: list of fields to select (e.g., []string{"time", "value"} or []string{"value"}) +// additionalConditions: additional WHERE clause conditions for label filtering (beyond job) +// All filters are embedded to be applied BEFORE UNNEST for optimal performance +func BuildSnapshotQuery(metricName, snapshotID string, selectFields []string, additionalConditions string) string { + // Build base query with job filter embedded before UNNEST + baseQuery := fmt.Sprintf( + "SELECT t._t AS time_millis, MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels FROM cbmonitor._default._default AS d UNNEST _timeseries(d, {'ts_ranges':[0, 9223372036854775807]}) AS t WHERE d.metric_name = '%s' AND d.labels.job = '%s'", + metricName, + snapshotID, + ) + + // Add additional label conditions if provided + if additionalConditions != "" { + baseQuery = fmt.Sprintf("%s AND %s", baseQuery, additionalConditions) + } + + // Build SELECT clause with d. prefix for each field + selectClause := buildSelectClause(selectFields) + + // Wrap to apply time filtering from snapshot metadata + return fmt.Sprintf( + "SELECT %s FROM (%s) AS d JOIN metadata._default._default AS s ON KEYS '%s' WHERE d.time_millis >= STR_TO_MILLIS(s.ts_start) AND d.time_millis <= STR_TO_MILLIS(s.ts_end)", + selectClause, + baseQuery, + snapshotID, + ) +} + +// BuildPhaseQuery builds a query with all label conditions embedded BEFORE UNNEST +// metricName: name of the metric +// snapshotID: snapshot/job identifier (always included as d.labels.job filter) +// phaseName: name of the phase +// selectFields: list of fields to select (e.g., []string{"time", "value"} or []string{"value"}) +// additionalConditions: additional WHERE clause conditions for label filtering (beyond job) +// All filters are embedded to be applied BEFORE UNNEST for optimal performance +func BuildPhaseQuery(metricName, snapshotID, phaseName string, selectFields []string, additionalConditions string) string { + // Build base query with job filter embedded before UNNEST + baseQuery := fmt.Sprintf( + "SELECT t._t AS time_millis, MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels FROM cbmonitor._default._default AS d UNNEST _timeseries(d, {'ts_ranges':[0, 9223372036854775807]}) AS t WHERE d.metric_name = '%s' AND d.labels.job = '%s'", + metricName, + snapshotID, + ) + + // Add additional label conditions if provided + if additionalConditions != "" { + baseQuery = fmt.Sprintf("%s AND %s", baseQuery, additionalConditions) + } + + // Build SELECT clause with d. prefix for each field + selectClause := buildSelectClause(selectFields) + + // Wrap to apply phase time filtering from snapshot metadata + return fmt.Sprintf( + "SELECT %s FROM (%s) AS d JOIN metadata._default._default AS s ON KEYS '%s' UNNEST s.phases AS p WHERE p.label = '%s' AND d.time_millis >= STR_TO_MILLIS(p.ts_start) AND d.time_millis <= STR_TO_MILLIS(p.ts_end)", + selectClause, + baseQuery, + snapshotID, + phaseName, + ) +} From b27cc254637a7a811f04e1d9561ee4cc7e37b344 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Tue, 2 Jun 2026 23:41:00 +0100 Subject: [PATCH 03/18] gateway: Add Prometheus and couchbase clients Co-authored-by: --- datasource-gateway/go.mod | 30 ++- datasource-gateway/go.sum | 146 +++++++++++- datasource-gateway/internal/api/handlers.go | 60 ++++- .../internal/couchbase/couchbase.go | 219 ++++++++++++++++++ .../internal/prometheus/prometheus.go | 73 ++++++ datasource-gateway/main.go | 29 ++- 6 files changed, 543 insertions(+), 14 deletions(-) create mode 100644 datasource-gateway/internal/couchbase/couchbase.go create mode 100644 datasource-gateway/internal/prometheus/prometheus.go diff --git a/datasource-gateway/go.mod b/datasource-gateway/go.mod index e0fe5ce..6b4ad09 100644 --- a/datasource-gateway/go.mod +++ b/datasource-gateway/go.mod @@ -2,14 +2,40 @@ module github.com/couchbase/datasource-gateway go 1.24.0 -require gopkg.in/yaml.v3 v3.0.1 +require ( + github.com/couchbase/gocb/v2 v2.11.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + cloud.google.com/go/compute/metadata v0.8.4 // indirect + github.com/couchbase/gocbcore/v10 v10.8.1 // indirect + github.com/couchbase/gocbcoreps v0.1.4 // indirect + github.com/couchbase/goprotostellar v1.0.2 // indirect + github.com/couchbaselabs/gocbconnstr/v2 v2.0.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.44.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 // indirect + google.golang.org/grpc v1.75.1 // indirect +) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dennwc/varint v1.0.0 // indirect github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect - github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/datasource-gateway/go.sum b/datasource-gateway/go.sum index 82de7c9..7c05ebe 100644 --- a/datasource-gateway/go.sum +++ b/datasource-gateway/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= @@ -12,6 +14,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDo github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI= github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I= @@ -42,25 +45,59 @@ github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/couchbase/gocb/v2 v2.11.1 h1:xWDco7Qk/XSvGUjbUWRaXi0V35nsMijJnm4vHXN/rqY= +github.com/couchbase/gocb/v2 v2.11.1/go.mod h1:aSh1Cmd1sPRpYyiBD5iWPehPWaTVF/oYhrtOAITWb/4= +github.com/couchbase/gocbcore/v10 v10.8.1 h1:i4SnH0DH9APGC4GS2vS2m+3u08V7oJwviamOXdgAZOQ= +github.com/couchbase/gocbcore/v10 v10.8.1/go.mod h1:OWKfU9R5Nm5V3QZBtfdZl5qCfgxtxTqOgXiNr4pn9/c= +github.com/couchbase/gocbcoreps v0.1.4 h1:/iZVHMpuEw3lyNz9mIahMQffJOurl/opXyOGads/JbI= +github.com/couchbase/gocbcoreps v0.1.4/go.mod h1:hBFpDNPnRno6HH5cRXExhqXYRmTsFJlFHQx7vztcXPk= +github.com/couchbase/goprotostellar v1.0.2 h1:yoPbAL9sCtcyZ5e/DcU5PRMOEFaJrF9awXYu3VPfGls= +github.com/couchbase/goprotostellar v1.0.2/go.mod h1:5/yqVnZlW2/NSbAWu1hPJCFBEwjxgpe0PFFOlRixnp4= +github.com/couchbaselabs/gocaves/client v0.0.0-20250107114554-f96479220ae8 h1:MQfvw4BiLTuyR69FuA5Kex+tXUeLkH+/ucJfVL1/hkM= +github.com/couchbaselabs/gocaves/client v0.0.0-20250107114554-f96479220ae8/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY= +github.com/couchbaselabs/gocbconnstr/v2 v2.0.0 h1:HU9DlAYYWR69jQnLN6cpg0fh0hxW/8d5hnglCXXjW78= +github.com/couchbaselabs/gocbconnstr/v2 v2.0.0/go.mod h1:o7T431UOfFVHDNvMBUmUxpHnhivwv7BziUao/nMl81E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= @@ -71,14 +108,24 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -90,12 +137,16 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI= @@ -108,55 +159,144 @@ github.com/prometheus/prometheus v0.307.3 h1:zGIN3EpiKacbMatcUL2i6wC26eRWXdoXfNP github.com/prometheus/prometheus v0.307.3/go.mod h1:sPbNW+KTS7WmzFIafC3Inzb6oZVaGLnSvwqTdz2jxRQ= github.com/prometheus/sigv4 v0.2.1 h1:hl8D3+QEzU9rRmbKIRwMKRwaFGyLkbPdH5ZerglRHY0= github.com/prometheus/sigv4 v0.2.1/go.mod h1:ySk6TahIlsR2sxADuHy4IBFhwEjRGGsfbbLGhFYFj6Q= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20250808145144-a408d31f581a h1:Y+7uR/b1Mw2iSXZ3G//1haIiSElDQZ8KWh0h+sZPG90= golang.org/x/exp v0.0.0-20250808145144-a408d31f581a/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.250.0 h1:qvkwrf/raASj82UegU2RSDGWi/89WkLckn4LuO4lVXM= google.golang.org/api v0.250.0/go.mod h1:Y9Uup8bDLJJtMzJyQnu+rLRJLA0wn+wTtc6vTlOvfXo= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 h1:V1jCN2HBa8sySkR5vLcCSqJSTMv093Rw9EJefhQGP7M= google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= diff --git a/datasource-gateway/internal/api/handlers.go b/datasource-gateway/internal/api/handlers.go index 99e0870..4fb523f 100644 --- a/datasource-gateway/internal/api/handlers.go +++ b/datasource-gateway/internal/api/handlers.go @@ -1,18 +1,37 @@ package api import ( + "context" "encoding/json" "net/http" + "time" ) -// Handler holds the gateway HTTP handlers. It is intentionally minimal for -// now; the Prometheus-compatible query routes (/api/v1/*) and the routing / -// translation dependencies are added in subsequent tasks. -type Handler struct{} +// couchbaseHealth is the slice of the Couchbase client the health endpoint +// needs. Behind an interface so the api package stays decoupled from the +// concrete client (and testable). +type couchbaseHealth interface { + Enabled() bool + Ready() bool +} + +// prometheusHealth is the slice of the Prometheus client the health endpoint +// needs. +type prometheusHealth interface { + URL() string + Reachable(ctx context.Context) bool +} + +// Handler holds the gateway HTTP handlers. The Prometheus-compatible query +// routes (/api/v1/*) attach to this in a later task. +type Handler struct { + couchbase couchbaseHealth + prometheus prometheusHealth +} // NewHandler constructs the gateway HTTP handler. -func NewHandler() *Handler { - return &Handler{} +func NewHandler(couchbase couchbaseHealth, prometheus prometheusHealth) *Handler { + return &Handler{couchbase: couchbase, prometheus: prometheus} } // Register wires the handler's routes onto the given mux. This is the seam @@ -21,13 +40,38 @@ func (h *Handler) Register(mux *http.ServeMux) { mux.HandleFunc("/healthz", h.Health) } -// Health reports service liveness. +// Health reports liveness (always HTTP 200) with per-dependency status in the +// body, mirroring the plugin's connection-healthcheck convention. func (h *Handler) Health(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } + + ctx, cancel := context.WithTimeout(r.Context(), 4*time.Second) + defer cancel() + + resp := map[string]any{ + "status": "ok", + "couchbase": couchbaseStatus(h.couchbase), + "prometheus": prometheusStatus(ctx, h.prometheus), + } + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) + _ = json.NewEncoder(w).Encode(resp) +} + +func couchbaseStatus(cb couchbaseHealth) map[string]any { + if cb == nil || !cb.Enabled() { + return map[string]any{"enabled": false} + } + return map[string]any{"enabled": true, "ready": cb.Ready()} +} + +func prometheusStatus(ctx context.Context, prom prometheusHealth) map[string]any { + if prom == nil { + return map[string]any{"configured": false} + } + return map[string]any{"url": prom.URL(), "reachable": prom.Reachable(ctx)} } diff --git a/datasource-gateway/internal/couchbase/couchbase.go b/datasource-gateway/internal/couchbase/couchbase.go new file mode 100644 index 0000000..0bc25be --- /dev/null +++ b/datasource-gateway/internal/couchbase/couchbase.go @@ -0,0 +1,219 @@ +package couchbase + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/couchbase/gocb/v2" +) + +const queryTimeout = 30 * time.Second + +// Config holds the Couchbase connection settings the gateway needs. +type Config struct { + Enabled bool + ConnectionString string // e.g. "couchbase://localhost" + Username string + Password string + MetadataBucket string // snapshot metadata documents (default collection) + MetricsBucket string // metrics keyspace + MetricsScope string + MetricsCollection string +} + +// Metadata is the subset of a snapshot's metadata document the gateway uses +// for routing and time-window resolution. +type Metadata struct { + ID string + TSStart string + TSEnd string + Store string // optional routing hint: "couchbase" | "prometheus" + Products []string + Phases []Phase +} + +// Phase is a labelled sub-window within a snapshot. +type Phase struct { + Label string + TSStart string + TSEnd string +} + +// Client is the gateway's Couchbase client. It connects without blocking +// startup (readiness is verified in the background) and degrades cleanly: +// when disabled or unavailable, Ready() is false and queries return a clear +// error instead of panicking. +type Client struct { + cfg Config + cluster *gocb.Cluster + metadataColl *gocb.Collection + metricsScope *gocb.Scope + ready atomic.Bool +} + +// New constructs the client. When cfg.Enabled is false it returns a disabled +// stub. On a connect error it returns a degraded (not-ready) client together +// with the error, so the caller can log and keep serving the Prometheus path. +func New(cfg Config) (*Client, error) { + c := &Client{cfg: cfg} + if !cfg.Enabled { + return c, nil + } + + cluster, err := gocb.Connect(cfg.ConnectionString, gocb.ClusterOptions{ + Authenticator: gocb.PasswordAuthenticator{ + Username: cfg.Username, + Password: cfg.Password, + }, + }) + if err != nil { + return c, fmt.Errorf("connect to Couchbase: %w", err) + } + + scopeName := cfg.MetricsScope + if scopeName == "" { + scopeName = "_default" + } + + metaBucket := cluster.Bucket(cfg.MetadataBucket) + metricsBucket := cluster.Bucket(cfg.MetricsBucket) + + c.cluster = cluster + c.metadataColl = metaBucket.DefaultCollection() + c.metricsScope = metricsBucket.Scope(scopeName) + + // Verify readiness in the background. gocb.Connect/Bucket do no network + // I/O; only WaitUntilReady blocks. Keeping it off the startup path means + // the gateway serves /healthz immediately and queries queue until ready. + go c.waitUntilReady(metaBucket, metricsBucket) + + return c, nil +} + +func (c *Client) waitUntilReady(buckets ...*gocb.Bucket) { + seen := make(map[string]bool) + for _, b := range buckets { + if b == nil || seen[b.Name()] { + continue + } + seen[b.Name()] = true + if err := b.WaitUntilReady(30*time.Second, nil); err != nil { + // Stays not-ready; the health endpoint reflects it. + return + } + } + c.ready.Store(true) +} + +// Enabled reports whether the Couchbase path is configured on. +func (c *Client) Enabled() bool { return c.cfg.Enabled } + +// Ready reports whether the buckets have become reachable. +func (c *Client) Ready() bool { return c.ready.Load() } + +// GetSnapshotMetadata fetches and parses the subset of a snapshot's metadata +// document used for routing and time-window resolution. +func (c *Client) GetSnapshotMetadata(ctx context.Context, snapshotID string) (*Metadata, error) { + if c.metadataColl == nil { + return nil, fmt.Errorf("couchbase metadata is unavailable") + } + res, err := c.metadataColl.Get(snapshotID, &gocb.GetOptions{Context: ctx, Timeout: queryTimeout}) + if err != nil { + if err == gocb.ErrDocumentNotFound { + return nil, fmt.Errorf("snapshot not found: %s", snapshotID) + } + return nil, fmt.Errorf("fetch snapshot metadata: %w", err) + } + + var raw map[string]interface{} + if err := res.Content(&raw); err != nil { + return nil, fmt.Errorf("decode snapshot metadata: %w", err) + } + + md := &Metadata{ID: snapshotID} + if v, ok := raw["ts_start"].(string); ok { + md.TSStart = v + } + if v, ok := raw["ts_end"].(string); ok { + md.TSEnd = v + } + if v, ok := raw["store"].(string); ok { + md.Store = v + } + if arr, ok := raw["products"].([]interface{}); ok { + for _, p := range arr { + if s, ok := p.(string); ok { + md.Products = append(md.Products, s) + } + } + } + if arr, ok := raw["phases"].([]interface{}); ok { + for _, p := range arr { + pm, ok := p.(map[string]interface{}) + if !ok { + continue + } + ph := Phase{} + if s, ok := pm["label"].(string); ok { + ph.Label = s + } + if s, ok := pm["ts_start"].(string); ok { + ph.TSStart = s + } + if s, ok := pm["ts_end"].(string); ok { + ph.TSEnd = s + } + md.Phases = append(md.Phases, ph) + } + } + return md, nil +} + +// ExecuteQuery runs a SQL++ statement under the configured metrics scope. +func (c *Client) ExecuteQuery(ctx context.Context, query string) ([]map[string]interface{}, error) { + if c.metricsScope == nil { + return nil, fmt.Errorf("couchbase metrics is unavailable") + } + results, err := c.metricsScope.Query(query, &gocb.QueryOptions{Context: ctx, Timeout: queryTimeout}) + if err != nil { + return nil, fmt.Errorf("execute query: %w", err) + } + defer results.Close() + + var rows []map[string]interface{} + for results.Next() { + var row map[string]interface{} + if err := results.Row(&row); err != nil { + continue + } + rows = append(rows, row) + } + if err := results.Err(); err != nil { + return nil, fmt.Errorf("query error: %w", err) + } + return rows, nil +} + +// Close releases the cluster connection. +func (c *Client) Close() error { + if c.cluster != nil { + return c.cluster.Close(nil) + } + return nil +} + +// BuildConnectionString prepends the couchbase:// scheme to a bare host. A +// value that already carries a scheme (couchbase://, couchbases://) is +// returned unchanged. +func BuildConnectionString(host string) string { + if host == "" { + return "" + } + if strings.Contains(host, "://") { + return host + } + return "couchbase://" + host +} diff --git a/datasource-gateway/internal/prometheus/prometheus.go b/datasource-gateway/internal/prometheus/prometheus.go new file mode 100644 index 0000000..86ae920 --- /dev/null +++ b/datasource-gateway/internal/prometheus/prometheus.go @@ -0,0 +1,73 @@ +package prometheus + +import ( + "context" + "net" + "net/http" + "strings" + "time" +) + +// Client is a thin HTTP client for an upstream Prometheus-compatible store. +// It backs the passthrough path; the connection-pooling (keep-alive) +// transport keeps the added gateway hop's overhead negligible versus +// querying the upstream directly. +type Client struct { + baseURL string + http *http.Client +} + +// New builds a client for the given upstream base URL with a keep-alive +// transport. +func New(baseURL string) *Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + http: &http.Client{Transport: transport, Timeout: 30 * time.Second}, + } +} + +// URL returns the upstream base URL. +func (c *Client) URL() string { return c.baseURL } + +// HTTPClient exposes the keep-alive client for the passthrough path (added in +// a later task). +func (c *Client) HTTPClient() *http.Client { return c.http } + +// Reachable probes the upstream with a trivial instant query. Used by the +// health endpoint; returns false on any error or non-200 response. +func (c *Client) Reachable(ctx context.Context) bool { + if c.baseURL == "" { + return false + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/query?query=1", nil) + if err != nil { + return false + } + resp, err := c.http.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK +} + +// Close releases idle connections. +func (c *Client) Close() error { + c.http.CloseIdleConnections() + return nil +} diff --git a/datasource-gateway/main.go b/datasource-gateway/main.go index dc5faeb..3c34a95 100644 --- a/datasource-gateway/main.go +++ b/datasource-gateway/main.go @@ -13,7 +13,9 @@ import ( "github.com/couchbase/datasource-gateway/internal/api" "github.com/couchbase/datasource-gateway/internal/config" + "github.com/couchbase/datasource-gateway/internal/couchbase" "github.com/couchbase/datasource-gateway/internal/logger" + "github.com/couchbase/datasource-gateway/internal/prometheus" ) const shutdownTimeout = 10 * time.Second @@ -55,7 +57,27 @@ func main() { "couchbase_metrics_bucket", cfg.Couchbase.MetricsBucket, ) - handler := api.NewHandler() + // Upstream Prometheus client (passthrough path) — always constructed. + promClient := prometheus.New(cfg.Prometheus.URL) + + // Couchbase client (metadata + metrics). Non-blocking: a connect error or + // an unreachable cluster degrades to the Prometheus-only path rather than + // failing startup. + cbClient, cbErr := couchbase.New(couchbase.Config{ + Enabled: cfg.Couchbase.Enabled, + ConnectionString: couchbase.BuildConnectionString(cfg.Couchbase.Host), + Username: cfg.Couchbase.Username, + Password: cfg.Couchbase.Password, + MetadataBucket: cfg.Couchbase.MetadataBucket, + MetricsBucket: cfg.Couchbase.MetricsBucket, + MetricsScope: cfg.Couchbase.MetricsScope, + MetricsCollection: cfg.Couchbase.MetricsCollection, + }) + if cbErr != nil { + logger.Error("Couchbase client init degraded; serving Prometheus path only", "error", cbErr) + } + + handler := api.NewHandler(cbClient, promClient) mux := http.NewServeMux() handler.Register(mux) @@ -90,5 +112,10 @@ func main() { os.Exit(1) } + _ = promClient.Close() + if err := cbClient.Close(); err != nil { + logger.Error("Error closing Couchbase client", "error", err) + } + logger.Info("Server exited") } From 7effe4381c698ed017a27acfc72432ac0e099e23 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Tue, 2 Jun 2026 23:58:00 +0100 Subject: [PATCH 04/18] gateway: Add prometheus http API support Co-authored-by: --- datasource-gateway/internal/api/handlers.go | 28 +++++--- .../internal/prometheus/prometheus.go | 63 +++++++++++++++-- .../internal/prometheus/prometheus_test.go | 70 +++++++++++++++++++ 3 files changed, 146 insertions(+), 15 deletions(-) create mode 100644 datasource-gateway/internal/prometheus/prometheus_test.go diff --git a/datasource-gateway/internal/api/handlers.go b/datasource-gateway/internal/api/handlers.go index 4fb523f..48d9472 100644 --- a/datasource-gateway/internal/api/handlers.go +++ b/datasource-gateway/internal/api/handlers.go @@ -15,29 +15,37 @@ type couchbaseHealth interface { Ready() bool } -// prometheusHealth is the slice of the Prometheus client the health endpoint -// needs. -type prometheusHealth interface { +// prometheusGateway is the slice of the Prometheus client the gateway needs: +// health probing plus the reverse-proxy handler for the passthrough path. +type prometheusGateway interface { URL() string Reachable(ctx context.Context) bool + ReverseProxy() http.Handler } -// Handler holds the gateway HTTP handlers. The Prometheus-compatible query -// routes (/api/v1/*) attach to this in a later task. +// Handler holds the gateway HTTP handlers: the health endpoint and the +// Prometheus-compatible API surface (/api/v1/*), forwarded to the upstream by +// a streaming reverse proxy. type Handler struct { couchbase couchbaseHealth - prometheus prometheusHealth + prometheus prometheusGateway } // NewHandler constructs the gateway HTTP handler. -func NewHandler(couchbase couchbaseHealth, prometheus prometheusHealth) *Handler { +func NewHandler(couchbase couchbaseHealth, prometheus prometheusGateway) *Handler { return &Handler{couchbase: couchbase, prometheus: prometheus} } -// Register wires the handler's routes onto the given mux. This is the seam -// the Prometheus-API routes attach to in a later task. +// Register wires the handler's routes onto the given mux: the gateway's own +// /healthz, plus the Prometheus-compatible API under /api/v1/ (forwarded to +// the upstream by a streaming reverse proxy). Per-snapshot routing and the +// Couchbase translation path attach to more specific /api/v1/query[_range] +// patterns in later tasks. func (h *Handler) Register(mux *http.ServeMux) { mux.HandleFunc("/healthz", h.Health) + if h.prometheus != nil { + mux.Handle("/api/v1/", h.prometheus.ReverseProxy()) + } } // Health reports liveness (always HTTP 200) with per-dependency status in the @@ -69,7 +77,7 @@ func couchbaseStatus(cb couchbaseHealth) map[string]any { return map[string]any{"enabled": true, "ready": cb.Ready()} } -func prometheusStatus(ctx context.Context, prom prometheusHealth) map[string]any { +func prometheusStatus(ctx context.Context, prom prometheusGateway) map[string]any { if prom == nil { return map[string]any{"configured": false} } diff --git a/datasource-gateway/internal/prometheus/prometheus.go b/datasource-gateway/internal/prometheus/prometheus.go index 86ae920..a6f7833 100644 --- a/datasource-gateway/internal/prometheus/prometheus.go +++ b/datasource-gateway/internal/prometheus/prometheus.go @@ -2,8 +2,12 @@ package prometheus import ( "context" + "encoding/json" + "fmt" "net" "net/http" + "net/http/httputil" + "net/url" "strings" "time" ) @@ -15,10 +19,11 @@ import ( type Client struct { baseURL string http *http.Client + proxy http.Handler } // New builds a client for the given upstream base URL with a keep-alive -// transport. +// transport and a reverse proxy to the upstream's Prometheus API. func New(baseURL string) *Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -33,19 +38,37 @@ func New(baseURL string) *Client { TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } - return &Client{ - baseURL: strings.TrimRight(baseURL, "/"), + trimmed := strings.TrimRight(baseURL, "/") + c := &Client{ + baseURL: trimmed, http: &http.Client{Transport: transport, Timeout: 30 * time.Second}, } + if u, err := url.Parse(trimmed); err == nil && u.Scheme != "" && u.Host != "" { + c.proxy = newReverseProxy(u, transport) + } + return c } // URL returns the upstream base URL. func (c *Client) URL() string { return c.baseURL } -// HTTPClient exposes the keep-alive client for the passthrough path (added in -// a later task). +// HTTPClient exposes the keep-alive client for callers that need to issue +// their own upstream requests. func (c *Client) HTTPClient() *http.Client { return c.http } +// ReverseProxy returns a streaming reverse-proxy handler to the upstream +// Prometheus API. Requests (e.g. /api/v1/*) are forwarded as-is over the +// keep-alive transport. Later tasks add the snapshot time-window rewrite and +// per-snapshot routing in front of this. +func (c *Client) ReverseProxy() http.Handler { + if c.proxy == nil { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writePromError(w, http.StatusBadGateway, "bad_gateway", "upstream Prometheus URL is not configured") + }) + } + return c.proxy +} + // Reachable probes the upstream with a trivial instant query. Used by the // health endpoint; returns false on any error or non-200 response. func (c *Client) Reachable(ctx context.Context) bool { @@ -71,3 +94,33 @@ func (c *Client) Close() error { c.http.CloseIdleConnections() return nil } + +// newReverseProxy builds a single-host reverse proxy to target over the given +// transport. The request path is joined onto the target's path (so an upstream +// behind a prefix like /prometheus works), and upstream failures are rendered +// as a Prometheus error envelope rather than plain text. +func newReverseProxy(target *url.URL, transport http.RoundTripper) *httputil.ReverseProxy { + rp := httputil.NewSingleHostReverseProxy(target) + rp.Transport = transport + defaultDirector := rp.Director + rp.Director = func(req *http.Request) { + defaultDirector(req) + req.Host = target.Host + } + rp.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { + writePromError(w, http.StatusBadGateway, "bad_gateway", fmt.Sprintf("upstream request failed: %v", err)) + } + return rp +} + +// writePromError writes a Prometheus-API error envelope so clients (including +// Grafana's Prometheus datasource) get a parseable response on failure. +func writePromError(w http.ResponseWriter, code int, errorType, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{ + "status": "error", + "errorType": errorType, + "error": msg, + }) +} diff --git a/datasource-gateway/internal/prometheus/prometheus_test.go b/datasource-gateway/internal/prometheus/prometheus_test.go new file mode 100644 index 0000000..9caa7eb --- /dev/null +++ b/datasource-gateway/internal/prometheus/prometheus_test.go @@ -0,0 +1,70 @@ +package prometheus + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestReverseProxyForwardsToUpstream verifies the /api/v1 surface forwards +// requests to the upstream Prometheus, preserving the path (joined onto the +// upstream's prefix) and query, and relaying the response. +func TestReverseProxyForwardsToUpstream(t *testing.T) { + var gotPath, gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotQuery = r.URL.RawQuery + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":{"resultType":"scalar","result":[1700000000,"1"]}}`) + })) + defer upstream.Close() + + // Upstream behind a /prometheus prefix, mimicking Mimir. + c := New(upstream.URL + "/prometheus") + + req := httptest.NewRequest(http.MethodGet, "/api/v1/query?query=1", nil) + rec := httptest.NewRecorder() + c.ReverseProxy().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if want := "/prometheus/api/v1/query"; gotPath != want { + t.Errorf("upstream path = %q, want %q", gotPath, want) + } + if !strings.Contains(gotQuery, "query=1") { + t.Errorf("upstream query = %q, want it to contain query=1", gotQuery) + } + var resp map[string]any + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode body: %v", err) + } + if resp["status"] != "success" { + t.Errorf("relayed status = %v, want success", resp["status"]) + } +} + +// TestReverseProxyUpstreamDownReturnsPromError verifies that an unreachable +// upstream produces a Prometheus error envelope (not plain text), so Grafana's +// Prometheus datasource gets a parseable response. +func TestReverseProxyUpstreamDownReturnsPromError(t *testing.T) { + c := New("http://127.0.0.1:0/prometheus") // unroutable + + req := httptest.NewRequest(http.MethodGet, "/api/v1/query?query=1", nil) + rec := httptest.NewRecorder() + c.ReverseProxy().ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502", rec.Code) + } + var resp map[string]string + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode body: %v", err) + } + if resp["status"] != "error" { + t.Errorf("status = %q, want error", resp["status"]) + } +} From cc46cd5e60363eadf07e4f2a1e5caa3c0f0bff9f Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Wed, 3 Jun 2026 09:12:44 +0100 Subject: [PATCH 05/18] gateway: Ensure passthrough rewrites snapshots time window --- datasource-gateway/internal/api/handlers.go | 29 ++-- datasource-gateway/internal/api/query.go | 125 +++++++++++++++++ datasource-gateway/internal/api/query_test.go | 127 ++++++++++++++++++ 3 files changed, 268 insertions(+), 13 deletions(-) create mode 100644 datasource-gateway/internal/api/query.go create mode 100644 datasource-gateway/internal/api/query_test.go diff --git a/datasource-gateway/internal/api/handlers.go b/datasource-gateway/internal/api/handlers.go index 48d9472..66f7b9a 100644 --- a/datasource-gateway/internal/api/handlers.go +++ b/datasource-gateway/internal/api/handlers.go @@ -5,14 +5,17 @@ import ( "encoding/json" "net/http" "time" + + "github.com/couchbase/datasource-gateway/internal/couchbase" ) -// couchbaseHealth is the slice of the Couchbase client the health endpoint -// needs. Behind an interface so the api package stays decoupled from the -// concrete client (and testable). -type couchbaseHealth interface { +// couchbaseGateway is the slice of the Couchbase client the gateway needs: +// health reporting plus snapshot-metadata lookup (for time-window rewriting +// and, later, routing). +type couchbaseGateway interface { Enabled() bool Ready() bool + GetSnapshotMetadata(ctx context.Context, snapshotID string) (*couchbase.Metadata, error) } // prometheusGateway is the slice of the Prometheus client the gateway needs: @@ -24,26 +27,26 @@ type prometheusGateway interface { } // Handler holds the gateway HTTP handlers: the health endpoint and the -// Prometheus-compatible API surface (/api/v1/*), forwarded to the upstream by -// a streaming reverse proxy. +// Prometheus-compatible API surface (/api/v1/*). type Handler struct { - couchbase couchbaseHealth + couchbase couchbaseGateway prometheus prometheusGateway } // NewHandler constructs the gateway HTTP handler. -func NewHandler(couchbase couchbaseHealth, prometheus prometheusGateway) *Handler { +func NewHandler(couchbase couchbaseGateway, prometheus prometheusGateway) *Handler { return &Handler{couchbase: couchbase, prometheus: prometheus} } // Register wires the handler's routes onto the given mux: the gateway's own -// /healthz, plus the Prometheus-compatible API under /api/v1/ (forwarded to -// the upstream by a streaming reverse proxy). Per-snapshot routing and the -// Couchbase translation path attach to more specific /api/v1/query[_range] -// patterns in later tasks. +// /healthz; /api/v1/query_range (which rewrites the time window to the +// snapshot's before forwarding); and a catch-all /api/v1/ streaming reverse +// proxy for every other Prometheus-API endpoint. Per-snapshot routing and the +// Couchbase translation path attach to the query routes in later tasks. func (h *Handler) Register(mux *http.ServeMux) { mux.HandleFunc("/healthz", h.Health) if h.prometheus != nil { + mux.HandleFunc("/api/v1/query_range", h.handleQueryRange) mux.Handle("/api/v1/", h.prometheus.ReverseProxy()) } } @@ -70,7 +73,7 @@ func (h *Handler) Health(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(resp) } -func couchbaseStatus(cb couchbaseHealth) map[string]any { +func couchbaseStatus(cb couchbaseGateway) map[string]any { if cb == nil || !cb.Enabled() { return map[string]any{"enabled": false} } diff --git a/datasource-gateway/internal/api/query.go b/datasource-gateway/internal/api/query.go new file mode 100644 index 0000000..4054a76 --- /dev/null +++ b/datasource-gateway/internal/api/query.go @@ -0,0 +1,125 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/couchbase/datasource-gateway/internal/couchbase" +) + +// jobSelectorRe captures the value of a positive job matcher (job="..." or +// job=~"...") in a PromQL query. Negative matchers (!=, !~) don't match +// because the '!' breaks the `job\s*=` prefix. +var jobSelectorRe = regexp.MustCompile(`job\s*=~?\s*"([^"]*)"`) + +// handleQueryRange serves /api/v1/query_range. For a single-snapshot query it +// rewrites start/end to the snapshot's stored time window (from Couchbase +// metadata) before forwarding upstream, so a historical snapshot's panels +// resolve against the snapshot's absolute window regardless of the dashboard's +// time picker. Multi-snapshot (overlap) queries, queries with no job matcher, +// and failed lookups are forwarded with their original time range. +func (h *Handler) handleQueryRange(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + writePromError(w, http.StatusBadRequest, "bad_data", "failed to parse request: "+err.Error()) + return + } + + h.applySnapshotWindow(r.Context(), r.Form) + + // Re-emit the (possibly rewritten) params onto the URL so the reverse + // proxy forwards them uniformly. This also reconstructs the request after + // ParseForm drained a POST body. + forwardForm(r) + + h.prometheus.ReverseProxy().ServeHTTP(w, r) +} + +// applySnapshotWindow rewrites form's start/end to the snapshot window when the +// query targets a single snapshot and Couchbase metadata is available. +func (h *Handler) applySnapshotWindow(ctx context.Context, form url.Values) { + if h.couchbase == nil || !h.couchbase.Enabled() { + return + } + job := singleJob(form.Get("query")) + if job == "" { + return + } + md, err := h.couchbase.GetSnapshotMetadata(ctx, job) + if err != nil { + // Forward unchanged; the upstream still answers with the original range. + return + } + if start, end, ok := snapshotWindowUnix(md); ok { + form.Set("start", start) + form.Set("end", end) + } +} + +// singleJob returns the snapshot ID from a single-snapshot job matcher, or "" +// when the query has no job matcher or targets multiple snapshots (overlap, +// signalled by a '|' in the matcher value). +func singleJob(query string) string { + m := jobSelectorRe.FindStringSubmatch(query) + if len(m) < 2 { + return "" + } + val := strings.TrimSpace(m[1]) + if val == "" || strings.Contains(val, "|") { + return "" + } + return val +} + +// forwardForm moves the merged form params onto the URL query and empties the +// body, so the reverse proxy forwards the (possibly rewritten) params +// uniformly for both GET and POST. +func forwardForm(r *http.Request) { + r.URL.RawQuery = r.Form.Encode() + r.Body = http.NoBody + r.ContentLength = 0 + r.Header.Del("Content-Type") + r.Header.Del("Content-Length") +} + +// snapshotWindowUnix returns the snapshot's [start,end] as Unix-second strings. +func snapshotWindowUnix(md *couchbase.Metadata) (string, string, bool) { + start, ok1 := parseSnapshotTime(md.TSStart) + end, ok2 := parseSnapshotTime(md.TSEnd) + if !ok1 || !ok2 { + return "", "", false + } + return strconv.FormatInt(start.Unix(), 10), strconv.FormatInt(end.Unix(), 10), true +} + +// parseSnapshotTime parses the ISO timestamps stored in snapshot metadata, +// tolerating RFC3339, a zone-less layout, and the space-separated variant the +// proxy accepts. +func parseSnapshotTime(s string) (time.Time, bool) { + layouts := []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02 15:04:05"} + for _, l := range layouts { + if t, err := time.Parse(l, s); err == nil { + return t, true + } + } + if t, err := time.Parse(time.RFC3339, strings.Replace(s, " ", "T", 1)); err == nil { + return t, true + } + return time.Time{}, false +} + +// writePromError writes a Prometheus-API error envelope. +func writePromError(w http.ResponseWriter, code int, errorType, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{ + "status": "error", + "errorType": errorType, + "error": msg, + }) +} diff --git a/datasource-gateway/internal/api/query_test.go b/datasource-gateway/internal/api/query_test.go new file mode 100644 index 0000000..7117f92 --- /dev/null +++ b/datasource-gateway/internal/api/query_test.go @@ -0,0 +1,127 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/couchbase/datasource-gateway/internal/couchbase" +) + +type fakeCouchbase struct { + enabled bool + md *couchbase.Metadata + err error + calls int +} + +func (f *fakeCouchbase) Enabled() bool { return f.enabled } +func (f *fakeCouchbase) Ready() bool { return true } +func (f *fakeCouchbase) GetSnapshotMetadata(_ context.Context, _ string) (*couchbase.Metadata, error) { + f.calls++ + if f.err != nil { + return nil, f.err + } + return f.md, nil +} + +type fakeProm struct{ proxy http.Handler } + +func (f *fakeProm) URL() string { return "http://upstream" } +func (f *fakeProm) Reachable(_ context.Context) bool { return true } +func (f *fakeProm) ReverseProxy() http.Handler { return f.proxy } + +// recorder captures the params the reverse proxy received from the handler. +type recorder struct{ start, end, query string } + +func (rc *recorder) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + rc.start = r.Form.Get("start") + rc.end = r.Form.Get("end") + rc.query = r.Form.Get("query") + w.WriteHeader(http.StatusOK) + }) +} + +func unixOf(t *testing.T, rfc string) string { + t.Helper() + ts, err := time.Parse(time.RFC3339, rfc) + if err != nil { + t.Fatalf("parse %q: %v", rfc, err) + } + return strconv.FormatInt(ts.Unix(), 10) +} + +func postQueryRange(h *Handler, query, start, end string) *recorder { + rc := &recorder{} + // Re-point the handler's proxy at the recorder for this call. + h.prometheus = &fakeProm{proxy: rc.handler()} + body := url.Values{"query": {query}, "start": {start}, "end": {end}, "step": {"15"}}.Encode() + req := httptest.NewRequest(http.MethodPost, "/api/v1/query_range", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + h.handleQueryRange(httptest.NewRecorder(), req) + return rc +} + +func TestQueryRangeRewritesSingleSnapshotWindow(t *testing.T) { + cb := &fakeCouchbase{ + enabled: true, + md: &couchbase.Metadata{ID: "snap-1", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}, + } + h := NewHandler(cb, &fakeProm{}) + + // Deliberately-wrong dashboard range (1..2); expect it rewritten to the window. + rc := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") + + if want := unixOf(t, "2024-01-02T00:00:00Z"); rc.start != want { + t.Errorf("start = %q, want %q", rc.start, want) + } + if want := unixOf(t, "2024-01-02T01:00:00Z"); rc.end != want { + t.Errorf("end = %q, want %q", rc.end, want) + } + if !strings.Contains(rc.query, "kv_ops") { + t.Errorf("query not forwarded: %q", rc.query) + } + if cb.calls != 1 { + t.Errorf("metadata lookups = %d, want 1", cb.calls) + } +} + +func TestQueryRangeMultiSnapshotForwardedUnchanged(t *testing.T) { + cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + h := NewHandler(cb, &fakeProm{}) + + // Overlap query (multiple jobs) → no rewrite; original range + query survive + // the POST-body reconstruction. + rc := postQueryRange(h, `rate(kv_ops{job=~"snap-1|snap-2"}[5m])`, "1000", "2000") + + if rc.start != "1000" || rc.end != "2000" { + t.Errorf("range rewritten for overlap: start=%q end=%q, want 1000/2000", rc.start, rc.end) + } + if !strings.Contains(rc.query, "snap-1|snap-2") { + t.Errorf("query not forwarded intact: %q", rc.query) + } + if cb.calls != 0 { + t.Errorf("metadata looked up for overlap query (%d calls); should be skipped", cb.calls) + } +} + +func TestQueryRangeCouchbaseDisabledForwardsUnchanged(t *testing.T) { + cb := &fakeCouchbase{enabled: false} + h := NewHandler(cb, &fakeProm{}) + + rc := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1000", "2000") + + if rc.start != "1000" || rc.end != "2000" { + t.Errorf("range rewritten while disabled: start=%q end=%q", rc.start, rc.end) + } + if cb.calls != 0 { + t.Errorf("metadata looked up while disabled (%d calls)", cb.calls) + } +} From 06d71ee8319e011d61c82de5ac2fba4ca4c830b1 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Wed, 3 Jun 2026 09:35:41 +0100 Subject: [PATCH 06/18] gateway: Manage per snapshot routing based on the data-source --- datasource-gateway/go.mod | 2 + datasource-gateway/go.sum | 2 + datasource-gateway/internal/api/handlers.go | 25 ++-- datasource-gateway/internal/api/query.go | 86 ++++------- datasource-gateway/internal/api/query_test.go | 97 ++++++++----- datasource-gateway/internal/router/router.go | 133 ++++++++++++++++++ .../internal/router/router_test.go | 119 ++++++++++++++++ datasource-gateway/main.go | 7 +- 8 files changed, 358 insertions(+), 113 deletions(-) create mode 100644 datasource-gateway/internal/router/router.go create mode 100644 datasource-gateway/internal/router/router_test.go diff --git a/datasource-gateway/go.mod b/datasource-gateway/go.mod index 6b4ad09..e0e05bb 100644 --- a/datasource-gateway/go.mod +++ b/datasource-gateway/go.mod @@ -4,6 +4,8 @@ go 1.24.0 require ( github.com/couchbase/gocb/v2 v2.11.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 + golang.org/x/sync v0.17.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/datasource-gateway/go.sum b/datasource-gateway/go.sum index 7c05ebe..a6784f2 100644 --- a/datasource-gateway/go.sum +++ b/datasource-gateway/go.sum @@ -114,6 +114,8 @@ github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7E github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/datasource-gateway/internal/api/handlers.go b/datasource-gateway/internal/api/handlers.go index 66f7b9a..8d000df 100644 --- a/datasource-gateway/internal/api/handlers.go +++ b/datasource-gateway/internal/api/handlers.go @@ -6,16 +6,14 @@ import ( "net/http" "time" - "github.com/couchbase/datasource-gateway/internal/couchbase" + "github.com/couchbase/datasource-gateway/internal/router" ) -// couchbaseGateway is the slice of the Couchbase client the gateway needs: -// health reporting plus snapshot-metadata lookup (for time-window rewriting -// and, later, routing). -type couchbaseGateway interface { +// couchbaseHealth is the slice of the Couchbase client the health endpoint +// needs. Snapshot-metadata lookups for routing live behind the router. +type couchbaseHealth interface { Enabled() bool Ready() bool - GetSnapshotMetadata(ctx context.Context, snapshotID string) (*couchbase.Metadata, error) } // prometheusGateway is the slice of the Prometheus client the gateway needs: @@ -29,20 +27,19 @@ type prometheusGateway interface { // Handler holds the gateway HTTP handlers: the health endpoint and the // Prometheus-compatible API surface (/api/v1/*). type Handler struct { - couchbase couchbaseGateway + couchbase couchbaseHealth prometheus prometheusGateway + router *router.Router } // NewHandler constructs the gateway HTTP handler. -func NewHandler(couchbase couchbaseGateway, prometheus prometheusGateway) *Handler { - return &Handler{couchbase: couchbase, prometheus: prometheus} +func NewHandler(couchbase couchbaseHealth, prometheus prometheusGateway, router *router.Router) *Handler { + return &Handler{couchbase: couchbase, prometheus: prometheus, router: router} } // Register wires the handler's routes onto the given mux: the gateway's own -// /healthz; /api/v1/query_range (which rewrites the time window to the -// snapshot's before forwarding); and a catch-all /api/v1/ streaming reverse -// proxy for every other Prometheus-API endpoint. Per-snapshot routing and the -// Couchbase translation path attach to the query routes in later tasks. +// /healthz; /api/v1/query_range (routed per snapshot); and a catch-all +// /api/v1/ streaming reverse proxy for every other Prometheus-API endpoint. func (h *Handler) Register(mux *http.ServeMux) { mux.HandleFunc("/healthz", h.Health) if h.prometheus != nil { @@ -73,7 +70,7 @@ func (h *Handler) Health(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(resp) } -func couchbaseStatus(cb couchbaseGateway) map[string]any { +func couchbaseStatus(cb couchbaseHealth) map[string]any { if cb == nil || !cb.Enabled() { return map[string]any{"enabled": false} } diff --git a/datasource-gateway/internal/api/query.go b/datasource-gateway/internal/api/query.go index 4054a76..8680cb2 100644 --- a/datasource-gateway/internal/api/query.go +++ b/datasource-gateway/internal/api/query.go @@ -1,16 +1,13 @@ package api import ( - "context" "encoding/json" "net/http" - "net/url" "regexp" "strconv" "strings" - "time" - "github.com/couchbase/datasource-gateway/internal/couchbase" + "github.com/couchbase/datasource-gateway/internal/router" ) // jobSelectorRe captures the value of a positive job matcher (job="..." or @@ -18,47 +15,40 @@ import ( // because the '!' breaks the `job\s*=` prefix. var jobSelectorRe = regexp.MustCompile(`job\s*=~?\s*"([^"]*)"`) -// handleQueryRange serves /api/v1/query_range. For a single-snapshot query it -// rewrites start/end to the snapshot's stored time window (from Couchbase -// metadata) before forwarding upstream, so a historical snapshot's panels -// resolve against the snapshot's absolute window regardless of the dashboard's -// time picker. Multi-snapshot (overlap) queries, queries with no job matcher, -// and failed lookups are forwarded with their original time range. +// handleQueryRange serves /api/v1/query_range. It resolves the snapshot's route +// (cached) and forks: Prometheus-backed snapshots are forwarded to the upstream +// with start/end rewritten to the snapshot's stored window; Couchbase-backed +// snapshots are translated and executed (wired in a later task). Multi-snapshot +// (overlap) and job-less queries resolve to a plain passthrough. func (h *Handler) handleQueryRange(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { writePromError(w, http.StatusBadRequest, "bad_data", "failed to parse request: "+err.Error()) return } - h.applySnapshotWindow(r.Context(), r.Form) + route := h.router.Resolve(r.Context(), singleJob(r.Form.Get("query"))) - // Re-emit the (possibly rewritten) params onto the URL so the reverse - // proxy forwards them uniformly. This also reconstructs the request after - // ParseForm drained a POST body. - forwardForm(r) + if route.Store == router.StoreCouchbase { + h.serveCouchbaseQueryRange(w, r, route) + return + } + // Prometheus-backed: rewrite the window (when known) and pass through. + if route.HasWindow { + r.Form.Set("start", strconv.FormatInt(route.Start.Unix(), 10)) + r.Form.Set("end", strconv.FormatInt(route.End.Unix(), 10)) + } + forwardForm(r) h.prometheus.ReverseProxy().ServeHTTP(w, r) } -// applySnapshotWindow rewrites form's start/end to the snapshot window when the -// query targets a single snapshot and Couchbase metadata is available. -func (h *Handler) applySnapshotWindow(ctx context.Context, form url.Values) { - if h.couchbase == nil || !h.couchbase.Enabled() { - return - } - job := singleJob(form.Get("query")) - if job == "" { - return - } - md, err := h.couchbase.GetSnapshotMetadata(ctx, job) - if err != nil { - // Forward unchanged; the upstream still answers with the original range. - return - } - if start, end, ok := snapshotWindowUnix(md); ok { - form.Set("start", start) - form.Set("end", end) - } +// serveCouchbaseQueryRange will translate the PromQL to SQL++, execute it +// against Couchbase, and shape the result as Prometheus JSON. Wired in a later +// task; for now it returns a clear, parseable error so the routing decision is +// observable without silently returning empty data. +func (h *Handler) serveCouchbaseQueryRange(w http.ResponseWriter, _ *http.Request, route router.Route) { + writePromError(w, http.StatusNotImplemented, "not_implemented", + "Couchbase-backed query execution for snapshot "+route.Snapshot+" is not yet wired") } // singleJob returns the snapshot ID from a single-snapshot job matcher, or "" @@ -78,7 +68,7 @@ func singleJob(query string) string { // forwardForm moves the merged form params onto the URL query and empties the // body, so the reverse proxy forwards the (possibly rewritten) params -// uniformly for both GET and POST. +// uniformly for GET and POST. func forwardForm(r *http.Request) { r.URL.RawQuery = r.Form.Encode() r.Body = http.NoBody @@ -87,32 +77,6 @@ func forwardForm(r *http.Request) { r.Header.Del("Content-Length") } -// snapshotWindowUnix returns the snapshot's [start,end] as Unix-second strings. -func snapshotWindowUnix(md *couchbase.Metadata) (string, string, bool) { - start, ok1 := parseSnapshotTime(md.TSStart) - end, ok2 := parseSnapshotTime(md.TSEnd) - if !ok1 || !ok2 { - return "", "", false - } - return strconv.FormatInt(start.Unix(), 10), strconv.FormatInt(end.Unix(), 10), true -} - -// parseSnapshotTime parses the ISO timestamps stored in snapshot metadata, -// tolerating RFC3339, a zone-less layout, and the space-separated variant the -// proxy accepts. -func parseSnapshotTime(s string) (time.Time, bool) { - layouts := []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02 15:04:05"} - for _, l := range layouts { - if t, err := time.Parse(l, s); err == nil { - return t, true - } - } - if t, err := time.Parse(time.RFC3339, strings.Replace(s, " ", "T", 1)); err == nil { - return t, true - } - return time.Time{}, false -} - // writePromError writes a Prometheus-API error envelope. func writePromError(w http.ResponseWriter, code int, errorType, msg string) { w.Header().Set("Content-Type", "application/json") diff --git a/datasource-gateway/internal/api/query_test.go b/datasource-gateway/internal/api/query_test.go index 7117f92..c6cc6d0 100644 --- a/datasource-gateway/internal/api/query_test.go +++ b/datasource-gateway/internal/api/query_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/couchbase/datasource-gateway/internal/couchbase" + "github.com/couchbase/datasource-gateway/internal/router" ) type fakeCouchbase struct { @@ -32,15 +33,19 @@ func (f *fakeCouchbase) GetSnapshotMetadata(_ context.Context, _ string) (*couch type fakeProm struct{ proxy http.Handler } -func (f *fakeProm) URL() string { return "http://upstream" } -func (f *fakeProm) Reachable(_ context.Context) bool { return true } -func (f *fakeProm) ReverseProxy() http.Handler { return f.proxy } +func (f *fakeProm) URL() string { return "http://upstream" } +func (f *fakeProm) Reachable(_ context.Context) bool { return true } +func (f *fakeProm) ReverseProxy() http.Handler { return f.proxy } // recorder captures the params the reverse proxy received from the handler. -type recorder struct{ start, end, query string } +type recorder struct { + called bool + start, end, query string +} func (rc *recorder) handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rc.called = true _ = r.ParseForm() rc.start = r.Form.Get("start") rc.end = r.Form.Get("end") @@ -49,6 +54,19 @@ func (rc *recorder) handler() http.Handler { }) } +func newTestHandler(cb *fakeCouchbase, rc *recorder) *Handler { + return NewHandler(cb, &fakeProm{proxy: rc.handler()}, router.New(cb)) +} + +func postQueryRange(h *Handler, query, start, end string) *httptest.ResponseRecorder { + body := url.Values{"query": {query}, "start": {start}, "end": {end}, "step": {"15"}}.Encode() + req := httptest.NewRequest(http.MethodPost, "/api/v1/query_range", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + w := httptest.NewRecorder() + h.handleQueryRange(w, req) + return w +} + func unixOf(t *testing.T, rfc string) string { t.Helper() ts, err := time.Parse(time.RFC3339, rfc) @@ -58,27 +76,17 @@ func unixOf(t *testing.T, rfc string) string { return strconv.FormatInt(ts.Unix(), 10) } -func postQueryRange(h *Handler, query, start, end string) *recorder { +func TestQueryRangePrometheusBackedRewritesWindow(t *testing.T) { rc := &recorder{} - // Re-point the handler's proxy at the recorder for this call. - h.prometheus = &fakeProm{proxy: rc.handler()} - body := url.Values{"query": {query}, "start": {start}, "end": {end}, "step": {"15"}}.Encode() - req := httptest.NewRequest(http.MethodPost, "/api/v1/query_range", strings.NewReader(body)) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - h.handleQueryRange(httptest.NewRecorder(), req) - return rc -} - -func TestQueryRangeRewritesSingleSnapshotWindow(t *testing.T) { - cb := &fakeCouchbase{ - enabled: true, - md: &couchbase.Metadata{ID: "snap-1", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}, - } - h := NewHandler(cb, &fakeProm{}) + cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "prometheus", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + h := newTestHandler(cb, rc) // Deliberately-wrong dashboard range (1..2); expect it rewritten to the window. - rc := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") + postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") + if !rc.called { + t.Fatal("expected passthrough to upstream for a Prometheus-backed snapshot") + } if want := unixOf(t, "2024-01-02T00:00:00Z"); rc.start != want { t.Errorf("start = %q, want %q", rc.start, want) } @@ -88,40 +96,55 @@ func TestQueryRangeRewritesSingleSnapshotWindow(t *testing.T) { if !strings.Contains(rc.query, "kv_ops") { t.Errorf("query not forwarded: %q", rc.query) } - if cb.calls != 1 { - t.Errorf("metadata lookups = %d, want 1", cb.calls) +} + +func TestQueryRangeCouchbaseBackedReturnsStub(t *testing.T) { + rc := &recorder{} + cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "couchbase", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + h := newTestHandler(cb, rc) + + w := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") + + if rc.called { + t.Error("Couchbase-backed query should not hit the passthrough") + } + if w.Code != http.StatusNotImplemented { + t.Errorf("code = %d, want 501", w.Code) } } -func TestQueryRangeMultiSnapshotForwardedUnchanged(t *testing.T) { - cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} - h := NewHandler(cb, &fakeProm{}) +func TestQueryRangeOverlapForwardedUnchanged(t *testing.T) { + rc := &recorder{} + cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "couchbase"}} + h := newTestHandler(cb, rc) - // Overlap query (multiple jobs) → no rewrite; original range + query survive - // the POST-body reconstruction. - rc := postQueryRange(h, `rate(kv_ops{job=~"snap-1|snap-2"}[5m])`, "1000", "2000") + postQueryRange(h, `rate(kv_ops{job=~"snap-1|snap-2"}[5m])`, "1000", "2000") - if rc.start != "1000" || rc.end != "2000" { - t.Errorf("range rewritten for overlap: start=%q end=%q, want 1000/2000", rc.start, rc.end) + if !rc.called { + t.Fatal("overlap query should pass through") } - if !strings.Contains(rc.query, "snap-1|snap-2") { - t.Errorf("query not forwarded intact: %q", rc.query) + if rc.start != "1000" || rc.end != "2000" { + t.Errorf("overlap window rewritten: start=%q end=%q, want 1000/2000", rc.start, rc.end) } if cb.calls != 0 { - t.Errorf("metadata looked up for overlap query (%d calls); should be skipped", cb.calls) + t.Errorf("metadata consulted for overlap query: %d calls", cb.calls) } } func TestQueryRangeCouchbaseDisabledForwardsUnchanged(t *testing.T) { + rc := &recorder{} cb := &fakeCouchbase{enabled: false} - h := NewHandler(cb, &fakeProm{}) + h := newTestHandler(cb, rc) - rc := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1000", "2000") + postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1000", "2000") + if !rc.called { + t.Fatal("disabled-Couchbase query should pass through") + } if rc.start != "1000" || rc.end != "2000" { t.Errorf("range rewritten while disabled: start=%q end=%q", rc.start, rc.end) } if cb.calls != 0 { - t.Errorf("metadata looked up while disabled (%d calls)", cb.calls) + t.Errorf("metadata consulted while disabled: %d calls", cb.calls) } } diff --git a/datasource-gateway/internal/router/router.go b/datasource-gateway/internal/router/router.go new file mode 100644 index 0000000..0f19db0 --- /dev/null +++ b/datasource-gateway/internal/router/router.go @@ -0,0 +1,133 @@ +package router + +import ( + "context" + "strings" + "time" + + lru "github.com/hashicorp/golang-lru/v2/expirable" + "golang.org/x/sync/singleflight" + + "github.com/couchbase/datasource-gateway/internal/couchbase" +) + +// Store identifies which backend holds a snapshot's metrics. +type Store string + +const ( + StoreCouchbase Store = "couchbase" + StorePrometheus Store = "prometheus" +) + +// Route is the cached routing decision and time window for a snapshot. +type Route struct { + Snapshot string + Store Store + Start time.Time + End time.Time + HasWindow bool +} + +// metadataSource is the slice of the Couchbase client the router needs. +type metadataSource interface { + Enabled() bool + GetSnapshotMetadata(ctx context.Context, snapshotID string) (*couchbase.Metadata, error) +} + +const ( + cacheSize = 4096 + cacheTTL = 6 * time.Hour +) + +// Router decides, per snapshot, whether its metrics come from Couchbase or the +// upstream Prometheus, and resolves the snapshot's absolute time window. A +// snapshot's metadata is fetched at most once (snapshots are immutable) and +// cached; concurrent resolves for the same snapshot share a single fetch. +type Router struct { + cb metadataSource + cache *lru.LRU[string, Route] + group singleflight.Group +} + +// New constructs a Router over the given metadata source. +func New(cb metadataSource) *Router { + return &Router{ + cb: cb, + cache: lru.NewLRU[string, Route](cacheSize, nil, cacheTTL), + } +} + +// Resolve returns the routing decision for a snapshot. Queries with no snapshot +// (empty id) or when Couchbase is disabled fall back to Prometheus passthrough. +// A failed metadata lookup also falls back to Prometheus and is not cached, so +// a transient error doesn't pin a wrong decision for the cache TTL. +func (r *Router) Resolve(ctx context.Context, snapshotID string) Route { + if snapshotID == "" || r.cb == nil || !r.cb.Enabled() { + return Route{Snapshot: snapshotID, Store: StorePrometheus} + } + if rt, ok := r.cache.Get(snapshotID); ok { + return rt + } + v, _, _ := r.group.Do(snapshotID, func() (any, error) { + if rt, ok := r.cache.Get(snapshotID); ok { + return rt, nil + } + rt, cacheable := r.resolveUncached(ctx, snapshotID) + if cacheable { + r.cache.Add(snapshotID, rt) + } + return rt, nil + }) + return v.(Route) +} + +func (r *Router) resolveUncached(ctx context.Context, snapshotID string) (Route, bool) { + md, err := r.cb.GetSnapshotMetadata(ctx, snapshotID) + if err != nil { + // No metadata doc (or transient error): fall back to passthrough and + // don't cache, so the next request retries the lookup. + return Route{Snapshot: snapshotID, Store: StorePrometheus}, false + } + rt := Route{Snapshot: snapshotID, Store: storeFromMetadata(md)} + if start, end, ok := parseWindow(md); ok { + rt.Start, rt.End, rt.HasWindow = start, end, true + } + return rt, true +} + +// storeFromMetadata honours an explicit `store` field; absent that, a present +// metadata document implies the metrics live in Couchbase too. +func storeFromMetadata(md *couchbase.Metadata) Store { + switch Store(strings.ToLower(strings.TrimSpace(md.Store))) { + case StorePrometheus: + return StorePrometheus + case StoreCouchbase: + return StoreCouchbase + default: + return StoreCouchbase + } +} + +// parseWindow parses the snapshot's [start,end] from its metadata timestamps, +// tolerating RFC3339, a zone-less layout, and the space-separated variant. +func parseWindow(md *couchbase.Metadata) (time.Time, time.Time, bool) { + start, ok1 := parseTime(md.TSStart) + end, ok2 := parseTime(md.TSEnd) + if !ok1 || !ok2 { + return time.Time{}, time.Time{}, false + } + return start, end, true +} + +func parseTime(s string) (time.Time, bool) { + layouts := []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02 15:04:05"} + for _, l := range layouts { + if t, err := time.Parse(l, s); err == nil { + return t, true + } + } + if t, err := time.Parse(time.RFC3339, strings.Replace(s, " ", "T", 1)); err == nil { + return t, true + } + return time.Time{}, false +} diff --git a/datasource-gateway/internal/router/router_test.go b/datasource-gateway/internal/router/router_test.go new file mode 100644 index 0000000..c3701fd --- /dev/null +++ b/datasource-gateway/internal/router/router_test.go @@ -0,0 +1,119 @@ +package router + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/couchbase/datasource-gateway/internal/couchbase" +) + +type fakeSource struct { + enabled bool + md *couchbase.Metadata + err error + block chan struct{} // if non-nil, the fetch blocks on it + + mu sync.Mutex + calls int +} + +func (f *fakeSource) Enabled() bool { return f.enabled } + +func (f *fakeSource) GetSnapshotMetadata(_ context.Context, _ string) (*couchbase.Metadata, error) { + f.mu.Lock() + f.calls++ + f.mu.Unlock() + if f.block != nil { + <-f.block + } + if f.err != nil { + return nil, f.err + } + return f.md, nil +} + +func (f *fakeSource) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +func TestResolveExplicitPrometheusStore(t *testing.T) { + f := &fakeSource{enabled: true, md: &couchbase.Metadata{Store: "prometheus", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + rt := New(f).Resolve(context.Background(), "snap-1") + if rt.Store != StorePrometheus { + t.Errorf("store = %q, want prometheus", rt.Store) + } + if !rt.HasWindow { + t.Error("expected a resolved window") + } +} + +func TestResolveDefaultsToCouchbaseWhenStoreUnset(t *testing.T) { + f := &fakeSource{enabled: true, md: &couchbase.Metadata{TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + rt := New(f).Resolve(context.Background(), "snap-1") + if rt.Store != StoreCouchbase { + t.Errorf("store = %q, want couchbase", rt.Store) + } +} + +func TestResolveDisabledFallsBackToPrometheus(t *testing.T) { + f := &fakeSource{enabled: false} + rt := New(f).Resolve(context.Background(), "snap-1") + if rt.Store != StorePrometheus { + t.Errorf("store = %q, want prometheus", rt.Store) + } + if f.callCount() != 0 { + t.Errorf("metadata fetched while disabled: %d", f.callCount()) + } +} + +func TestResolveCachesSuccessfulLookups(t *testing.T) { + f := &fakeSource{enabled: true, md: &couchbase.Metadata{Store: "couchbase", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + r := New(f) + for i := 0; i < 5; i++ { + r.Resolve(context.Background(), "snap-1") + } + if f.callCount() != 1 { + t.Errorf("metadata fetched %d times, want 1 (cached)", f.callCount()) + } +} + +func TestResolveDoesNotCacheErrors(t *testing.T) { + f := &fakeSource{enabled: true, err: errors.New("not found")} + r := New(f) + rt := r.Resolve(context.Background(), "snap-1") + r.Resolve(context.Background(), "snap-1") + if f.callCount() != 2 { + t.Errorf("error result was cached: calls = %d, want 2 (retried)", f.callCount()) + } + if rt.Store != StorePrometheus { + t.Errorf("error fallback store = %q, want prometheus", rt.Store) + } +} + +func TestResolveConcurrentSingleFetch(t *testing.T) { + block := make(chan struct{}) + f := &fakeSource{enabled: true, md: &couchbase.Metadata{Store: "couchbase"}, block: block} + r := New(f) + + const n = 20 + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + go func() { + defer wg.Done() + r.Resolve(context.Background(), "snap-x") + }() + } + time.Sleep(50 * time.Millisecond) // let all resolves reach the in-flight fetch + close(block) // release the single fetch + wg.Wait() + + if f.callCount() != 1 { + t.Errorf("concurrent resolves triggered %d fetches, want 1", f.callCount()) + } +} diff --git a/datasource-gateway/main.go b/datasource-gateway/main.go index 3c34a95..6f6e2e2 100644 --- a/datasource-gateway/main.go +++ b/datasource-gateway/main.go @@ -16,6 +16,7 @@ import ( "github.com/couchbase/datasource-gateway/internal/couchbase" "github.com/couchbase/datasource-gateway/internal/logger" "github.com/couchbase/datasource-gateway/internal/prometheus" + "github.com/couchbase/datasource-gateway/internal/router" ) const shutdownTimeout = 10 * time.Second @@ -77,7 +78,11 @@ func main() { logger.Error("Couchbase client init degraded; serving Prometheus path only", "error", cbErr) } - handler := api.NewHandler(cbClient, promClient) + // Per-snapshot router: caches each snapshot's store + time window so + // routing is a map lookup after the first query. + metricRouter := router.New(cbClient) + + handler := api.NewHandler(cbClient, promClient, metricRouter) mux := http.NewServeMux() handler.Register(mux) From ea99ccdf3593ebe318c75f5731a3efa17263cf8c Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Wed, 3 Jun 2026 11:11:51 +0100 Subject: [PATCH 07/18] gateway: Fix matches and where clause in the translator --- datasource-gateway/internal/promql/parser.go | 1 + datasource-gateway/internal/promql/planner.go | 12 +- .../internal/promql/sqlbuilder.go | 349 +++++------------- .../internal/promql/sqlbuilder_test.go | 141 +++++++ .../internal/querybuilder/querybuilder.go | 5 +- 5 files changed, 237 insertions(+), 271 deletions(-) create mode 100644 datasource-gateway/internal/promql/sqlbuilder_test.go diff --git a/datasource-gateway/internal/promql/parser.go b/datasource-gateway/internal/promql/parser.go index 15c769b..07af416 100644 --- a/datasource-gateway/internal/promql/parser.go +++ b/datasource-gateway/internal/promql/parser.go @@ -28,6 +28,7 @@ type QueryContext struct { Step time.Duration IsRange bool SnapshotID string // Optional snapshot ID from context + Keyspace string // Metrics keyspace (bucket.scope.collection); empty → default } // ParseQueryContext parses query parameters into a QueryContext diff --git a/datasource-gateway/internal/promql/planner.go b/datasource-gateway/internal/promql/planner.go index 7233ddd..0728447 100644 --- a/datasource-gateway/internal/promql/planner.go +++ b/datasource-gateway/internal/promql/planner.go @@ -73,12 +73,14 @@ func (v *queryPlanner) Visit(node parser.Node, path []parser.Node) (parser.Visit v.plan.SeriesQueries = append(v.plan.SeriesQueries, seriesQuery) case *parser.AggregateExpr: - // Extract aggregation - v.plan.Aggregation = &AggregationPlan{ - Operation: n.Op.String(), - By: v.extractLabels(n.Grouping), - Without: v.extractLabels(n.Grouping), // Simplified - would need to check Without flag + // Extract aggregation, honouring the by/without flag. + agg := &AggregationPlan{Operation: n.Op.String()} + if n.Without { + agg.Without = v.extractLabels(n.Grouping) + } else { + agg.By = v.extractLabels(n.Grouping) } + v.plan.Aggregation = agg case *parser.Call: // Extract function call diff --git a/datasource-gateway/internal/promql/sqlbuilder.go b/datasource-gateway/internal/promql/sqlbuilder.go index da7c749..a3e7a8f 100644 --- a/datasource-gateway/internal/promql/sqlbuilder.go +++ b/datasource-gateway/internal/promql/sqlbuilder.go @@ -8,312 +8,133 @@ import ( "github.com/couchbase/datasource-gateway/internal/querybuilder" ) -// SQLBuilder builds SQL++ queries from query plans +const ( + // defaultKeyspace is used when the query context doesn't specify one; the + // real keyspace (bucket.scope.collection) is supplied per query. + defaultKeyspace = "cbmonitor._default._default" + + // instantLookback is how far back an instant query reads to find the most + // recent sample. + instantLookback = 5 * time.Minute +) + +// SQLBuilder builds SQL++ queries from a query plan. It emits the raw, +// time-bounded, per-series samples (one query per selector). rate/irate/ +// increase and aggregation are applied in Go (see transformer.go) after the +// samples are fetched — rate must be computed per series before any +// aggregation, so neither can be pushed into SQL. type SQLBuilder struct { - plan *QueryPlan - queryCtx *QueryContext - useBatching bool + plan *QueryPlan + queryCtx *QueryContext } -// NewSQLBuilder creates a new SQL++ query builder +// NewSQLBuilder creates a new SQL++ query builder. func NewSQLBuilder(plan *QueryPlan, queryCtx *QueryContext) *SQLBuilder { - return &SQLBuilder{ - plan: plan, - queryCtx: queryCtx, - useBatching: plan.ShouldBatch(), - } + return &SQLBuilder{plan: plan, queryCtx: queryCtx} } -// Build generates SQL++ query string(s) from the plan +// Build generates one SQL++ query per series selector in the plan. func (b *SQLBuilder) Build() ([]string, error) { - if b.useBatching { - return b.buildBatchedQueries() + if len(b.plan.SeriesQueries) == 0 { + return nil, fmt.Errorf("query has no series selector") } - return b.buildIndividualQueries() -} - -// buildIndividualQueries builds separate queries for each series -func (b *SQLBuilder) buildIndividualQueries() ([]string, error) { - var queries []string - - for _, seriesQuery := range b.plan.SeriesQueries { - query, err := b.buildSeriesQuery(seriesQuery) - if err != nil { - return nil, err - } - queries = append(queries, query) + queries := make([]string, 0, len(b.plan.SeriesQueries)) + for _, sq := range b.plan.SeriesQueries { + queries = append(queries, b.buildSeriesQuery(sq)) } - return queries, nil } -// buildBatchedQueries builds batched queries using UNION ALL -func (b *SQLBuilder) buildBatchedQueries() ([]string, error) { - batched := b.plan.GetBatchedQueries() - var queries []string +// buildSeriesQuery selects a single metric's samples. metric_name, job, and the +// label matchers filter the document BEFORE UNNEST (efficient); the time window +// is bound into the _timeseries range so only in-window samples are expanded. +func (b *SQLBuilder) buildSeriesQuery(sq SeriesQuery) string { + fromMillis, toMillis := b.timeRangeMillis() - for _, seriesQueries := range batched { - if len(seriesQueries) == 1 { - // Single series, no need to batch - query, err := b.buildSeriesQuery(seriesQueries[0]) - if err != nil { - return nil, err - } - queries = append(queries, query) - continue - } - - // Build UNION ALL query - var unionParts []string - for _, sq := range seriesQueries { - part := b.buildSeriesQueryPart(sq) - unionParts = append(unionParts, part) - } - - // Combine with UNION ALL and add aggregation if needed - unionQuery := strings.Join(unionParts, " UNION ALL ") - - // If aggregation is needed, wrap in aggregation query - if b.plan.Aggregation != nil { - unionQuery = b.wrapWithAggregation(unionQuery) - } - - queries = append(queries, unionQuery) + conditions := []string{fmt.Sprintf("d.metric_name = '%s'", escapeSQLString(sq.MetricName))} + if labelWhere := b.buildLabelWhereClause(sq); labelWhere != "" { + conditions = append(conditions, labelWhere) } - return queries, nil -} - -// buildSeriesQuery builds a SQL++ query for a single series -func (b *SQLBuilder) buildSeriesQuery(seriesQuery SeriesQuery) (string, error) { - baseQuery := b.buildSeriesQueryPart(seriesQuery) - - // Apply function transformation if needed - if b.plan.Function != nil { - baseQuery = b.applyFunction(baseQuery) - } - - // Apply aggregation if needed - if b.plan.Aggregation != nil { - baseQuery = b.wrapWithAggregation(baseQuery) - } - - return baseQuery, nil -} - -// buildSeriesQueryPart builds the base SQL++ query part for a series -func (b *SQLBuilder) buildSeriesQueryPart(seriesQuery SeriesQuery) string { - // Build metrics_filter function call - metricFilter := b.buildMetricFilter(seriesQuery) - - // Build time range - fromMillis := b.queryCtx.StartTime.UnixMilli() - toMillis := b.queryCtx.EndTime.UnixMilli() - if !b.queryCtx.IsRange { - // For instant queries, use a small range around the time (~30s) - fromMillis = b.queryCtx.Time.UnixMilli() - 30000 - toMillis = b.queryCtx.Time.UnixMilli() + 1000 - } - - // Build SELECT clause - selectClause := b.buildSelectClause(seriesQuery) - - // Build time range condition - // Note: label filters are already embedded in metricFilter (before UNNEST) - // Time filtering happens after UNNEST but before final output - timeCondition := fmt.Sprintf("d.time_millis >= %d AND d.time_millis <= %d", fromMillis, toMillis) - - query := fmt.Sprintf( - "SELECT %s FROM %s AS d WHERE %s", - selectClause, - metricFilter, - timeCondition, + return fmt.Sprintf( + "SELECT MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels "+ + "FROM %s AS d UNNEST _timeseries(d, {'ts_ranges':[%d, %d]}) AS t WHERE %s", + b.keyspace(), fromMillis, toMillis, strings.Join(conditions, " AND "), ) - - return query } -// buildMetricFilter builds the query with all label conditions embedded BEFORE UNNEST -// This ensures optimal performance by filtering at the earliest possible point -func (b *SQLBuilder) buildMetricFilter(seriesQuery SeriesQuery) string { - // Build all label conditions including job/snapshot - labelConditions := b.buildLabelWhereClause(seriesQuery) - - // Construct query inline with conditions embedded - filters BEFORE UNNEST - baseQuery := fmt.Sprintf( - "SELECT t._t AS time_millis, MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels FROM cbmonitor._default._default AS d UNNEST _timeseries(d, {'ts_ranges':[0, 9223372036854775807]}) AS t WHERE d.metric_name = '%s'", - seriesQuery.MetricName, - ) - - if labelConditions != "" { - baseQuery = fmt.Sprintf("%s AND %s", baseQuery, labelConditions) +// keyspace returns the configured metrics keyspace (bucket.scope.collection), +// falling back to the default when unset. +func (b *SQLBuilder) keyspace() string { + if b.queryCtx != nil && b.queryCtx.Keyspace != "" { + return b.queryCtx.Keyspace } - - return fmt.Sprintf("(%s)", baseQuery) + return defaultKeyspace } -// buildSelectClause builds the SELECT clause -func (b *SQLBuilder) buildSelectClause(seriesQuery SeriesQuery) string { - parts := []string{ - "d.time", - "d.`value`", - } - - // Add label fields if needed - if len(seriesQuery.Labels) > 0 || b.plan.Aggregation != nil { - // Include labels for grouping/aggregation - parts = append(parts, "d.labels") +// timeRangeMillis returns the [from,to] window in Unix milliseconds. Range +// queries use the requested window; instant queries read back instantLookback +// from the evaluation time. +func (b *SQLBuilder) timeRangeMillis() (int64, int64) { + if b.queryCtx.IsRange { + return b.queryCtx.StartTime.UnixMilli(), b.queryCtx.EndTime.UnixMilli() } - - return strings.Join(parts, ", ") + return b.queryCtx.Time.Add(-instantLookback).UnixMilli(), b.queryCtx.Time.UnixMilli() } -// buildLabelWhereClause builds the WHERE clause for label filters including job/snapshot -func (b *SQLBuilder) buildLabelWhereClause(seriesQuery SeriesQuery) string { - // Convert seriesQuery.Labels to LabelFilter slice for shared utility +// buildLabelWhereClause builds the job/snapshot + label-matcher conditions +// (everything except metric_name) via the shared query builder. +func (b *SQLBuilder) buildLabelWhereClause(sq SeriesQuery) string { var filters []querybuilder.LabelFilter - // Check if job label is already in Labels (could be with operators like !=, =~, !~) - hasJobInLabels := false - for labelName := range seriesQuery.Labels { - // Remove operator prefixes to get actual label name - actualLabel := labelName - if strings.HasPrefix(labelName, "!") { - actualLabel = strings.TrimPrefix(labelName, "!") - } else if strings.HasPrefix(labelName, "=~") { - actualLabel = strings.TrimPrefix(labelName, "=~") - } else if strings.HasPrefix(labelName, "!~") { - actualLabel = strings.TrimPrefix(labelName, "!~") - } - if actualLabel == "job" || actualLabel == "snapshot" || actualLabel == "snapshot_id" { - hasJobInLabels = true - break - } - } - - // Add job/snapshot filter if not already present in Labels - if !hasJobInLabels { - snapshot := seriesQuery.Snapshot + // Add the job/snapshot equality filter unless the selector already carried + // an explicit job matcher (preserved below with its own operator). + if !hasJobMatcher(sq.Labels) { + snapshot := sq.Snapshot if snapshot == "" { snapshot = b.queryCtx.SnapshotID } if snapshot != "" { - // Add job label filter (snapshot is stored as job label) - filters = append(filters, querybuilder.LabelFilter{ - Name: "job", - Value: snapshot, - Op: "=", - }) + filters = append(filters, querybuilder.LabelFilter{Name: "job", Value: snapshot, Op: "="}) } } - // Process all label filters from Labels - for labelName, labelValue := range seriesQuery.Labels { - if strings.HasPrefix(labelName, "!") { - // NOT EQUAL - actualLabel := strings.TrimPrefix(labelName, "!") - filters = append(filters, querybuilder.LabelFilter{ - Name: actualLabel, - Value: labelValue, - Op: "!=", - }) - } else if strings.HasPrefix(labelName, "=~") { - // REGEX MATCH - actualLabel := strings.TrimPrefix(labelName, "=~") - filters = append(filters, querybuilder.LabelFilter{ - Name: actualLabel, - Value: labelValue, - Op: "=~", - }) - } else if strings.HasPrefix(labelName, "!~") { - // NOT REGEX MATCH - actualLabel := strings.TrimPrefix(labelName, "!~") - filters = append(filters, querybuilder.LabelFilter{ - Name: actualLabel, - Value: labelValue, - Op: "!~", - }) - } else { - // EQUAL - process all labels (job will be handled above if not in Labels) - filters = append(filters, querybuilder.LabelFilter{ - Name: labelName, - Value: labelValue, - Op: "=", - }) + for labelName, labelValue := range sq.Labels { + switch { + case strings.HasPrefix(labelName, "!~"): + filters = append(filters, querybuilder.LabelFilter{Name: strings.TrimPrefix(labelName, "!~"), Value: labelValue, Op: "!~"}) + case strings.HasPrefix(labelName, "=~"): + filters = append(filters, querybuilder.LabelFilter{Name: strings.TrimPrefix(labelName, "=~"), Value: labelValue, Op: "=~"}) + case strings.HasPrefix(labelName, "!"): + filters = append(filters, querybuilder.LabelFilter{Name: strings.TrimPrefix(labelName, "!"), Value: labelValue, Op: "!="}) + default: + filters = append(filters, querybuilder.LabelFilter{Name: labelName, Value: labelValue, Op: "="}) } } return querybuilder.BuildLabelWhereClauseFromFilters(filters) } -// applyFunction applies PromQL function transformations -func (b *SQLBuilder) applyFunction(query string) string { - if b.plan.Function == nil { - return query - } - - switch b.plan.Function.Name { - case "rate", "irate", "increase": - // These need to be calculated post-query for now - // We'll mark the query for post-processing - return query - default: - return query - } -} - -// wrapWithAggregation wraps query with aggregation -func (b *SQLBuilder) wrapWithAggregation(query string) string { - if b.plan.Aggregation == nil { - return query - } - - // Build GROUP BY clause - var groupBy []string - if len(b.plan.Aggregation.By) > 0 { - for _, label := range b.plan.Aggregation.By { - groupBy = append(groupBy, fmt.Sprintf("d.labels.%s", querybuilder.EscapeLabel(label))) +// hasJobMatcher reports whether the selector already constrains job/snapshot +// (with any operator), so the default equality job filter isn't added twice. +func hasJobMatcher(labels map[string]string) bool { + for name := range labels { + actual := name + switch { + case strings.HasPrefix(actual, "!~"): + actual = strings.TrimPrefix(actual, "!~") + case strings.HasPrefix(actual, "=~"): + actual = strings.TrimPrefix(actual, "=~") + case strings.HasPrefix(actual, "!"): + actual = strings.TrimPrefix(actual, "!") } - } - groupBy = append(groupBy, "d.time") // Always group by time - - // Build aggregation function - aggFunc := strings.ToUpper(b.plan.Aggregation.Operation) - if aggFunc == "" { - aggFunc = "SUM" - } - - // Wrap query - wrapped := fmt.Sprintf( - "SELECT time, %s(value) AS value %s FROM (%s) AS subq GROUP BY %s ORDER BY time", - aggFunc, - b.buildLabelSelect(groupBy), - query, - strings.Join(groupBy, ", "), - ) - - return wrapped -} - -// buildLabelSelect builds label selection for GROUP BY -func (b *SQLBuilder) buildLabelSelect(groupBy []string) string { - var labels []string - for _, gb := range groupBy { - if strings.HasPrefix(gb, "d.labels.") { - labels = append(labels, gb) + if actual == "job" || actual == "snapshot" || actual == "snapshot_id" { + return true } } - if len(labels) > 0 { - return ", " + strings.Join(labels, ", ") - } - return "" + return false } -// GetTimeRange returns the time range for the query -func (b *SQLBuilder) GetTimeRange() (time.Time, time.Time) { - if b.queryCtx.IsRange { - return b.queryCtx.StartTime, b.queryCtx.EndTime - } - // For instant queries, return a small range - return b.queryCtx.Time.Add(-time.Second), b.queryCtx.Time.Add(time.Second) +func escapeSQLString(s string) string { + return strings.ReplaceAll(s, "'", "''") } diff --git a/datasource-gateway/internal/promql/sqlbuilder_test.go b/datasource-gateway/internal/promql/sqlbuilder_test.go new file mode 100644 index 0000000..139fcde --- /dev/null +++ b/datasource-gateway/internal/promql/sqlbuilder_test.go @@ -0,0 +1,141 @@ +package promql + +import ( + "strings" + "testing" +) + +func translateOne(t *testing.T, query string, ctx *QueryContext) string { + t.Helper() + expr, err := ParseQuery(query) + if err != nil { + t.Fatalf("ParseQuery(%q): %v", query, err) + } + plan, err := PlanQuery(expr, "") + if err != nil { + t.Fatalf("PlanQuery: %v", err) + } + queries, err := NewSQLBuilder(plan, ctx).Build() + if err != nil { + t.Fatalf("Build: %v", err) + } + if len(queries) != 1 { + t.Fatalf("expected 1 query, got %d: %v", len(queries), queries) + } + return queries[0] +} + +// rangeCtx is a 1h range-query context [1700000000, 1700003600] (Unix seconds). +func rangeCtx(keyspace string) *QueryContext { + c, err := ParseQueryContext("", "", "1700000000", "1700003600", "15s", "") + if err != nil { + panic(err) + } + c.Keyspace = keyspace + return c +} + +func mustContainAll(t *testing.T, sql string, wants ...string) { + t.Helper() + for _, w := range wants { + if !strings.Contains(sql, w) { + t.Errorf("SQL missing %q\nSQL: %s", w, sql) + } + } +} + +func mustNotContain(t *testing.T, sql string, notWants ...string) { + t.Helper() + for _, w := range notWants { + if strings.Contains(sql, w) { + t.Errorf("SQL unexpectedly contains %q\nSQL: %s", w, sql) + } + } +} + +func TestBuildSelectorWithJobAndWindow(t *testing.T) { + sql := translateOne(t, `kv_curr_items{job="snap-1"}`, rangeCtx("bkt.scp.col")) + mustContainAll(t, sql, + "FROM bkt.scp.col AS d", + "d.metric_name = 'kv_curr_items'", + "d.labels.`job` = 'snap-1'", + "_timeseries(d, {'ts_ranges':[1700000000000, 1700003600000]})", + "MILLIS_TO_STR(t._t) AS time", + "t._v0 AS `value`", + "d.labels AS labels", + ) +} + +func TestBuildDefaultKeyspace(t *testing.T) { + sql := translateOne(t, `up{job="s"}`, rangeCtx("")) + mustContainAll(t, sql, "FROM cbmonitor._default._default AS d") +} + +func TestBuildLabelMatchers(t *testing.T) { + sql := translateOne(t, `kv_ops{job="s",proc="memcached",mode!="idle"}`, rangeCtx("k")) + mustContainAll(t, sql, + "d.labels.`proc` = 'memcached'", + "d.labels.`mode` != 'idle'", + ) +} + +func TestBuildRegexMatchers(t *testing.T) { + sql := translateOne(t, `kv_ops{job="s",bucket=~"a.*",node!~"x.*"}`, rangeCtx("k")) + mustContainAll(t, sql, + "REGEXP_MATCHES(d.labels.`bucket`, '^(a.*)$')", + "NOT REGEXP_MATCHES(d.labels.`node`, '^(x.*)$')", + ) + mustNotContain(t, sql, "LIKE") +} + +func TestBuildRateHasNoSQLRate(t *testing.T) { + // rate is computed in Go; the SQL is just the underlying selector. + sql := translateOne(t, `rate(kv_ops{job="s"}[5m])`, rangeCtx("k")) + mustContainAll(t, sql, "d.metric_name = 'kv_ops'") + mustNotContain(t, sql, "rate(", "RATE(") +} + +func TestBuildAggregationHasNoSQLGroupBy(t *testing.T) { + // aggregation is computed in Go; the SQL is just the underlying selector. + sql := translateOne(t, `sum by (instance) (kv_ops{job="s"})`, rangeCtx("k")) + mustContainAll(t, sql, "d.metric_name = 'kv_ops'") + mustNotContain(t, sql, "GROUP BY", "SUM(") +} + +func TestBuildInstantQueryWindow(t *testing.T) { + c, err := ParseQueryContext(`up{job="s"}`, "1700000000", "", "", "", "") + if err != nil { + t.Fatalf("ParseQueryContext: %v", err) + } + c.Keyspace = "k" + sql := translateOne(t, `up{job="s"}`, c) + // Instant query: window ends at the eval time (1700000000000) and looks + // back instantLookback (5m = 300000ms) → [1699999700000, 1700000000000]. + mustContainAll(t, sql, "_timeseries(d, {'ts_ranges':[1699999700000, 1700000000000]})") +} + +func TestPlannerByWithout(t *testing.T) { + byExpr, err := ParseQuery(`sum by (instance) (up{job="s"})`) + if err != nil { + t.Fatalf("parse by: %v", err) + } + byPlan, err := PlanQuery(byExpr, "") + if err != nil { + t.Fatalf("plan by: %v", err) + } + if byPlan.Aggregation == nil || len(byPlan.Aggregation.By) != 1 || byPlan.Aggregation.By[0] != "instance" || len(byPlan.Aggregation.Without) != 0 { + t.Errorf("by-plan = %+v, want By=[instance] Without=[]", byPlan.Aggregation) + } + + withoutExpr, err := ParseQuery(`sum without (le) (up{job="s"})`) + if err != nil { + t.Fatalf("parse without: %v", err) + } + withoutPlan, err := PlanQuery(withoutExpr, "") + if err != nil { + t.Fatalf("plan without: %v", err) + } + if withoutPlan.Aggregation == nil || len(withoutPlan.Aggregation.Without) != 1 || withoutPlan.Aggregation.Without[0] != "le" || len(withoutPlan.Aggregation.By) != 0 { + t.Errorf("without-plan = %+v, want Without=[le] By=[]", withoutPlan.Aggregation) + } +} diff --git a/datasource-gateway/internal/querybuilder/querybuilder.go b/datasource-gateway/internal/querybuilder/querybuilder.go index 3315d15..6be8eb5 100644 --- a/datasource-gateway/internal/querybuilder/querybuilder.go +++ b/datasource-gateway/internal/querybuilder/querybuilder.go @@ -58,9 +58,10 @@ func BuildLabelWhereClauseFromFilters(filters []LabelFilter) string { case "!=": condition = fmt.Sprintf(`d.labels.%s != '%s'`, escapedLabel, escapedValue) case "=~": - condition = fmt.Sprintf(`d.labels.%s LIKE '%s'`, escapedLabel, escapedValue) + // PromQL =~ is a full-string regex match, not a SQL glob. + condition = fmt.Sprintf(`REGEXP_MATCHES(d.labels.%s, '^(%s)$')`, escapedLabel, escapedValue) case "!~": - condition = fmt.Sprintf(`d.labels.%s NOT LIKE '%s'`, escapedLabel, escapedValue) + condition = fmt.Sprintf(`NOT REGEXP_MATCHES(d.labels.%s, '^(%s)$')`, escapedLabel, escapedValue) default: // "=" condition = fmt.Sprintf(`d.labels.%s = '%s'`, escapedLabel, escapedValue) } From d3671aa6076082961b6267e5f3cfdcd7254e3797 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Wed, 3 Jun 2026 13:14:27 +0100 Subject: [PATCH 08/18] gateway: Implement cb query evaluator Use prometheus query engine to run promql functions, only selectors generate slq++ --- datasource-gateway/go.mod | 3 + datasource-gateway/go.sum | 5 + datasource-gateway/internal/cbeval/cbeval.go | 336 ++++++++++++++++++ .../internal/cbeval/cbeval_test.go | 143 ++++++++ 4 files changed, 487 insertions(+) create mode 100644 datasource-gateway/internal/cbeval/cbeval.go create mode 100644 datasource-gateway/internal/cbeval/cbeval_test.go diff --git a/datasource-gateway/go.mod b/datasource-gateway/go.mod index e0e05bb..5a06d8c 100644 --- a/datasource-gateway/go.mod +++ b/datasource-gateway/go.mod @@ -15,8 +15,11 @@ require ( github.com/couchbase/gocbcoreps v0.1.4 // indirect github.com/couchbase/goprotostellar v1.0.2 // indirect github.com/couchbaselabs/gocbconnstr/v2 v2.0.0 // indirect + github.com/edsrzf/mmap-go v1.2.0 // indirect + github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect diff --git a/datasource-gateway/go.sum b/datasource-gateway/go.sum index a6784f2..07ccb35 100644 --- a/datasource-gateway/go.sum +++ b/datasource-gateway/go.sum @@ -71,10 +71,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= +github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= +github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= +github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -85,6 +89,7 @@ github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= diff --git a/datasource-gateway/internal/cbeval/cbeval.go b/datasource-gateway/internal/cbeval/cbeval.go new file mode 100644 index 0000000..b0b8fd7 --- /dev/null +++ b/datasource-gateway/internal/cbeval/cbeval.go @@ -0,0 +1,336 @@ +// Package cbeval evaluates PromQL against samples stored in Couchbase by +// running the real Prometheus query engine over a storage adapter. The adapter +// translates each selector to SQL++ and returns the raw samples; the engine +// computes rate/irate/increase, aggregation, and every other PromQL construct +// with exact Prometheus semantics — the same engine Mimir uses — so results +// match a Prometheus/Mimir passthrough by construction. +package cbeval + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/util/annotations" + + "github.com/couchbase/datasource-gateway/internal/querybuilder" +) + +const ( + defaultMaxSamples = 50_000_000 + defaultTimeout = 2 * time.Minute +) + +// RowQuerier executes a SQL++ statement and returns the rows. The Couchbase +// client satisfies this. +type RowQuerier interface { + ExecuteQuery(ctx context.Context, query string) ([]map[string]interface{}, error) +} + +// Result is the Prometheus-API response shape produced by the evaluator. +type Result struct { + Status string `json:"status"` + Data ResultData `json:"data"` +} + +// ResultData is the data envelope of a Prometheus query response. +type ResultData struct { + ResultType string `json:"resultType"` + Result []SeriesJSON `json:"result"` +} + +// SeriesJSON is one matrix series in Prometheus JSON form. +type SeriesJSON struct { + Metric map[string]string `json:"metric"` + Values [][]interface{} `json:"values,omitempty"` +} + +// Evaluator runs the Prometheus query engine over Couchbase-backed samples. +type Evaluator struct { + engine *promql.Engine + querier RowQuerier + keyspace string +} + +// NewEvaluator builds an evaluator that fetches samples via querier from the +// given metrics keyspace (bucket.scope.collection). +func NewEvaluator(querier RowQuerier, keyspace string) *Evaluator { + engine := promql.NewEngine(promql.EngineOpts{ + MaxSamples: defaultMaxSamples, + Timeout: defaultTimeout, + EnableAtModifier: true, + EnableNegativeOffset: true, + }) + return &Evaluator{engine: engine, querier: querier, keyspace: keyspace} +} + +// RangeQuery evaluates a PromQL range query against Couchbase-backed samples +// and returns the matrix result in Prometheus JSON form. +func (e *Evaluator) RangeQuery(ctx context.Context, query string, start, end time.Time, step time.Duration) (*Result, error) { + q, err := e.engine.NewRangeQuery(ctx, e.queryable(), nil, query, start, end, step) + if err != nil { + return nil, err + } + defer q.Close() + + res := q.Exec(ctx) + if res.Err != nil { + return nil, res.Err + } + matrix, ok := res.Value.(promql.Matrix) + if !ok { + return nil, fmt.Errorf("unexpected result type %s for range query", res.Value.Type()) + } + return matrixToResult(matrix), nil +} + +func (e *Evaluator) queryable() storage.Queryable { + return &couchbaseQueryable{querier: e.querier, keyspace: e.keyspace} +} + +func matrixToResult(m promql.Matrix) *Result { + out := &Result{Status: "success"} + out.Data.ResultType = "matrix" + out.Data.Result = make([]SeriesJSON, 0, len(m)) + for _, s := range m { + sj := SeriesJSON{Metric: s.Metric.Map(), Values: make([][]interface{}, 0, len(s.Floats))} + for _, p := range s.Floats { + sj.Values = append(sj.Values, []interface{}{ + float64(p.T) / 1000, + strconv.FormatFloat(p.F, 'f', -1, 64), + }) + } + out.Data.Result = append(out.Data.Result, sj) + } + return out +} + +// --- storage adapter --- + +type couchbaseQueryable struct { + querier RowQuerier + keyspace string +} + +func (q *couchbaseQueryable) Querier(mint, maxt int64) (storage.Querier, error) { + return &couchbaseQuerier{querier: q.querier, keyspace: q.keyspace, mint: mint, maxt: maxt}, nil +} + +type couchbaseQuerier struct { + querier RowQuerier + keyspace string + mint, maxt int64 +} + +func (q *couchbaseQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + from, to := q.mint, q.maxt + if hints != nil { + from, to = hints.Start, hints.End + } + sql, err := buildSelectorSQL(matchers, q.keyspace, from, to) + if err != nil { + return storage.ErrSeriesSet(err) + } + rows, err := q.querier.ExecuteQuery(ctx, sql) + if err != nil { + return storage.ErrSeriesSet(fmt.Errorf("couchbase query failed: %w", err)) + } + series := rowsToSeries(rows, metricName(matchers)) + if sortSeries { + sort.Slice(series, func(i, j int) bool { + return labels.Compare(series[i].Labels(), series[j].Labels()) < 0 + }) + } + return &sliceSeriesSet{series: series, idx: -1} +} + +func (q *couchbaseQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { + return nil, nil, nil +} + +func (q *couchbaseQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { + return nil, nil, nil +} + +func (q *couchbaseQuerier) Close() error { return nil } + +// --- row parsing --- + +func metricName(matchers []*labels.Matcher) string { + for _, m := range matchers { + if m.Name == labels.MetricName { + return m.Value + } + } + return "" +} + +func rowsToSeries(rows []map[string]interface{}, metric string) []storage.Series { + grouped := map[string]*seriesBuilder{} + var order []string + for _, row := range rows { + lbls := rowLabels(row, metric) + key := lbls.String() + sb, ok := grouped[key] + if !ok { + sb = &seriesBuilder{lbls: lbls} + grouped[key] = sb + order = append(order, key) + } + ts, ok := rowMillis(row["time"]) + if !ok { + continue + } + v, ok := rowFloat(row["value"]) + if !ok { + continue + } + sb.samples = append(sb.samples, floatSample{t: ts, v: v}) + } + + series := make([]storage.Series, 0, len(order)) + for _, key := range order { + sb := grouped[key] + sort.Slice(sb.samples, func(i, j int) bool { return sb.samples[i].t < sb.samples[j].t }) + cs := make([]chunks.Sample, len(sb.samples)) + for i := range sb.samples { + cs[i] = sb.samples[i] + } + series = append(series, storage.NewListSeries(sb.lbls, cs)) + } + return series +} + +type seriesBuilder struct { + lbls labels.Labels + samples []floatSample +} + +func rowLabels(row map[string]interface{}, metric string) labels.Labels { + b := labels.NewBuilder(labels.EmptyLabels()) + if metric != "" { + b.Set(labels.MetricName, metric) + } + if raw, ok := row["labels"].(map[string]interface{}); ok { + for k, v := range raw { + b.Set(k, fmt.Sprintf("%v", v)) + } + } + return b.Labels() +} + +func rowMillis(v interface{}) (int64, bool) { + switch t := v.(type) { + case string: + if ts, err := time.Parse(time.RFC3339, t); err == nil { + return ts.UnixMilli(), true + } + if ms, err := strconv.ParseInt(t, 10, 64); err == nil { + return ms, true + } + case float64: + return int64(t), true + case int64: + return t, true + } + return 0, false +} + +func rowFloat(v interface{}) (float64, bool) { + switch n := v.(type) { + case float64: + return n, true + case int64: + return float64(n), true + case int: + return float64(n), true + case string: + if f, err := strconv.ParseFloat(n, 64); err == nil { + return f, true + } + } + return 0, false +} + +// --- minimal chunks.Sample + storage.SeriesSet implementations --- + +type floatSample struct { + t int64 + v float64 +} + +func (s floatSample) T() int64 { return s.t } +func (s floatSample) F() float64 { return s.v } +func (s floatSample) H() *histogram.Histogram { return nil } +func (s floatSample) FH() *histogram.FloatHistogram { return nil } +func (s floatSample) Type() chunkenc.ValueType { return chunkenc.ValFloat } +func (s floatSample) Copy() chunks.Sample { return s } + +type sliceSeriesSet struct { + series []storage.Series + idx int +} + +func (s *sliceSeriesSet) Next() bool { s.idx++; return s.idx < len(s.series) } +func (s *sliceSeriesSet) At() storage.Series { return s.series[s.idx] } +func (s *sliceSeriesSet) Err() error { return nil } +func (s *sliceSeriesSet) Warnings() annotations.Annotations { return nil } + +// --- selector SQL --- + +// buildSelectorSQL turns one vector selector's label matchers into the SQL++ +// that fetches its raw samples over [fromMillis, toMillis]. The metric name +// filters the document; label matchers (=, !=, =~, !~) reuse the shared label +// clause builder; the window is bound into the _timeseries range. +func buildSelectorSQL(matchers []*labels.Matcher, keyspace string, fromMillis, toMillis int64) (string, error) { + var metric string + var filters []querybuilder.LabelFilter + for _, m := range matchers { + if m.Name == labels.MetricName { + if m.Type != labels.MatchEqual { + return "", fmt.Errorf("metric name must use an equality matcher, got %q", m.String()) + } + metric = m.Value + continue + } + filters = append(filters, querybuilder.LabelFilter{Name: m.Name, Value: m.Value, Op: matchOp(m.Type)}) + } + if metric == "" { + return "", fmt.Errorf("selector has no metric name") + } + + conds := []string{fmt.Sprintf("d.metric_name = '%s'", escapeSQL(metric))} + if lw := querybuilder.BuildLabelWhereClauseFromFilters(filters); lw != "" { + conds = append(conds, lw) + } + + return fmt.Sprintf( + "SELECT MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels "+ + "FROM %s AS d UNNEST _timeseries(d, {'ts_ranges':[%d, %d]}) AS t WHERE %s", + keyspace, fromMillis, toMillis, strings.Join(conds, " AND "), + ), nil +} + +func matchOp(t labels.MatchType) string { + switch t { + case labels.MatchNotEqual: + return "!=" + case labels.MatchRegexp: + return "=~" + case labels.MatchNotRegexp: + return "!~" + default: + return "=" + } +} + +func escapeSQL(s string) string { return strings.ReplaceAll(s, "'", "''") } diff --git a/datasource-gateway/internal/cbeval/cbeval_test.go b/datasource-gateway/internal/cbeval/cbeval_test.go new file mode 100644 index 0000000..8a7807c --- /dev/null +++ b/datasource-gateway/internal/cbeval/cbeval_test.go @@ -0,0 +1,143 @@ +package cbeval + +import ( + "context" + "strconv" + "strings" + "testing" + "time" + + "github.com/prometheus/prometheus/model/labels" +) + +var base = time.Unix(1700000000, 0).UTC() + +// fakeRowQuerier returns canned rows regardless of the SQL — the test controls +// the dataset and the query so the returned rows are exactly the series the +// engine should evaluate. +type fakeRowQuerier struct { + rows []map[string]interface{} + lastSQL string +} + +func (f *fakeRowQuerier) ExecuteQuery(_ context.Context, sql string) ([]map[string]interface{}, error) { + f.lastSQL = sql + return f.rows, nil +} + +// counterRows builds samples for one series: value = offsetSeconds * slope, +// every 15s for the given count, so a slope of 1 yields a 1/s counter. +func counterRows(job, instance string, count int, slope float64) []map[string]interface{} { + rows := make([]map[string]interface{}, 0, count) + for i := 0; i < count; i++ { + offset := i * 15 + rows = append(rows, map[string]interface{}{ + "time": base.Add(time.Duration(offset) * time.Second).Format(time.RFC3339), + "value": float64(offset) * slope, + "labels": map[string]interface{}{"job": job, "instance": instance}, + }) + } + return rows +} + +func TestRangeQueryRate(t *testing.T) { + // One linear counter (value == seconds) → rate == 1/s. + q := &fakeRowQuerier{rows: counterRows("s", "n1", 14, 1)} + e := NewEvaluator(q, "k") + + res, err := e.RangeQuery(context.Background(), + `rate(c{job="s"}[1m])`, + base.Add(60*time.Second), base.Add(180*time.Second), 30*time.Second) + if err != nil { + t.Fatalf("RangeQuery: %v", err) + } + if res.Data.ResultType != "matrix" { + t.Fatalf("resultType = %q, want matrix", res.Data.ResultType) + } + if len(res.Data.Result) != 1 { + t.Fatalf("got %d series, want 1", len(res.Data.Result)) + } + pts := res.Data.Result[0].Values + if len(pts) == 0 { + t.Fatal("no points in rate series") + } + for _, p := range pts { + v, err := strconv.ParseFloat(p[1].(string), 64) + if err != nil { + t.Fatalf("parse value %v: %v", p[1], err) + } + if v < 0.8 || v > 1.2 { + t.Errorf("rate = %v, want ~1.0 (per-second)", v) + } + } +} + +func TestRangeQueryAggregation(t *testing.T) { + // Two instances; sum by(job) collapses them into one series. + rows := append(counterRows("s", "n1", 14, 1), counterRows("s", "n2", 14, 2)...) + q := &fakeRowQuerier{rows: rows} + e := NewEvaluator(q, "k") + + res, err := e.RangeQuery(context.Background(), + `sum by (job) (c{job="s"})`, + base.Add(60*time.Second), base.Add(180*time.Second), 30*time.Second) + if err != nil { + t.Fatalf("RangeQuery: %v", err) + } + if len(res.Data.Result) != 1 { + t.Fatalf("got %d series, want 1 (grouped by job)", len(res.Data.Result)) + } + metric := res.Data.Result[0].Metric + if len(metric) != 1 || metric["job"] != "s" { + t.Errorf("metric = %v, want {job: s}", metric) + } + // At the last step (base+180s) the latest samples are n1=180, n2=360 → 540. + pts := res.Data.Result[0].Values + last := pts[len(pts)-1] + v, err := strconv.ParseFloat(last[1].(string), 64) + if err != nil { + t.Fatalf("parse last value: %v", err) + } + if v < 539 || v > 541 { + t.Errorf("last sum = %v, want ~540", v) + } +} + +func TestBuildSelectorSQL(t *testing.T) { + mk := func(typ labels.MatchType, n, v string) *labels.Matcher { + m, err := labels.NewMatcher(typ, n, v) + if err != nil { + t.Fatalf("matcher: %v", err) + } + return m + } + matchers := []*labels.Matcher{ + mk(labels.MatchEqual, labels.MetricName, "kv_ops"), + mk(labels.MatchEqual, "job", "snap-1"), + mk(labels.MatchRegexp, "bucket", "a.*"), + mk(labels.MatchNotEqual, "mode", "idle"), + } + sql, err := buildSelectorSQL(matchers, "bkt.scp.col", 1000, 2000) + if err != nil { + t.Fatalf("buildSelectorSQL: %v", err) + } + for _, want := range []string{ + "FROM bkt.scp.col AS d", + "d.metric_name = 'kv_ops'", + "d.labels.`job` = 'snap-1'", + "REGEXP_MATCHES(d.labels.`bucket`, '^(a.*)$')", + "d.labels.`mode` != 'idle'", + "_timeseries(d, {'ts_ranges':[1000, 2000]})", + } { + if !strings.Contains(sql, want) { + t.Errorf("SQL missing %q\nSQL: %s", want, sql) + } + } +} + +func TestBuildSelectorSQLRequiresMetricName(t *testing.T) { + m, _ := labels.NewMatcher(labels.MatchEqual, "job", "s") + if _, err := buildSelectorSQL([]*labels.Matcher{m}, "k", 0, 1); err == nil { + t.Error("expected an error when no metric name is present") + } +} From bda5674b396673f39e306604e4c24b4271c5fd07 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Wed, 3 Jun 2026 15:29:13 +0100 Subject: [PATCH 09/18] gateway: Evaluate snapshot metadata through the gateway --- datasource-gateway/internal/api/handlers.go | 12 ++- datasource-gateway/internal/api/query.go | 67 ++++++++++++++-- datasource-gateway/internal/api/query_test.go | 78 ++++++++++++++++--- datasource-gateway/main.go | 9 ++- 4 files changed, 144 insertions(+), 22 deletions(-) diff --git a/datasource-gateway/internal/api/handlers.go b/datasource-gateway/internal/api/handlers.go index 8d000df..21aa943 100644 --- a/datasource-gateway/internal/api/handlers.go +++ b/datasource-gateway/internal/api/handlers.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/couchbase/datasource-gateway/internal/cbeval" "github.com/couchbase/datasource-gateway/internal/router" ) @@ -24,17 +25,24 @@ type prometheusGateway interface { ReverseProxy() http.Handler } +// couchbaseEvaluator runs a PromQL range query against Couchbase-backed samples +// (via the Prometheus engine) and returns the matrix result. +type couchbaseEvaluator interface { + RangeQuery(ctx context.Context, query string, start, end time.Time, step time.Duration) (*cbeval.Result, error) +} + // Handler holds the gateway HTTP handlers: the health endpoint and the // Prometheus-compatible API surface (/api/v1/*). type Handler struct { couchbase couchbaseHealth prometheus prometheusGateway router *router.Router + evaluator couchbaseEvaluator } // NewHandler constructs the gateway HTTP handler. -func NewHandler(couchbase couchbaseHealth, prometheus prometheusGateway, router *router.Router) *Handler { - return &Handler{couchbase: couchbase, prometheus: prometheus, router: router} +func NewHandler(couchbase couchbaseHealth, prometheus prometheusGateway, router *router.Router, evaluator couchbaseEvaluator) *Handler { + return &Handler{couchbase: couchbase, prometheus: prometheus, router: router, evaluator: evaluator} } // Register wires the handler's routes onto the given mux: the gateway's own diff --git a/datasource-gateway/internal/api/query.go b/datasource-gateway/internal/api/query.go index 8680cb2..0ccc361 100644 --- a/datasource-gateway/internal/api/query.go +++ b/datasource-gateway/internal/api/query.go @@ -3,9 +3,11 @@ package api import ( "encoding/json" "net/http" + "net/url" "regexp" "strconv" "strings" + "time" "github.com/couchbase/datasource-gateway/internal/router" ) @@ -42,13 +44,64 @@ func (h *Handler) handleQueryRange(w http.ResponseWriter, r *http.Request) { h.prometheus.ReverseProxy().ServeHTTP(w, r) } -// serveCouchbaseQueryRange will translate the PromQL to SQL++, execute it -// against Couchbase, and shape the result as Prometheus JSON. Wired in a later -// task; for now it returns a clear, parseable error so the routing decision is -// observable without silently returning empty data. -func (h *Handler) serveCouchbaseQueryRange(w http.ResponseWriter, _ *http.Request, route router.Route) { - writePromError(w, http.StatusNotImplemented, "not_implemented", - "Couchbase-backed query execution for snapshot "+route.Snapshot+" is not yet wired") +// serveCouchbaseQueryRange evaluates the PromQL against Couchbase-backed +// samples via the Prometheus engine and writes the matrix result. It evaluates +// over the snapshot's stored window (from the router) so the panel resolves +// against the snapshot regardless of the dashboard's time picker. +func (h *Handler) serveCouchbaseQueryRange(w http.ResponseWriter, r *http.Request, route router.Route) { + start, end, ok := evalRange(route, r.Form) + if !ok { + writePromError(w, http.StatusBadRequest, "bad_data", "no snapshot window and missing/invalid start/end") + return + } + + result, err := h.evaluator.RangeQuery(r.Context(), r.Form.Get("query"), start, end, parseStepParam(r.Form.Get("step"))) + if err != nil { + writePromError(w, http.StatusUnprocessableEntity, "execution", err.Error()) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(result) +} + +// evalRange picks the evaluation window: the snapshot's stored window when +// known, else the request's own start/end (Unix seconds, Prometheus-style). +func evalRange(route router.Route, form url.Values) (time.Time, time.Time, bool) { + if route.HasWindow { + return route.Start, route.End, true + } + start, ok1 := parseUnixSeconds(form.Get("start")) + end, ok2 := parseUnixSeconds(form.Get("end")) + if !ok1 || !ok2 { + return time.Time{}, time.Time{}, false + } + return start, end, true +} + +// parseStepParam parses the query_range step, accepting a Go duration ("15s") +// or bare seconds ("15"); defaults to 15s. +func parseStepParam(s string) time.Duration { + if s == "" { + return 15 * time.Second + } + if d, err := time.ParseDuration(s); err == nil && d > 0 { + return d + } + if f, err := strconv.ParseFloat(s, 64); err == nil && f > 0 { + return time.Duration(f * float64(time.Second)) + } + return 15 * time.Second +} + +func parseUnixSeconds(s string) (time.Time, bool) { + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return time.Time{}, false + } + sec := int64(f) + return time.Unix(sec, int64((f-float64(sec))*1e9)), true } // singleJob returns the snapshot ID from a single-snapshot job matcher, or "" diff --git a/datasource-gateway/internal/api/query_test.go b/datasource-gateway/internal/api/query_test.go index c6cc6d0..6340bcf 100644 --- a/datasource-gateway/internal/api/query_test.go +++ b/datasource-gateway/internal/api/query_test.go @@ -2,6 +2,8 @@ package api import ( "context" + "encoding/json" + "errors" "net/http" "net/http/httptest" "net/url" @@ -10,6 +12,7 @@ import ( "testing" "time" + "github.com/couchbase/datasource-gateway/internal/cbeval" "github.com/couchbase/datasource-gateway/internal/couchbase" "github.com/couchbase/datasource-gateway/internal/router" ) @@ -37,10 +40,26 @@ func (f *fakeProm) URL() string { return "http://upstream" func (f *fakeProm) Reachable(_ context.Context) bool { return true } func (f *fakeProm) ReverseProxy() http.Handler { return f.proxy } +type fakeEvaluator struct { + result *cbeval.Result + err error + called bool + gotQuery string + gotStart time.Time + gotEnd time.Time + gotStep time.Duration +} + +func (f *fakeEvaluator) RangeQuery(_ context.Context, query string, start, end time.Time, step time.Duration) (*cbeval.Result, error) { + f.called = true + f.gotQuery, f.gotStart, f.gotEnd, f.gotStep = query, start, end, step + return f.result, f.err +} + // recorder captures the params the reverse proxy received from the handler. type recorder struct { - called bool - start, end, query string + called bool + start, end, query string } func (rc *recorder) handler() http.Handler { @@ -54,8 +73,8 @@ func (rc *recorder) handler() http.Handler { }) } -func newTestHandler(cb *fakeCouchbase, rc *recorder) *Handler { - return NewHandler(cb, &fakeProm{proxy: rc.handler()}, router.New(cb)) +func newTestHandler(cb *fakeCouchbase, rc *recorder, ev couchbaseEvaluator) *Handler { + return NewHandler(cb, &fakeProm{proxy: rc.handler()}, router.New(cb), ev) } func postQueryRange(h *Handler, query, start, end string) *httptest.ResponseRecorder { @@ -79,9 +98,8 @@ func unixOf(t *testing.T, rfc string) string { func TestQueryRangePrometheusBackedRewritesWindow(t *testing.T) { rc := &recorder{} cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "prometheus", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} - h := newTestHandler(cb, rc) + h := newTestHandler(cb, rc, &fakeEvaluator{}) - // Deliberately-wrong dashboard range (1..2); expect it rewritten to the window. postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") if !rc.called { @@ -98,25 +116,61 @@ func TestQueryRangePrometheusBackedRewritesWindow(t *testing.T) { } } -func TestQueryRangeCouchbaseBackedReturnsStub(t *testing.T) { +func TestQueryRangeCouchbaseBackedRunsEvaluator(t *testing.T) { rc := &recorder{} cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "couchbase", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} - h := newTestHandler(cb, rc) + ev := &fakeEvaluator{result: &cbeval.Result{Status: "success"}} + ev.result.Data.ResultType = "matrix" + h := newTestHandler(cb, rc, ev) w := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") if rc.called { t.Error("Couchbase-backed query should not hit the passthrough") } - if w.Code != http.StatusNotImplemented { - t.Errorf("code = %d, want 501", w.Code) + if !ev.called { + t.Fatal("evaluator not called for a Couchbase-backed snapshot") + } + if w.Code != http.StatusOK { + t.Errorf("code = %d, want 200", w.Code) + } + if !strings.Contains(ev.gotQuery, "kv_ops") { + t.Errorf("query = %q", ev.gotQuery) + } + // Evaluated over the snapshot window, not the dashboard range (1..2). + wantStart, _ := time.Parse(time.RFC3339, "2024-01-02T00:00:00Z") + wantEnd, _ := time.Parse(time.RFC3339, "2024-01-02T01:00:00Z") + if !ev.gotStart.Equal(wantStart) || !ev.gotEnd.Equal(wantEnd) { + t.Errorf("window = [%v, %v], want [%v, %v]", ev.gotStart, ev.gotEnd, wantStart, wantEnd) + } + if ev.gotStep != 15*time.Second { + t.Errorf("step = %v, want 15s", ev.gotStep) + } +} + +func TestQueryRangeCouchbaseEvaluatorErrorRendersEnvelope(t *testing.T) { + cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "couchbase", TSStart: "2024-01-02T00:00:00Z", TSEnd: "2024-01-02T01:00:00Z"}} + ev := &fakeEvaluator{err: errors.New("boom")} + h := newTestHandler(cb, &recorder{}, ev) + + w := postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1", "2") + + if w.Code != http.StatusUnprocessableEntity { + t.Errorf("code = %d, want 422", w.Code) + } + var resp map[string]string + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp["status"] != "error" { + t.Errorf("status = %q, want error", resp["status"]) } } func TestQueryRangeOverlapForwardedUnchanged(t *testing.T) { rc := &recorder{} cb := &fakeCouchbase{enabled: true, md: &couchbase.Metadata{Store: "couchbase"}} - h := newTestHandler(cb, rc) + h := newTestHandler(cb, rc, &fakeEvaluator{}) postQueryRange(h, `rate(kv_ops{job=~"snap-1|snap-2"}[5m])`, "1000", "2000") @@ -134,7 +188,7 @@ func TestQueryRangeOverlapForwardedUnchanged(t *testing.T) { func TestQueryRangeCouchbaseDisabledForwardsUnchanged(t *testing.T) { rc := &recorder{} cb := &fakeCouchbase{enabled: false} - h := newTestHandler(cb, rc) + h := newTestHandler(cb, rc, &fakeEvaluator{}) postQueryRange(h, `rate(kv_ops{job="snap-1"}[5m])`, "1000", "2000") diff --git a/datasource-gateway/main.go b/datasource-gateway/main.go index 6f6e2e2..93b4e2f 100644 --- a/datasource-gateway/main.go +++ b/datasource-gateway/main.go @@ -12,6 +12,7 @@ import ( "time" "github.com/couchbase/datasource-gateway/internal/api" + "github.com/couchbase/datasource-gateway/internal/cbeval" "github.com/couchbase/datasource-gateway/internal/config" "github.com/couchbase/datasource-gateway/internal/couchbase" "github.com/couchbase/datasource-gateway/internal/logger" @@ -82,7 +83,13 @@ func main() { // routing is a map lookup after the first query. metricRouter := router.New(cbClient) - handler := api.NewHandler(cbClient, promClient, metricRouter) + // Couchbase query evaluator: runs the Prometheus engine over samples + // fetched from the metrics keyspace (bucket.scope.collection). + metricsKeyspace := fmt.Sprintf("`%s`.`%s`.`%s`", + cfg.Couchbase.MetricsBucket, cfg.Couchbase.MetricsScope, cfg.Couchbase.MetricsCollection) + evaluator := cbeval.NewEvaluator(cbClient, metricsKeyspace) + + handler := api.NewHandler(cbClient, promClient, metricRouter, evaluator) mux := http.NewServeMux() handler.Register(mux) From 53d2de8474b6c613641f6d21818a466b7a1d4877 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Wed, 3 Jun 2026 15:32:02 +0100 Subject: [PATCH 10/18] gateway: Remove the unused promql package --- datasource-gateway/internal/promql/parser.go | 114 ---- datasource-gateway/internal/promql/planner.go | 212 -------- .../internal/promql/promql_test.go | 86 --- .../internal/promql/sqlbuilder.go | 140 ----- .../internal/promql/sqlbuilder_test.go | 141 ----- .../internal/promql/transformer.go | 510 ------------------ 6 files changed, 1203 deletions(-) delete mode 100644 datasource-gateway/internal/promql/parser.go delete mode 100644 datasource-gateway/internal/promql/planner.go delete mode 100644 datasource-gateway/internal/promql/promql_test.go delete mode 100644 datasource-gateway/internal/promql/sqlbuilder.go delete mode 100644 datasource-gateway/internal/promql/sqlbuilder_test.go delete mode 100644 datasource-gateway/internal/promql/transformer.go diff --git a/datasource-gateway/internal/promql/parser.go b/datasource-gateway/internal/promql/parser.go deleted file mode 100644 index 07af416..0000000 --- a/datasource-gateway/internal/promql/parser.go +++ /dev/null @@ -1,114 +0,0 @@ -package promql - -import ( - "context" - "fmt" - "strconv" - "time" - - "github.com/prometheus/prometheus/promql/parser" -) - -// ParseQuery parses a PromQL query string into an AST -func ParseQuery(query string) (parser.Expr, error) { - expr, err := parser.ParseExpr(query) - if err != nil { - return nil, fmt.Errorf("failed to parse PromQL query: %w", err) - } - return expr, nil -} - -// QueryContext holds context for query execution -type QueryContext struct { - Context context.Context - Query string - Time time.Time - StartTime time.Time - EndTime time.Time - Step time.Duration - IsRange bool - SnapshotID string // Optional snapshot ID from context - Keyspace string // Metrics keyspace (bucket.scope.collection); empty → default -} - -// ParseQueryContext parses query parameters into a QueryContext -func ParseQueryContext(query string, timeStr string, startStr, endStr string, stepStr string, snapshotID string) (*QueryContext, error) { - ctx := &QueryContext{ - Query: query, - SnapshotID: snapshotID, - } - - // Parse time (for instant queries) - if timeStr != "" { - t, err := parseTime(timeStr) - if err != nil { - return nil, fmt.Errorf("invalid time parameter: %w", err) - } - ctx.Time = t - ctx.IsRange = false - } - - // Parse time range (for range queries) - if startStr != "" && endStr != "" { - start, err := parseTime(startStr) - if err != nil { - return nil, fmt.Errorf("invalid start time: %w", err) - } - end, err := parseTime(endStr) - if err != nil { - return nil, fmt.Errorf("invalid end time: %w", err) - } - ctx.StartTime = start - ctx.EndTime = end - ctx.IsRange = true - ctx.Time = end // Use end time as the evaluation time - } - - // Parse step (for range queries) - if stepStr != "" { - step, err := parseDuration(stepStr) - if err != nil { - return nil, fmt.Errorf("invalid step parameter: %w", err) - } - ctx.Step = step - } - - // Default to current time if no time specified - if ctx.Time.IsZero() { - ctx.Time = time.Now() - } - - return ctx, nil -} - -// parseTime parses a time string (Unix timestamp or RFC3339) -func parseTime(timeStr string) (time.Time, error) { - // Try RFC3339 first (most common format from Prometheus API) - // This includes formats like: 2025-11-26T14:54:31.661792773Z - if t, err := time.Parse(time.RFC3339, timeStr); err == nil { - return t, nil - } - - // Try RFC3339Nano for nanosecond precision - if t, err := time.Parse(time.RFC3339Nano, timeStr); err == nil { - return t, nil - } - - // Try Unix timestamp (only if the entire string is numeric) - // Use strconv.ParseFloat which requires the entire string to be a number - if timestamp, err := strconv.ParseFloat(timeStr, 64); err == nil { - // Check if it's in seconds or milliseconds - if timestamp > 1e10 { - // Likely milliseconds - return time.Unix(0, int64(timestamp*1e6)), nil - } - return time.Unix(int64(timestamp), 0), nil - } - - return time.Time{}, fmt.Errorf("unable to parse time: %s (expected RFC3339 format or Unix timestamp)", timeStr) -} - -// parseDuration parses a duration string (e.g., "5m", "1h", "30s") -func parseDuration(durationStr string) (time.Duration, error) { - return time.ParseDuration(durationStr) -} diff --git a/datasource-gateway/internal/promql/planner.go b/datasource-gateway/internal/promql/planner.go deleted file mode 100644 index 0728447..0000000 --- a/datasource-gateway/internal/promql/planner.go +++ /dev/null @@ -1,212 +0,0 @@ -package promql - -import ( - "fmt" - "strings" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql/parser" -) - -// QueryPlan represents an optimized query execution plan -type QueryPlan struct { - BaseExpr parser.Expr - SeriesQueries []SeriesQuery - Aggregation *AggregationPlan - Function *FunctionPlan -} - -// SeriesQuery represents a single series query to Couchbase -type SeriesQuery struct { - MetricName string - Snapshot string // Extracted from 'job' label in PromQL - Labels map[string]string // Additional labels to filter (including node, instance, etc.) -} - -// AggregationPlan represents aggregation operations -type AggregationPlan struct { - Operation string // sum, avg, max, min, count - By []string - Without []string -} - -// FunctionPlan represents function operations -type FunctionPlan struct { - Name string // rate, increase, irate, etc. - Range string // For range functions like rate[5m] - Parameter string // Optional parameter -} - -// PlanQuery creates an optimized query plan from a PromQL expression -func PlanQuery(expr parser.Expr, snapshotID string) (*QueryPlan, error) { - plan := &QueryPlan{ - BaseExpr: expr, - } - - // Walk the AST to extract series selectors and operations - err := parser.Walk(&queryPlanner{ - plan: plan, - snapshotID: snapshotID, - }, expr, nil) - - if err != nil { - return nil, fmt.Errorf("failed to plan query: %w", err) - } - - return plan, nil -} - -// queryPlanner implements parser.Visitor to walk the AST -type queryPlanner struct { - plan *QueryPlan - snapshotID string -} - -func (v *queryPlanner) Visit(node parser.Node, path []parser.Node) (parser.Visitor, error) { - switch n := node.(type) { - case *parser.VectorSelector: - // Extract series selector - seriesQuery, err := v.extractSeriesSelector(n) - if err != nil { - return nil, err - } - v.plan.SeriesQueries = append(v.plan.SeriesQueries, seriesQuery) - - case *parser.AggregateExpr: - // Extract aggregation, honouring the by/without flag. - agg := &AggregationPlan{Operation: n.Op.String()} - if n.Without { - agg.Without = v.extractLabels(n.Grouping) - } else { - agg.By = v.extractLabels(n.Grouping) - } - v.plan.Aggregation = agg - - case *parser.Call: - // Extract function call - v.plan.Function = &FunctionPlan{ - Name: n.Func.Name, - } - // Extract range parameter if it's a range function - if len(n.Args) > 0 { - if matrixSelector, ok := n.Args[0].(*parser.MatrixSelector); ok { - v.plan.Function.Range = matrixSelector.Range.String() - } - } - } - - return v, nil -} - -// extractSeriesSelector extracts series information from a vector selector -func (v *queryPlanner) extractSeriesSelector(selector *parser.VectorSelector) (SeriesQuery, error) { - query := SeriesQuery{ - MetricName: selector.Name, - Snapshot: v.snapshotID, // Will be overridden by 'job' label if present - Labels: make(map[string]string), - } - - // Extract labels from matchers - for _, matcher := range selector.LabelMatchers { - labelName := matcher.Name - labelValue := matcher.Value - - switch matcher.Type { - case labels.MatchEqual: - // Map known labels to query parameters - switch labelName { - case "job", "snapshot", "snapshot_id": - // 'job' label maps to snapshot in get_metric_for function - query.Snapshot = labelValue - default: - // Store as additional label filter (node, instance, bucket, etc.) - query.Labels[labelName] = labelValue - } - - case labels.MatchNotEqual: - // For !=, we'll need to filter in WHERE clause - query.Labels["!"+labelName] = labelValue - - case labels.MatchRegexp: - // For =~, we'll need regex matching in WHERE clause - query.Labels["=~"+labelName] = labelValue - - case labels.MatchNotRegexp: - // For !~, we'll need negative regex matching - query.Labels["!~"+labelName] = labelValue - } - } - - // Default snapshot if not provided (from job label or context) - if query.Snapshot == "" { - query.Snapshot = v.snapshotID - } - - return query, nil -} - -// extractLabels extracts label names from grouping -func (v *queryPlanner) extractLabels(grouping []string) []string { - return grouping -} - -// GetMetricName extracts the metric name from the expression -func GetMetricName(expr parser.Expr) string { - switch e := expr.(type) { - case *parser.VectorSelector: - return e.Name - case *parser.MatrixSelector: - return GetMetricName(e.VectorSelector) - case *parser.Call: - if len(e.Args) > 0 { - return GetMetricName(e.Args[0]) - } - } - return "" -} - -// HasMultipleSeries checks if the query involves multiple series -func (p *QueryPlan) HasMultipleSeries() bool { - return len(p.SeriesQueries) > 1 -} - -// ShouldBatch determines if queries should be batched -func (p *QueryPlan) ShouldBatch() bool { - // Batch if we have multiple series with the same metric name - if len(p.SeriesQueries) <= 1 { - return false - } - - // Check if all series have the same metric name - firstMetric := p.SeriesQueries[0].MetricName - for _, sq := range p.SeriesQueries[1:] { - if sq.MetricName != firstMetric { - return false - } - } - - // Batch if we have 5-20 series (adaptive threshold) - return len(p.SeriesQueries) >= 5 && len(p.SeriesQueries) <= 20 -} - -// GetBatchedQueries groups queries by metric name for batching -func (p *QueryPlan) GetBatchedQueries() map[string][]SeriesQuery { - batched := make(map[string][]SeriesQuery) - for _, sq := range p.SeriesQueries { - batched[sq.MetricName] = append(batched[sq.MetricName], sq) - } - return batched -} - -// String returns a human-readable representation of the plan -func (p *QueryPlan) String() string { - var parts []string - parts = append(parts, fmt.Sprintf("SeriesQueries: %d", len(p.SeriesQueries))) - if p.Aggregation != nil { - parts = append(parts, fmt.Sprintf("Aggregation: %s", p.Aggregation.Operation)) - } - if p.Function != nil { - parts = append(parts, fmt.Sprintf("Function: %s", p.Function.Name)) - } - return strings.Join(parts, ", ") -} diff --git a/datasource-gateway/internal/promql/promql_test.go b/datasource-gateway/internal/promql/promql_test.go deleted file mode 100644 index b214a62..0000000 --- a/datasource-gateway/internal/promql/promql_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package promql - -import ( - "strings" - "testing" -) - -// TestTranslateSmoke proves the relocated PromQL->SQL++ translator compiles -// and runs end-to-end (parse -> plan -> SQL) inside the gateway module. Full -// subset coverage and correctness are hardened in a later task (G7/G8). -func TestTranslateSmoke(t *testing.T) { - cases := []struct { - name string - query string - wantInSQL []string - }{ - { - name: "vector selector with job", - query: `kv_curr_items{job="snap-1"}`, - wantInSQL: []string{"kv_curr_items", "snap-1"}, - }, - { - name: "aggregation over rate", - query: `sum by (instance) (rate(kv_ops{job="snap-1",bucket="default"}[5m]))`, - wantInSQL: []string{"kv_ops"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - ctx, err := ParseQueryContext(tc.query, "", "1700000000", "1700003600", "15s", "") - if err != nil { - t.Fatalf("ParseQueryContext: %v", err) - } - - expr, err := ParseQuery(tc.query) - if err != nil { - t.Fatalf("ParseQuery: %v", err) - } - - plan, err := PlanQuery(expr, "") - if err != nil { - t.Fatalf("PlanQuery: %v", err) - } - if len(plan.SeriesQueries) == 0 { - t.Fatalf("expected at least one series query, got plan: %s", plan.String()) - } - - queries, err := NewSQLBuilder(plan, ctx).Build() - if err != nil { - t.Fatalf("Build: %v", err) - } - if len(queries) == 0 { - t.Fatal("expected at least one SQL++ query") - } - sql := strings.Join(queries, " ") - for _, want := range tc.wantInSQL { - if !strings.Contains(sql, want) { - t.Errorf("generated SQL missing %q\nSQL: %s", want, sql) - } - } - }) - } -} - -// TestExtractJobAsSnapshot verifies the `job` label is mapped to the snapshot -// the generated SQL filters on. -func TestExtractJobAsSnapshot(t *testing.T) { - expr, err := ParseQuery(`kv_ops{job="snap-42"}`) - if err != nil { - t.Fatalf("ParseQuery: %v", err) - } - plan, err := PlanQuery(expr, "") - if err != nil { - t.Fatalf("PlanQuery: %v", err) - } - if len(plan.SeriesQueries) != 1 { - t.Fatalf("expected 1 series query, got %d", len(plan.SeriesQueries)) - } - if got := plan.SeriesQueries[0].Snapshot; got != "snap-42" { - t.Errorf("snapshot = %q, want %q", got, "snap-42") - } - if got := plan.SeriesQueries[0].MetricName; got != "kv_ops" { - t.Errorf("metric = %q, want %q", got, "kv_ops") - } -} diff --git a/datasource-gateway/internal/promql/sqlbuilder.go b/datasource-gateway/internal/promql/sqlbuilder.go deleted file mode 100644 index a3e7a8f..0000000 --- a/datasource-gateway/internal/promql/sqlbuilder.go +++ /dev/null @@ -1,140 +0,0 @@ -package promql - -import ( - "fmt" - "strings" - "time" - - "github.com/couchbase/datasource-gateway/internal/querybuilder" -) - -const ( - // defaultKeyspace is used when the query context doesn't specify one; the - // real keyspace (bucket.scope.collection) is supplied per query. - defaultKeyspace = "cbmonitor._default._default" - - // instantLookback is how far back an instant query reads to find the most - // recent sample. - instantLookback = 5 * time.Minute -) - -// SQLBuilder builds SQL++ queries from a query plan. It emits the raw, -// time-bounded, per-series samples (one query per selector). rate/irate/ -// increase and aggregation are applied in Go (see transformer.go) after the -// samples are fetched — rate must be computed per series before any -// aggregation, so neither can be pushed into SQL. -type SQLBuilder struct { - plan *QueryPlan - queryCtx *QueryContext -} - -// NewSQLBuilder creates a new SQL++ query builder. -func NewSQLBuilder(plan *QueryPlan, queryCtx *QueryContext) *SQLBuilder { - return &SQLBuilder{plan: plan, queryCtx: queryCtx} -} - -// Build generates one SQL++ query per series selector in the plan. -func (b *SQLBuilder) Build() ([]string, error) { - if len(b.plan.SeriesQueries) == 0 { - return nil, fmt.Errorf("query has no series selector") - } - queries := make([]string, 0, len(b.plan.SeriesQueries)) - for _, sq := range b.plan.SeriesQueries { - queries = append(queries, b.buildSeriesQuery(sq)) - } - return queries, nil -} - -// buildSeriesQuery selects a single metric's samples. metric_name, job, and the -// label matchers filter the document BEFORE UNNEST (efficient); the time window -// is bound into the _timeseries range so only in-window samples are expanded. -func (b *SQLBuilder) buildSeriesQuery(sq SeriesQuery) string { - fromMillis, toMillis := b.timeRangeMillis() - - conditions := []string{fmt.Sprintf("d.metric_name = '%s'", escapeSQLString(sq.MetricName))} - if labelWhere := b.buildLabelWhereClause(sq); labelWhere != "" { - conditions = append(conditions, labelWhere) - } - - return fmt.Sprintf( - "SELECT MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels "+ - "FROM %s AS d UNNEST _timeseries(d, {'ts_ranges':[%d, %d]}) AS t WHERE %s", - b.keyspace(), fromMillis, toMillis, strings.Join(conditions, " AND "), - ) -} - -// keyspace returns the configured metrics keyspace (bucket.scope.collection), -// falling back to the default when unset. -func (b *SQLBuilder) keyspace() string { - if b.queryCtx != nil && b.queryCtx.Keyspace != "" { - return b.queryCtx.Keyspace - } - return defaultKeyspace -} - -// timeRangeMillis returns the [from,to] window in Unix milliseconds. Range -// queries use the requested window; instant queries read back instantLookback -// from the evaluation time. -func (b *SQLBuilder) timeRangeMillis() (int64, int64) { - if b.queryCtx.IsRange { - return b.queryCtx.StartTime.UnixMilli(), b.queryCtx.EndTime.UnixMilli() - } - return b.queryCtx.Time.Add(-instantLookback).UnixMilli(), b.queryCtx.Time.UnixMilli() -} - -// buildLabelWhereClause builds the job/snapshot + label-matcher conditions -// (everything except metric_name) via the shared query builder. -func (b *SQLBuilder) buildLabelWhereClause(sq SeriesQuery) string { - var filters []querybuilder.LabelFilter - - // Add the job/snapshot equality filter unless the selector already carried - // an explicit job matcher (preserved below with its own operator). - if !hasJobMatcher(sq.Labels) { - snapshot := sq.Snapshot - if snapshot == "" { - snapshot = b.queryCtx.SnapshotID - } - if snapshot != "" { - filters = append(filters, querybuilder.LabelFilter{Name: "job", Value: snapshot, Op: "="}) - } - } - - for labelName, labelValue := range sq.Labels { - switch { - case strings.HasPrefix(labelName, "!~"): - filters = append(filters, querybuilder.LabelFilter{Name: strings.TrimPrefix(labelName, "!~"), Value: labelValue, Op: "!~"}) - case strings.HasPrefix(labelName, "=~"): - filters = append(filters, querybuilder.LabelFilter{Name: strings.TrimPrefix(labelName, "=~"), Value: labelValue, Op: "=~"}) - case strings.HasPrefix(labelName, "!"): - filters = append(filters, querybuilder.LabelFilter{Name: strings.TrimPrefix(labelName, "!"), Value: labelValue, Op: "!="}) - default: - filters = append(filters, querybuilder.LabelFilter{Name: labelName, Value: labelValue, Op: "="}) - } - } - - return querybuilder.BuildLabelWhereClauseFromFilters(filters) -} - -// hasJobMatcher reports whether the selector already constrains job/snapshot -// (with any operator), so the default equality job filter isn't added twice. -func hasJobMatcher(labels map[string]string) bool { - for name := range labels { - actual := name - switch { - case strings.HasPrefix(actual, "!~"): - actual = strings.TrimPrefix(actual, "!~") - case strings.HasPrefix(actual, "=~"): - actual = strings.TrimPrefix(actual, "=~") - case strings.HasPrefix(actual, "!"): - actual = strings.TrimPrefix(actual, "!") - } - if actual == "job" || actual == "snapshot" || actual == "snapshot_id" { - return true - } - } - return false -} - -func escapeSQLString(s string) string { - return strings.ReplaceAll(s, "'", "''") -} diff --git a/datasource-gateway/internal/promql/sqlbuilder_test.go b/datasource-gateway/internal/promql/sqlbuilder_test.go deleted file mode 100644 index 139fcde..0000000 --- a/datasource-gateway/internal/promql/sqlbuilder_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package promql - -import ( - "strings" - "testing" -) - -func translateOne(t *testing.T, query string, ctx *QueryContext) string { - t.Helper() - expr, err := ParseQuery(query) - if err != nil { - t.Fatalf("ParseQuery(%q): %v", query, err) - } - plan, err := PlanQuery(expr, "") - if err != nil { - t.Fatalf("PlanQuery: %v", err) - } - queries, err := NewSQLBuilder(plan, ctx).Build() - if err != nil { - t.Fatalf("Build: %v", err) - } - if len(queries) != 1 { - t.Fatalf("expected 1 query, got %d: %v", len(queries), queries) - } - return queries[0] -} - -// rangeCtx is a 1h range-query context [1700000000, 1700003600] (Unix seconds). -func rangeCtx(keyspace string) *QueryContext { - c, err := ParseQueryContext("", "", "1700000000", "1700003600", "15s", "") - if err != nil { - panic(err) - } - c.Keyspace = keyspace - return c -} - -func mustContainAll(t *testing.T, sql string, wants ...string) { - t.Helper() - for _, w := range wants { - if !strings.Contains(sql, w) { - t.Errorf("SQL missing %q\nSQL: %s", w, sql) - } - } -} - -func mustNotContain(t *testing.T, sql string, notWants ...string) { - t.Helper() - for _, w := range notWants { - if strings.Contains(sql, w) { - t.Errorf("SQL unexpectedly contains %q\nSQL: %s", w, sql) - } - } -} - -func TestBuildSelectorWithJobAndWindow(t *testing.T) { - sql := translateOne(t, `kv_curr_items{job="snap-1"}`, rangeCtx("bkt.scp.col")) - mustContainAll(t, sql, - "FROM bkt.scp.col AS d", - "d.metric_name = 'kv_curr_items'", - "d.labels.`job` = 'snap-1'", - "_timeseries(d, {'ts_ranges':[1700000000000, 1700003600000]})", - "MILLIS_TO_STR(t._t) AS time", - "t._v0 AS `value`", - "d.labels AS labels", - ) -} - -func TestBuildDefaultKeyspace(t *testing.T) { - sql := translateOne(t, `up{job="s"}`, rangeCtx("")) - mustContainAll(t, sql, "FROM cbmonitor._default._default AS d") -} - -func TestBuildLabelMatchers(t *testing.T) { - sql := translateOne(t, `kv_ops{job="s",proc="memcached",mode!="idle"}`, rangeCtx("k")) - mustContainAll(t, sql, - "d.labels.`proc` = 'memcached'", - "d.labels.`mode` != 'idle'", - ) -} - -func TestBuildRegexMatchers(t *testing.T) { - sql := translateOne(t, `kv_ops{job="s",bucket=~"a.*",node!~"x.*"}`, rangeCtx("k")) - mustContainAll(t, sql, - "REGEXP_MATCHES(d.labels.`bucket`, '^(a.*)$')", - "NOT REGEXP_MATCHES(d.labels.`node`, '^(x.*)$')", - ) - mustNotContain(t, sql, "LIKE") -} - -func TestBuildRateHasNoSQLRate(t *testing.T) { - // rate is computed in Go; the SQL is just the underlying selector. - sql := translateOne(t, `rate(kv_ops{job="s"}[5m])`, rangeCtx("k")) - mustContainAll(t, sql, "d.metric_name = 'kv_ops'") - mustNotContain(t, sql, "rate(", "RATE(") -} - -func TestBuildAggregationHasNoSQLGroupBy(t *testing.T) { - // aggregation is computed in Go; the SQL is just the underlying selector. - sql := translateOne(t, `sum by (instance) (kv_ops{job="s"})`, rangeCtx("k")) - mustContainAll(t, sql, "d.metric_name = 'kv_ops'") - mustNotContain(t, sql, "GROUP BY", "SUM(") -} - -func TestBuildInstantQueryWindow(t *testing.T) { - c, err := ParseQueryContext(`up{job="s"}`, "1700000000", "", "", "", "") - if err != nil { - t.Fatalf("ParseQueryContext: %v", err) - } - c.Keyspace = "k" - sql := translateOne(t, `up{job="s"}`, c) - // Instant query: window ends at the eval time (1700000000000) and looks - // back instantLookback (5m = 300000ms) → [1699999700000, 1700000000000]. - mustContainAll(t, sql, "_timeseries(d, {'ts_ranges':[1699999700000, 1700000000000]})") -} - -func TestPlannerByWithout(t *testing.T) { - byExpr, err := ParseQuery(`sum by (instance) (up{job="s"})`) - if err != nil { - t.Fatalf("parse by: %v", err) - } - byPlan, err := PlanQuery(byExpr, "") - if err != nil { - t.Fatalf("plan by: %v", err) - } - if byPlan.Aggregation == nil || len(byPlan.Aggregation.By) != 1 || byPlan.Aggregation.By[0] != "instance" || len(byPlan.Aggregation.Without) != 0 { - t.Errorf("by-plan = %+v, want By=[instance] Without=[]", byPlan.Aggregation) - } - - withoutExpr, err := ParseQuery(`sum without (le) (up{job="s"})`) - if err != nil { - t.Fatalf("parse without: %v", err) - } - withoutPlan, err := PlanQuery(withoutExpr, "") - if err != nil { - t.Fatalf("plan without: %v", err) - } - if withoutPlan.Aggregation == nil || len(withoutPlan.Aggregation.Without) != 1 || withoutPlan.Aggregation.Without[0] != "le" || len(withoutPlan.Aggregation.By) != 0 { - t.Errorf("without-plan = %+v, want Without=[le] By=[]", withoutPlan.Aggregation) - } -} diff --git a/datasource-gateway/internal/promql/transformer.go b/datasource-gateway/internal/promql/transformer.go deleted file mode 100644 index 3f10c53..0000000 --- a/datasource-gateway/internal/promql/transformer.go +++ /dev/null @@ -1,510 +0,0 @@ -package promql - -import ( - "fmt" - "math" - "sort" - "strconv" - "strings" - "time" -) - -// PrometheusResult represents a Prometheus API result -type PrometheusResult struct { - Status string `json:"status"` - Data ResultData `json:"data"` - Error string `json:"error,omitempty"` - ErrorType string `json:"errorType,omitempty"` - // Warnings carries non-fatal issues encountered while serving the query - // (e.g. some sub-queries failed but enough succeeded to return data). - // Matches the Prometheus HTTP API spec. - Warnings []string `json:"warnings,omitempty"` -} - -// ResultData represents the data portion of Prometheus response -type ResultData struct { - ResultType string `json:"resultType"` - Result []interface{} `json:"result"` -} - -// Sample represents a single time-value pair -type Sample struct { - Timestamp float64 `json:"timestamp"` // Unix timestamp in seconds - Value string `json:"value"` // String representation of float -} - -// Series represents a time series with labels and samples -type Series struct { - Metric map[string]string `json:"metric"` - Values []Sample `json:"values,omitempty"` // For range queries - Value Sample `json:"value,omitempty"` // For instant queries -} - -// QueryResult represents raw query results from Couchbase -type QueryResult struct { - Time string `json:"time"` - Value interface{} `json:"value"` - Labels map[string]interface{} `json:"labels,omitempty"` -} - -// TransformResults transforms Couchbase query results to Prometheus format -func TransformResults(results []QueryResult, plan *QueryPlan, queryCtx *QueryContext) (*PrometheusResult, error) { - if len(results) == 0 { - return &PrometheusResult{ - Status: "success", - Data: ResultData{ - ResultType: getResultType(plan, queryCtx), - Result: []interface{}{}, - }, - }, nil - } - - // Group results by labels to form series - seriesMap := make(map[string]*Series) - - for _, result := range results { - // Extract labels - labels := extractLabels(result, plan) - labelKey := buildLabelKey(labels) - - // Get or create series - series, exists := seriesMap[labelKey] - if !exists { - series = &Series{ - Metric: labels, - } - seriesMap[labelKey] = series - } - - // Parse time and value - timestamp, err := parseTimeFromResult(result.Time) - if err != nil { - return nil, fmt.Errorf("failed to parse time: %w", err) - } - - value, err := parseValue(result.Value) - if err != nil { - return nil, fmt.Errorf("failed to parse value: %w", err) - } - - // Create sample - sample := Sample{ - Timestamp: float64(timestamp.Unix()), - Value: formatValue(value), - } - - // Add to series - if queryCtx.IsRange { - series.Values = append(series.Values, sample) - } else { - series.Value = sample - } - } - - // Convert map to slice and sort - var seriesList []interface{} - for _, series := range seriesMap { - // Sort values by timestamp - if queryCtx.IsRange { - sort.Slice(series.Values, func(i, j int) bool { - return series.Values[i].Timestamp < series.Values[j].Timestamp - }) - } - seriesList = append(seriesList, series) - } - - // Apply function transformations - if plan.Function != nil { - seriesList, err := applyFunction(seriesList, plan.Function, queryCtx) - if err != nil { - return nil, fmt.Errorf("failed to apply function: %w", err) - } - // Update seriesList with transformed results - _ = seriesList // Will be used after implementing function application - } - - // Apply aggregation if needed - if plan.Aggregation != nil { - seriesList = applyAggregation(seriesList, plan.Aggregation) - } - - return &PrometheusResult{ - Status: "success", - Data: ResultData{ - ResultType: getResultType(plan, queryCtx), - Result: seriesList, - }, - }, nil -} - -// extractLabels extracts labels from query result -func extractLabels(result QueryResult, plan *QueryPlan) map[string]string { - labels := make(map[string]string) - - // Add metric name from plan - if len(plan.SeriesQueries) > 0 { - metricName := plan.SeriesQueries[0].MetricName - if metricName != "" { - labels["__name__"] = metricName - } - } - - // Extract labels from result - if result.Labels != nil { - for key, value := range result.Labels { - if strValue, ok := value.(string); ok { - labels[key] = strValue - } else if strValue := fmt.Sprintf("%v", value); strValue != "" { - labels[key] = strValue - } - } - } - - return labels -} - -// buildLabelKey creates a unique key for a label set -func buildLabelKey(labels map[string]string) string { - // Sort keys for consistent ordering - var keys []string - for k := range labels { - keys = append(keys, k) - } - sort.Strings(keys) - - var parts []string - for _, k := range keys { - parts = append(parts, fmt.Sprintf("%s=%s", k, labels[k])) - } - return fmt.Sprintf("{%s}", strings.Join(parts, ",")) -} - -// parseTimeFromResult parses time from result string -func parseTimeFromResult(timeStr string) (time.Time, error) { - // Try RFC3339 first - if t, err := time.Parse(time.RFC3339, timeStr); err == nil { - return t, nil - } - - // Try Unix timestamp - if timestamp, err := strconv.ParseFloat(timeStr, 64); err == nil { - if timestamp > 1e10 { - // Milliseconds - return time.Unix(0, int64(timestamp*1e6)), nil - } - return time.Unix(int64(timestamp), 0), nil - } - - return time.Time{}, fmt.Errorf("unable to parse time: %s", timeStr) -} - -// parseValue parses value from result -func parseValue(value interface{}) (float64, error) { - switch v := value.(type) { - case float64: - return v, nil - case int: - return float64(v), nil - case int64: - return float64(v), nil - case string: - f, err := strconv.ParseFloat(v, 64) - if err != nil { - return 0, fmt.Errorf("invalid value: %s", v) - } - return f, nil - default: - return 0, fmt.Errorf("unsupported value type: %T", value) - } -} - -// formatValue formats a float value as string (Prometheus format) -func formatValue(value float64) string { - // Handle special values - if math.IsNaN(value) { - return "NaN" - } - if math.IsInf(value, 1) { - return "+Inf" - } - if math.IsInf(value, -1) { - return "-Inf" - } - - // Format with appropriate precision - return strconv.FormatFloat(value, 'f', -1, 64) -} - -// getResultType determines the result type based on query -func getResultType(plan *QueryPlan, queryCtx *QueryContext) string { - if queryCtx.IsRange { - return "matrix" - } - return "vector" -} - -// applyFunction applies PromQL function transformations -func applyFunction(seriesList []interface{}, function *FunctionPlan, queryCtx *QueryContext) ([]interface{}, error) { - switch function.Name { - case "rate": - return applyRate(seriesList, function.Range) - case "irate": - return applyIRate(seriesList) - case "increase": - return applyIncrease(seriesList, function.Range) - default: - // Unknown function, return as-is - return seriesList, nil - } -} - -// applyRate calculates rate (per-second average rate) -func applyRate(seriesList []interface{}, rangeStr string) ([]interface{}, error) { - // Parse range duration (for future use in range-based rate calculation) - _, err := time.ParseDuration(rangeStr) - if err != nil && rangeStr != "" { - return nil, fmt.Errorf("invalid range: %w", err) - } - - // Apply rate to each series - var result []interface{} - for _, item := range seriesList { - series, ok := item.(*Series) - if !ok { - continue - } - - if len(series.Values) < 2 { - // Need at least 2 points for rate - continue - } - - // Calculate rate for each interval - newValues := make([]Sample, 0, len(series.Values)-1) - for i := 1; i < len(series.Values); i++ { - prev := series.Values[i-1] - curr := series.Values[i] - - prevVal, _ := strconv.ParseFloat(prev.Value, 64) - currVal, _ := strconv.ParseFloat(curr.Value, 64) - - timeDiff := curr.Timestamp - prev.Timestamp - if timeDiff <= 0 { - continue - } - - // Rate = (value change) / (time change in seconds) - rate := (currVal - prevVal) / timeDiff - - newValues = append(newValues, Sample{ - Timestamp: curr.Timestamp, - Value: formatValue(rate), - }) - } - - // Update series - newSeries := *series - newSeries.Values = newValues - result = append(result, &newSeries) - } - - return result, nil -} - -// applyIRate calculates instant rate (per-second rate from last two points) -func applyIRate(seriesList []interface{}) ([]interface{}, error) { - // Similar to rate but only uses last two points - return applyRate(seriesList, "") -} - -// applyIncrease calculates increase over time range -func applyIncrease(seriesList []interface{}, rangeStr string) ([]interface{}, error) { - // Similar to rate but returns absolute increase - // For now, use rate implementation - return applyRate(seriesList, rangeStr) -} - -// applyAggregation applies aggregation operations -func applyAggregation(seriesList []interface{}, agg *AggregationPlan) []interface{} { - // Group series by labels (excluding aggregation labels) - grouped := make(map[string][]*Series) - - for _, item := range seriesList { - series, ok := item.(*Series) - if !ok { - continue - } - - // Build group key - groupKey := buildAggregationKey(series.Metric, agg) - grouped[groupKey] = append(grouped[groupKey], series) - } - - // Apply aggregation to each group - var result []interface{} - for _, group := range grouped { - aggSeries := aggregateSeries(group, agg) - if aggSeries != nil { - result = append(result, aggSeries) - } - } - - return result -} - -// buildAggregationKey builds key for grouping -func buildAggregationKey(labels map[string]string, agg *AggregationPlan) string { - var keys []string - for k, v := range labels { - // Skip __name__ and aggregation-excluded labels - if k == "__name__" { - continue - } - // Include only labels in "by" or exclude those in "without" - keys = append(keys, fmt.Sprintf("%s=%s", k, v)) - } - sort.Strings(keys) - return strings.Join(keys, ",") -} - -// aggregateSeries aggregates multiple series into one -func aggregateSeries(seriesList []*Series, agg *AggregationPlan) *Series { - if len(seriesList) == 0 { - return nil - } - - // Use first series as base - result := &Series{ - Metric: make(map[string]string), - } - - // Copy labels from first series (excluding aggregation-excluded labels) - for k, v := range seriesList[0].Metric { - if k != "__name__" { - result.Metric[k] = v - } - } - - // Aggregate values - if len(seriesList[0].Values) > 0 { - // Range query - aggregate across series for each timestamp - result.Values = aggregateValues(seriesList, agg) - } else { - // Instant query - aggregate single values - result.Value = aggregateValue(seriesList, agg) - } - - return result -} - -// aggregateValues aggregates values across series for range queries -func aggregateValues(seriesList []*Series, agg *AggregationPlan) []Sample { - // Find all unique timestamps - timestampMap := make(map[float64]bool) - for _, series := range seriesList { - for _, sample := range series.Values { - timestampMap[sample.Timestamp] = true - } - } - - // Sort timestamps - var timestamps []float64 - for ts := range timestampMap { - timestamps = append(timestamps, ts) - } - sort.Float64s(timestamps) - - // Aggregate for each timestamp - var result []Sample - for _, ts := range timestamps { - var values []float64 - for _, series := range seriesList { - for _, sample := range series.Values { - if sample.Timestamp == ts { - if val, err := strconv.ParseFloat(sample.Value, 64); err == nil { - values = append(values, val) - } - break - } - } - } - - if len(values) > 0 { - aggValue := aggregateFloat(values, agg.Operation) - result = append(result, Sample{ - Timestamp: ts, - Value: formatValue(aggValue), - }) - } - } - - return result -} - -// aggregateValue aggregates single values for instant queries -func aggregateValue(seriesList []*Series, agg *AggregationPlan) Sample { - var values []float64 - var timestamp float64 - - for _, series := range seriesList { - if val, err := strconv.ParseFloat(series.Value.Value, 64); err == nil { - values = append(values, val) - if timestamp == 0 { - timestamp = series.Value.Timestamp - } - } - } - - aggValue := aggregateFloat(values, agg.Operation) - return Sample{ - Timestamp: timestamp, - Value: formatValue(aggValue), - } -} - -// aggregateFloat aggregates float values based on operation -func aggregateFloat(values []float64, operation string) float64 { - if len(values) == 0 { - return 0 - } - - switch strings.ToUpper(operation) { - case "SUM": - sum := 0.0 - for _, v := range values { - sum += v - } - return sum - case "AVG", "AVERAGE": - sum := 0.0 - for _, v := range values { - sum += v - } - return sum / float64(len(values)) - case "MAX": - max := values[0] - for _, v := range values { - if v > max { - max = v - } - } - return max - case "MIN": - min := values[0] - for _, v := range values { - if v < min { - min = v - } - } - return min - case "COUNT": - return float64(len(values)) - default: - // Default to sum - sum := 0.0 - for _, v := range values { - sum += v - } - return sum - } -} From a7bffbff24c3a0113b0c0ce17be34d29c849080f Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 17:46:12 +0100 Subject: [PATCH 11/18] gateway: Add plugin reconsiler and settings --- cbmonitor/pkg/plugin/datasources.go | 59 ++++----- cbmonitor/pkg/plugin/datasources_test.go | 118 +++++++++--------- cbmonitor/pkg/plugin/resources.go | 3 + cbmonitor/pkg/plugin/settings.go | 26 ++++ cbmonitor/pkg/plugin/settings_test.go | 44 +++++++ .../provisioning/datasources/datasource.yaml | 23 +--- 6 files changed, 158 insertions(+), 115 deletions(-) diff --git a/cbmonitor/pkg/plugin/datasources.go b/cbmonitor/pkg/plugin/datasources.go index 397a2d4..5a02eac 100644 --- a/cbmonitor/pkg/plugin/datasources.go +++ b/cbmonitor/pkg/plugin/datasources.go @@ -27,6 +27,12 @@ const ( // Any UID in this set that isn't in the current `desired` list during a // reconcile pass gets DELETEd from Grafana — that's how toggling a // feature off in app settings cleans up its datasource. +// +// dsUIDCouchbase is retained here even though desiredDatasources no longer +// produces it: the gateway replaced the standalone couchbase-datasource, so +// the reconciler must DELETE any pre-existing cbdatasource left over from +// before the gateway. Drop it (and the constant + IAM scopes) in a future +// release once deployments have reconciled it away. var appManagedUIDs = []string{dsUIDPrometheus, dsUIDCouchbase} // reconcileTimeout caps a single reconciliation pass. Long enough to handle @@ -364,9 +370,10 @@ func readBodySnippet(body io.Reader) string { // uses this to distinguish "feature disabled — clean up the DS" from // "feature enabled but misconfigured — leave the existing DS alone". func (s *PluginSettings) claimedDatasources() map[string]bool { + // cbdatasource is intentionally absent: it's retired, so it's never + // "claimed" and the reconciler's delete phase always removes any leftover. return map[string]bool{ dsUIDPrometheus: s.PrometheusDatasource.Enabled, - dsUIDCouchbase: s.CouchbaseDatasource.Enabled, } } @@ -376,13 +383,23 @@ func (s *PluginSettings) claimedDatasources() map[string]bool { // as "skipped", not as a hard error. func (s *PluginSettings) desiredDatasources() []DesiredDatasource { out := []DesiredDatasource{} - if s.PrometheusDatasource.Enabled && s.PrometheusDatasource.URL != "" { + + // The single Prometheus-typed datasource points at the gateway sidecar when + // the gateway is enabled (so it can translate Couchbase-backed snapshots and + // serve overlap); otherwise straight at the upstream Prometheus/Mimir + // (pure-Prometheus mode). Either way it's a prometheus-typed datasource + // speaking the Prometheus HTTP API, so the jsonData below is unchanged. + promURL := s.PrometheusDatasource.URL + if s.Gateway.Enabled { + promURL = s.Gateway.URL + } + if s.PrometheusDatasource.Enabled && promURL != "" { out = append(out, DesiredDatasource{ UID: dsUIDPrometheus, Name: "Prometheus", Type: "prometheus", Access: "proxy", - URL: s.PrometheusDatasource.URL, + URL: promURL, IsDefault: s.PrometheusDatasource.IsDefault, JSONData: map[string]any{ "cacheLevel": "High", @@ -392,38 +409,8 @@ func (s *PluginSettings) desiredDatasources() []DesiredDatasource { }, }) } - if s.CouchbaseDatasource.Enabled && s.CouchbaseServer.ConnectionString != "" { - ds := s.CouchbaseDatasource - // The couchbase-datasource plugin (and any UDFs the panels call - // through) read bucket/scope/collection from jsonData. Forward - // every field the app captures so panel queries don't need to - // hardcode location info. - jsonData := map[string]any{ - "host": s.CouchbaseServer.ConnectionString, - "username": s.CouchbaseServer.Username, - "bucket": ds.Bucket, - } - if ds.Scope != "" { - jsonData["scope"] = ds.Scope - } - if ds.Collection != "" { - jsonData["collection"] = ds.Collection - } - out = append(out, DesiredDatasource{ - UID: dsUIDCouchbase, - Name: "cbdatasource", - Type: "couchbase-datasource", - Access: "proxy", - // The couchbase-datasource plugin reads `host` from jsonData, - // not the top-level `url` — but Grafana's API requires `url` - // to be set on all datasources. Send the connection string in - // both so the wire object is well-formed. - URL: s.CouchbaseServer.ConnectionString, - JSONData: jsonData, - SecureJSONData: map[string]string{ - "password": s.CouchbaseServer.Password, - }, - }) - } + // cbdatasource is no longer created: Couchbase-backed snapshots are served + // by the gateway (the prometheus DS above points at it). Any pre-existing + // cbdatasource is orphan-deleted via appManagedUIDs. return out } diff --git a/cbmonitor/pkg/plugin/datasources_test.go b/cbmonitor/pkg/plugin/datasources_test.go index 2e683fc..da84e60 100644 --- a/cbmonitor/pkg/plugin/datasources_test.go +++ b/cbmonitor/pkg/plugin/datasources_test.go @@ -469,42 +469,38 @@ func TestReconcile_PreservesClaimedDatasourceWithMissingConfig(t *testing.T) { } func TestClaimedDatasources_ReflectsEnabledFlags(t *testing.T) { - // claimedDatasources() ignores URL/connection-string presence — it's - // purely about user intent. Verifies the hardening pivot: "did the - // user ask for this feature to be on?" rather than "is it currently - // reconcilable?". + // claimedDatasources() ignores URL presence — it's purely about user + // intent ("did the user ask for this feature on?"). cbdatasource is retired + // and must never be claimed, so its leftover always gets orphan-deleted. cases := []struct { - name string - s PluginSettings - want map[string]bool + name string + s PluginSettings + wantProm bool }{ { - name: "both enabled, both URLs/strings empty", - s: PluginSettings{ - PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true}, - CouchbaseDatasource: CouchbaseDatasourceSettings{Enabled: true}, - }, - want: map[string]bool{dsUIDPrometheus: true, dsUIDCouchbase: true}, + name: "prom enabled, url empty", + s: PluginSettings{PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true}}, + wantProm: true, }, { - name: "prom enabled, couchbase disabled", - s: PluginSettings{ - PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true, URL: "http://x"}, - CouchbaseDatasource: CouchbaseDatasourceSettings{Enabled: false}, - }, - want: map[string]bool{dsUIDPrometheus: true, dsUIDCouchbase: false}, + name: "prom enabled with url", + s: PluginSettings{PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true, URL: "http://x"}}, + wantProm: true, }, { - name: "neither enabled", - s: PluginSettings{}, - want: map[string]bool{dsUIDPrometheus: false, dsUIDCouchbase: false}, + name: "prom disabled", + s: PluginSettings{}, + wantProm: false, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := tc.s.claimedDatasources() - if got[dsUIDPrometheus] != tc.want[dsUIDPrometheus] || got[dsUIDCouchbase] != tc.want[dsUIDCouchbase] { - t.Errorf("claimedDatasources() = %v, want %v", got, tc.want) + if got[dsUIDPrometheus] != tc.wantProm { + t.Errorf("claimed[prometheus] = %v, want %v", got[dsUIDPrometheus], tc.wantProm) + } + if got[dsUIDCouchbase] { + t.Error("cbdatasource must not be claimed (retired)") } }) } @@ -531,18 +527,19 @@ func TestDesiredDatasources_OnlyIncludesEnabledWithRequiredFields(t *testing.T) wantUIDs: []string{}, }, { - name: "both enabled with required fields", + name: "both features enabled — only the prometheus DS (cbdatasource retired)", s: PluginSettings{ PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true, URL: "http://prom"}, CouchbaseDatasource: CouchbaseDatasourceSettings{Enabled: true, Bucket: "cbmonitor"}, CouchbaseServer: CouchbaseServerSettings{ConnectionString: "couchbase://x"}, }, - wantUIDs: []string{"prometheus", "cbdatasource"}, + wantUIDs: []string{"prometheus"}, }, { - name: "couchbase enabled but connection string missing — skipped", + name: "couchbase enabled, prometheus off — no datasource (cbdatasource retired)", s: PluginSettings{ CouchbaseDatasource: CouchbaseDatasourceSettings{Enabled: true, Bucket: "cbmonitor"}, + CouchbaseServer: CouchbaseServerSettings{ConnectionString: "couchbase://x"}, }, wantUIDs: []string{}, }, @@ -561,11 +558,12 @@ func TestDesiredDatasources_OnlyIncludesEnabledWithRequiredFields(t *testing.T) } } -func TestDesiredDatasources_PropagatesBucketScopeCollectionToCbdatasource(t *testing.T) { - // The whole point of these settings is for panel queries (and any - // UDFs) to know where the timeseries data lives. Verify they actually - // land in the cbdatasource JSONData the reconciler sends to Grafana. +func TestDesiredDatasources_NeverCreatesCbdatasource(t *testing.T) { + // cbdatasource is retired: even with the Couchbase datasource feature fully + // enabled, no couchbase-datasource entry is produced — the gateway serves + // Couchbase-backed snapshots through the single prometheus datasource. s := PluginSettings{ + PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true, URL: "http://prom:9090"}, CouchbaseDatasource: CouchbaseDatasourceSettings{ Enabled: true, Bucket: "metrics-bucket", @@ -577,44 +575,40 @@ func TestDesiredDatasources_PropagatesBucketScopeCollectionToCbdatasource(t *tes Username: "u", }, } - got := s.desiredDatasources() - if len(got) != 1 || got[0].UID != "cbdatasource" { - t.Fatalf("expected exactly one cbdatasource desired entry, got %#v", got) + for _, d := range s.desiredDatasources() { + if d.Type == "couchbase-datasource" || d.UID == dsUIDCouchbase { + t.Errorf("cbdatasource should no longer be produced, got %#v", d) + } } - jd := got[0].JSONData - if jd["bucket"] != "metrics-bucket" { - t.Errorf("bucket not propagated, got %v", jd["bucket"]) +} + +func TestDesiredDatasources_PrometheusTargetsGatewayWhenEnabled(t *testing.T) { + base := PluginSettings{ + PrometheusDatasource: PrometheusDatasourceSettings{Enabled: true, URL: "http://mimir:9009/prometheus", IsDefault: true}, } - if jd["scope"] != "metrics-scope" { - t.Errorf("scope not propagated, got %v", jd["scope"]) + + // Pure-Prometheus mode: the datasource targets the upstream Mimir directly. + got := base.desiredDatasources() + if len(got) != 1 || got[0].UID != dsUIDPrometheus { + t.Fatalf("expected one prometheus DS, got %#v", got) } - if jd["collection"] != "timeseries" { - t.Errorf("collection not propagated, got %v", jd["collection"]) + if got[0].URL != "http://mimir:9009/prometheus" { + t.Errorf("pure-Prometheus URL = %q, want the Mimir URL", got[0].URL) } -} -func TestDesiredDatasources_OmitsEmptyScopeCollection(t *testing.T) { - // When scope/collection are blank (user wants defaults), the - // JSONData shouldn't carry empty strings — the couchbase-datasource - // plugin / UDFs should see them as absent, not as "" (which can - // trigger different code paths). - s := PluginSettings{ - CouchbaseDatasource: CouchbaseDatasourceSettings{ - Enabled: true, - Bucket: "metrics-bucket", - }, - CouchbaseServer: CouchbaseServerSettings{ - ConnectionString: "couchbase://cb", - Username: "u", - }, + // Gateway enabled: the same datasource now targets the gateway URL, still + // as a prometheus-typed datasource. + withGW := base + withGW.Gateway = GatewaySettings{Enabled: true, URL: "http://datasource-gateway:8090"} + got = withGW.desiredDatasources() + if len(got) != 1 || got[0].UID != dsUIDPrometheus { + t.Fatalf("expected one prometheus DS, got %#v", got) } - got := s.desiredDatasources() - jd := got[0].JSONData - if _, ok := jd["scope"]; ok { - t.Errorf("scope should be omitted when blank, got %v", jd["scope"]) + if got[0].URL != "http://datasource-gateway:8090" { + t.Errorf("gateway-mode URL = %q, want the gateway URL", got[0].URL) } - if _, ok := jd["collection"]; ok { - t.Errorf("collection should be omitted when blank, got %v", jd["collection"]) + if got[0].Type != "prometheus" { + t.Errorf("DS type = %q, want prometheus", got[0].Type) } } diff --git a/cbmonitor/pkg/plugin/resources.go b/cbmonitor/pkg/plugin/resources.go index ad8d5e8..3bc0cc5 100644 --- a/cbmonitor/pkg/plugin/resources.go +++ b/cbmonitor/pkg/plugin/resources.go @@ -63,6 +63,9 @@ func (a *App) handleGetDatasourceConfig(w http.ResponseWriter, req *http.Request "defaultDataSource": a.settings.DefaultDataSource(), "prometheusAvailable": a.settings.PrometheusDatasource.Enabled, "couchbaseAvailable": a.settings.CouchbaseDatasource.Enabled, + "gatewayEnabled": a.settings.Gateway.Enabled, + "gatewayUrl": a.settings.Gateway.URL, + "overlapEnabled": a.settings.Gateway.Enabled && a.settings.Gateway.Overlap, "reconciliation": a.getReconcileState(), "settings": settingsBlock, } diff --git a/cbmonitor/pkg/plugin/settings.go b/cbmonitor/pkg/plugin/settings.go index da10aa9..ef321f1 100644 --- a/cbmonitor/pkg/plugin/settings.go +++ b/cbmonitor/pkg/plugin/settings.go @@ -16,6 +16,7 @@ type PluginSettings struct { Snapshots SnapshotsSettings `json:"snapshots"` CouchbaseDatasource CouchbaseDatasourceSettings `json:"couchbaseDatasource"` PrometheusDatasource PrometheusDatasourceSettings `json:"prometheusDatasource"` + Gateway GatewaySettings `json:"gateway"` } type CouchbaseServerSettings struct { @@ -47,6 +48,19 @@ type PrometheusDatasourceSettings struct { URL string `json:"url"` } +// GatewaySettings configures the datasource-gateway sidecar. When Enabled, the +// reconciler points the single Prometheus datasource at URL (instead of at the +// upstream Prometheus/Mimir) so the gateway can translate Couchbase-backed +// snapshots and serve overlap. When disabled, the plugin runs as plain +// Prometheus — the datasource targets PrometheusDatasource.URL directly. +type GatewaySettings struct { + Enabled bool `json:"enabled"` + URL string `json:"url"` + // Overlap reports whether snapshot overlap/time-padding is available via the + // gateway; surfaced to the UI for feature-gating the comparison view. + Overlap bool `json:"overlap"` +} + // secureFieldCouchbasePassword is the secureJsonData key that holds the // Couchbase server password. const secureFieldCouchbasePassword = "couchbasePassword" @@ -96,6 +110,9 @@ func defaultSettings() *PluginSettings { Enabled: true, IsDefault: true, }, + Gateway: GatewaySettings{ + Enabled: false, + }, } } @@ -121,6 +138,15 @@ func (s *PluginSettings) validate() error { return fmt.Errorf("prometheusDatasource.url must be an absolute URL (e.g. http://prometheus:9090)") } } + if s.Gateway.Enabled { + if s.Gateway.URL == "" { + return fmt.Errorf("gateway.url is required when the gateway is enabled") + } + u, err := url.Parse(s.Gateway.URL) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("gateway.url must be an absolute URL (e.g. http://datasource-gateway:8090)") + } + } return nil } diff --git a/cbmonitor/pkg/plugin/settings_test.go b/cbmonitor/pkg/plugin/settings_test.go index da4b3b3..7d4ec9a 100644 --- a/cbmonitor/pkg/plugin/settings_test.go +++ b/cbmonitor/pkg/plugin/settings_test.go @@ -154,3 +154,47 @@ func TestDefaultDataSource(t *testing.T) { }) } } + +func TestLoadSettings_GatewayParsed(t *testing.T) { + jsonData := []byte(`{ + "prometheusDatasource": {"enabled": true, "url": "http://prometheus:9090"}, + "gateway": {"enabled": true, "url": "http://datasource-gateway:8090", "overlap": true} + }`) + s, err := LoadSettings(backend.AppInstanceSettings{JSONData: jsonData}) + if err != nil { + t.Fatalf("LoadSettings: %v", err) + } + if !s.Gateway.Enabled { + t.Error("gateway.enabled not parsed") + } + if s.Gateway.URL != "http://datasource-gateway:8090" { + t.Errorf("gateway.url = %q", s.Gateway.URL) + } + if !s.Gateway.Overlap { + t.Error("gateway.overlap not parsed") + } +} + +func TestLoadSettings_GatewayDisabledByDefault(t *testing.T) { + s, err := LoadSettings(backend.AppInstanceSettings{}) + if err != nil { + t.Fatalf("LoadSettings: %v", err) + } + if s.Gateway.Enabled { + t.Error("gateway should be disabled by default (pure-Prometheus)") + } +} + +func TestLoadSettings_GatewayRequiresURLWhenEnabled(t *testing.T) { + jsonData := []byte(`{"gateway": {"enabled": true}}`) + if _, err := LoadSettings(backend.AppInstanceSettings{JSONData: jsonData}); err == nil { + t.Fatal("expected error when gateway enabled without url") + } +} + +func TestLoadSettings_GatewayRejectsMalformedURL(t *testing.T) { + jsonData := []byte(`{"gateway": {"enabled": true, "url": "not-a-url"}}`) + if _, err := LoadSettings(backend.AppInstanceSettings{JSONData: jsonData}); err == nil { + t.Fatal("expected error for malformed gateway url") + } +} diff --git a/cbmonitor/provisioning/datasources/datasource.yaml b/cbmonitor/provisioning/datasources/datasource.yaml index 778aa61..99f62c5 100644 --- a/cbmonitor/provisioning/datasources/datasource.yaml +++ b/cbmonitor/provisioning/datasources/datasource.yaml @@ -1,19 +1,8 @@ apiVersion: 1 -# The `prometheus` and `cbdatasource` datasources are managed by the -# cbmonitor app plugin itself (reconciled at runtime from app jsonData via -# the Grafana HTTP API). Don't add them back here or you'll get duplicates. -datasources: -- name: ProxyPrometheus - uid: proxyprometheus - type: prometheus - access: proxy - url: ${PROXY_PROMETHEUS_URL} - basicAuth: false - isDefault: false - jsonData: - cacheLevel: High - httpMethod: GET - prometheusType: Mimir - prometheusVersion: 2.9.1 - secureJsonData: {} +# The single `prometheus` datasource is managed by the cbmonitor app plugin +# itself (reconciled at runtime from app jsonData via the Grafana HTTP API). +# It points at the datasource-gateway sidecar when the gateway is enabled, +# otherwise at Prometheus/Mimir directly. Don't add it back here or you'll get +# duplicates. +datasources: [] From c1b5e0fa9e39812cb0c7cc5815351477be88194c Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 18:01:57 +0100 Subject: [PATCH 12/18] cbmonitor: Remove datasource toggle from the plugin --- .../DataSourceToggle/DataSourceToggle.tsx | 130 ------------------ .../SettingsDropdown/SettingsDropdown.tsx | 38 ----- .../SnapshotDisplay/comparisonInstance.ts | 1 - cbmonitor/src/pages/snapshotViewPage.ts | 8 -- cbmonitor/src/services/datasourceService.ts | 8 +- 5 files changed, 6 insertions(+), 179 deletions(-) delete mode 100644 cbmonitor/src/components/DataSourceToggle/DataSourceToggle.tsx diff --git a/cbmonitor/src/components/DataSourceToggle/DataSourceToggle.tsx b/cbmonitor/src/components/DataSourceToggle/DataSourceToggle.tsx deleted file mode 100644 index 663b7cf..0000000 --- a/cbmonitor/src/components/DataSourceToggle/DataSourceToggle.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { SceneObjectBase, SceneComponentProps, SceneObjectState } from '@grafana/scenes'; -import { SelectableValue } from '@grafana/data'; -import { Select, Tooltip } from '@grafana/ui'; -import { DataSourceConfig, DataSourceType } from 'types/datasource'; -import { dataSourceService } from 'services/datasourceService'; - -interface DataSourceToggleState extends SceneObjectState { - snapshotId: string; - onDataSourceChange?: () => void; -} - -export class DataSourceToggle extends SceneObjectBase { - static Component = DataSourceToggleRenderer; - - constructor(state: DataSourceToggleState) { - super(state); - } -} - -function DataSourceToggleRenderer({ model }: SceneComponentProps) { - const state = model.useState(); - const { onDataSourceChange, snapshotId } = state || {}; - const [dataSource, setDataSource] = useState(DataSourceType.Prometheus); - const [config, setConfig] = useState({ - defaultDataSource: DataSourceType.Prometheus, - prometheusAvailable: true, - couchbaseAvailable: false, - }); - - useEffect(() => { - let isActive = true; - - // Early return if no snapshotId yet - if (!snapshotId) { - return; - } - - const loadInitialState = async () => { - try { - const currentDs = dataSourceService.getCurrentDataSource(); - const fullCfg = await dataSourceService.getDataSourceConfig(); - - // Set initial datasource from service - if (isActive) { - setConfig(fullCfg); - - const preferredDataSource = currentDs || fullCfg.defaultDataSource; - const isPreferredAvailable = - (preferredDataSource === DataSourceType.Prometheus && fullCfg.prometheusAvailable) || - (preferredDataSource === DataSourceType.Couchbase && fullCfg.couchbaseAvailable); - const fallbackDataSource = fullCfg.prometheusAvailable - ? DataSourceType.Prometheus - : DataSourceType.Couchbase; - const nextDataSource = isPreferredAvailable ? preferredDataSource : fallbackDataSource; - - setDataSource(nextDataSource); - dataSourceService.setCurrentDataSource(nextDataSource); - } - } catch (error) { - console.error('[DataSourceToggle] Failed to load datasource config:', error); - } - }; - - loadInitialState(); - - // Subscribe to datasource changes - const unsubscribe = dataSourceService.subscribe((newDataSource: DataSourceType) => { - if (isActive) { - setDataSource(newDataSource); - } - }); - - return () => { - isActive = false; - unsubscribe(); - }; - }, [snapshotId]); - - const handleChange = (option: SelectableValue | null) => { - if (option?.value) { - if (option.value === dataSource) { - return; - } - dataSourceService.setCurrentDataSource(option.value); - // Trigger callback to reload dashboards - if (onDataSourceChange) { - onDataSourceChange(); - } - } - }; - - const options: Array> = [ - { - label: 'Prometheus (Default)', - value: DataSourceType.Prometheus, - description: 'Primary datasource', - }, - { - label: 'Couchbase SQL++', - value: DataSourceType.Couchbase, - description: 'Experimental', - }, - ].filter( - (option) => - (option.value === DataSourceType.Prometheus && config.prometheusAvailable) || - (option.value === DataSourceType.Couchbase && config.couchbaseAvailable) - ); - - if (options.length === 0) { - return null; - } - - return ( -
- - - -
- )} - {showClusterSection && clusters.length > 0 && (
Cluster Filter
diff --git a/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts b/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts index 1bb1c71..4b36514 100644 --- a/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts +++ b/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts @@ -66,7 +66,6 @@ function CompareHeaderContainer(props: CompareHeaderContainerProps) { sceneCacheService.clearAll(); invalidateComparisonTabs(); }, - showDataSourceSection: false, showClusterSection: false, showHideEmptySection: true, }), []); diff --git a/cbmonitor/src/pages/snapshotViewPage.ts b/cbmonitor/src/pages/snapshotViewPage.ts index 2ed9bcf..558d3e7 100644 --- a/cbmonitor/src/pages/snapshotViewPage.ts +++ b/cbmonitor/src/pages/snapshotViewPage.ts @@ -246,13 +246,6 @@ snapshotViewPage.addActivationHandler(() => { }); }; - const handleDataSourceChange = () => { - sceneCacheService.clearAll(); - snapshotViewPage.setState({ - tabs: getDashboardsForServices(metadata.services, snapshotId, metadata.custom_panels, tabOverrides, metadata.products), - }); - }; - const handleClusterChange = (clusterId: string | null) => { clusterFilterService.setCurrentCluster(clusterId); sceneCacheService.clearAll(); @@ -298,7 +291,6 @@ snapshotViewPage.addActivationHandler(() => { snapshotId, clusters: metadata.clusters || [], onLayoutChange: handleLayoutChange, - onDataSourceChange: handleDataSourceChange, onHideEmptyChange: handleHideEmptyChange, availableTabs, tabOverrides, diff --git a/cbmonitor/src/services/datasourceService.ts b/cbmonitor/src/services/datasourceService.ts index 06967ff..d830c03 100644 --- a/cbmonitor/src/services/datasourceService.ts +++ b/cbmonitor/src/services/datasourceService.ts @@ -59,9 +59,13 @@ class DataSourceService { return this.config; } - /** Get the currently selected datasource type */ + /** + * The active datasource is pinned to Prometheus: the single gateway + * datasource speaks PromQL, and the Couchbase-vs-Prometheus runtime toggle + * has been retired. (This service is removed entirely in a later step.) + */ getCurrentDataSource(): DataSourceType { - return this.currentDataSource; + return DataSourceType.Prometheus; } /** Switch the active datasource and notify all subscribers */ From c371be2d6efb565b992778f56a25886ec10aa99c Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 18:15:19 +0100 Subject: [PATCH 13/18] cbmonitor: Collapse panels to only deal with one datasource --- cbmonitor/src/components/App/App.tsx | 17 +- .../src/components/AppConfig/AppConfig.tsx | 10 +- .../DashboardHeader/actions/ExploreButton.tsx | 21 +-- .../actions/MetricsDrilldownButton.tsx | 24 +-- cbmonitor/src/constants.ts | 6 - cbmonitor/src/services/datasourceService.ts | 96 ---------- cbmonitor/src/services/instanceService.ts | 45 ++--- cbmonitor/src/types/datasource.ts | 20 -- cbmonitor/src/utils/utils.cbquery.ts | 178 ------------------ cbmonitor/src/utils/utils.panel.ts | 140 ++++---------- 10 files changed, 62 insertions(+), 495 deletions(-) delete mode 100644 cbmonitor/src/services/datasourceService.ts delete mode 100644 cbmonitor/src/types/datasource.ts delete mode 100644 cbmonitor/src/utils/utils.cbquery.ts diff --git a/cbmonitor/src/components/App/App.tsx b/cbmonitor/src/components/App/App.tsx index 7e8a9e1..c3c0c93 100644 --- a/cbmonitor/src/components/App/App.tsx +++ b/cbmonitor/src/components/App/App.tsx @@ -1,7 +1,7 @@ -import React, { useEffect } from 'react'; +import React from 'react'; import { AppRootProps } from '@grafana/data'; import { config } from '@grafana/runtime'; -import { CB_DATASOURCE_REF, PROM_DATASOURCE_REF } from '../../constants'; +import { PROM_DATASOURCE_REF } from '../../constants'; import { SceneApp, useSceneApp } from '@grafana/scenes'; import { Alert } from '@grafana/ui'; import { PluginPropsContext } from 'utils/utils.plugin'; @@ -9,7 +9,6 @@ import { AppNavHeader } from '../AppNavHeader/AppNavHeader'; import { snapshotSearchPage, snapshotViewPage } from '../../pages/snapshotViewPage'; import { preferencesPage } from '../../pages/preferencesPage'; import { comparisonPage } from '../../components/SnapshotDisplay/comparisonInstance'; -import { dataSourceService } from '../../services/datasourceService'; // Defines the app and its pages function getCBMonitorApp(){ @@ -31,22 +30,14 @@ function getCBMonitorApp(){ function CBMonitorHome() { const scene = useSceneApp(getCBMonitorApp); - // Initialize datasource configuration from backend when app mounts - useEffect(() => { - dataSourceService.initializeConfig().catch((error) => { - console.error('[App] Failed to initialize datasource config:', error); - }); - }, []); - const datasources = Object.values(config.datasources); - const haveCb = datasources.some((d) => d.uid === CB_DATASOURCE_REF.uid); const haveProm = datasources.some((d) => d.uid === PROM_DATASOURCE_REF.uid); return ( <> - {!haveCb && !haveProm && ( + {!haveProm && ( - {JSON.stringify(CB_DATASOURCE_REF)} or {JSON.stringify(PROM_DATASOURCE_REF)} datasource is required to use this app. + The {JSON.stringify(PROM_DATASOURCE_REF)} datasource is required to use this app. Available datasources:
    {datasources.map((datasource) => ( diff --git a/cbmonitor/src/components/AppConfig/AppConfig.tsx b/cbmonitor/src/components/AppConfig/AppConfig.tsx index 4847f29..6b6e1b2 100644 --- a/cbmonitor/src/components/AppConfig/AppConfig.tsx +++ b/cbmonitor/src/components/AppConfig/AppConfig.tsx @@ -16,9 +16,7 @@ import { useStyles2, } from '@grafana/ui'; import { testIds } from '../testIds'; -import { dataSourceService } from '../../services/datasourceService'; -import { DataSourceType } from '../../types/datasource'; -import { API_BASE_URL, CB_DATASOURCE_REF, PROM_DATASOURCE_REF } from '../../constants'; +import { API_BASE_URL, PROM_DATASOURCE_REF } from '../../constants'; // Mirror of the Go PluginSettings JSON shape, minus the password (which lives // in secureJsonData and is never readable from the frontend). @@ -176,11 +174,7 @@ const AppConfig = ({ plugin }: AppConfigProps) => { setCbDsResult(null); setDsResult(null); - const cfg = await dataSourceService.getDataSourceConfig(); - const uid = - cfg.defaultDataSource === DataSourceType.Couchbase - ? CB_DATASOURCE_REF.uid - : PROM_DATASOURCE_REF.uid; + const uid = PROM_DATASOURCE_REF.uid; setDsUid(uid); const probes: Array> = []; diff --git a/cbmonitor/src/components/DashboardHeader/actions/ExploreButton.tsx b/cbmonitor/src/components/DashboardHeader/actions/ExploreButton.tsx index 1e10802..1ccedfc 100644 --- a/cbmonitor/src/components/DashboardHeader/actions/ExploreButton.tsx +++ b/cbmonitor/src/components/DashboardHeader/actions/ExploreButton.tsx @@ -1,8 +1,6 @@ -import React, { useEffect, useState } from 'react'; +import React from 'react'; import { SceneTimeRange } from '@grafana/scenes'; import { ToolbarButton } from '@grafana/ui'; -import { dataSourceService } from '../../../services/datasourceService'; -import { DataSourceType } from '../../../types/datasource'; import { buildExploreUrl } from '../../../utils/exploreUrl'; import { openInNewTab } from '../../../utils/openInNewTab'; @@ -12,19 +10,9 @@ interface ExploreButtonProps { } export function ExploreButton({ snapshotId, timeRange }: ExploreButtonProps) { - const [ds, setDs] = useState(dataSourceService.getCurrentDataSource()); - - useEffect(() => { - return dataSourceService.subscribe(setDs); - }, []); - const { value } = timeRange.useState(); - const promAvailable = ds === DataSourceType.Prometheus; const onClick = () => { - if (!promAvailable) { - return; - } const url = buildExploreUrl({ snapshotIds: snapshotId, range: { from: String(value.raw.from), to: String(value.raw.to) }, @@ -35,14 +23,9 @@ export function ExploreButton({ snapshotId, timeRange }: ExploreButtonProps) { return ( Explore diff --git a/cbmonitor/src/components/DashboardHeader/actions/MetricsDrilldownButton.tsx b/cbmonitor/src/components/DashboardHeader/actions/MetricsDrilldownButton.tsx index c2f1351..2655635 100644 --- a/cbmonitor/src/components/DashboardHeader/actions/MetricsDrilldownButton.tsx +++ b/cbmonitor/src/components/DashboardHeader/actions/MetricsDrilldownButton.tsx @@ -1,9 +1,7 @@ -import React, { useEffect, useState } from 'react'; +import React from 'react'; import { SceneTimeRange } from '@grafana/scenes'; import { config } from '@grafana/runtime'; import { ToolbarButton } from '@grafana/ui'; -import { dataSourceService } from '../../../services/datasourceService'; -import { DataSourceType } from '../../../types/datasource'; import { buildMetricsDrilldownUrl, METRICS_DRILLDOWN_PLUGIN_ID, @@ -21,24 +19,13 @@ function isMetricsDrilldownAppInstalled(): boolean { } export function MetricsDrilldownButton({ snapshotId, timeRange }: MetricsDrilldownButtonProps) { - const [ds, setDs] = useState(dataSourceService.getCurrentDataSource()); - - useEffect(() => { - return dataSourceService.subscribe(setDs); - }, []); - const { value } = timeRange.useState(); - const promAvailable = ds === DataSourceType.Prometheus; - const appInstalled = isMetricsDrilldownAppInstalled(); - if (!appInstalled) { + if (!isMetricsDrilldownAppInstalled()) { return null; } const onClick = () => { - if (!promAvailable) { - return; - } const url = buildMetricsDrilldownUrl({ snapshotId, range: { from: String(value.raw.from), to: String(value.raw.to) }, @@ -49,14 +36,9 @@ export function MetricsDrilldownButton({ snapshotId, timeRange }: MetricsDrilldo return ( Drilldown diff --git a/cbmonitor/src/constants.ts b/cbmonitor/src/constants.ts index f45f073..dfbed0f 100644 --- a/cbmonitor/src/constants.ts +++ b/cbmonitor/src/constants.ts @@ -9,12 +9,6 @@ export const DASHBOARD_UIDS = { kv_basic: '12b2cb59-bdeb-4015-a7c6-7367a7ff3878' } as const; -// Couchbase Datasource Reference -export const CB_DATASOURCE_REF = { - uid: 'cbdatasource', - type: 'couchbase-datasource' -} as const; - // Prometheus Datasource Reference export const PROM_DATASOURCE_REF = { uid: 'prometheus', diff --git a/cbmonitor/src/services/datasourceService.ts b/cbmonitor/src/services/datasourceService.ts deleted file mode 100644 index d830c03..0000000 --- a/cbmonitor/src/services/datasourceService.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { DataSourceType, DataSourceConfig } from '../types/datasource'; -import { API_BASE_URL } from '../constants'; - -type DataSourceChangeListener = (dataSource: DataSourceType) => void; - -/** - * Singleton service for managing which datasource (Couchbase SQL++ vs Prometheus) is active. - */ -class DataSourceService { - private currentDataSource: DataSourceType = DataSourceType.Prometheus; - private config: DataSourceConfig = { - defaultDataSource: DataSourceType.Prometheus, - prometheusAvailable: true, - couchbaseAvailable: false, - }; - private listeners: Set = new Set(); - private configInitialized = false; - - /** - * Initialize datasource configuration from backend. - */ - async initializeConfig(): Promise { - if (this.configInitialized) { - return this.config; - } - - try { - const response = await fetch(`${API_BASE_URL}/config/datasources`); - if (!response.ok) { - console.warn(`[DataSourceService] Failed to fetch datasource config: ${response.status}. Using defaults.`); - this.configInitialized = true; - return this.config; - } - - const data = await response.json(); - - // Update config with backend values - this.config.prometheusAvailable = data.prometheusAvailable ?? true; - this.config.couchbaseAvailable = data.couchbaseAvailable ?? false; - this.config.defaultDataSource = data.defaultDataSource ?? DataSourceType.Prometheus; - - // Set initial datasource based on availability - const preferredDataSource = this.config.defaultDataSource; - const isPreferredAvailable = - (preferredDataSource === DataSourceType.Prometheus && this.config.prometheusAvailable) || - (preferredDataSource === DataSourceType.Couchbase && this.config.couchbaseAvailable); - const fallbackDataSource = this.config.prometheusAvailable - ? DataSourceType.Prometheus - : DataSourceType.Couchbase; - const nextDataSource = isPreferredAvailable ? preferredDataSource : fallbackDataSource; - - this.currentDataSource = nextDataSource; - this.configInitialized = true; - } catch (error) { - console.error('[DataSourceService] Failed to initialize config from backend; using defaults:', error); - this.configInitialized = true; - } - - return this.config; - } - - /** - * The active datasource is pinned to Prometheus: the single gateway - * datasource speaks PromQL, and the Couchbase-vs-Prometheus runtime toggle - * has been retired. (This service is removed entirely in a later step.) - */ - getCurrentDataSource(): DataSourceType { - return DataSourceType.Prometheus; - } - - /** Switch the active datasource and notify all subscribers */ - setCurrentDataSource(ds: DataSourceType): void { - if (this.currentDataSource !== ds) { - this.currentDataSource = ds; - this.listeners.forEach((fn) => fn(ds)); - } - } - - /** Subscribe to datasource changes. Returns an unsubscribe function. */ - subscribe(listener: DataSourceChangeListener): () => void { - this.listeners.add(listener); - return () => { - this.listeners.delete(listener); - }; - } - - /** - * Return datasource config used to determine which options are shown. - */ - async getDataSourceConfig(): Promise { - return this.config; - } -} - -/** Singleton instance used across the application */ -export const dataSourceService = new DataSourceService(); diff --git a/cbmonitor/src/services/instanceService.ts b/cbmonitor/src/services/instanceService.ts index 88c0b42..c08e867 100644 --- a/cbmonitor/src/services/instanceService.ts +++ b/cbmonitor/src/services/instanceService.ts @@ -1,48 +1,31 @@ import { SceneQueryRunner } from '@grafana/scenes'; -import { CBQueryBuilder } from '../utils/utils.cbquery'; -import { dataSourceService } from './datasourceService'; import { clusterFilterService } from './clusterFilterService'; import { instanceFilterService } from './instanceFilterService'; -import { DataSourceType } from '../types/datasource'; import { PROXY_PROM_DATASOURCE_REF, PROM_DATASOURCE_REF } from '../constants'; import { injectClusterFilter, injectInstanceFilter } from '../utils/utils.panel'; export function getInstancesFromMetricRunner(snapshotId: string, metricName = 'sys_cpu_utilization_rate'): SceneQueryRunner { - const ds = dataSourceService.getCurrentDataSource(); const clusterFilter = clusterFilterService.getCurrentCluster(); const instanceFilter = instanceFilterService.getCurrentInstance(); - if (ds === DataSourceType.Prometheus) { - // PromQL path: hardcoded expression, then apply active filters so the - // instance discovery query only returns nodes inside the active scope. - let expr = `group by (instance) (${metricName}{job="${snapshotId}"})`; - if (clusterFilter) { - expr = injectClusterFilter(expr, clusterFilter); - } - if (instanceFilter) { - expr = injectInstanceFilter(expr, instanceFilter); - } - return new SceneQueryRunner({ - datasource: PROM_DATASOURCE_REF, - queries: [{ - refId: 'instances', - expr, - legendFormat: '{{instance}}', - instant: true, - }], - }); - } - - // SQL++ path: use CBQueryBuilder - const builder = new CBQueryBuilder(snapshotId, metricName); - builder.setExtraFields(['d.labels.instance']); + // Discover instances via PromQL, applying active filters so the query only + // returns nodes inside the active scope. + let expr = `group by (instance) (${metricName}{job="${snapshotId}"})`; if (clusterFilter) { - builder.addLabelFilter('cluster', clusterFilter); + expr = injectClusterFilter(expr, clusterFilter); } if (instanceFilter) { - builder.addLabelFilter('instance', instanceFilter); + expr = injectInstanceFilter(expr, instanceFilter); } - return builder.buildQueryRunner(); + return new SceneQueryRunner({ + datasource: PROM_DATASOURCE_REF, + queries: [{ + refId: 'instances', + expr, + legendFormat: '{{instance}}', + instant: true, + }], + }); } /** diff --git a/cbmonitor/src/types/datasource.ts b/cbmonitor/src/types/datasource.ts deleted file mode 100644 index 6d17b8f..0000000 --- a/cbmonitor/src/types/datasource.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Datasource types supported by the application. - */ -export enum DataSourceType { - Couchbase = 'couchbase', - Prometheus = 'prometheus', - ProxyPrometheus = 'proxyprometheus', -} - -/** - * Configuration for dual-datasource support, including visibility per deployment. - */ -export interface DataSourceConfig { - /** The default datasource used unless overridden */ - defaultDataSource: DataSourceType; - /** Whether Couchbase is available in the UI selector */ - couchbaseAvailable: boolean; - /** Whether Prometheus is available in the UI selector */ - prometheusAvailable: boolean; -} diff --git a/cbmonitor/src/utils/utils.cbquery.ts b/cbmonitor/src/utils/utils.cbquery.ts deleted file mode 100644 index 3b8daeb..0000000 --- a/cbmonitor/src/utils/utils.cbquery.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { SceneQueryRunner } from "@grafana/scenes"; -import { CB_DATASOURCE_REF } from "../constants"; - -/** - * Builder class for constructing Couchbase query strings and query runner - * The class is used to create a query string and a query runner for a given metric name and snapshot id. - */ -export class CBQueryBuilder { - // Required parameters - protected snapshotId: string; - protected metricName: string; - protected labelFilters: Map = new Map(); - protected extraFields: string[] = ['d.labels.instance']; // Most panels will select the instance label by default - - - constructor(snapshotId: string, metricName: string) { - this.snapshotId = snapshotId; - this.metricName = metricName; - } - - // Add label filters - addLabelFilter(label: string, value: string | string[]): this { - this.labelFilters.set(label, value); - return this; - } - - // Set which extra fields to fetch - setExtraFields(fields: string[]): this { - this.extraFields = fields; - return this; - } - - - /* The default WHERE clause is the time_range(t._t) which is expanded by the datasource - * to include the time range variables $__from and $__to. - * - * For panels that need to filter on specific labels, use the addLabelFilter method. - */ - addExtraField(field: string): this { - if (!this.extraFields.includes(field)) { - this.extraFields.push(field); - } - return this; - } - - // Build the WHERE clause - protected buildWhereClause(): string { - const conditions: string[] = [ - `time_range(t._t)`, - ]; - - // Add label filters - for (const [label, value] of this.labelFilters.entries()) { - if (Array.isArray(value)) { - const valueList = value.map(v => `'${v}'`).join(', '); - conditions.push(`d.labels.\`${label}\` IN [${valueList}]`); - } else { - conditions.push(`d.labels.\`${label}\` = '${value}'`); - } - } - - return conditions.join(' AND '); - } - - // Build the SELECT clause - protected buildSelectClause(): string { - const fields = [ - 'MILLIS_TO_STR(t._t) AS time', - `t._v0 AS \`${this.metricName}\`` , // Use metric name as the label for the metric value so it can be displayed in the legend - ...this.extraFields - ]; - return fields.join(', '); - } - - // Build the complete query string using the select and where clauses - build(): string { - const selectClause = this.buildSelectClause(); - const whereClause = this.buildWhereClause(); - - let query = `SELECT ${selectClause} FROM get_metric_for('${this.metricName}', '${this.snapshotId}') AS d UNNEST _timeseries(d,{'ts_ranges':[\${__from}, \${__to}]}) AS t WHERE ${whereClause}`; - - // Ensure deterministic series ordering by label fields when the datasource materializes series - if (this.extraFields && this.extraFields.length > 0) { - const orderable = this.extraFields.filter(f => f.startsWith('d.labels')); - if (orderable.length > 0) { - query += ` ORDER BY ${orderable.join(', ')}`; - } - } - - return query; - } - - // Build and return a SceneQueryRunner - buildQueryRunner(): SceneQueryRunner { - return new SceneQueryRunner({ - datasource: CB_DATASOURCE_REF, - queries: [{ - refId: this.metricName, - query: this.build(), - }], - }); - } -} - -/** - * Builder that applies a timeseries transformation (e.g. rate) via a derived subquery. - * Output schema: t._t as time, t._v0 as value. - */ -export class AggregationQueryBuilder extends CBQueryBuilder { - private transformFunction = 'rate'; - private outerAlias = 'd2'; - private innerAlias = 'd'; - // Allowlist of supported transformation functions to avoid SQL injection via function name - private static readonly ALLOWED_TRANSFORMS: Set = new Set([ - 'rate', - 'irate', - 'increase' - ]); - - setTransformFunction(fnName: string): this { - const normalized = fnName.trim(); - if (!AggregationQueryBuilder.ALLOWED_TRANSFORMS.has(normalized)) { - throw new Error(`Invalid transform function: ${fnName}`); - } - this.transformFunction = normalized; - return this; - } - - - protected buildSelectClause(): string { - const alias = this.outerAlias; - const remappedExtras = this.extraFields.map(f => f.replace(/^d\./, `${alias}.`)); - const fields = [ - 'millis_to_str(t._t) AS time', - `t._v0 AS \`${this.metricName}\``, - ...remappedExtras, - ]; - return fields.join(', '); - } - - protected buildWhereClause(): string { - const conditions: string[] = [ - 'time_range(t._t)' - ]; - return conditions.join(' AND '); - } - - private buildInnerWhereClause(): string { - const innerConds: string[] = []; - for (const [label, value] of this.labelFilters.entries()) { - if (Array.isArray(value)) { - const valueList = value.map(v => `'${v}'`).join(', '); - innerConds.push(`${this.innerAlias}.labels.\`${label}\` IN [${valueList}]`); - } else { - innerConds.push(`${this.innerAlias}.labels.\`${label}\` = '${value}'`); - } - } - return innerConds.length ? ` WHERE ${innerConds.join(' AND ')}` : ''; - } - - build(): string { - const selectClause = this.buildSelectClause(); - const whereClause = this.buildWhereClause(); - const innerWhere = this.buildInnerWhereClause(); - let query = `SELECT ${selectClause} FROM ( SELECT RAW OBJECT_PUT(${this.innerAlias}, "ts_data", ${this.transformFunction}(${this.innerAlias}.ts_data, 40)) FROM get_metric_for('${this.metricName}', '${this.snapshotId}') AS ${this.innerAlias}${innerWhere} ) AS ${this.outerAlias} UNNEST _timeseries(${this.outerAlias},{'ts_ranges':[\${__from}, \${__to}]}) AS t WHERE ${whereClause}`; - - // Deterministic series ordering by label fields - if (this.extraFields && this.extraFields.length > 0) { - const orderable = this.extraFields - .map(f => f.replace(/^d\./, `${this.outerAlias}.`)) - .filter(f => f.startsWith(`${this.outerAlias}.labels`)); - if (orderable.length > 0) { - query += ` ORDER BY ${orderable.join(', ')}`; - } - } - return query; - } -} diff --git a/cbmonitor/src/utils/utils.panel.ts b/cbmonitor/src/utils/utils.panel.ts index cb4d702..cabedeb 100644 --- a/cbmonitor/src/utils/utils.panel.ts +++ b/cbmonitor/src/utils/utils.panel.ts @@ -1,15 +1,11 @@ -// Panel creation utilities with hardcoded PromQL expressions as the source of truth. -// SQL++ queries are still generated from parameters via CBQueryBuilder when that datasource is active. -// In the future, SQL++ queries will be derived from the PromQL expressions. +// Panel creation utilities. Panels are authored as PromQL expressions and +// queried against the single Prometheus datasource (the gateway). import { PanelBuilders, SceneDataTransformer, SceneFlexItem, SceneQueryRunner, SceneDataState } from '@grafana/scenes'; import { TooltipDisplayMode, LegendDisplayMode } from '@grafana/schema'; -import { CBQueryBuilder, AggregationQueryBuilder } from './utils.cbquery'; import { layoutService } from '../services/layoutService'; -import { dataSourceService } from '../services/datasourceService'; import { clusterFilterService } from '../services/clusterFilterService'; import { instanceFilterService } from '../services/instanceFilterService'; -import { DataSourceType } from '../types/datasource'; import { PROM_DATASOURCE_REF } from '../constants'; import { ROUTES, prefixRoute } from './utils.routing'; @@ -82,21 +78,6 @@ type PanelOptions = { height?: number; }; -// Apply label filters and extra fields to a CBQueryBuilder -function applyCBBuilderOptions( - builder: CBQueryBuilder, - options: PanelOptions -) { - if (options.labelFilters) { - for (const [label, value] of Object.entries(options.labelFilters)) { - builder.addLabelFilter(label, value); - } - } - if (options.extraFields) { - builder.setExtraFields(options.extraFields); - } -} - // Build legend template for Grafana panel display name override export function makeLegendTemplate(extraFields?: string[]): string { const ef = extraFields ?? []; @@ -164,15 +145,13 @@ export function injectInstanceFilter(expr: string, instance: string): string { } /** - * Create a metric panel with a hardcoded PromQL expression (source of truth). - * - * When the active datasource is PromQL, uses the hardcoded `expr` directly. - * When the active datasource is Couchbase SQL++, builds a query via CBQueryBuilder - * using `snapshotId`, `labelFilters`, `extraFields`, and optionally `transformFunction`. + * Create a metric panel from a PromQL expression, queried against the single + * Prometheus datasource. Active cluster/instance filters are injected into the + * expression. * * @param metricName - Metric / refId identifier * @param title - Panel display title - * @param options - Panel options (PromQL expr + SQL++ params + display settings) + * @param options - Panel options (PromQL expr + display settings) */ export function createMetricPanel( metricName: string, @@ -180,8 +159,6 @@ export function createMetricPanel( options: PanelOptions ): SceneFlexItem { const panelWidth = options.width ?? layoutService.getPanelWidth(); - const ds = dataSourceService.getCurrentDataSource(); - const isPrometheus = ds === DataSourceType.Prometheus; const panelBuilder = PanelBuilders.timeseries().setTitle(title); panelBuilder.setOption('tooltip', { mode: TooltipDisplayMode.Multi }); @@ -223,82 +200,39 @@ export function createMetricPanel( const clusterFilter = clusterFilterService.getCurrentCluster(); const instanceFilter = instanceFilterService.getCurrentInstance(); - if (isPrometheus) { - // --- PromQL path: use hardcoded expression directly --- - // Apply active filters in order: cluster first, then instance (so the - // node drilldown's instance scope overrides any per-instance hardcoded - // selectors in the dashboard expressions). - let finalExpr = options.expr; - if (clusterFilter) { - finalExpr = injectClusterFilter(finalExpr, clusterFilter); - } - if (instanceFilter) { - finalExpr = injectInstanceFilter(finalExpr, instanceFilter); - } - - queryRunner = new SceneQueryRunner({ - datasource: PROM_DATASOURCE_REF, - queries: [{ - refId: metricName, - expr: finalExpr, - }], - }); - - // Description - try { - const descriptionMd = [ - `**Metric:** ${metricName} \n`, - `**Datasource:** PromQL \n`, - clusterFilter ? `**Cluster:** ${clusterFilter} \n` : '', - instanceFilter ? `**Instance:** ${instanceFilter} \n` : '', - '', - '**Query:**', - '```promql', - finalExpr, - '```', - ].join('\n'); - panelBuilder.setDescription(descriptionMd); - } catch (_e) { /* skip */ } - } else { - // --- SQL++ path: build via CBQueryBuilder --- - let builder: CBQueryBuilder; - if (options.transformFunction) { - const aggBuilder = new AggregationQueryBuilder(options.snapshotId, metricName); - aggBuilder.setTransformFunction(options.transformFunction); - builder = aggBuilder; - } else { - builder = new CBQueryBuilder(options.snapshotId, metricName); - } - applyCBBuilderOptions(builder, options); - - // If cluster filter is active, add it to the query - if (clusterFilter) { - builder.addLabelFilter('cluster', clusterFilter); - } - // If instance filter is active, scope the SQL++ query to that node. - if (instanceFilter) { - builder.addLabelFilter('instance', instanceFilter); - } + // Apply active filters in order: cluster first, then instance (so the node + // drilldown's instance scope overrides any per-instance hardcoded selectors + // in the dashboard expressions). + let finalExpr = options.expr; + if (clusterFilter) { + finalExpr = injectClusterFilter(finalExpr, clusterFilter); + } + if (instanceFilter) { + finalExpr = injectInstanceFilter(finalExpr, instanceFilter); + } - queryRunner = builder.buildQueryRunner(); + queryRunner = new SceneQueryRunner({ + datasource: PROM_DATASOURCE_REF, + queries: [{ + refId: metricName, + expr: finalExpr, + }], + }); - // Description - try { - const queryText = builder.build(); - const extraDesc = options.transformFunction ? [`**Transform:** ${options.transformFunction}`] : []; - const descriptionMd = [ - `**Metric:** ${metricName} \n`, - `**Datasource:** Couchbase SQL++ (experimental) \n`, - ...extraDesc, - '', - '**Query:**', - '```sql', - queryText, - '```', - ].join('\n'); - panelBuilder.setDescription(descriptionMd); - } catch (_e) { /* skip */ } - } + // Description + try { + const descriptionMd = [ + `**Metric:** ${metricName} \n`, + clusterFilter ? `**Cluster:** ${clusterFilter} \n` : '', + instanceFilter ? `**Instance:** ${instanceFilter} \n` : '', + '', + '**Query:**', + '```promql', + finalExpr, + '```', + ].join('\n'); + panelBuilder.setDescription(descriptionMd); + } catch (_e) { /* skip */ } if (options.unit) { panelBuilder.setUnit(options.unit); From a1eae08847388c38fda9831c243cea129f7b81cf Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 18:42:43 +0100 Subject: [PATCH 14/18] cbmonitor: Simplify panels to only support promQL queries --- cbmonitor/src/dashboards/analytics.ts | 6 ---- cbmonitor/src/dashboards/clusterManager.ts | 11 -------- cbmonitor/src/dashboards/custom.ts | 1 - cbmonitor/src/dashboards/eventing.ts | 5 ---- cbmonitor/src/dashboards/fts.ts | 4 --- cbmonitor/src/dashboards/index.ts | 9 ------ cbmonitor/src/dashboards/kv.ts | 17 ----------- cbmonitor/src/dashboards/query.ts | 6 ---- cbmonitor/src/dashboards/sgw.ts | 3 -- cbmonitor/src/dashboards/system.ts | 11 -------- cbmonitor/src/dashboards/types.ts | 12 ++------ cbmonitor/src/dashboards/xdcr.ts | 13 --------- cbmonitor/src/utils/utils.panel.ts | 33 +++------------------- 13 files changed, 6 insertions(+), 125 deletions(-) diff --git a/cbmonitor/src/dashboards/analytics.ts b/cbmonitor/src/dashboards/analytics.ts index b485c65..ee95e5c 100644 --- a/cbmonitor/src/dashboards/analytics.ts +++ b/cbmonitor/src/dashboards/analytics.ts @@ -16,14 +16,11 @@ export const analyticsBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `Java CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="java"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'java' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `Java Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="java"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'java' }, unit: 'bytes', }), @@ -43,8 +40,6 @@ export const analyticsBuilder: ServiceBuilder = (ctx) => { ctx.panel('cbas_jobs_total', `Jobs/Sec${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy('result')}) (rate(cbas_jobs_total{${ctx.jobSelector}${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend('result'), - extraFields: ['d.labels.`result`', 'd.labels.`instance`'], - transformFunction: 'rate', unit: 'short', }), @@ -61,7 +56,6 @@ function simpleCbasPanel(ctx: MetricContext, metric: string, title: string, unit return ctx.panel(metric, `${title}${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (${series})`, legendFormat: ctx.legend(), - ...(rate ? { transformFunction: 'rate' } : {}), unit, }); } diff --git a/cbmonitor/src/dashboards/clusterManager.ts b/cbmonitor/src/dashboards/clusterManager.ts index 5b949a1..b3ba78d 100644 --- a/cbmonitor/src/dashboards/clusterManager.ts +++ b/cbmonitor/src/dashboards/clusterManager.ts @@ -17,27 +17,21 @@ export const clusterManagerBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `ns_server CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="ns_server"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'ns_server' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `ns_server Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="ns_server"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'ns_server' }, unit: 'bytes', }), ctx.panel('sysproc_cpu_seconds_total', `Prometheus CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="prometheus"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'prometheus' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `Prometheus Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="prometheus"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'prometheus' }, unit: 'bytes', }), // HTTP requests aggregated in overlap base; single mode emits @@ -60,9 +54,6 @@ export const clusterManagerBuilder: ServiceBuilder = (ctx) => { ctx.panel('cm_http_requests_total', `HTTP Requests/Sec (${i})`, { expr: `sum by (method) (rate(cm_http_requests_total{${ctx.jobSelector},instance="${i}"}[$__rate_interval]))`, legendFormat: '{{method}}', - labelFilters: { instance: i }, - extraFields: ['d.labels.method'], - transformFunction: 'rate', unit: 'short', }), ]; @@ -73,8 +64,6 @@ export const clusterManagerBuilder: ServiceBuilder = (ctx) => { ctx.panel('cm_http_requests_total', 'HTTP Requests/Sec', { expr: `rate(cm_http_requests_total{${ctx.jobSelector}}[$__rate_interval])`, legendFormat: '{{method}} , {{instance}}', - extraFields: ['d.labels.method', 'd.labels.instance'], - transformFunction: 'rate', unit: 'short', }), ]; diff --git a/cbmonitor/src/dashboards/custom.ts b/cbmonitor/src/dashboards/custom.ts index c55f2d0..555c023 100644 --- a/cbmonitor/src/dashboards/custom.ts +++ b/cbmonitor/src/dashboards/custom.ts @@ -51,7 +51,6 @@ function buildPanel( return ctx.panel(metric, title, { expr, legendFormat: override?.legendFormat ?? ctx.legend(), - ...(shouldRate ? { transformFunction: 'rate' } : {}), ...(override?.unit ? { unit: override.unit } : {}), }); } diff --git a/cbmonitor/src/dashboards/eventing.ts b/cbmonitor/src/dashboards/eventing.ts index 23fb680..8542147 100644 --- a/cbmonitor/src/dashboards/eventing.ts +++ b/cbmonitor/src/dashboards/eventing.ts @@ -16,21 +16,16 @@ export const eventingBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `Eventing CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="eventing-produc"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'eventing-produc' }, - extraFields: ['d.labels.`instance`', 'd.labels.`mode`'], - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `Eventing Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="eventing-produc"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'eventing-produc' }, unit: 'bytes', }), ctx.panel('eventing_worker_restart_count', `Worker Restart Count${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (eventing_worker_restart_count{${ctx.jobSelector}${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - extraFields: ['d.labels.instance'], unit: 'short', }), ]; diff --git a/cbmonitor/src/dashboards/fts.ts b/cbmonitor/src/dashboards/fts.ts index 23e5070..2bd73f2 100644 --- a/cbmonitor/src/dashboards/fts.ts +++ b/cbmonitor/src/dashboards/fts.ts @@ -15,14 +15,11 @@ export const ftsBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `Search CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="cbft"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'cbft' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `Search Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="cbft"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'cbft' }, unit: 'bytes', }), @@ -47,7 +44,6 @@ function simpleFtsPanel(ctx: MetricContext, metric: string, title: string, unit: return ctx.panel(metric, `${title}${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (${series})`, legendFormat: ctx.legend(), - ...(rate ? { transformFunction: 'rate' } : {}), unit, }); } diff --git a/cbmonitor/src/dashboards/index.ts b/cbmonitor/src/dashboards/index.ts index 1ccaba0..49f40d2 100644 --- a/cbmonitor/src/dashboards/index.ts +++ b/cbmonitor/src/dashboards/index.ts @@ -43,15 +43,11 @@ export const indexBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `Indexer CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="indexer"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'indexer' }, - extraFields: ['d.labels.`instance`', 'd.labels.`mode`'], - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `Indexer Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="indexer"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'indexer' }, unit: 'bytes', }), @@ -112,8 +108,6 @@ function singleMemoryUsedPanel(ctx: MetricContext, i: string): SceneFlexItem { return ctx.panel('index_memory_used', `Index Memory Used (${i})`, { expr: `index_memory_used{${ctx.jobSelector}, instance="${i}"}`, legendFormat: '{{bucket}} , {{index}} , {{scope}} , {{collection}}', - labelFilters: { instance: i }, - extraFields: ['d.labels.`bucket`', 'd.labels.`index`', 'd.labels.`scope`', 'd.labels.`collection`'], unit: 'bytes', }); } @@ -122,8 +116,6 @@ function singleDetailPanelForInstance(ctx: MetricContext, metric: string, title: return ctx.panel(metric, `${title} (${i})`, { expr: `${metric}{${ctx.jobSelector},instance="${i}"}`, legendFormat: '{{bucket}} , {{index}}', - labelFilters: { instance: i }, - extraFields: ['d.labels.`bucket`', 'd.labels.`index`'], unit, }); } @@ -132,7 +124,6 @@ function singleDetailPanelAggregated(ctx: MetricContext, metric: string, title: return ctx.panel(metric, title, { expr: `${metric}{${ctx.jobSelector}}`, legendFormat: '{{bucket}} , {{index}}', - extraFields: ['d.labels.`bucket`', 'd.labels.`index`'], unit, }); } diff --git a/cbmonitor/src/dashboards/kv.ts b/cbmonitor/src/dashboards/kv.ts index 8f66b0d..035a82f 100644 --- a/cbmonitor/src/dashboards/kv.ts +++ b/cbmonitor/src/dashboards/kv.ts @@ -20,14 +20,11 @@ export const kvBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `memcached CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="memcached"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'memcached' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `memcached Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="memcached"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'memcached' }, unit: 'bytes', }), // Resident ratio is reported per bucket and vBucket state @@ -36,8 +33,6 @@ export const kvBuilder: ServiceBuilder = (ctx) => { ctx.panel('kv_vb_perc_mem_resident_ratio', `vBucket Memory Resident Ratio (%)${ctx.titleSuffix}`, { expr: `avg by (${ctx.mode === 'overlap' ? 'job, bucket, state' : 'bucket, state'}) (kv_vb_perc_mem_resident_ratio{${ctx.jobSelector},state=~"active|replica"})`, legendFormat: ctx.mode === 'overlap' ? '{{job}} , {{bucket}} , {{state}}' : '{{bucket}} , {{state}}', - labelFilters: { state: ['active', 'replica'] }, - extraFields: ['d.labels.`bucket`', 'd.labels.`state`'], unit: 'percentunit', }), @@ -45,8 +40,6 @@ export const kvBuilder: ServiceBuilder = (ctx) => { ctx.panel('kv_vb_ops_get', `vBucket GET Ops/Sec${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy('bucket')}) (rate(kv_vb_ops_get{${ctx.jobSelector}${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend('bucket'), - transformFunction: 'rate', - extraFields: ['d.labels.`instance`', 'd.labels.`bucket`'], unit: 'short', }), @@ -58,8 +51,6 @@ export const kvBuilder: ServiceBuilder = (ctx) => { ctx.panel('kv_ops_by_type', 'KV Operations by Type (ops/sec)', { expr: `sum by (op) (rate(kv_ops{${ctx.jobSelector}}[$__rate_interval]))`, legendFormat: '{{op}}', - transformFunction: 'rate', - extraFields: ['d.labels.`op`'], unit: 'short', }), ]), @@ -106,9 +97,6 @@ export const kvBuilder: ServiceBuilder = (ctx) => { ctx.panel('kv_ops', `KV Operations/Sec (${i})`, { expr: `rate(kv_ops{${ctx.jobSelector},instance="${i}"}[$__rate_interval])`, legendFormat: '{{bucket}} , {{op}} , {{result}}', - labelFilters: { instance: i }, - extraFields: ['d.labels.`bucket`', 'd.labels.`op`', 'd.labels.`result`'], - transformFunction: 'rate', unit: 'short', }), ]; @@ -119,8 +107,6 @@ export const kvBuilder: ServiceBuilder = (ctx) => { ctx.panel('kv_ops', 'KV Operations/Sec', { expr: `rate(kv_ops{${ctx.jobSelector}}[$__rate_interval])`, legendFormat: '{{instance}} , {{bucket}} , {{op}} , {{result}}', - extraFields: ['d.labels.`instance`', 'd.labels.`bucket`', 'd.labels.`op`', 'd.labels.`result`'], - transformFunction: 'rate', unit: 'short', }), ]; @@ -132,7 +118,6 @@ function simpleKvPanel(ctx: MetricContext, metric: string, title: string, unit: return ctx.panel(metric, `${title}${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (${series})`, legendFormat: ctx.legend(), - ...(rate ? { transformFunction: 'rate' } : {}), unit, }); } @@ -154,8 +139,6 @@ function dcpPanel(ctx: MetricContext, metric: string, title: string, unit: strin return ctx.panel(metric, `${title}${ctx.titleSuffix}`, { expr, legendFormat: ctx.legend('bucket', 'connection_type'), - ...(rate ? { transformFunction: 'rate' } : {}), - extraFields: ['d.labels.`instance`', 'd.labels.`bucket`', 'd.labels.`connection_type`'], unit, }); } diff --git a/cbmonitor/src/dashboards/query.ts b/cbmonitor/src/dashboards/query.ts index 437b94e..100d665 100644 --- a/cbmonitor/src/dashboards/query.ts +++ b/cbmonitor/src/dashboards/query.ts @@ -25,15 +25,11 @@ export const queryBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `Query Engine CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="cbq-engine"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'cbq-engine' }, - extraFields: ['d.labels.`instance`', 'd.labels.`mode`'], - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `Query Engine Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="cbq-engine"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'cbq-engine' }, unit: 'bytes', }), @@ -86,7 +82,6 @@ export const queryBuilder: ServiceBuilder = (ctx) => { or label_replace(sum by (${ctx.sumBy()}) (n1ql_request_timer_p99{${ctx.jobSelector}}), "quantile", "p99", "", "") `.trim(), legendFormat: ctx.legend('quantile'), - extraFields: ['d.labels.`instance`', 'd.labels.`quantile`'], unit: 'ns', }), ctx.panel('n1ql_request_timer_mean_max', 'Query Request Time — mean / max', { @@ -95,7 +90,6 @@ export const queryBuilder: ServiceBuilder = (ctx) => { or label_replace(sum by (${ctx.sumBy()}) (n1ql_request_timer_max{${ctx.jobSelector}}), "stat", "max", "", "") `.trim(), legendFormat: ctx.legend('stat'), - extraFields: ['d.labels.`instance`', 'd.labels.`stat`'], unit: 'ns', }), ]), diff --git a/cbmonitor/src/dashboards/sgw.ts b/cbmonitor/src/dashboards/sgw.ts index 7f840c6..3972f63 100644 --- a/cbmonitor/src/dashboards/sgw.ts +++ b/cbmonitor/src/dashboards/sgw.ts @@ -172,7 +172,6 @@ function sgwResourcePanel(ctx: MetricContext, [metric, title, unit, rate]: SgwMe return ctx.panel(metric, `${title}${ctx.titleSuffix}`, { expr, legendFormat: ctx.legend(), - ...(rate ? { transformFunction: 'rate' } : {}), unit, }); } @@ -193,8 +192,6 @@ function sgwDatabasePanel(ctx: MetricContext, [metric, title, unit, rate]: SgwMe return ctx.panel(metric, `${title}${ctx.titleSuffix}`, { expr: `sum by (${groupBy}) (${series})`, legendFormat: '{{instance}} , {{database}}', - extraFields: ['d.labels.instance', 'd.labels.`database`'], - ...(rate ? { transformFunction: 'rate' } : {}), unit, }); } diff --git a/cbmonitor/src/dashboards/system.ts b/cbmonitor/src/dashboards/system.ts index 7f569fa..7a783a7 100644 --- a/cbmonitor/src/dashboards/system.ts +++ b/cbmonitor/src/dashboards/system.ts @@ -42,7 +42,6 @@ export const systemBuilder: ServiceBuilder = (ctx) => { ctx.panel('couch_docs_actual_disk_size', `Couch Docs Actual Disk Size (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy('bucket')}) (couch_docs_actual_disk_size{${ctx.jobSelector}${ctx.instanceFilter}})`, legendFormat: ctx.legend('bucket'), - extraFields: ['d.labels.`instance`', 'd.labels.`bucket`'], unit: 'bytes', }), // Disk read/write: overlap renders them aggregated in 'base'; @@ -69,17 +68,11 @@ export const systemBuilder: ServiceBuilder = (ctx) => { ctx.panel('sys_disk_read_bytes', `Rate Disk Read Bytes (${i})`, { expr: `rate(sys_disk_read_bytes{${ctx.jobSelector},instance="${i}"}[$__rate_interval])`, legendFormat: '{{disk}}', - labelFilters: { instance: i }, - extraFields: ['d.labels.`disk`'], - transformFunction: 'rate', unit: 'Bps', }), ctx.panel('sys_disk_write_bytes', `Rate Disk Write Bytes (${i})`, { expr: `rate(sys_disk_write_bytes{${ctx.jobSelector},instance="${i}"}[$__rate_interval])`, legendFormat: '{{disk}}', - labelFilters: { instance: i }, - extraFields: ['d.labels.`disk`'], - transformFunction: 'rate', unit: 'Bps', }), ]; @@ -90,15 +83,11 @@ export const systemBuilder: ServiceBuilder = (ctx) => { ctx.panel('sys_disk_read_bytes', 'Rate Disk Read Bytes', { expr: `rate(sys_disk_read_bytes{${ctx.jobSelector}}[$__rate_interval])`, legendFormat: '{{disk}}', - extraFields: ['d.labels.`disk`'], - transformFunction: 'rate', unit: 'Bps', }), ctx.panel('sys_disk_write_bytes', 'Rate Disk Write Bytes', { expr: `rate(sys_disk_write_bytes{${ctx.jobSelector}}[$__rate_interval])`, legendFormat: '{{disk}}', - extraFields: ['d.labels.`disk`'], - transformFunction: 'rate', unit: 'Bps', }), ]; diff --git a/cbmonitor/src/dashboards/types.ts b/cbmonitor/src/dashboards/types.ts index 2fb8b3e..89862a1 100644 --- a/cbmonitor/src/dashboards/types.ts +++ b/cbmonitor/src/dashboards/types.ts @@ -1,12 +1,8 @@ import { SceneFlexItem } from '@grafana/scenes'; /** - * Declarative spec for a single panel. Both modes accept the same shape. - * - * `labelFilters`, `extraFields`, `transformFunction` are SQL++ forwarding - * concerns consumed only by the single-mode `ctx.panel` (which routes to - * `createMetricPanel`). The overlap-mode `ctx.panel` is PromQL-only and - * silently ignores them. + * Declarative spec for a single panel: a PromQL expression plus display + * options. Both modes accept the same shape. */ export interface PanelSpec { expr: string; @@ -14,10 +10,6 @@ export interface PanelSpec { unit?: string; width?: string | number; height?: number; - - labelFilters?: Record; - extraFields?: string[]; - transformFunction?: string; } /** diff --git a/cbmonitor/src/dashboards/xdcr.ts b/cbmonitor/src/dashboards/xdcr.ts index 935e010..309312d 100644 --- a/cbmonitor/src/dashboards/xdcr.ts +++ b/cbmonitor/src/dashboards/xdcr.ts @@ -14,14 +14,11 @@ export const xdcrBuilder: ServiceBuilder = (ctx) => { ctx.panel('sysproc_cpu_seconds_total', `goxdcr CPU Usage (cores)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(sysproc_cpu_seconds_total{${ctx.jobSelector},proc="goxdcr"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { proc: 'goxdcr' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('sysproc_mem_resident', `goxdcr Resident Memory (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (sysproc_mem_resident{${ctx.jobSelector},proc="goxdcr"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { proc: 'goxdcr' }, unit: 'bytes', }), // NOTE: gauge, not a counter — do not wrap in rate(). Tracks documents @@ -29,46 +26,36 @@ export const xdcrBuilder: ServiceBuilder = (ctx) => { ctx.panel('xdcr_changes_left_total', `Changes Left (Pending Docs)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (xdcr_changes_left_total{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, unit: 'short', }), ctx.panel('xdcr_docs_cloned_total', `Documents Cloned/Sec${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(xdcr_docs_cloned_total{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('xdcr_docs_checked_total', `Documents Checked/Sec${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(xdcr_docs_checked_total{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('xdcr_docs_written_total', `Documents Written/Sec${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (rate(xdcr_docs_written_total{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}}[$__rate_interval]))`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, - transformFunction: 'rate', unit: 'short', }), ctx.panel('xdcr_wtavg_docs_latency_seconds', `Weighted Average Document Latency (Seconds)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (xdcr_wtavg_docs_latency_seconds{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, unit: 's', }), ctx.panel('xdcr_wtavg_meta_latency_seconds', `Weighted Average Metadata Latency (Seconds)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (xdcr_wtavg_meta_latency_seconds{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, unit: 's', }), ctx.panel('xdcr_data_replicated_bytes', `Data Replicated (Bytes)${ctx.titleSuffix}`, { expr: `sum by (${ctx.sumBy()}) (xdcr_data_replicated_bytes{${ctx.jobSelector},pipelineType="Main"${ctx.instanceFilter}})`, legendFormat: ctx.legend(), - labelFilters: { pipelineType: 'Main' }, unit: 'bytes', }), ]; diff --git a/cbmonitor/src/utils/utils.panel.ts b/cbmonitor/src/utils/utils.panel.ts index cabedeb..9b06141 100644 --- a/cbmonitor/src/utils/utils.panel.ts +++ b/cbmonitor/src/utils/utils.panel.ts @@ -61,44 +61,19 @@ export function getNewTimeSeriesDataTransformer(queryRunner: SceneQueryRunner) { // Options for panel creation type PanelOptions = { - // PromQL expression — hardcoded, source of truth + // PromQL expression — the source of truth expr: string; // Prometheus legend format, e.g. '{{instance}}'. Default: '{{instance}}' legendFormat?: string; - - // SQL++ parameters for CBQueryBuilder (used when Couchbase datasource is active) + // Snapshot ID, baked into the node-drilldown data link. snapshotId: string; - labelFilters?: Record; - extraFields?: string[]; - transformFunction?: string; // e.g., 'rate', 'irate', 'increase' — uses AggregationQueryBuilder - // Shared display options + // Display options unit?: string; width?: string | number; height?: number; }; -// Build legend template for Grafana panel display name override -export function makeLegendTemplate(extraFields?: string[]): string { - const ef = extraFields ?? []; - const labelKeys = ef - .map((f) => { - const matchBackticks = f.match(/d\.labels\.\`([^`]+)\`/); - if (matchBackticks) { - return matchBackticks[1]; - } - const matchDot = f.match(/d\.labels\.([^`\.]+)/); - return matchDot ? matchDot[1] : undefined; - }) - .filter((v): v is string => Boolean(v)); - - if (labelKeys.length > 0) { - const parts = labelKeys.map((k) => '${__field.labels.' + k + '}'); - return parts.join(' , '); - } - return '${__field.labels.instance}'; -} - /** * Inject cluster filter into a PromQL expression. * Adds `cluster_uuid=""` label selector to metric selectors that already have labels. @@ -174,7 +149,7 @@ export function createMetricPanel( // so the legend style is consistent regardless of datasource. const legendTemplate = options.legendFormat ? options.legendFormat.replace(/\{\{(\w+)\}\}/g, '${__field.labels.$1}') - : makeLegendTemplate(options.extraFields); + : '${__field.labels.instance}'; // Data link to the node drilldown: rendered in the tooltip + legend // context menu whenever a series has an `instance` label. Grafana // resolves `${__field.labels.instance}` per-series, so non-instance From bae30406ecd35b203d93b59e6abca9a3ac1ed6a2 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 18:58:37 +0100 Subject: [PATCH 15/18] cbmonitor: Use the same datasource for overlap too --- cbmonitor/src/constants.ts | 5 ---- cbmonitor/src/services/instanceService.ts | 30 ++++++++--------------- cbmonitor/src/utils/instanceScene.ts | 4 +-- cbmonitor/src/utils/utils.panelOverlap.ts | 7 +++--- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/cbmonitor/src/constants.ts b/cbmonitor/src/constants.ts index dfbed0f..714beae 100644 --- a/cbmonitor/src/constants.ts +++ b/cbmonitor/src/constants.ts @@ -15,11 +15,6 @@ export const PROM_DATASOURCE_REF = { type: 'prometheus' } as const; -export const PROXY_PROM_DATASOURCE_REF = { - uid: 'proxyprometheus', - type: 'prometheus' -} as const; - export type DashboardId = keyof typeof DASHBOARD_UIDS; // Helper function to get UID from dashboard name diff --git a/cbmonitor/src/services/instanceService.ts b/cbmonitor/src/services/instanceService.ts index c08e867..f6d3cb3 100644 --- a/cbmonitor/src/services/instanceService.ts +++ b/cbmonitor/src/services/instanceService.ts @@ -1,7 +1,7 @@ import { SceneQueryRunner } from '@grafana/scenes'; import { clusterFilterService } from './clusterFilterService'; import { instanceFilterService } from './instanceFilterService'; -import { PROXY_PROM_DATASOURCE_REF, PROM_DATASOURCE_REF } from '../constants'; +import { PROM_DATASOURCE_REF } from '../constants'; import { injectClusterFilter, injectInstanceFilter } from '../utils/utils.panel'; export function getInstancesFromMetricRunner(snapshotId: string, metricName = 'sys_cpu_utilization_rate'): SceneQueryRunner { @@ -29,9 +29,8 @@ export function getInstancesFromMetricRunner(snapshotId: string, metricName = 's } /** - * Convenience util to parse instances from a data frame response. - * Supports Prometheus responses where instance is a field label, - * and SQL++ responses where instance is a column. + * Parse the set of `instance` label values from a Prometheus data-frame + * response (instance is carried as a field label). */ export function parseInstancesFromFrames(frames: any[]): string[] { if (!Array.isArray(frames) || frames.length === 0) {return [];} @@ -39,33 +38,24 @@ export function parseInstancesFromFrames(frames: any[]): string[] { for (const frame of frames) { const fields = frame?.fields ?? []; for (const f of fields) { - // Extract from Prometheus field labels const instance = f?.labels?.instance; if (instance) { set.add(String(instance)); } } - // Fallback: try SQL++ style (named instance column) - const instanceField = fields.find((f: any) => f?.name === 'instance') - ?? fields.find((f: any) => (f?.type?.name === 'string') || f?.type === 'string'); - if (instanceField) { - const rawValues = instanceField?.values ?? []; - const arr = (rawValues && typeof rawValues.toArray === 'function') ? rawValues.toArray() : Array.from(rawValues); - for (const v of arr) { - if (v) {set.add(String(v));} - } - } } return Array.from(set); } -export function getInstancesFromProxyPromMetricRunner(snapshotId: string, metricName = 'sys_cpu_utilization_rate'): SceneQueryRunner { - // This is a bit of a hack to get instance lists from Proxy Prometheus without needing to define a new query language or builder. - // We leverage the fact that Proxy Prom supports PromQL syntax and just use a special datasource reference to route it correctly. - const expr = `group by (instance) (${metricName}{job=~"${snapshotId}"})`; +export function getInstancesFromOverlapMetricRunner(snapshotIds: string, metricName = 'sys_cpu_utilization_rate'): SceneQueryRunner { + // Overlap discovers instances across every compared snapshot at once via a + // regex job matcher (`job=~"a|b"`). The gateway recognises the multi-snapshot + // matcher and routes it through the overlap seam; the single Prometheus + // datasource is the only query target. + const expr = `group by (instance) (${metricName}{job=~"${snapshotIds}"})`; return new SceneQueryRunner({ - datasource: PROXY_PROM_DATASOURCE_REF, + datasource: PROM_DATASOURCE_REF, queries: [{ refId: 'instances', expr, diff --git a/cbmonitor/src/utils/instanceScene.ts b/cbmonitor/src/utils/instanceScene.ts index dd2c741..cccdcf6 100644 --- a/cbmonitor/src/utils/instanceScene.ts +++ b/cbmonitor/src/utils/instanceScene.ts @@ -1,5 +1,5 @@ import { EmbeddedScene, SceneFlexLayout, SceneFlexItem, SceneDataLayerSet } from '@grafana/scenes'; -import { getInstancesFromMetricRunner, getInstancesFromProxyPromMetricRunner, parseInstancesFromFrames } from 'services/instanceService'; +import { getInstancesFromMetricRunner, getInstancesFromOverlapMetricRunner, parseInstancesFromFrames } from 'services/instanceService'; import { layoutService } from '../services/layoutService'; import { SnapshotPhaseRegionsLayer } from '../layers/SnapshotPhaseRegionsLayer'; import { createOverlapMetricPanel } from './utils.panelOverlap'; @@ -165,7 +165,7 @@ export function createInstanceAwareOverlapScene( children: [], }); - const instancesRunner = getInstancesFromProxyPromMetricRunner(snapshotIds, options.instanceMetric); + const instancesRunner = getInstancesFromOverlapMetricRunner(snapshotIds, options.instanceMetric); layout.setState({ $data: instancesRunner }); (instancesRunner as any).run?.(); diff --git a/cbmonitor/src/utils/utils.panelOverlap.ts b/cbmonitor/src/utils/utils.panelOverlap.ts index 1dedeff..ad04507 100644 --- a/cbmonitor/src/utils/utils.panelOverlap.ts +++ b/cbmonitor/src/utils/utils.panelOverlap.ts @@ -1,7 +1,7 @@ import { PanelBuilders, SceneDataTransformer, SceneFlexItem, SceneQueryRunner } from "@grafana/scenes"; import { FieldType } from "@grafana/data"; import { hasDataValues } from "./utils.panel"; -import { PROXY_PROM_DATASOURCE_REF } from "../constants"; +import { PROM_DATASOURCE_REF } from "../constants"; import { LegendDisplayMode, TooltipDisplayMode } from "@grafana/schema"; import { layoutService } from "services/layoutService"; import { NoUrlSyncTimeRange } from "./timeRange"; @@ -87,7 +87,7 @@ function getCachedOverlapQueryRunner(metricName: string, options: OverlapPanelOp return overlapQueryCacheService.getOrCreateRunner({ keyParts: [ 'overlap-panel', - PROXY_PROM_DATASOURCE_REF.uid, + PROM_DATASOURCE_REF.uid, metricName, options.expr, options.legendFormat, @@ -97,7 +97,7 @@ function getCachedOverlapQueryRunner(metricName: string, options: OverlapPanelOp createRunner: () => { const runner = new SceneQueryRunner({ $timeRange: timeRange, - datasource: PROXY_PROM_DATASOURCE_REF, + datasource: PROM_DATASOURCE_REF, queries: [{ refId: metricName, expr: options.expr, @@ -149,7 +149,6 @@ export function createOverlapMetricPanel( try { const descriptionMd = [ `**Metric:** ${metricName} \n`, - `**Datasource:** Proxy Prometheus \n`, '', '**Query:**', '```promql', From bdae7850eefbc9656d3b44b568ef03205848e625 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 19:09:02 +0100 Subject: [PATCH 16/18] ui: Gate special features only when special capabilities exist --- .../SnapshotDisplay/comparisonInstance.ts | 34 ++++++- .../src/services/datasourceCapabilities.ts | 89 +++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 cbmonitor/src/services/datasourceCapabilities.ts diff --git a/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts b/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts index 4b36514..9581c8b 100644 --- a/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts +++ b/cbmonitor/src/components/SnapshotDisplay/comparisonInstance.ts @@ -13,6 +13,7 @@ import { InputScene } from '../SceneComponents/InputScene'; import { SettingsDropdown } from '../SettingsDropdown/SettingsDropdown'; import { CompareDashboardHeader } from '../DashboardHeader/CompareDashboardHeader'; import { OverlapToggle } from '../DashboardHeader/actions/OverlapToggle'; +import { datasourceCapabilitiesService } from '../../services/datasourceCapabilities'; // Global overlap mode (when true, hide columns and show placeholders) let overlapMode = false; @@ -54,6 +55,25 @@ interface CompareHeaderContainerProps { function CompareHeaderContainer(props: CompareHeaderContainerProps) { const [overlap, setOverlap] = React.useState(isOverlapModeEnabled()); + const [overlapAvailable, setOverlapAvailable] = React.useState( + datasourceCapabilitiesService.isOverlapEnabled(), + ); + + // The overlap path routes `job=~"a|b"` to the gateway's overlap seam, so + // the toggle is only offered when the gateway reports overlap support. + React.useEffect(() => { + datasourceCapabilitiesService.load(); + return datasourceCapabilitiesService.subscribe((caps) => setOverlapAvailable(caps.overlapEnabled)); + }, []); + + // If overlap support is absent but the mode was somehow left on, force it + // off so tabs rebuild in side-by-side mode. + React.useEffect(() => { + if (!overlapAvailable && overlap) { + setOverlap(false); + setOverlapMode(false); + } + }, [overlapAvailable, overlap]); const compareSettingsDropdown = React.useMemo(() => new SettingsDropdown({ snapshotId: '', @@ -78,6 +98,12 @@ function CompareHeaderContainer(props: CompareHeaderContainerProps) { }); }, []); + const actions = [ + ...(overlapAvailable + ? [{ key: 'overlap', render: () => React.createElement(OverlapToggle, { active: overlap, onToggle: onToggleOverlap }) }] + : []), + ]; + return React.createElement(CompareDashboardHeader, { items: props.items, commonPhases: props.commonPhases, @@ -85,9 +111,7 @@ function CompareHeaderContainer(props: CompareHeaderContainerProps) { onSelectFullRange: props.onSelectFullRange, overlapEnabled: overlap, settingsDropdown: compareSettingsDropdown, - actions: [ - { key: 'overlap', render: () => React.createElement(OverlapToggle, { active: overlap, onToggle: onToggleOverlap }) }, - ], + actions, }); } @@ -142,6 +166,10 @@ export function getComparisonTimeRanges() { // Add activation handler to fetch and validate snapshots comparisonPage.addActivationHandler(() => { + // Resolve gateway capabilities early so the overlap affordance is gated + // before the compare header mounts. + datasourceCapabilitiesService.load(); + const initialParams = locationService.getSearchObject(); if (initialParams.refresh) { locationService.partial({ refresh: null }, true); diff --git a/cbmonitor/src/services/datasourceCapabilities.ts b/cbmonitor/src/services/datasourceCapabilities.ts new file mode 100644 index 0000000..cdd5376 --- /dev/null +++ b/cbmonitor/src/services/datasourceCapabilities.ts @@ -0,0 +1,89 @@ +import { getBackendSrv } from '@grafana/runtime'; +import { API_BASE_URL } from '../constants'; + +/** + * Optional gateway-provided capabilities, sourced from the plugin's + * `/config/datasources` resource endpoint. These reflect deployment-time + * plugin settings, not per-request state, so they are fetched once and cached. + * + * `overlapEnabled` gates the multi-snapshot overlap path (`job=~"a|b"` routed + * to the gateway's overlap seam). It is false in pure-Prometheus deployments + * (no gateway) and in gateway deployments with overlap turned off — in both + * cases the overlap affordance must stay hidden. + */ +export interface DatasourceCapabilities { + gatewayEnabled: boolean; + overlapEnabled: boolean; +} + +// Degrade closed: until proven otherwise, assume no gateway features so the +// UI never offers an affordance the backend can't serve. +const DEFAULT_CAPABILITIES: DatasourceCapabilities = { + gatewayEnabled: false, + overlapEnabled: false, +}; + +class DatasourceCapabilitiesService { + private capabilities: DatasourceCapabilities = DEFAULT_CAPABILITIES; + private loaded = false; + private inflight: Promise | null = null; + private listeners = new Set<(caps: DatasourceCapabilities) => void>(); + + /** Synchronous snapshot of the last-known capabilities. */ + get(): DatasourceCapabilities { + return this.capabilities; + } + + isOverlapEnabled(): boolean { + return this.capabilities.overlapEnabled; + } + + /** + * Fetch capabilities once and cache them. Safe to call repeatedly — a + * second call while a fetch is in flight reuses the same promise, and + * calls after a successful load resolve immediately. + */ + load(): Promise { + if (this.loaded) { + return Promise.resolve(this.capabilities); + } + if (this.inflight) { + return this.inflight; + } + this.inflight = getBackendSrv() + .get(`${API_BASE_URL}/config/datasources`) + .then((response: any) => { + this.capabilities = { + gatewayEnabled: Boolean(response?.gatewayEnabled), + overlapEnabled: Boolean(response?.overlapEnabled), + }; + this.finishLoad(); + return this.capabilities; + }) + .catch(() => { + // Endpoint unavailable: keep the closed defaults so no + // gateway-only affordance is offered. + this.capabilities = DEFAULT_CAPABILITIES; + this.finishLoad(); + return this.capabilities; + }); + return this.inflight; + } + + subscribe(listener: (caps: DatasourceCapabilities) => void): () => void { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + } + + private finishLoad() { + this.loaded = true; + this.inflight = null; + for (const listener of this.listeners) { + listener(this.capabilities); + } + } +} + +export const datasourceCapabilitiesService = new DatasourceCapabilitiesService(); From 8fec746b1977295f630f95c2077ab6ef7f580aad Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 21:21:16 +0100 Subject: [PATCH 17/18] cbmonitor: Remove promql logic from datasource backend --- Makefile | 17 +- cbmonitor/.config/docker-compose-base.yaml | 6 +- cbmonitor/pkg/handlers/prometheus.go | 279 ----------- cbmonitor/pkg/handlers/prometheus_test.go | 89 ---- cbmonitor/pkg/plugin/resources.go | 17 - cbmonitor/pkg/plugin/resources_test.go | 14 +- cbmonitor/pkg/promql/parser.go | 113 ----- cbmonitor/pkg/promql/planner.go | 210 -------- cbmonitor/pkg/promql/sqlbuilder.go | 319 ------------ cbmonitor/pkg/promql/transformer.go | 510 -------------------- cbmonitor/provisioning/plugins/plugins.yaml | 3 +- cbmonitor/src/plugin.json | 5 - 12 files changed, 14 insertions(+), 1568 deletions(-) delete mode 100644 cbmonitor/pkg/handlers/prometheus.go delete mode 100644 cbmonitor/pkg/handlers/prometheus_test.go delete mode 100644 cbmonitor/pkg/promql/parser.go delete mode 100644 cbmonitor/pkg/promql/planner.go delete mode 100644 cbmonitor/pkg/promql/sqlbuilder.go delete mode 100644 cbmonitor/pkg/promql/transformer.go diff --git a/Makefile b/Makefile index 45585bc..6de4299 100644 --- a/Makefile +++ b/Makefile @@ -37,28 +37,15 @@ build-plugin: mage # Build the grafana-app plugin docker image -build-plugin-docker: build-plugin move-datasource-build-artifacts +build-plugin-docker: build-plugin @echo "Building cbmonitor grafana-app plugin docker image..." @cd cbmonitor && npm run server # Rebuild plugin locally and restart container to pick up changes -reload-plugin: build-plugin move-datasource-build-artifacts +reload-plugin: build-plugin @echo "Restarting cbmonitor container to load plugin changes..." @docker restart cbmonitor -# Build the datasource plugin -build-couchbase-datasource: - @cd mfork-grafana-plugin && cd couchbase-datasource && \ - yarn install && \ - yarn build && \ - mage -v - -# Move the datasource build artifacts to the cbmonitor dist directory -move-datasource-build-artifacts: - @echo "Moving datasource build artifacts..." - @mkdir -p cbmonitor/dist/couchbase-datasource/ - @cp -r mfork-grafana-plugin/couchbase-datasource/dist/* cbmonitor/dist/couchbase-datasource/ - # Clean build artifacts clean-cm: @echo "Cleaning config-manager service build artifacts..." diff --git a/cbmonitor/.config/docker-compose-base.yaml b/cbmonitor/.config/docker-compose-base.yaml index f18cfe0..17848b6 100644 --- a/cbmonitor/.config/docker-compose-base.yaml +++ b/cbmonitor/.config/docker-compose-base.yaml @@ -25,7 +25,6 @@ services: - ../dist:/var/lib/grafana/plugins/cbmonitor - ../provisioning:/etc/grafana/provisioning - ..:/root/cbmonitor - - ../../couchbase-datasource/couchbase-datasource/dist:/var/lib/grafana/plugins/couchbase-datasource - ../../test-dashboards:/var/lib/grafana/dashboards environment: @@ -43,9 +42,8 @@ services: GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED: 'true' GF_FEATURE_TOGGLES_ENABLE: ${GF_FEATURE_TOGGLES_ENABLE:-externalServiceAccounts} - # Couchbase credentials are consumed by: - # 1. provisioning/datasources/datasource.yaml (couchbase-datasource plugin) - # 2. provisioning/plugins/apps.yaml + # Couchbase credentials are consumed by provisioning/plugins/apps.yaml + # (the cbmonitor app jsonData). COUCHBASE_CONNECTION_STRING: ${COUCHBASE_CONNECTION_STRING:-couchbase://host.docker.internal:8091} COUCHBASE_USERNAME: ${COUCHBASE_USERNAME:-Administrator} COUCHBASE_PASSWORD: ${COUCHBASE_PASSWORD:-password} diff --git a/cbmonitor/pkg/handlers/prometheus.go b/cbmonitor/pkg/handlers/prometheus.go deleted file mode 100644 index f7ccbd9..0000000 --- a/cbmonitor/pkg/handlers/prometheus.go +++ /dev/null @@ -1,279 +0,0 @@ -package handlers - -import ( - "context" - "encoding/json" - "fmt" - "log" - "net/http" - "time" - - "github.com/couchbase/cbmonitor/pkg/promql" - "github.com/couchbase/cbmonitor/pkg/services" -) - -// couchbaseQuerier captures just the bit of services.CouchbaseService the -// PromQL handler needs. Behind an interface so tests can inject failing -// stubs without standing up a real gocb cluster. -type couchbaseQuerier interface { - ExecuteQuery(ctx context.Context, query string) ([]map[string]interface{}, error) -} - -// PromQLHandler handles Prometheus Query API requests -// https://prometheus.io/docs/prometheus/latest/querying/api/ -type PromQLHandler struct { - couchbaseService couchbaseQuerier -} - -// NewPromQLHandler creates a new PromQL handler. A nil svc is permitted — -// requests will return a clean error rather than panic — so the plugin -// can boot even when the underlying Couchbase service failed to init. -func NewPromQLHandler(svc *services.CouchbaseService) *PromQLHandler { - h := &PromQLHandler{} - if svc != nil { - h.couchbaseService = svc - } - return h -} - -// HandleQuery handles GET /query (instant query) -func (h *PromQLHandler) HandleQuery(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse query parameters - query := req.URL.Query().Get("query") - if query == "" { - h.sendErrorResponse(w, "query parameter is required", http.StatusBadRequest) - return - } - - timeStr := req.URL.Query().Get("time") - if timeStr == "" { - timeStr = fmt.Sprintf("%d", time.Now().Unix()) - } - - // Parse query context (snapshot will come from 'job' label in PromQL) - queryCtx, err := promql.ParseQueryContext(query, timeStr, "", "", "", "") - if err != nil { - h.sendErrorResponse(w, fmt.Sprintf("Invalid query parameters: %v", err), http.StatusBadRequest) - return - } - queryCtx.Context = req.Context() - - // Execute query - result, err := h.executeQuery(queryCtx) - if err != nil { - log.Printf("Query execution error: %v", err) - h.sendErrorResponse(w, fmt.Sprintf("Query execution failed: %v", err), http.StatusInternalServerError) - return - } - - // Send response - h.sendJSONResponse(w, result, http.StatusOK) -} - -// HandleQueryRange handles GET /query_range (range query) -func (h *PromQLHandler) HandleQueryRange(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse query parameters - query := req.URL.Query().Get("query") - if query == "" { - h.sendErrorResponse(w, "query parameter is required", http.StatusBadRequest) - return - } - - startStr := req.URL.Query().Get("start") - endStr := req.URL.Query().Get("end") - stepStr := req.URL.Query().Get("step") - - if startStr == "" || endStr == "" { - h.sendErrorResponse(w, "start and end parameters are required", http.StatusBadRequest) - return - } - - if stepStr == "" { - stepStr = "15s" // Default step - } - - // Parse query context (snapshot will come from 'job' label in PromQL) - queryCtx, err := promql.ParseQueryContext(query, "", startStr, endStr, stepStr, "") - if err != nil { - h.sendErrorResponse(w, fmt.Sprintf("Invalid query parameters: %v", err), http.StatusBadRequest) - return - } - queryCtx.Context = req.Context() - - // Execute query - result, err := h.executeQuery(queryCtx) - if err != nil { - log.Printf("Query range execution error: %v", err) - h.sendErrorResponse(w, fmt.Sprintf("Query execution failed: %v", err), http.StatusInternalServerError) - return - } - - // Send response - h.sendJSONResponse(w, result, http.StatusOK) -} - -// HandleSeries handles GET /api/v1/series (series discovery) -func (h *PromQLHandler) HandleSeries(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse query parameters - match := req.URL.Query()["match[]"] // Can be multiple - _ = req.URL.Query().Get("start") // TODO: Use for series discovery - _ = req.URL.Query().Get("end") // TODO: Use for series discovery - - if len(match) == 0 { - h.sendErrorResponse(w, "match[] parameter is required", http.StatusBadRequest) - return - } - - // For now, return a simple response indicating series discovery is not fully implemented - // This would require querying Couchbase metadata to discover available series - result := promql.PrometheusResult{ - Status: "success", - Data: promql.ResultData{ - ResultType: "series", - Result: []interface{}{}, - }, - } - - h.sendJSONResponse(w, result, http.StatusOK) -} - -// executeQuery executes a PromQL query and returns Prometheus-formatted results -func (h *PromQLHandler) executeQuery(queryCtx *promql.QueryContext) (*promql.PrometheusResult, error) { - if h.couchbaseService == nil { - return nil, fmt.Errorf("couchbase metrics service is unavailable (initialization may have failed; check plugin settings and Couchbase connectivity)") - } - - // Parse PromQL query - expr, err := promql.ParseQuery(queryCtx.Query) - if err != nil { - return nil, fmt.Errorf("failed to parse PromQL: %w", err) - } - - // Create query plan (snapshot will be extracted from 'job' label in PromQL) - plan, err := promql.PlanQuery(expr, "") - if err != nil { - return nil, fmt.Errorf("failed to plan query: %w", err) - } - - log.Printf("Query plan: %s", plan.String()) - - // Build SQL++ queries - sqlBuilder := promql.NewSQLBuilder(plan, queryCtx) - sqlQueries, err := sqlBuilder.Build() - if err != nil { - return nil, fmt.Errorf("failed to build SQL++ queries: %w", err) - } - - log.Printf("Generated %d SQL++ queries", len(sqlQueries)) - - // Execute queries against Couchbase, tracking which sub-queries fail. - // Partial failure produces warnings on the response so the client knows - // the data is incomplete; total failure surfaces as an error. - var ( - allResults []promql.QueryResult - failures []error - ) - for i, sqlQuery := range sqlQueries { - log.Printf("Executing query %d: %s", i+1, sqlQuery) - - results, err := h.couchbaseService.ExecuteQuery(queryCtx.Context, sqlQuery) - if err != nil { - log.Printf("Query execution error: %v", err) - failures = append(failures, fmt.Errorf("sub-query %d: %w", i+1, err)) - continue - } - - // Convert results to QueryResult format - for _, row := range results { - queryResult := promql.QueryResult{ - Time: getStringValue(row, "time"), - Value: getValue(row, "value"), - } - - // Extract labels if present - if labels, ok := row["labels"].(map[string]interface{}); ok { - queryResult.Labels = labels - } - - allResults = append(allResults, queryResult) - } - } - - // All sub-queries failed → surface a real error instead of returning - // a misleading empty success that renders as "no data" in panels. - if len(failures) > 0 && len(failures) == len(sqlQueries) { - return nil, fmt.Errorf("all %d sub-queries failed; first error: %w", len(failures), failures[0]) - } - - // Transform results to Prometheus format - result, err := promql.TransformResults(allResults, plan, queryCtx) - if err != nil { - return nil, fmt.Errorf("failed to transform results: %w", err) - } - - // Partial failure → return data but warn the client some sub-queries - // failed, per Prometheus HTTP API conventions. - for _, f := range failures { - result.Warnings = append(result.Warnings, f.Error()) - } - - return result, nil -} - -// Helper functions for extracting values from query results -func getStringValue(row map[string]interface{}, key string) string { - if val, ok := row[key]; ok { - if str, ok := val.(string); ok { - return str - } - return fmt.Sprintf("%v", val) - } - return "" -} - -func getValue(row map[string]interface{}, key string) interface{} { - if val, ok := row[key]; ok { - return val - } - return nil -} - -// sendJSONResponse sends a JSON response -func (h *PromQLHandler) sendJSONResponse(w http.ResponseWriter, data interface{}, statusCode int) { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type") - - w.WriteHeader(statusCode) - - if err := json.NewEncoder(w).Encode(data); err != nil { - log.Printf("Error encoding JSON response: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - } -} - -// sendErrorResponse sends an error response -func (h *PromQLHandler) sendErrorResponse(w http.ResponseWriter, message string, statusCode int) { - response := promql.PrometheusResult{ - Status: "error", - Error: message, - ErrorType: "bad_data", - } - h.sendJSONResponse(w, response, statusCode) -} diff --git a/cbmonitor/pkg/handlers/prometheus_test.go b/cbmonitor/pkg/handlers/prometheus_test.go deleted file mode 100644 index 038bcd8..0000000 --- a/cbmonitor/pkg/handlers/prometheus_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package handlers - -import ( - "context" - "errors" - "strings" - "testing" - - "github.com/couchbase/cbmonitor/pkg/promql" -) - -// fakeQuerier is a couchbaseQuerier stub: each invocation pops the next -// scripted result from results, with errs supplying the matching errors. -// Used to exercise the PromQL handler's per-sub-query failure tracking -// without standing up a real gocb cluster. -type fakeQuerier struct { - calls int - results [][]map[string]interface{} - errs []error -} - -func (f *fakeQuerier) ExecuteQuery(ctx context.Context, query string) ([]map[string]interface{}, error) { - i := f.calls - f.calls++ - var rows []map[string]interface{} - var err error - if i < len(f.results) { - rows = f.results[i] - } - if i < len(f.errs) { - err = f.errs[i] - } - return rows, err -} - -// runExecuteQuery exercises the handler's executeQuery with a fake -// querier. Returns the result + error verbatim so callers can assert on either path. -func runExecuteQuery(t *testing.T, q couchbaseQuerier, promQL string) (*promql.PrometheusResult, error) { - t.Helper() - h := &PromQLHandler{couchbaseService: q} - // PromQL parsing requires a real metric name + timestamp; use a - // simple instant query. - queryCtx, err := promql.ParseQueryContext(promQL, "1700000000", "", "", "", "") - if err != nil { - t.Fatalf("ParseQueryContext: %v", err) - } - queryCtx.Context = context.Background() - return h.executeQuery(queryCtx) -} - -func TestPromQLExecute_AllSubQueriesFailedReturnsError(t *testing.T) { - // Force the underlying querier to fail every call. The handler must - // not return a misleading "success with empty data" — it should - // surface the failure so panels show an error instead of silently - // rendering "no data". - q := &fakeQuerier{ - errs: []error{ - errors.New("boom1"), - errors.New("boom2"), - errors.New("boom3"), - }, - } - _, err := runExecuteQuery(t, q, `up`) - if err == nil { - t.Fatalf("expected error when all sub-queries fail; got nil") - } - if !strings.Contains(err.Error(), "sub-queries failed") { - t.Errorf("error should mention sub-query failures, got: %v", err) - } -} - -func TestPromQLExecute_NilServiceReturnsError(t *testing.T) { - // A nil-couchbase-service handler (init failed but routes are still - // registered because the feature toggle is on) must return a clear - // error rather than panic with a nil-pointer dereference. - h := &PromQLHandler{couchbaseService: nil} - queryCtx, err := promql.ParseQueryContext(`up`, "1700000000", "", "", "", "") - if err != nil { - t.Fatalf("ParseQueryContext: %v", err) - } - queryCtx.Context = context.Background() - _, err = h.executeQuery(queryCtx) - if err == nil { - t.Fatalf("expected error when couchbaseService is nil") - } - if !strings.Contains(err.Error(), "unavailable") { - t.Errorf("error should explain the service is unavailable, got: %v", err) - } -} diff --git a/cbmonitor/pkg/plugin/resources.go b/cbmonitor/pkg/plugin/resources.go index 3bc0cc5..2894a82 100644 --- a/cbmonitor/pkg/plugin/resources.go +++ b/cbmonitor/pkg/plugin/resources.go @@ -130,10 +130,6 @@ func (a *App) registerRoutes(mux *http.ServeMux) { mux.HandleFunc("/admin/reconcile-dashboards", a.handleReconcileDashboards) mux.HandleFunc("/healthcheck/connection", a.handleHealthCheckConnection) - if a.settings.CouchbaseDatasource.Enabled { - a.setupPrometheusRoutes(mux) - } - if a.settings.Snapshots.Enabled || a.settings.PrometheusDatasource.Enabled { a.setupSnapshotRoutes(mux) } @@ -189,19 +185,6 @@ func (a *App) setupSnapshotRoutes(mux *http.ServeMux) { log.Printf("Snapshot routes registered: /snapshots/{id}, /snapshots/{id}/metric-names, /snapshots/{id}/metrics/{metric}, /snapshots/{id}/metrics/{metric}/phases/{phase}, /snapshots/{id}/annotations/sync") } -// setupPrometheusRoutes registers the Prometheus Query API routes backed -// by the Couchbase datasource bucket. Only called when -// CouchbaseDatasource.Enabled is true. Service is owned by App.initServices. -func (a *App) setupPrometheusRoutes(mux *http.ServeMux) { - promQLHandler := handlers.NewPromQLHandler(a.couchbaseService) - - mux.HandleFunc("/query", promQLHandler.HandleQuery) - mux.HandleFunc("/query_range", promQLHandler.HandleQueryRange) - mux.HandleFunc("/series", promQLHandler.HandleSeries) - - log.Printf("PromQL Query API routes registered: /query, /query_range, /series") -} - // handleHealthCheckConnection probes the Couchbase buckets each enabled // feature relies on (Snapshots metadata bucket + CouchbaseDatasource // metrics bucket). Always responds HTTP 200; per-bucket state lives in diff --git a/cbmonitor/pkg/plugin/resources_test.go b/cbmonitor/pkg/plugin/resources_test.go index e2a448e..528ca33 100644 --- a/cbmonitor/pkg/plugin/resources_test.go +++ b/cbmonitor/pkg/plugin/resources_test.go @@ -127,15 +127,19 @@ func TestCallResource_DatasourceConfigReflectsSettings(t *testing.T) { } } -func TestCallResource_DatasourceRoutesGatedByToggle(t *testing.T) { - // Couchbase datasource OFF → /query, /query_range, /series should 404. - app := newAppWithSettings(t, defaultSettings()) +func TestCallResource_QueryRoutesNotServedByPlugin(t *testing.T) { + // The in-plugin PromQL-over-Couchbase query API was removed; those + // queries are served by the standalone datasource-gateway. The routes + // must 404 even when the Couchbase datasource is enabled. + settings := defaultSettings() + settings.CouchbaseDatasource = CouchbaseDatasourceSettings{Enabled: true, Bucket: "cbmonitor"} + app := newAppWithSettings(t, settings) for _, path := range []string{"query", "query_range", "series"} { - t.Run("off/"+path, func(t *testing.T) { + t.Run(path, func(t *testing.T) { resp := call(t, app, http.MethodGet, path, nil) if resp.Status != http.StatusNotFound { - t.Errorf("%s with Couchbase DS off: status = %d, want 404", path, resp.Status) + t.Errorf("%s: status = %d, want 404 (query API moved to gateway)", path, resp.Status) } }) } diff --git a/cbmonitor/pkg/promql/parser.go b/cbmonitor/pkg/promql/parser.go deleted file mode 100644 index 15c769b..0000000 --- a/cbmonitor/pkg/promql/parser.go +++ /dev/null @@ -1,113 +0,0 @@ -package promql - -import ( - "context" - "fmt" - "strconv" - "time" - - "github.com/prometheus/prometheus/promql/parser" -) - -// ParseQuery parses a PromQL query string into an AST -func ParseQuery(query string) (parser.Expr, error) { - expr, err := parser.ParseExpr(query) - if err != nil { - return nil, fmt.Errorf("failed to parse PromQL query: %w", err) - } - return expr, nil -} - -// QueryContext holds context for query execution -type QueryContext struct { - Context context.Context - Query string - Time time.Time - StartTime time.Time - EndTime time.Time - Step time.Duration - IsRange bool - SnapshotID string // Optional snapshot ID from context -} - -// ParseQueryContext parses query parameters into a QueryContext -func ParseQueryContext(query string, timeStr string, startStr, endStr string, stepStr string, snapshotID string) (*QueryContext, error) { - ctx := &QueryContext{ - Query: query, - SnapshotID: snapshotID, - } - - // Parse time (for instant queries) - if timeStr != "" { - t, err := parseTime(timeStr) - if err != nil { - return nil, fmt.Errorf("invalid time parameter: %w", err) - } - ctx.Time = t - ctx.IsRange = false - } - - // Parse time range (for range queries) - if startStr != "" && endStr != "" { - start, err := parseTime(startStr) - if err != nil { - return nil, fmt.Errorf("invalid start time: %w", err) - } - end, err := parseTime(endStr) - if err != nil { - return nil, fmt.Errorf("invalid end time: %w", err) - } - ctx.StartTime = start - ctx.EndTime = end - ctx.IsRange = true - ctx.Time = end // Use end time as the evaluation time - } - - // Parse step (for range queries) - if stepStr != "" { - step, err := parseDuration(stepStr) - if err != nil { - return nil, fmt.Errorf("invalid step parameter: %w", err) - } - ctx.Step = step - } - - // Default to current time if no time specified - if ctx.Time.IsZero() { - ctx.Time = time.Now() - } - - return ctx, nil -} - -// parseTime parses a time string (Unix timestamp or RFC3339) -func parseTime(timeStr string) (time.Time, error) { - // Try RFC3339 first (most common format from Prometheus API) - // This includes formats like: 2025-11-26T14:54:31.661792773Z - if t, err := time.Parse(time.RFC3339, timeStr); err == nil { - return t, nil - } - - // Try RFC3339Nano for nanosecond precision - if t, err := time.Parse(time.RFC3339Nano, timeStr); err == nil { - return t, nil - } - - // Try Unix timestamp (only if the entire string is numeric) - // Use strconv.ParseFloat which requires the entire string to be a number - if timestamp, err := strconv.ParseFloat(timeStr, 64); err == nil { - // Check if it's in seconds or milliseconds - if timestamp > 1e10 { - // Likely milliseconds - return time.Unix(0, int64(timestamp*1e6)), nil - } - return time.Unix(int64(timestamp), 0), nil - } - - return time.Time{}, fmt.Errorf("unable to parse time: %s (expected RFC3339 format or Unix timestamp)", timeStr) -} - -// parseDuration parses a duration string (e.g., "5m", "1h", "30s") -func parseDuration(durationStr string) (time.Duration, error) { - return time.ParseDuration(durationStr) -} diff --git a/cbmonitor/pkg/promql/planner.go b/cbmonitor/pkg/promql/planner.go deleted file mode 100644 index 7233ddd..0000000 --- a/cbmonitor/pkg/promql/planner.go +++ /dev/null @@ -1,210 +0,0 @@ -package promql - -import ( - "fmt" - "strings" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql/parser" -) - -// QueryPlan represents an optimized query execution plan -type QueryPlan struct { - BaseExpr parser.Expr - SeriesQueries []SeriesQuery - Aggregation *AggregationPlan - Function *FunctionPlan -} - -// SeriesQuery represents a single series query to Couchbase -type SeriesQuery struct { - MetricName string - Snapshot string // Extracted from 'job' label in PromQL - Labels map[string]string // Additional labels to filter (including node, instance, etc.) -} - -// AggregationPlan represents aggregation operations -type AggregationPlan struct { - Operation string // sum, avg, max, min, count - By []string - Without []string -} - -// FunctionPlan represents function operations -type FunctionPlan struct { - Name string // rate, increase, irate, etc. - Range string // For range functions like rate[5m] - Parameter string // Optional parameter -} - -// PlanQuery creates an optimized query plan from a PromQL expression -func PlanQuery(expr parser.Expr, snapshotID string) (*QueryPlan, error) { - plan := &QueryPlan{ - BaseExpr: expr, - } - - // Walk the AST to extract series selectors and operations - err := parser.Walk(&queryPlanner{ - plan: plan, - snapshotID: snapshotID, - }, expr, nil) - - if err != nil { - return nil, fmt.Errorf("failed to plan query: %w", err) - } - - return plan, nil -} - -// queryPlanner implements parser.Visitor to walk the AST -type queryPlanner struct { - plan *QueryPlan - snapshotID string -} - -func (v *queryPlanner) Visit(node parser.Node, path []parser.Node) (parser.Visitor, error) { - switch n := node.(type) { - case *parser.VectorSelector: - // Extract series selector - seriesQuery, err := v.extractSeriesSelector(n) - if err != nil { - return nil, err - } - v.plan.SeriesQueries = append(v.plan.SeriesQueries, seriesQuery) - - case *parser.AggregateExpr: - // Extract aggregation - v.plan.Aggregation = &AggregationPlan{ - Operation: n.Op.String(), - By: v.extractLabels(n.Grouping), - Without: v.extractLabels(n.Grouping), // Simplified - would need to check Without flag - } - - case *parser.Call: - // Extract function call - v.plan.Function = &FunctionPlan{ - Name: n.Func.Name, - } - // Extract range parameter if it's a range function - if len(n.Args) > 0 { - if matrixSelector, ok := n.Args[0].(*parser.MatrixSelector); ok { - v.plan.Function.Range = matrixSelector.Range.String() - } - } - } - - return v, nil -} - -// extractSeriesSelector extracts series information from a vector selector -func (v *queryPlanner) extractSeriesSelector(selector *parser.VectorSelector) (SeriesQuery, error) { - query := SeriesQuery{ - MetricName: selector.Name, - Snapshot: v.snapshotID, // Will be overridden by 'job' label if present - Labels: make(map[string]string), - } - - // Extract labels from matchers - for _, matcher := range selector.LabelMatchers { - labelName := matcher.Name - labelValue := matcher.Value - - switch matcher.Type { - case labels.MatchEqual: - // Map known labels to query parameters - switch labelName { - case "job", "snapshot", "snapshot_id": - // 'job' label maps to snapshot in get_metric_for function - query.Snapshot = labelValue - default: - // Store as additional label filter (node, instance, bucket, etc.) - query.Labels[labelName] = labelValue - } - - case labels.MatchNotEqual: - // For !=, we'll need to filter in WHERE clause - query.Labels["!"+labelName] = labelValue - - case labels.MatchRegexp: - // For =~, we'll need regex matching in WHERE clause - query.Labels["=~"+labelName] = labelValue - - case labels.MatchNotRegexp: - // For !~, we'll need negative regex matching - query.Labels["!~"+labelName] = labelValue - } - } - - // Default snapshot if not provided (from job label or context) - if query.Snapshot == "" { - query.Snapshot = v.snapshotID - } - - return query, nil -} - -// extractLabels extracts label names from grouping -func (v *queryPlanner) extractLabels(grouping []string) []string { - return grouping -} - -// GetMetricName extracts the metric name from the expression -func GetMetricName(expr parser.Expr) string { - switch e := expr.(type) { - case *parser.VectorSelector: - return e.Name - case *parser.MatrixSelector: - return GetMetricName(e.VectorSelector) - case *parser.Call: - if len(e.Args) > 0 { - return GetMetricName(e.Args[0]) - } - } - return "" -} - -// HasMultipleSeries checks if the query involves multiple series -func (p *QueryPlan) HasMultipleSeries() bool { - return len(p.SeriesQueries) > 1 -} - -// ShouldBatch determines if queries should be batched -func (p *QueryPlan) ShouldBatch() bool { - // Batch if we have multiple series with the same metric name - if len(p.SeriesQueries) <= 1 { - return false - } - - // Check if all series have the same metric name - firstMetric := p.SeriesQueries[0].MetricName - for _, sq := range p.SeriesQueries[1:] { - if sq.MetricName != firstMetric { - return false - } - } - - // Batch if we have 5-20 series (adaptive threshold) - return len(p.SeriesQueries) >= 5 && len(p.SeriesQueries) <= 20 -} - -// GetBatchedQueries groups queries by metric name for batching -func (p *QueryPlan) GetBatchedQueries() map[string][]SeriesQuery { - batched := make(map[string][]SeriesQuery) - for _, sq := range p.SeriesQueries { - batched[sq.MetricName] = append(batched[sq.MetricName], sq) - } - return batched -} - -// String returns a human-readable representation of the plan -func (p *QueryPlan) String() string { - var parts []string - parts = append(parts, fmt.Sprintf("SeriesQueries: %d", len(p.SeriesQueries))) - if p.Aggregation != nil { - parts = append(parts, fmt.Sprintf("Aggregation: %s", p.Aggregation.Operation)) - } - if p.Function != nil { - parts = append(parts, fmt.Sprintf("Function: %s", p.Function.Name)) - } - return strings.Join(parts, ", ") -} diff --git a/cbmonitor/pkg/promql/sqlbuilder.go b/cbmonitor/pkg/promql/sqlbuilder.go deleted file mode 100644 index 995b05b..0000000 --- a/cbmonitor/pkg/promql/sqlbuilder.go +++ /dev/null @@ -1,319 +0,0 @@ -package promql - -import ( - "fmt" - "strings" - "time" - - "github.com/couchbase/cbmonitor/pkg/querybuilder" -) - -// SQLBuilder builds SQL++ queries from query plans -type SQLBuilder struct { - plan *QueryPlan - queryCtx *QueryContext - useBatching bool -} - -// NewSQLBuilder creates a new SQL++ query builder -func NewSQLBuilder(plan *QueryPlan, queryCtx *QueryContext) *SQLBuilder { - return &SQLBuilder{ - plan: plan, - queryCtx: queryCtx, - useBatching: plan.ShouldBatch(), - } -} - -// Build generates SQL++ query string(s) from the plan -func (b *SQLBuilder) Build() ([]string, error) { - if b.useBatching { - return b.buildBatchedQueries() - } - return b.buildIndividualQueries() -} - -// buildIndividualQueries builds separate queries for each series -func (b *SQLBuilder) buildIndividualQueries() ([]string, error) { - var queries []string - - for _, seriesQuery := range b.plan.SeriesQueries { - query, err := b.buildSeriesQuery(seriesQuery) - if err != nil { - return nil, err - } - queries = append(queries, query) - } - - return queries, nil -} - -// buildBatchedQueries builds batched queries using UNION ALL -func (b *SQLBuilder) buildBatchedQueries() ([]string, error) { - batched := b.plan.GetBatchedQueries() - var queries []string - - for _, seriesQueries := range batched { - if len(seriesQueries) == 1 { - // Single series, no need to batch - query, err := b.buildSeriesQuery(seriesQueries[0]) - if err != nil { - return nil, err - } - queries = append(queries, query) - continue - } - - // Build UNION ALL query - var unionParts []string - for _, sq := range seriesQueries { - part := b.buildSeriesQueryPart(sq) - unionParts = append(unionParts, part) - } - - // Combine with UNION ALL and add aggregation if needed - unionQuery := strings.Join(unionParts, " UNION ALL ") - - // If aggregation is needed, wrap in aggregation query - if b.plan.Aggregation != nil { - unionQuery = b.wrapWithAggregation(unionQuery) - } - - queries = append(queries, unionQuery) - } - - return queries, nil -} - -// buildSeriesQuery builds a SQL++ query for a single series -func (b *SQLBuilder) buildSeriesQuery(seriesQuery SeriesQuery) (string, error) { - baseQuery := b.buildSeriesQueryPart(seriesQuery) - - // Apply function transformation if needed - if b.plan.Function != nil { - baseQuery = b.applyFunction(baseQuery) - } - - // Apply aggregation if needed - if b.plan.Aggregation != nil { - baseQuery = b.wrapWithAggregation(baseQuery) - } - - return baseQuery, nil -} - -// buildSeriesQueryPart builds the base SQL++ query part for a series -func (b *SQLBuilder) buildSeriesQueryPart(seriesQuery SeriesQuery) string { - // Build metrics_filter function call - metricFilter := b.buildMetricFilter(seriesQuery) - - // Build time range - fromMillis := b.queryCtx.StartTime.UnixMilli() - toMillis := b.queryCtx.EndTime.UnixMilli() - if !b.queryCtx.IsRange { - // For instant queries, use a small range around the time (~30s) - fromMillis = b.queryCtx.Time.UnixMilli() - 30000 - toMillis = b.queryCtx.Time.UnixMilli() + 1000 - } - - // Build SELECT clause - selectClause := b.buildSelectClause(seriesQuery) - - // Build time range condition - // Note: label filters are already embedded in metricFilter (before UNNEST) - // Time filtering happens after UNNEST but before final output - timeCondition := fmt.Sprintf("d.time_millis >= %d AND d.time_millis <= %d", fromMillis, toMillis) - - query := fmt.Sprintf( - "SELECT %s FROM %s AS d WHERE %s", - selectClause, - metricFilter, - timeCondition, - ) - - return query -} - -// buildMetricFilter builds the query with all label conditions embedded BEFORE UNNEST -// This ensures optimal performance by filtering at the earliest possible point -func (b *SQLBuilder) buildMetricFilter(seriesQuery SeriesQuery) string { - // Build all label conditions including job/snapshot - labelConditions := b.buildLabelWhereClause(seriesQuery) - - // Construct query inline with conditions embedded - filters BEFORE UNNEST - baseQuery := fmt.Sprintf( - "SELECT t._t AS time_millis, MILLIS_TO_STR(t._t) AS time, t._v0 AS `value`, d.labels AS labels FROM cbmonitor._default._default AS d UNNEST _timeseries(d, {'ts_ranges':[0, 9223372036854775807]}) AS t WHERE d.metric_name = '%s'", - seriesQuery.MetricName, - ) - - if labelConditions != "" { - baseQuery = fmt.Sprintf("%s AND %s", baseQuery, labelConditions) - } - - return fmt.Sprintf("(%s)", baseQuery) -} - -// buildSelectClause builds the SELECT clause -func (b *SQLBuilder) buildSelectClause(seriesQuery SeriesQuery) string { - parts := []string{ - "d.time", - "d.`value`", - } - - // Add label fields if needed - if len(seriesQuery.Labels) > 0 || b.plan.Aggregation != nil { - // Include labels for grouping/aggregation - parts = append(parts, "d.labels") - } - - return strings.Join(parts, ", ") -} - -// buildLabelWhereClause builds the WHERE clause for label filters including job/snapshot -func (b *SQLBuilder) buildLabelWhereClause(seriesQuery SeriesQuery) string { - // Convert seriesQuery.Labels to LabelFilter slice for shared utility - var filters []querybuilder.LabelFilter - - // Check if job label is already in Labels (could be with operators like !=, =~, !~) - hasJobInLabels := false - for labelName := range seriesQuery.Labels { - // Remove operator prefixes to get actual label name - actualLabel := labelName - if strings.HasPrefix(labelName, "!") { - actualLabel = strings.TrimPrefix(labelName, "!") - } else if strings.HasPrefix(labelName, "=~") { - actualLabel = strings.TrimPrefix(labelName, "=~") - } else if strings.HasPrefix(labelName, "!~") { - actualLabel = strings.TrimPrefix(labelName, "!~") - } - if actualLabel == "job" || actualLabel == "snapshot" || actualLabel == "snapshot_id" { - hasJobInLabels = true - break - } - } - - // Add job/snapshot filter if not already present in Labels - if !hasJobInLabels { - snapshot := seriesQuery.Snapshot - if snapshot == "" { - snapshot = b.queryCtx.SnapshotID - } - if snapshot != "" { - // Add job label filter (snapshot is stored as job label) - filters = append(filters, querybuilder.LabelFilter{ - Name: "job", - Value: snapshot, - Op: "=", - }) - } - } - - // Process all label filters from Labels - for labelName, labelValue := range seriesQuery.Labels { - if strings.HasPrefix(labelName, "!") { - // NOT EQUAL - actualLabel := strings.TrimPrefix(labelName, "!") - filters = append(filters, querybuilder.LabelFilter{ - Name: actualLabel, - Value: labelValue, - Op: "!=", - }) - } else if strings.HasPrefix(labelName, "=~") { - // REGEX MATCH - actualLabel := strings.TrimPrefix(labelName, "=~") - filters = append(filters, querybuilder.LabelFilter{ - Name: actualLabel, - Value: labelValue, - Op: "=~", - }) - } else if strings.HasPrefix(labelName, "!~") { - // NOT REGEX MATCH - actualLabel := strings.TrimPrefix(labelName, "!~") - filters = append(filters, querybuilder.LabelFilter{ - Name: actualLabel, - Value: labelValue, - Op: "!~", - }) - } else { - // EQUAL - process all labels (job will be handled above if not in Labels) - filters = append(filters, querybuilder.LabelFilter{ - Name: labelName, - Value: labelValue, - Op: "=", - }) - } - } - - return querybuilder.BuildLabelWhereClauseFromFilters(filters) -} - -// applyFunction applies PromQL function transformations -func (b *SQLBuilder) applyFunction(query string) string { - if b.plan.Function == nil { - return query - } - - switch b.plan.Function.Name { - case "rate", "irate", "increase": - // These need to be calculated post-query for now - // We'll mark the query for post-processing - return query - default: - return query - } -} - -// wrapWithAggregation wraps query with aggregation -func (b *SQLBuilder) wrapWithAggregation(query string) string { - if b.plan.Aggregation == nil { - return query - } - - // Build GROUP BY clause - var groupBy []string - if len(b.plan.Aggregation.By) > 0 { - for _, label := range b.plan.Aggregation.By { - groupBy = append(groupBy, fmt.Sprintf("d.labels.%s", querybuilder.EscapeLabel(label))) - } - } - groupBy = append(groupBy, "d.time") // Always group by time - - // Build aggregation function - aggFunc := strings.ToUpper(b.plan.Aggregation.Operation) - if aggFunc == "" { - aggFunc = "SUM" - } - - // Wrap query - wrapped := fmt.Sprintf( - "SELECT time, %s(value) AS value %s FROM (%s) AS subq GROUP BY %s ORDER BY time", - aggFunc, - b.buildLabelSelect(groupBy), - query, - strings.Join(groupBy, ", "), - ) - - return wrapped -} - -// buildLabelSelect builds label selection for GROUP BY -func (b *SQLBuilder) buildLabelSelect(groupBy []string) string { - var labels []string - for _, gb := range groupBy { - if strings.HasPrefix(gb, "d.labels.") { - labels = append(labels, gb) - } - } - if len(labels) > 0 { - return ", " + strings.Join(labels, ", ") - } - return "" -} - -// GetTimeRange returns the time range for the query -func (b *SQLBuilder) GetTimeRange() (time.Time, time.Time) { - if b.queryCtx.IsRange { - return b.queryCtx.StartTime, b.queryCtx.EndTime - } - // For instant queries, return a small range - return b.queryCtx.Time.Add(-time.Second), b.queryCtx.Time.Add(time.Second) -} diff --git a/cbmonitor/pkg/promql/transformer.go b/cbmonitor/pkg/promql/transformer.go deleted file mode 100644 index 3f10c53..0000000 --- a/cbmonitor/pkg/promql/transformer.go +++ /dev/null @@ -1,510 +0,0 @@ -package promql - -import ( - "fmt" - "math" - "sort" - "strconv" - "strings" - "time" -) - -// PrometheusResult represents a Prometheus API result -type PrometheusResult struct { - Status string `json:"status"` - Data ResultData `json:"data"` - Error string `json:"error,omitempty"` - ErrorType string `json:"errorType,omitempty"` - // Warnings carries non-fatal issues encountered while serving the query - // (e.g. some sub-queries failed but enough succeeded to return data). - // Matches the Prometheus HTTP API spec. - Warnings []string `json:"warnings,omitempty"` -} - -// ResultData represents the data portion of Prometheus response -type ResultData struct { - ResultType string `json:"resultType"` - Result []interface{} `json:"result"` -} - -// Sample represents a single time-value pair -type Sample struct { - Timestamp float64 `json:"timestamp"` // Unix timestamp in seconds - Value string `json:"value"` // String representation of float -} - -// Series represents a time series with labels and samples -type Series struct { - Metric map[string]string `json:"metric"` - Values []Sample `json:"values,omitempty"` // For range queries - Value Sample `json:"value,omitempty"` // For instant queries -} - -// QueryResult represents raw query results from Couchbase -type QueryResult struct { - Time string `json:"time"` - Value interface{} `json:"value"` - Labels map[string]interface{} `json:"labels,omitempty"` -} - -// TransformResults transforms Couchbase query results to Prometheus format -func TransformResults(results []QueryResult, plan *QueryPlan, queryCtx *QueryContext) (*PrometheusResult, error) { - if len(results) == 0 { - return &PrometheusResult{ - Status: "success", - Data: ResultData{ - ResultType: getResultType(plan, queryCtx), - Result: []interface{}{}, - }, - }, nil - } - - // Group results by labels to form series - seriesMap := make(map[string]*Series) - - for _, result := range results { - // Extract labels - labels := extractLabels(result, plan) - labelKey := buildLabelKey(labels) - - // Get or create series - series, exists := seriesMap[labelKey] - if !exists { - series = &Series{ - Metric: labels, - } - seriesMap[labelKey] = series - } - - // Parse time and value - timestamp, err := parseTimeFromResult(result.Time) - if err != nil { - return nil, fmt.Errorf("failed to parse time: %w", err) - } - - value, err := parseValue(result.Value) - if err != nil { - return nil, fmt.Errorf("failed to parse value: %w", err) - } - - // Create sample - sample := Sample{ - Timestamp: float64(timestamp.Unix()), - Value: formatValue(value), - } - - // Add to series - if queryCtx.IsRange { - series.Values = append(series.Values, sample) - } else { - series.Value = sample - } - } - - // Convert map to slice and sort - var seriesList []interface{} - for _, series := range seriesMap { - // Sort values by timestamp - if queryCtx.IsRange { - sort.Slice(series.Values, func(i, j int) bool { - return series.Values[i].Timestamp < series.Values[j].Timestamp - }) - } - seriesList = append(seriesList, series) - } - - // Apply function transformations - if plan.Function != nil { - seriesList, err := applyFunction(seriesList, plan.Function, queryCtx) - if err != nil { - return nil, fmt.Errorf("failed to apply function: %w", err) - } - // Update seriesList with transformed results - _ = seriesList // Will be used after implementing function application - } - - // Apply aggregation if needed - if plan.Aggregation != nil { - seriesList = applyAggregation(seriesList, plan.Aggregation) - } - - return &PrometheusResult{ - Status: "success", - Data: ResultData{ - ResultType: getResultType(plan, queryCtx), - Result: seriesList, - }, - }, nil -} - -// extractLabels extracts labels from query result -func extractLabels(result QueryResult, plan *QueryPlan) map[string]string { - labels := make(map[string]string) - - // Add metric name from plan - if len(plan.SeriesQueries) > 0 { - metricName := plan.SeriesQueries[0].MetricName - if metricName != "" { - labels["__name__"] = metricName - } - } - - // Extract labels from result - if result.Labels != nil { - for key, value := range result.Labels { - if strValue, ok := value.(string); ok { - labels[key] = strValue - } else if strValue := fmt.Sprintf("%v", value); strValue != "" { - labels[key] = strValue - } - } - } - - return labels -} - -// buildLabelKey creates a unique key for a label set -func buildLabelKey(labels map[string]string) string { - // Sort keys for consistent ordering - var keys []string - for k := range labels { - keys = append(keys, k) - } - sort.Strings(keys) - - var parts []string - for _, k := range keys { - parts = append(parts, fmt.Sprintf("%s=%s", k, labels[k])) - } - return fmt.Sprintf("{%s}", strings.Join(parts, ",")) -} - -// parseTimeFromResult parses time from result string -func parseTimeFromResult(timeStr string) (time.Time, error) { - // Try RFC3339 first - if t, err := time.Parse(time.RFC3339, timeStr); err == nil { - return t, nil - } - - // Try Unix timestamp - if timestamp, err := strconv.ParseFloat(timeStr, 64); err == nil { - if timestamp > 1e10 { - // Milliseconds - return time.Unix(0, int64(timestamp*1e6)), nil - } - return time.Unix(int64(timestamp), 0), nil - } - - return time.Time{}, fmt.Errorf("unable to parse time: %s", timeStr) -} - -// parseValue parses value from result -func parseValue(value interface{}) (float64, error) { - switch v := value.(type) { - case float64: - return v, nil - case int: - return float64(v), nil - case int64: - return float64(v), nil - case string: - f, err := strconv.ParseFloat(v, 64) - if err != nil { - return 0, fmt.Errorf("invalid value: %s", v) - } - return f, nil - default: - return 0, fmt.Errorf("unsupported value type: %T", value) - } -} - -// formatValue formats a float value as string (Prometheus format) -func formatValue(value float64) string { - // Handle special values - if math.IsNaN(value) { - return "NaN" - } - if math.IsInf(value, 1) { - return "+Inf" - } - if math.IsInf(value, -1) { - return "-Inf" - } - - // Format with appropriate precision - return strconv.FormatFloat(value, 'f', -1, 64) -} - -// getResultType determines the result type based on query -func getResultType(plan *QueryPlan, queryCtx *QueryContext) string { - if queryCtx.IsRange { - return "matrix" - } - return "vector" -} - -// applyFunction applies PromQL function transformations -func applyFunction(seriesList []interface{}, function *FunctionPlan, queryCtx *QueryContext) ([]interface{}, error) { - switch function.Name { - case "rate": - return applyRate(seriesList, function.Range) - case "irate": - return applyIRate(seriesList) - case "increase": - return applyIncrease(seriesList, function.Range) - default: - // Unknown function, return as-is - return seriesList, nil - } -} - -// applyRate calculates rate (per-second average rate) -func applyRate(seriesList []interface{}, rangeStr string) ([]interface{}, error) { - // Parse range duration (for future use in range-based rate calculation) - _, err := time.ParseDuration(rangeStr) - if err != nil && rangeStr != "" { - return nil, fmt.Errorf("invalid range: %w", err) - } - - // Apply rate to each series - var result []interface{} - for _, item := range seriesList { - series, ok := item.(*Series) - if !ok { - continue - } - - if len(series.Values) < 2 { - // Need at least 2 points for rate - continue - } - - // Calculate rate for each interval - newValues := make([]Sample, 0, len(series.Values)-1) - for i := 1; i < len(series.Values); i++ { - prev := series.Values[i-1] - curr := series.Values[i] - - prevVal, _ := strconv.ParseFloat(prev.Value, 64) - currVal, _ := strconv.ParseFloat(curr.Value, 64) - - timeDiff := curr.Timestamp - prev.Timestamp - if timeDiff <= 0 { - continue - } - - // Rate = (value change) / (time change in seconds) - rate := (currVal - prevVal) / timeDiff - - newValues = append(newValues, Sample{ - Timestamp: curr.Timestamp, - Value: formatValue(rate), - }) - } - - // Update series - newSeries := *series - newSeries.Values = newValues - result = append(result, &newSeries) - } - - return result, nil -} - -// applyIRate calculates instant rate (per-second rate from last two points) -func applyIRate(seriesList []interface{}) ([]interface{}, error) { - // Similar to rate but only uses last two points - return applyRate(seriesList, "") -} - -// applyIncrease calculates increase over time range -func applyIncrease(seriesList []interface{}, rangeStr string) ([]interface{}, error) { - // Similar to rate but returns absolute increase - // For now, use rate implementation - return applyRate(seriesList, rangeStr) -} - -// applyAggregation applies aggregation operations -func applyAggregation(seriesList []interface{}, agg *AggregationPlan) []interface{} { - // Group series by labels (excluding aggregation labels) - grouped := make(map[string][]*Series) - - for _, item := range seriesList { - series, ok := item.(*Series) - if !ok { - continue - } - - // Build group key - groupKey := buildAggregationKey(series.Metric, agg) - grouped[groupKey] = append(grouped[groupKey], series) - } - - // Apply aggregation to each group - var result []interface{} - for _, group := range grouped { - aggSeries := aggregateSeries(group, agg) - if aggSeries != nil { - result = append(result, aggSeries) - } - } - - return result -} - -// buildAggregationKey builds key for grouping -func buildAggregationKey(labels map[string]string, agg *AggregationPlan) string { - var keys []string - for k, v := range labels { - // Skip __name__ and aggregation-excluded labels - if k == "__name__" { - continue - } - // Include only labels in "by" or exclude those in "without" - keys = append(keys, fmt.Sprintf("%s=%s", k, v)) - } - sort.Strings(keys) - return strings.Join(keys, ",") -} - -// aggregateSeries aggregates multiple series into one -func aggregateSeries(seriesList []*Series, agg *AggregationPlan) *Series { - if len(seriesList) == 0 { - return nil - } - - // Use first series as base - result := &Series{ - Metric: make(map[string]string), - } - - // Copy labels from first series (excluding aggregation-excluded labels) - for k, v := range seriesList[0].Metric { - if k != "__name__" { - result.Metric[k] = v - } - } - - // Aggregate values - if len(seriesList[0].Values) > 0 { - // Range query - aggregate across series for each timestamp - result.Values = aggregateValues(seriesList, agg) - } else { - // Instant query - aggregate single values - result.Value = aggregateValue(seriesList, agg) - } - - return result -} - -// aggregateValues aggregates values across series for range queries -func aggregateValues(seriesList []*Series, agg *AggregationPlan) []Sample { - // Find all unique timestamps - timestampMap := make(map[float64]bool) - for _, series := range seriesList { - for _, sample := range series.Values { - timestampMap[sample.Timestamp] = true - } - } - - // Sort timestamps - var timestamps []float64 - for ts := range timestampMap { - timestamps = append(timestamps, ts) - } - sort.Float64s(timestamps) - - // Aggregate for each timestamp - var result []Sample - for _, ts := range timestamps { - var values []float64 - for _, series := range seriesList { - for _, sample := range series.Values { - if sample.Timestamp == ts { - if val, err := strconv.ParseFloat(sample.Value, 64); err == nil { - values = append(values, val) - } - break - } - } - } - - if len(values) > 0 { - aggValue := aggregateFloat(values, agg.Operation) - result = append(result, Sample{ - Timestamp: ts, - Value: formatValue(aggValue), - }) - } - } - - return result -} - -// aggregateValue aggregates single values for instant queries -func aggregateValue(seriesList []*Series, agg *AggregationPlan) Sample { - var values []float64 - var timestamp float64 - - for _, series := range seriesList { - if val, err := strconv.ParseFloat(series.Value.Value, 64); err == nil { - values = append(values, val) - if timestamp == 0 { - timestamp = series.Value.Timestamp - } - } - } - - aggValue := aggregateFloat(values, agg.Operation) - return Sample{ - Timestamp: timestamp, - Value: formatValue(aggValue), - } -} - -// aggregateFloat aggregates float values based on operation -func aggregateFloat(values []float64, operation string) float64 { - if len(values) == 0 { - return 0 - } - - switch strings.ToUpper(operation) { - case "SUM": - sum := 0.0 - for _, v := range values { - sum += v - } - return sum - case "AVG", "AVERAGE": - sum := 0.0 - for _, v := range values { - sum += v - } - return sum / float64(len(values)) - case "MAX": - max := values[0] - for _, v := range values { - if v > max { - max = v - } - } - return max - case "MIN": - min := values[0] - for _, v := range values { - if v < min { - min = v - } - } - return min - case "COUNT": - return float64(len(values)) - default: - // Default to sum - sum := 0.0 - for _, v := range values { - sum += v - } - return sum - } -} diff --git a/cbmonitor/provisioning/plugins/plugins.yaml b/cbmonitor/provisioning/plugins/plugins.yaml index 4db95e5..357091b 100644 --- a/cbmonitor/provisioning/plugins/plugins.yaml +++ b/cbmonitor/provisioning/plugins/plugins.yaml @@ -1,6 +1,5 @@ apiVersion: 1 # No plugins to enable here. The cbmonitor app plugin itself is provisioned -# via plugins/apps.yaml; datasource plugins (prometheus, couchbase-datasource) -# come from Grafana's built-ins / bundled includes. +# via plugins/apps.yaml; the Prometheus datasource comes from Grafana's built-ins. plugins: [] diff --git a/cbmonitor/src/plugin.json b/cbmonitor/src/plugin.json index 14c5fab..fdf7b77 100644 --- a/cbmonitor/src/plugin.json +++ b/cbmonitor/src/plugin.json @@ -53,11 +53,6 @@ "path": "/plugins/%PLUGIN_ID%", "role": "Admin", "addToNav": true - }, - { - "type": "datasource", - "name": "couchbase-datasource", - "path": "couchbase-datasource/plugin.json" } ], "iam": { From 570194c57bdf7475977245ba3ac1ff0850c9a609 Mon Sep 17 00:00:00 2001 From: Salim Salim Date: Fri, 19 Jun 2026 21:32:45 +0100 Subject: [PATCH 18/18] cbmonitor: deregister cb-datasource --- .gitmodules | 4 ---- mfork-grafana-plugin | 1 - 2 files changed, 5 deletions(-) delete mode 100644 .gitmodules delete mode 160000 mfork-grafana-plugin diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 9500d07..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "mfork-grafana-plugin"] - path = mfork-grafana-plugin - url = https://github.com/couchbaselabs/mfork-grafana-plugin.git - ignore = dirty diff --git a/mfork-grafana-plugin b/mfork-grafana-plugin deleted file mode 160000 index a3c38a8..0000000 --- a/mfork-grafana-plugin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a3c38a8719dea1867f4cf25c35c850be6f7587c2