diff --git a/charts/nebari-llm-serving/dashboards/per-user-token-usage.json b/charts/nebari-llm-serving/dashboards/per-user-token-usage.json new file mode 100644 index 0000000..bd4e25d --- /dev/null +++ b/charts/nebari-llm-serving/dashboards/per-user-token-usage.json @@ -0,0 +1,75 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "graphTooltip": 0, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Total LLM tokens (input + output) attributed to each external API-key owner over the last 30 days. Source: gen_ai_client_token_usage, labeled with user.id via the AI Gateway.", + "fieldConfig": { + "defaults": { + "custom": { "align": "auto", "cellOptions": { "type": "auto" } }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 18, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Tokens (30d)", "desc": true }] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=\"\"}[30d]))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Tokens used per user (last 30 days)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "indexByName": {}, + "renameByName": { "user_id": "User", "Value": "Tokens (30d)" } + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": ["nebari", "llm", "token-usage"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { "from": "now-30d", "to": "now" }, + "timepicker": {}, + "title": "LLM Per-User Token Usage", + "uid": "nebari-llm-token-usage", + "version": 1 +} diff --git a/charts/nebari-llm-serving/templates/token-usage-dashboard.yaml b/charts/nebari-llm-serving/templates/token-usage-dashboard.yaml new file mode 100644 index 0000000..d1004e6 --- /dev/null +++ b/charts/nebari-llm-serving/templates/token-usage-dashboard.yaml @@ -0,0 +1,22 @@ +{{- if .Values.observability.dashboard.enabled }} +# Per-user LLM token-usage Grafana dashboard. +# +# Labeled `grafana_dashboard: "1"` so the LGTM pack's grafana-sc-dashboard +# sidecar auto-loads it (no Grafana restart needed). Placed in the monitoring +# namespace (observability.dashboard.namespace) where the sidecar watches, +# which differs from this chart's release namespace, so the namespace is set +# explicitly. The panel sums gen_ai_client_token_usage per user_id over 30d; +# the user_id label is populated by the AI Gateway's +# metricsRequestHeaderAttributes mapping of the forwarded x-client-id header. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nebari-llm-serving.fullname" . }}-token-usage-dashboard + namespace: {{ .Values.observability.dashboard.namespace }} + labels: + {{- include "nebari-llm-serving.labels" . | nindent 4 }} + grafana_dashboard: "1" +data: + per-user-token-usage.json: |- + {{- .Files.Get "dashboards/per-user-token-usage.json" | nindent 4 }} +{{- end }} diff --git a/charts/nebari-llm-serving/values.yaml b/charts/nebari-llm-serving/values.yaml index b7e596a..bdaf081 100644 --- a/charts/nebari-llm-serving/values.yaml +++ b/charts/nebari-llm-serving/values.yaml @@ -152,3 +152,20 @@ defaults: storageClassName: "" monitoring: enabled: true + +# observability.dashboard provisions a Grafana dashboard showing per-user +# LLM token usage (one row per API-key owner, total tokens over 30 days), +# built on the LGTM pack already running in the cluster. The dashboard is +# shipped as a ConfigMap labeled `grafana_dashboard: "1"`, which the Grafana +# sidecar (grafana-sc-dashboard) auto-loads. Requires the AI Gateway value +# controller.metricsRequestHeaderAttributes: "x-client-id:user.id" (see +# examples/envoy-ai-gateway.yaml) so the gen_ai_client_token_usage metric +# carries the user_id label. +observability: + dashboard: + # enabled: provision the per-user token-usage dashboard ConfigMap. + enabled: true + # namespace: where the LGTM Grafana sidecar watches for dashboard + # ConfigMaps. Must match the namespace Grafana runs in. Defaults to the + # standard Nebari LGTM namespace. + namespace: monitoring diff --git a/dev/Makefile b/dev/Makefile index b5dbd9c..1726038 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -15,7 +15,7 @@ setup: ## Create kind cluster and install dependencies # GIE CRDs (v1.4.0, includes graduated k8s.io group) kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.4.0/manifests.yaml # Envoy Gateway - helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.3.0 -n envoy-gateway-system --create-namespace + helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.6.2 -n envoy-gateway-system --create-namespace kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s # Envoy AI Gateway helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v0.5.0 -n envoy-ai-gateway-system --create-namespace diff --git a/docs/install-production.md b/docs/install-production.md index c4e76a7..2c1da45 100644 --- a/docs/install-production.md +++ b/docs/install-production.md @@ -1739,3 +1739,160 @@ kubectl rollout restart deploy/ai-gateway-controller -n envoy-ai-gateway-system Confirm recovery: the controller logs should show only the live model's `InferencePool` being reconciled, with no panic. + +## 14. Per-user token usage in Grafana + +This feature attributes LLM token usage to the API-key owner who made +each request and renders it as a Grafana table — one row per user, total +tokens used in the last 30 days — built entirely on the LGTM pack +(Grafana + Mimir) already running in the cluster. No Langfuse, no extra +datastores, no secrets in your GitOps repo. + +**Scope:** This covers the **external (API-key) access path only.** The +internal JWT path is not attributed in this version. + +### 14.1 How it works + +The pipeline has four stages: + +1. **Envoy Gateway credential forwarding.** The operator's external + `SecurityPolicy` sets `apiKeyAuth.forwardClientIDHeader: x-client-id` + and `apiKeyAuth.sanitize: true`. When Envoy Gateway matches a presented + API key it forwards the matched credential's name — the key-manager + **clientID** (e.g. `user-chuck-1`) — downstream as the `x-client-id` + header. `sanitize: true` strips the raw API key from the request before + it is proxied upstream. **These fields require Envoy Gateway v1.5.1+** + (verified present in v1.6.2, the pack's pinned version; absent in v1.3). + +2. **Header → metric label mapping.** The AI Gateway controller value + `controller.metricsRequestHeaderAttributes: "x-client-id:user.id"` + (set in `examples/envoy-ai-gateway.yaml`) maps the forwarded header + onto the GenAI token-usage metric as the `user.id` attribute, which + Prometheus exposes as the label `user_id`. The extProc already records + `gen_ai_client_token_usage` (input and output token counts) on every + request. + +3. **Scrape into Mimir.** The `gen_ai_*` metrics are exposed on the extProc + sidecar's own admin port — `:1064` at `/metrics` — **not** on envoy's + `:19001 /stats/prometheus`. The proxy pod's default `prometheus.io/scrape` + annotation only covers `:19001`, and annotation-based discovery scrapes a + single port, so a **dedicated scrape job** targeting the gateway proxy pods + on `:1064` must be added to the LGTM OpenTelemetry Collector (see §14.2). + That job writes the `gen_ai_*` series to Mimir. No gateway-side OTLP export + is configured. + +4. **Grafana table.** The pack ships a dashboard ConfigMap (labeled + `grafana_dashboard: "1"`) that the Grafana sidecar auto-loads. Its + single Table panel runs: + + ```promql + sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=""}[30d])) + ``` + + Summing across `gen_ai_token_type` (input + output) yields total tokens + per user over the last 30 days; `user_id != ""` drops unattributed + (e.g. internal) traffic. + +**Granularity note:** `user_id` is the verbatim clientID (e.g. +`user-chuck-1`), so attribution is **per-key**, not per-human-aggregated. +Because clientIDs embed the username, the data is still effectively +per-user, but a user with multiple keys appears as multiple rows. True +per-user rollup across keys is a deferred follow-up. + +> **Streaming caveat (verified on a live cluster):** for **streaming** +> requests (`"stream": true`), the token-usage metric is recorded **only if +> the client sends `stream_options: {"include_usage": true}`** — that is what +> makes the model emit the final usage chunk the extProc reads from. Streaming +> requests without it complete normally (HTTP 200) but contribute **zero** to +> the per-user totals. Non-streaming requests always carry `usage` and are +> always counted. There is no gateway-side way to force `include_usage`, so +> document this for users/SDKs, or treat streaming-without-usage traffic as a +> known undercount. (Tested: 10 streaming requests with `include_usage` +> incremented the metric by exactly their usage; 5 without it incremented by +> nothing.) + +### 14.2 What you deploy + +Three pieces — the metric label, the scrape job, and the dashboard: + +**1. Metric label** — the one-line `controller.metricsRequestHeaderAttributes` +value in `examples/envoy-ai-gateway.yaml` (already set). The controller passes +this to the extProc sidecar as `-metricsRequestHeaderAttributes`, so the +sidecar must be (re)created after the value changes. Roll the gateway proxy +(the extProc is injected into it), which re-injects the sidecar with the new +flag: +`kubectl rollout restart deploy -n envoy-gateway-system `. + +**2. Scrape job for `:1064`** — add a scrape job to the LGTM OpenTelemetry +Collector so the `gen_ai_*` metrics on the extProc admin port reach Mimir. The +collector config is single-owner (see ADR-0005 in nebari-infrastructure-core); +add this to the collector-override ConfigMap rather than editing the base. The +job discovers the gateway proxy pods and rewrites the scrape target to `:1064`: + +```yaml +# OpenTelemetry Collector prometheus receiver — scrape extProc gen_ai metrics. +receivers: + prometheus/ai-gateway-genai: + config: + scrape_configs: + - job_name: ai-gateway-genai + metrics_path: /metrics + kubernetes_sd_configs: + - role: pod + namespaces: + names: [envoy-gateway-system] + relabel_configs: + # keep only the Envoy AI Gateway proxy pods. The regex must match + # your shared gateway's name (platform.gateway.external.name in the + # chart values; "nebari-gateway" by default). Use ".+" to scrape the + # extProc on every Envoy Gateway proxy regardless of name. + - source_labels: + - __meta_kubernetes_pod_label_gateway_envoyproxy_io_owning_gateway_name + regex: nebari-gateway + action: keep + # scrape the extProc admin port, not envoy's :19001 + - source_labels: [__address__] + regex: (.+?)(?::\d+)? + target_label: __address__ + replacement: $${1}:1064 +``` + +Then add `prometheus/ai-gateway-genai` to the collector's `metrics` pipeline +`receivers:` list so it flows through the existing `otlphttp/mimir` exporter. + +> **OTel escaping gotcha (verified):** the relabel `replacement` is `$${1}`, +> not `${1}`. The OpenTelemetry Collector expands `${...}` as an environment +> variable at config-load time, so an unescaped `${1}` crashes the collector +> with `environment variable "1" has invalid name`. Doubling the dollar sign +> (`$${1}`) passes a literal `${1}` through to the Prometheus relabel engine. +> (In a *raw* Prometheus `scrape_config` — not via the OTel collector — use the +> normal `${1}`.) + +**3. Dashboard** — ships as a chart-managed ConfigMap, controlled by +`observability.dashboard.enabled` (default `true`) and placed in +`observability.dashboard.namespace` (default `monitoring`). Set the namespace +to wherever your LGTM Grafana sidecar watches for dashboards. + +**Prerequisites:** the LGTM pack (Grafana + Mimir) installed with the OTel +collector forwarding to Mimir, and Mimir retention ≥ 30 days (the LGTM +default). + +### 14.3 Verification + +1. Mint an API key via the key-manager UI (see section 11.2). +2. Call the external endpoint with that key a few times (see section 11.3). +3. Query Mimir directly to confirm the labeled metric landed: + + ```promql + sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=""}[30d])) + ``` + + Expect a row whose `user_id` is the clientID of the key you minted + (`user--N`) with a non-zero token count. +4. Open Grafana, find the **LLM Per-User Token Usage** dashboard, and + confirm the table renders the same row. The dashboard appears without a + Grafana restart (the sidecar loads the ConfigMap live). + +> **Note:** Per-model and input/output split breakdowns, time-series/rate +> panels, alerting, and cost estimation are deliberate follow-ups and are +> not part of v1. diff --git a/examples/envoy-ai-gateway.yaml b/examples/envoy-ai-gateway.yaml index 211a41e..d23e80f 100644 --- a/examples/envoy-ai-gateway.yaml +++ b/examples/envoy-ai-gateway.yaml @@ -75,6 +75,27 @@ spec: chart: ai-gateway-helm repoURL: oci://docker.io/envoyproxy targetRevision: v0.5.0 + helm: + values: | + # Per-user token usage as a Grafana metric label (no Langfuse, no OTLP). + # + # controller.metricsRequestHeaderAttributes maps an HTTP request header + # onto a label on the GenAI token-usage metric (format "header:label"). + # We copy the x-client-id header onto gen_ai_client_token_usage as + # user.id (Prometheus label `user_id`). That header is the matched + # API-key clientID (e.g. "user-chuck-1"), forwarded by + # apiKeyAuth.forwardClientIDHeader on the operator-generated external + # SecurityPolicy. + # + # COLLECTION: the gen_ai_* metrics are exposed on the extproc sidecar's + # admin port (:1064 /metrics), NOT on envoy's :19001 /stats/prometheus. + # The proxy pod's default prometheus.io/scrape annotation targets :19001 + # only, so a DEDICATED scrape job for :1064 must be added to the LGTM + # OTel collector (see docs/install-production.md §14.2 for the + # scrape_config). Then a Grafana table sums tokens per user. + # See also the chart's observability.dashboard value. + controller: + metricsRequestHeaderAttributes: "x-client-id:user.id" destination: server: https://kubernetes.default.svc diff --git a/examples/envoy-gateway.yaml b/examples/envoy-gateway.yaml index f65d0b3..a64a8dd 100644 --- a/examples/envoy-gateway.yaml +++ b/examples/envoy-gateway.yaml @@ -42,7 +42,7 @@ spec: source: chart: gateway-helm repoURL: docker.io/envoyproxy - targetRevision: v1.6.2 # or later; pin to a tested version + targetRevision: v1.6.2 # apiKeyAuth.forwardClientIDHeader/sanitize (per-user tracing) need v1.5.1+; v1.6.2 is the validated pin helm: releaseName: envoy-gateway values: | diff --git a/operator/internal/controller/reconcilers/auth.go b/operator/internal/controller/reconcilers/auth.go index f52b84e..581972d 100644 --- a/operator/internal/controller/reconcilers/auth.go +++ b/operator/internal/controller/reconcilers/auth.go @@ -132,9 +132,17 @@ func buildAPIKeyAuthSecurityPolicy(name, namespace string, labels map[string]int }, }, }, - // TODO: Add sanitize:true and forwardClientIDHeader when minimum - // Envoy Gateway version is bumped to v1.7+ (these fields are not - // in v1.3 SecurityPolicy CRD) + // Forward the matched credential's name (the key-manager clientID, + // e.g. "user-chuck-1") downstream as the x-client-id header. The + // AI Gateway controller copies this header onto the GenAI + // token-usage metric as the label user.id (via + // controller.metricsRequestHeaderAttributes), which surfaces in + // Grafana as per-user token usage. sanitize removes the extracted + // API key from the request before it is proxied upstream. Both + // require Envoy Gateway v1.5.1+ (present in the pack's pinned + // v1.6.2; absent in v1.3). + "forwardClientIDHeader": "x-client-id", + "sanitize": true, }, }, }, diff --git a/operator/internal/controller/reconcilers/auth_test.go b/operator/internal/controller/reconcilers/auth_test.go index d0ca98b..bc7c74e 100644 --- a/operator/internal/controller/reconcilers/auth_test.go +++ b/operator/internal/controller/reconcilers/auth_test.go @@ -214,8 +214,31 @@ func TestBuildAuthResources(t *testing.T) { //nolint:gocyclo // table-driven tes } }, }, - // NOTE: sanitize and forwardClientIDHeader are Envoy Gateway v1.7+ features. - // Tests for these fields will be added when minimum EG version is bumped. + { + name: "External SecurityPolicy: apiKeyAuth forwards clientID and sanitizes", + model: defaultAuthModel(), + cfg: defaultAuthConfig(), + check: func(t *testing.T, result *AuthResources, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.ExternalSecurityPolicy == nil { + t.Fatal("expected ExternalSecurityPolicy to be non-nil") + } + spec := result.ExternalSecurityPolicy.Object["spec"].(map[string]interface{}) + apiKeyAuth := spec["apiKeyAuth"].(map[string]interface{}) + if got := apiKeyAuth["forwardClientIDHeader"]; got != "x-client-id" { + t.Errorf("expected forwardClientIDHeader=x-client-id, got %v", got) + } + if got := apiKeyAuth["sanitize"]; got != true { + t.Errorf("expected sanitize=true, got %v", got) + } + }, + }, + // NOTE: forwardClientIDHeader and sanitize are now rendered on the external + // SecurityPolicy. Both fields require Envoy Gateway v1.5.1+ (present in the + // pack's pinned v1.6.2; absent in v1.3). Operators on an older Envoy Gateway + // must upgrade or the fields are rejected/ignored by the SecurityPolicy CRD. { name: "External SecurityPolicy: extractFrom headers includes Authorization", model: defaultAuthModel(),