feat: per-user token usage as a Grafana table (LGTM, no Langfuse)

charts/nebari-llm-serving/dashboards/per-user-token-usage.json

@@ -0,0 +1,75 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "graphTooltip": 0,
+  "panels": [
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "description": "Total LLM tokens (input + output) attributed to each external API-key owner over the last 30 days. Source: gen_ai_client_token_usage, labeled with user.id via the AI Gateway.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "align": "auto", "cellOptions": { "type": "auto" } },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 18, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "showHeader": true,
+        "sortBy": [{ "displayName": "Tokens (30d)", "desc": true }]
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${datasource}" },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=\"\"}[30d]))",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Tokens used per user (last 30 days)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true },
+            "indexByName": {},
+            "renameByName": { "user_id": "User", "Value": "Tokens (30d)" }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 39,
+  "tags": ["nebari", "llm", "token-usage"],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Data source",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": { "from": "now-30d", "to": "now" },
+  "timepicker": {},
+  "title": "LLM Per-User Token Usage",
+  "uid": "nebari-llm-token-usage",
+  "version": 1
+}

charts/nebari-llm-serving/templates/token-usage-dashboard.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.observability.dashboard.enabled }}
+# Per-user LLM token-usage Grafana dashboard.
+#
+# Labeled `grafana_dashboard: "1"` so the LGTM pack's grafana-sc-dashboard
+# sidecar auto-loads it (no Grafana restart needed). Placed in the monitoring
+# namespace (observability.dashboard.namespace) where the sidecar watches,
+# which differs from this chart's release namespace, so the namespace is set
+# explicitly. The panel sums gen_ai_client_token_usage per user_id over 30d;
+# the user_id label is populated by the AI Gateway's
+# metricsRequestHeaderAttributes mapping of the forwarded x-client-id header.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nebari-llm-serving.fullname" . }}-token-usage-dashboard
+  namespace: {{ .Values.observability.dashboard.namespace }}
+  labels:
+    {{- include "nebari-llm-serving.labels" . | nindent 4 }}
+    grafana_dashboard: "1"
+data:
+  per-user-token-usage.json: |-
+    {{- .Files.Get "dashboards/per-user-token-usage.json" | nindent 4 }}
+{{- end }}

charts/nebari-llm-serving/values.yaml

@@ -152,3 +152,20 @@ defaults:
     storageClassName: ""
   monitoring:
     enabled: true
+
+# observability.dashboard provisions a Grafana dashboard showing per-user
+# LLM token usage (one row per API-key owner, total tokens over 30 days),
+# built on the LGTM pack already running in the cluster. The dashboard is
+# shipped as a ConfigMap labeled `grafana_dashboard: "1"`, which the Grafana
+# sidecar (grafana-sc-dashboard) auto-loads. Requires the AI Gateway value
+# controller.metricsRequestHeaderAttributes: "x-client-id:user.id" (see
+# examples/envoy-ai-gateway.yaml) so the gen_ai_client_token_usage metric
+# carries the user_id label.
+observability:
+  dashboard:
+    # enabled: provision the per-user token-usage dashboard ConfigMap.
+    enabled: true
+    # namespace: where the LGTM Grafana sidecar watches for dashboard
+    # ConfigMaps. Must match the namespace Grafana runs in. Defaults to the
+    # standard Nebari LGTM namespace.
+    namespace: monitoring

dev/Makefile

@@ -15,7 +15,7 @@ setup: ## Create kind cluster and install dependencies
 	# GIE CRDs (v1.4.0, includes graduated k8s.io group)
 	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.4.0/manifests.yaml
 	# Envoy Gateway
-	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.3.0 -n envoy-gateway-system --create-namespace
+	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.6.2 -n envoy-gateway-system --create-namespace
 	kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s
 	# Envoy AI Gateway
 	helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v0.5.0 -n envoy-ai-gateway-system --create-namespace

docs/install-production.md

@@ -1739,3 +1739,160 @@ kubectl rollout restart deploy/ai-gateway-controller -n envoy-ai-gateway-system
 
 Confirm recovery: the controller logs should show only the live
 model's `InferencePool` being reconciled, with no panic.
+
+## 14. Per-user token usage in Grafana
+
+This feature attributes LLM token usage to the API-key owner who made
+each request and renders it as a Grafana table — one row per user, total
+tokens used in the last 30 days — built entirely on the LGTM pack
+(Grafana + Mimir) already running in the cluster. No Langfuse, no extra
+datastores, no secrets in your GitOps repo.
+
+**Scope:** This covers the **external (API-key) access path only.** The
+internal JWT path is not attributed in this version.
+
+### 14.1 How it works
+
+The pipeline has four stages:
+
+1. **Envoy Gateway credential forwarding.** The operator's external
+   `SecurityPolicy` sets `apiKeyAuth.forwardClientIDHeader: x-client-id`
+   and `apiKeyAuth.sanitize: true`. When Envoy Gateway matches a presented
+   API key it forwards the matched credential's name — the key-manager
+   **clientID** (e.g. `user-chuck-1`) — downstream as the `x-client-id`
+   header. `sanitize: true` strips the raw API key from the request before
+   it is proxied upstream. **These fields require Envoy Gateway v1.5.1+**
+   (verified present in v1.6.2, the pack's pinned version; absent in v1.3).
+
+2. **Header → metric label mapping.** The AI Gateway controller value
+   `controller.metricsRequestHeaderAttributes: "x-client-id:user.id"`
+   (set in `examples/envoy-ai-gateway.yaml`) maps the forwarded header
+   onto the GenAI token-usage metric as the `user.id` attribute, which
+   Prometheus exposes as the label `user_id`. The extProc already records
+   `gen_ai_client_token_usage` (input and output token counts) on every
+   request.
+
+3. **Scrape into Mimir.** The `gen_ai_*` metrics are exposed on the extProc
+   sidecar's own admin port — `:1064` at `/metrics` — **not** on envoy's
+   `:19001 /stats/prometheus`. The proxy pod's default `prometheus.io/scrape`
+   annotation only covers `:19001`, and annotation-based discovery scrapes a
+   single port, so a **dedicated scrape job** targeting the gateway proxy pods
+   on `:1064` must be added to the LGTM OpenTelemetry Collector (see §14.2).
+   That job writes the `gen_ai_*` series to Mimir. No gateway-side OTLP export
+   is configured.
+
+4. **Grafana table.** The pack ships a dashboard ConfigMap (labeled
+   `grafana_dashboard: "1"`) that the Grafana sidecar auto-loads. Its
+   single Table panel runs:
+
+   ```promql
+   sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=""}[30d]))
+   ```
+
+   Summing across `gen_ai_token_type` (input + output) yields total tokens
+   per user over the last 30 days; `user_id != ""` drops unattributed
+   (e.g. internal) traffic.
+
+**Granularity note:** `user_id` is the verbatim clientID (e.g.
+`user-chuck-1`), so attribution is **per-key**, not per-human-aggregated.
+Because clientIDs embed the username, the data is still effectively
+per-user, but a user with multiple keys appears as multiple rows. True
+per-user rollup across keys is a deferred follow-up.
+
+> **Streaming caveat (verified on a live cluster):** for **streaming**
+> requests (`"stream": true`), the token-usage metric is recorded **only if
+> the client sends `stream_options: {"include_usage": true}`** — that is what
+> makes the model emit the final usage chunk the extProc reads from. Streaming
+> requests without it complete normally (HTTP 200) but contribute **zero** to
+> the per-user totals. Non-streaming requests always carry `usage` and are
+> always counted. There is no gateway-side way to force `include_usage`, so
+> document this for users/SDKs, or treat streaming-without-usage traffic as a
+> known undercount. (Tested: 10 streaming requests with `include_usage`
+> incremented the metric by exactly their usage; 5 without it incremented by
+> nothing.)
+
+### 14.2 What you deploy
+
+Three pieces — the metric label, the scrape job, and the dashboard:
+
+**1. Metric label** — the one-line `controller.metricsRequestHeaderAttributes`
+value in `examples/envoy-ai-gateway.yaml` (already set). The controller passes
+this to the extProc sidecar as `-metricsRequestHeaderAttributes`, so the
+sidecar must be (re)created after the value changes. Roll the gateway proxy
+(the extProc is injected into it), which re-injects the sidecar with the new
+flag:
+`kubectl rollout restart deploy -n envoy-gateway-system <gateway-proxy-deploy>`.
+
+**2. Scrape job for `:1064`** — add a scrape job to the LGTM OpenTelemetry
+Collector so the `gen_ai_*` metrics on the extProc admin port reach Mimir. The
+collector config is single-owner (see ADR-0005 in nebari-infrastructure-core);
+add this to the collector-override ConfigMap rather than editing the base. The
+job discovers the gateway proxy pods and rewrites the scrape target to `:1064`:
+
+```yaml
+# OpenTelemetry Collector prometheus receiver — scrape extProc gen_ai metrics.
+receivers:
+  prometheus/ai-gateway-genai:
+    config:
+      scrape_configs:
+        - job_name: ai-gateway-genai
+          metrics_path: /metrics
+          kubernetes_sd_configs:
+            - role: pod
+              namespaces:
+                names: [envoy-gateway-system]
+          relabel_configs:
+            # keep only the Envoy AI Gateway proxy pods. The regex must match
+            # your shared gateway's name (platform.gateway.external.name in the
+            # chart values; "nebari-gateway" by default). Use ".+" to scrape the
+            # extProc on every Envoy Gateway proxy regardless of name.
+            - source_labels:
+                - __meta_kubernetes_pod_label_gateway_envoyproxy_io_owning_gateway_name
+              regex: nebari-gateway
+              action: keep
+            # scrape the extProc admin port, not envoy's :19001
+            - source_labels: [__address__]
+              regex: (.+?)(?::\d+)?
+              target_label: __address__
+              replacement: $${1}:1064
+```
+
+Then add `prometheus/ai-gateway-genai` to the collector's `metrics` pipeline
+`receivers:` list so it flows through the existing `otlphttp/mimir` exporter.
+
+> **OTel escaping gotcha (verified):** the relabel `replacement` is `$${1}`,
+> not `${1}`. The OpenTelemetry Collector expands `${...}` as an environment
+> variable at config-load time, so an unescaped `${1}` crashes the collector
+> with `environment variable "1" has invalid name`. Doubling the dollar sign
+> (`$${1}`) passes a literal `${1}` through to the Prometheus relabel engine.
+> (In a *raw* Prometheus `scrape_config` — not via the OTel collector — use the
+> normal `${1}`.)
+
+**3. Dashboard** — ships as a chart-managed ConfigMap, controlled by
+`observability.dashboard.enabled` (default `true`) and placed in
+`observability.dashboard.namespace` (default `monitoring`). Set the namespace
+to wherever your LGTM Grafana sidecar watches for dashboards.
+
+**Prerequisites:** the LGTM pack (Grafana + Mimir) installed with the OTel
+collector forwarding to Mimir, and Mimir retention ≥ 30 days (the LGTM
+default).
+
+### 14.3 Verification
+
+1. Mint an API key via the key-manager UI (see section 11.2).
+2. Call the external endpoint with that key a few times (see section 11.3).
+3. Query Mimir directly to confirm the labeled metric landed:
+
+   ```promql
+   sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=""}[30d]))
+   ```
+
+   Expect a row whose `user_id` is the clientID of the key you minted
+   (`user-<name>-N`) with a non-zero token count.
+4. Open Grafana, find the **LLM Per-User Token Usage** dashboard, and
+   confirm the table renders the same row. The dashboard appears without a
+   Grafana restart (the sidecar loads the ConfigMap live).
+
+> **Note:** Per-model and input/output split breakdowns, time-series/rate
+> panels, alerting, and cost estimation are deliberate follow-ups and are
+> not part of v1.

examples/envoy-ai-gateway.yaml

@@ -75,6 +75,27 @@ spec:
     chart: ai-gateway-helm
     repoURL: oci://docker.io/envoyproxy
     targetRevision: v0.5.0
+    helm:
+      values: |
+        # Per-user token usage as a Grafana metric label (no Langfuse, no OTLP).
+        #
+        # controller.metricsRequestHeaderAttributes maps an HTTP request header
+        # onto a label on the GenAI token-usage metric (format "header:label").
+        # We copy the x-client-id header onto gen_ai_client_token_usage as
+        # user.id (Prometheus label `user_id`). That header is the matched
+        # API-key clientID (e.g. "user-chuck-1"), forwarded by
+        # apiKeyAuth.forwardClientIDHeader on the operator-generated external
+        # SecurityPolicy.
+        #
+        # COLLECTION: the gen_ai_* metrics are exposed on the extproc sidecar's
+        # admin port (:1064 /metrics), NOT on envoy's :19001 /stats/prometheus.
+        # The proxy pod's default prometheus.io/scrape annotation targets :19001
+        # only, so a DEDICATED scrape job for :1064 must be added to the LGTM
+        # OTel collector (see docs/install-production.md §14.2 for the
+        # scrape_config). Then a Grafana table sums tokens per user.
+        # See also the chart's observability.dashboard value.
+        controller:
+          metricsRequestHeaderAttributes: "x-client-id:user.id"
 
   destination:
     server: https://kubernetes.default.svc

examples/envoy-gateway.yaml

@@ -42,7 +42,7 @@ spec:
   source:
     chart: gateway-helm
     repoURL: docker.io/envoyproxy
-    targetRevision: v1.6.2       # or later; pin to a tested version
+    targetRevision: v1.6.2       # apiKeyAuth.forwardClientIDHeader/sanitize (per-user tracing) need v1.5.1+; v1.6.2 is the validated pin
     helm:
       releaseName: envoy-gateway
       values: |

operator/internal/controller/reconcilers/auth.go

@@ -132,9 +132,17 @@ func buildAPIKeyAuthSecurityPolicy(name, namespace string, labels map[string]int
 							},
 						},
 					},
-					// TODO: Add sanitize:true and forwardClientIDHeader when minimum
-					// Envoy Gateway version is bumped to v1.7+ (these fields are not
-					// in v1.3 SecurityPolicy CRD)
+					// Forward the matched credential's name (the key-manager clientID,
+					// e.g. "user-chuck-1") downstream as the x-client-id header. The
+					// AI Gateway controller copies this header onto the GenAI
+					// token-usage metric as the label user.id (via
+					// controller.metricsRequestHeaderAttributes), which surfaces in
+					// Grafana as per-user token usage. sanitize removes the extracted
+					// API key from the request before it is proxied upstream. Both
+					// require Envoy Gateway v1.5.1+ (present in the pack's pinned
+					// v1.6.2; absent in v1.3).
+					"forwardClientIDHeader": "x-client-id",
+					"sanitize":              true,
 				},
 			},
 		},

operator/internal/controller/reconcilers/auth_test.go

@@ -214,8 +214,31 @@ func TestBuildAuthResources(t *testing.T) { //nolint:gocyclo // table-driven tes
 				}
 			},
 		},
-		// NOTE: sanitize and forwardClientIDHeader are Envoy Gateway v1.7+ features.
-		// Tests for these fields will be added when minimum EG version is bumped.
+		{
+			name:  "External SecurityPolicy: apiKeyAuth forwards clientID and sanitizes",
+			model: defaultAuthModel(),
+			cfg:   defaultAuthConfig(),
+			check: func(t *testing.T, result *AuthResources, err error) {
+				if err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				if result.ExternalSecurityPolicy == nil {
+					t.Fatal("expected ExternalSecurityPolicy to be non-nil")
+				}
+				spec := result.ExternalSecurityPolicy.Object["spec"].(map[string]interface{})
+				apiKeyAuth := spec["apiKeyAuth"].(map[string]interface{})
+				if got := apiKeyAuth["forwardClientIDHeader"]; got != "x-client-id" {
+					t.Errorf("expected forwardClientIDHeader=x-client-id, got %v", got)
+				}
+				if got := apiKeyAuth["sanitize"]; got != true {
+					t.Errorf("expected sanitize=true, got %v", got)
+				}
+			},
+		},
+		// NOTE: forwardClientIDHeader and sanitize are now rendered on the external
+		// SecurityPolicy. Both fields require Envoy Gateway v1.5.1+ (present in the
+		// pack's pinned v1.6.2; absent in v1.3). Operators on an older Envoy Gateway
+		// must upgrade or the fields are rejected/ignored by the SecurityPolicy CRD.
 		{
 			name:  "External SecurityPolicy: extractFrom headers includes Authorization",
 			model: defaultAuthModel(),