From 02f52cc6e699e531167d46b5181c6da5526d4d06 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 14:57:39 -0500
Subject: [PATCH 01/20] feat(operator): forward API-key clientID header for
 per-user tracing

---
 .../internal/controller/reconcilers/auth.go   | 10 ++++---
 .../controller/reconcilers/auth_test.go       | 27 +++++++++++++++++--
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/operator/internal/controller/reconcilers/auth.go b/operator/internal/controller/reconcilers/auth.go
index f52b84e..336a567 100644
--- a/operator/internal/controller/reconcilers/auth.go
+++ b/operator/internal/controller/reconcilers/auth.go
@@ -132,9 +132,13 @@ func buildAPIKeyAuthSecurityPolicy(name, namespace string, labels map[string]int
 							},
 						},
 					},
-					// TODO: Add sanitize:true and forwardClientIDHeader when minimum
-					// Envoy Gateway version is bumped to v1.7+ (these fields are not
-					// in v1.3 SecurityPolicy CRD)
+					// Forward the matched credential's name (the key-manager clientID,
+					// e.g. "user-chuck-1") downstream so the AI Gateway tracer can stamp
+					// it onto the GenAI span as user.id for per-user usage in Langfuse.
+					// sanitize removes the extracted API key from the request before
+					// it is proxied upstream. Both require Envoy Gateway v1.7+.
+					"forwardClientIDHeader": "x-client-id",
+					"sanitize":              true,
 				},
 			},
 		},
diff --git a/operator/internal/controller/reconcilers/auth_test.go b/operator/internal/controller/reconcilers/auth_test.go
index d0ca98b..fd9c75e 100644
--- a/operator/internal/controller/reconcilers/auth_test.go
+++ b/operator/internal/controller/reconcilers/auth_test.go
@@ -214,8 +214,31 @@ func TestBuildAuthResources(t *testing.T) { //nolint:gocyclo // table-driven tes
 				}
 			},
 		},
-		// NOTE: sanitize and forwardClientIDHeader are Envoy Gateway v1.7+ features.
-		// Tests for these fields will be added when minimum EG version is bumped.
+		{
+			name:  "External SecurityPolicy: apiKeyAuth forwards clientID and sanitizes",
+			model: defaultAuthModel(),
+			cfg:   defaultAuthConfig(),
+			check: func(t *testing.T, result *AuthResources, err error) {
+				if err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				if result.ExternalSecurityPolicy == nil {
+					t.Fatal("expected ExternalSecurityPolicy to be non-nil")
+				}
+				spec := result.ExternalSecurityPolicy.Object["spec"].(map[string]interface{})
+				apiKeyAuth := spec["apiKeyAuth"].(map[string]interface{})
+				if got := apiKeyAuth["forwardClientIDHeader"]; got != "x-client-id" {
+					t.Errorf("expected forwardClientIDHeader=x-client-id, got %v", got)
+				}
+				if got := apiKeyAuth["sanitize"]; got != true {
+					t.Errorf("expected sanitize=true, got %v", got)
+				}
+			},
+		},
+		// NOTE: forwardClientIDHeader and sanitize are now rendered on the external
+		// SecurityPolicy. Both fields require Envoy Gateway v1.7+. Operators must
+		// ensure their Envoy Gateway install meets this minimum; the SecurityPolicy
+		// will be rejected by older CRDs.
 		{
 			name:  "External SecurityPolicy: extractFrom headers includes Authorization",
 			model: defaultAuthModel(),

From 2370ccdbd82c0dc66f3f4a8c97904f7146bd563e Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:16:55 -0500
Subject: [PATCH 02/20] chore: require Envoy Gateway v1.7+ for apiKeyAuth
 client-id forwarding

---
 dev/Makefile                | 2 +-
 examples/envoy-gateway.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/Makefile b/dev/Makefile
index b5dbd9c..54889f6 100644
--- a/dev/Makefile
+++ b/dev/Makefile
@@ -15,7 +15,7 @@ setup: ## Create kind cluster and install dependencies
 	# GIE CRDs (v1.4.0, includes graduated k8s.io group)
 	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.4.0/manifests.yaml
 	# Envoy Gateway
-	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.3.0 -n envoy-gateway-system --create-namespace
+	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.7.0 -n envoy-gateway-system --create-namespace
 	kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s
 	# Envoy AI Gateway
 	helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v0.5.0 -n envoy-ai-gateway-system --create-namespace
diff --git a/examples/envoy-gateway.yaml b/examples/envoy-gateway.yaml
index f65d0b3..630989d 100644
--- a/examples/envoy-gateway.yaml
+++ b/examples/envoy-gateway.yaml
@@ -42,7 +42,7 @@ spec:
   source:
     chart: gateway-helm
     repoURL: docker.io/envoyproxy
-    targetRevision: v1.6.2       # or later; pin to a tested version
+    targetRevision: v1.7.0       # v1.7+ REQUIRED for apiKeyAuth.forwardClientIDHeader/sanitize (per-user tracing); pin to a tested patch
     helm:
       releaseName: envoy-gateway
       values: |

From 3be7a2ba1fd104d6ee1cd0abfba31b7a1b0548d9 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:16:55 -0500
Subject: [PATCH 03/20] feat: export AI Gateway GenAI traces to Langfuse with
 per-user user.id

---
 examples/envoy-ai-gateway.yaml | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/examples/envoy-ai-gateway.yaml b/examples/envoy-ai-gateway.yaml
index 211a41e..236b444 100644
--- a/examples/envoy-ai-gateway.yaml
+++ b/examples/envoy-ai-gateway.yaml
@@ -75,6 +75,40 @@ spec:
     chart: ai-gateway-helm
     repoURL: oci://docker.io/envoyproxy
     targetRevision: v0.5.0
+    helm:
+      values: |
+        # GenAI distributed tracing -> self-hosted Langfuse (OTLP).
+        #
+        # controller.spanRequestHeaderAttributes maps an HTTP request header to
+        # an OTel span attribute (format "header:attribute"). We copy the
+        # x-client-id header onto the GenAI span as user.id. That header is the
+        # matched API-key clientID (e.g. "user-chuck-1"), forwarded by
+        # apiKeyAuth.forwardClientIDHeader on the operator-generated external
+        # SecurityPolicy. Langfuse then attributes token usage per API-key owner
+        # (per-key granularity; see docs/install-production.md).
+        #
+        # The extProc OpenInference tracer exports OTLP to Langfuse. The endpoint
+        # and Basic-auth header are read from the langfuse-otlp-credentials
+        # Secret, which MUST exist in this namespace (envoy-ai-gateway-system).
+        # See examples/langfuse-otlp-credentials.example.yaml.
+        controller:
+          spanRequestHeaderAttributes: "x-client-id:user.id"
+        extProc:
+          extraEnvVars:
+            - name: OTEL_TRACES_EXPORTER
+              value: "otlp"
+            - name: OTEL_SERVICE_NAME
+              value: "nebari-llm-ai-gateway"
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: langfuse-otlp-credentials
+                  key: endpoint
+            - name: OTEL_EXPORTER_OTLP_HEADERS
+              valueFrom:
+                secretKeyRef:
+                  name: langfuse-otlp-credentials
+                  key: otlp-headers
 
   destination:
     server: https://kubernetes.default.svc

From c4b5a1a9bb627c16465ca9de9acecc4194e0fc7e Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:16:55 -0500
Subject: [PATCH 04/20] docs: add Langfuse OTLP interface Secret template

---
 .../langfuse-otlp-credentials.example.yaml    | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 examples/langfuse-otlp-credentials.example.yaml

diff --git a/examples/langfuse-otlp-credentials.example.yaml b/examples/langfuse-otlp-credentials.example.yaml
new file mode 100644
index 0000000..05bb2d3
--- /dev/null
+++ b/examples/langfuse-otlp-credentials.example.yaml
@@ -0,0 +1,28 @@
+# Interface Secret consumed by the Envoy AI Gateway extProc (see
+# examples/envoy-ai-gateway.yaml) to export GenAI traces to Langfuse.
+# Create it in the AI Gateway's namespace (envoy-ai-gateway-system).
+#
+#   endpoint     - Langfuse OTLP ingest URL, e.g.
+#                  https://langfuse.<baseDomain>/api/public/otel
+#   otlp-headers - OTLP exporter headers in W3C key=value form. Langfuse
+#                  authenticates its OTLP endpoint with HTTP Basic using a
+#                  project's public:secret API-key pair. Build it with:
+#                      printf 'pk-lf-...:sk-lf-...' | base64
+#                  then set: Authorization=Basic <that-base64>
+#
+# The project keys only exist after you create an org + project in the Langfuse
+# UI; see the "Langfuse project-key bootstrap" runbook in
+# docs/install-production.md.
+#
+# Replace both values, then:
+#   kubectl apply -n envoy-ai-gateway-system -f langfuse-otlp-credentials.yaml
+#   kubectl rollout restart deploy -n envoy-ai-gateway-system -l app.kubernetes.io/name=ai-gateway-extproc
+apiVersion: v1
+kind: Secret
+metadata:
+  name: langfuse-otlp-credentials
+  namespace: envoy-ai-gateway-system
+type: Opaque
+stringData:
+  endpoint: "https://langfuse.REPLACE-baseDomain/api/public/otel"
+  otlp-headers: "Authorization=Basic REPLACE-base64-of-pk:sk"

From 8a6eb8540d30114a1ccb8fbd992d63ccde3ac9ea Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:16:55 -0500
Subject: [PATCH 05/20] feat: add self-hosted Langfuse ArgoCD app with
 OIDC-gated UI

---
 examples/langfuse.yaml | 150 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)
 create mode 100644 examples/langfuse.yaml

diff --git a/examples/langfuse.yaml b/examples/langfuse.yaml
new file mode 100644
index 0000000..1c8c6d1
--- /dev/null
+++ b/examples/langfuse.yaml
@@ -0,0 +1,150 @@
+# Self-hosted Langfuse for per-user LLM token-usage observability.
+#
+# Langfuse is the OTLP trace backend for this pack's GenAI tracing (see
+# examples/envoy-ai-gateway.yaml). This file deploys Langfuse v3 (chart
+# langfuse/langfuse 1.5.36, appVersion 3.x) via ArgoCD, with bundled Postgres,
+# ClickHouse, Redis, and MinIO (S3). PVCs use the cluster's default
+# StorageClass (Longhorn on a NIC deploy).
+#
+# v1 scope: single-replica datastores. HA, external managed datastores, and
+# backups are out of scope.
+#
+# ── PREREQUISITE: create the app-secrets Secret (do NOT commit real values) ──
+#
+#   kubectl create namespace langfuse
+#   kubectl -n langfuse create secret generic langfuse-app-secrets \
+#     --from-literal=nextauth-secret="$(openssl rand -base64 32)" \
+#     --from-literal=salt="$(openssl rand -base64 32)" \
+#     --from-literal=encryption-key="$(openssl rand -hex 32)" \
+#     --from-literal=postgres-password="$(openssl rand -hex 24)" \
+#     --from-literal=clickhouse-password="$(openssl rand -hex 24)" \
+#     --from-literal=redis-password="$(openssl rand -hex 24)" \
+#     --from-literal=minio-root-password="$(openssl rand -hex 24)"
+#
+# encryption-key MUST be 64 hex chars (openssl rand -hex 32). The ArgoCD app
+# below references this Secret; it never contains secret material itself.
+#
+# Replace every REPLACE-baseDomain before applying.
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: langfuse
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: foundational
+
+  source:
+    chart: langfuse
+    repoURL: https://langfuse.github.io/langfuse-k8s
+    targetRevision: 1.5.36
+    helm:
+      releaseName: langfuse
+      values: |
+        # Stable resource names: web Service becomes "langfuse-web".
+        fullnameOverride: langfuse
+        langfuse:
+          # Canonical site URL — used by NextAuth for callbacks. Must match the
+          # NebariApp hostname below.
+          nextauth:
+            url: https://langfuse.REPLACE-baseDomain
+            secret:
+              secretKeyRef:
+                name: langfuse-app-secrets
+                key: nextauth-secret
+          salt:
+            secretKeyRef:
+              name: langfuse-app-secrets
+              key: salt
+          encryptionKey:
+            secretKeyRef:
+              name: langfuse-app-secrets
+              key: encryption-key
+        postgresql:
+          deploy: true
+          auth:
+            existingSecret: langfuse-app-secrets
+            secretKeys:
+              userPasswordKey: postgres-password
+              adminPasswordKey: postgres-password
+        clickhouse:
+          deploy: true
+          auth:
+            existingSecret: langfuse-app-secrets
+            existingSecretKey: clickhouse-password
+        redis:
+          deploy: true
+          auth:
+            existingSecret: langfuse-app-secrets
+            existingSecretPasswordKey: redis-password
+        s3:
+          deploy: true
+          auth:
+            existingSecret: langfuse-app-secrets
+            rootPasswordSecretKey: minio-root-password
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: langfuse
+
+  syncPolicy:
+    # Opt the langfuse namespace into Nebari management so the NebariApp below
+    # (which exposes the UI) is reconciled by the nebari-operator rather than
+    # rejected with NamespaceNotOptedIn.
+    managedNamespaceMetadata:
+      labels:
+        nebari.dev/managed: "true"
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
+---
+# Exposes the Langfuse web UI via the shared gateway with Keycloak OIDC auth and
+# a Nebari landing-page tile. Mirrors the key-manager NebariApp pattern. Lives
+# in the langfuse namespace so its same-namespace service reference resolves;
+# the namespace is opted into Nebari management by the Application above.
+apiVersion: reconcilers.nebari.dev/v1
+kind: NebariApp
+metadata:
+  name: langfuse
+  namespace: langfuse
+spec:
+  hostname: langfuse.REPLACE-baseDomain
+  gateway: nebari-gateway
+  service:
+    name: langfuse-web
+    port: 3000
+  routing:
+    publicRoutes: []
+  auth:
+    enabled: true
+    provider: keycloak
+    provisionClient: true
+    scopes:
+      - openid
+      - profile
+      - email
+  landingPage:
+    enabled: true
+    displayName: Langfuse
+    description: LLM observability — per-user token usage and traces
+    category: Monitoring
+    healthCheck:
+      enabled: true
+      path: /api/public/health
+      port: 3000
+      intervalSeconds: 30
+      timeoutSeconds: 5

From 04c98b7a003d2992c354440707eec9ca890050dd Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:16:55 -0500
Subject: [PATCH 06/20] docs: document per-user token usage in Langfuse

---
 docs/install-production.md | 122 +++++++++++++++++++++++++++++++++++++
 1 file changed, 122 insertions(+)

diff --git a/docs/install-production.md b/docs/install-production.md
index c4e76a7..2b128da 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1739,3 +1739,125 @@ kubectl rollout restart deploy/ai-gateway-controller -n envoy-ai-gateway-system
 
 Confirm recovery: the controller logs should show only the live
 model's `InferencePool` being reconciled, with no panic.
+
+## 14. Per-user token usage in Langfuse
+
+This feature attributes LLM token usage to the API-key owner who made
+each request, visible per-user in a self-hosted Langfuse instance.
+
+**Scope:** This covers the **external (API-key) access path only.**
+The internal JWT path is not traced to Langfuse in this version.
+
+### 14.1 How it works
+
+The pipeline has four stages:
+
+1. **Envoy Gateway credential forwarding.** The operator's external
+   `SecurityPolicy` sets `apiKeyAuth.forwardClientIDHeader: x-client-id`
+   and `apiKeyAuth.sanitize: true`. When Envoy Gateway matches a
+   presented API key it forwards the matched credential's name — the
+   key-manager **clientID** (e.g. `user-chuck-1`) — downstream as the
+   `x-client-id` header. `sanitize: true` strips the raw API key from
+   the request before it is proxied upstream. **These fields require
+   Envoy Gateway v1.7+**; the `examples/envoy-gateway.yaml` pin and
+   `dev/Makefile` were bumped to v1.7.0. Older Envoy Gateway will
+   reject the SecurityPolicy.
+
+2. **Header → span attribute mapping.** The AI Gateway controller value
+   `controller.spanRequestHeaderAttributes: "x-client-id:user.id"`
+   (set in `examples/envoy-ai-gateway.yaml`) maps the forwarded header
+   onto the OpenInference GenAI span as the `user.id` attribute. The
+   extProc OpenInference tracer already records token usage on that
+   span.
+
+3. **OTLP export to Langfuse.** The extProc exports OTLP directly to
+   Langfuse — there is no OTel Collector in the path. The endpoint and
+   credentials are read from a Secret named
+   `langfuse-otlp-credentials` in the `envoy-ai-gateway-system`
+   namespace via these extProc env vars:
+
+   | Env var | Secret key |
+   |---|---|
+   | `OTEL_TRACES_EXPORTER=otlp` | (literal) |
+   | `OTEL_SERVICE_NAME` | (literal) |
+   | `OTEL_EXPORTER_OTLP_ENDPOINT` | `endpoint` |
+   | `OTEL_EXPORTER_OTLP_HEADERS` | `otlp-headers` |
+
+4. **Langfuse ingestion.** Langfuse ingests OTLP at
+   `/api/public/otel`, reads token usage from the span, and uses
+   `user.id` to populate its per-user view.
+
+**Granularity note:** `user.id` is the verbatim clientID
+(e.g. `user-chuck-1`), which means attribution is **per-key**, not
+per-human-aggregated. Because clientIDs embed the username, the data
+is still effectively per-user and groupable in Langfuse, but a user
+with multiple keys will appear as multiple distinct user IDs. True
+per-user rollup across keys is a deferred follow-up.
+
+### 14.2 Deploying self-hosted Langfuse
+
+Langfuse is deployed via ArgoCD using `examples/langfuse.yaml` (chart
+`langfuse/langfuse` 1.5.36). The chart bundles Postgres, ClickHouse,
+Redis, and MinIO; PVCs use the default StorageClass (Longhorn on
+standard Nebari clusters). This is a single-replica v1 deployment —
+HA, external datastores, and backup configuration are out of scope.
+
+**Prerequisites:**
+
+- A `langfuse-app-secrets` Kubernetes Secret must exist before ArgoCD
+  syncs the application. The exact `kubectl create secret generic`
+  command (with all required keys) is in the comment header of
+  `examples/langfuse.yaml` — follow that file rather than duplicating
+  the command here.
+
+**DNS:** The UI is exposed via a `NebariApp` at
+`langfuse.<baseDomain>` with Keycloak OIDC and a landing-page tile.
+A `langfuse.<baseDomain>` DNS record is required; a wildcard CNAME on
+the base domain covers it.
+
+### 14.3 Langfuse project-key bootstrap
+
+OTLP authentication uses a Langfuse **project's** public and secret
+API keys. These keys only exist after a project is created in the UI,
+so this one-time bootstrap must be performed before the extProc can
+export traces.
+
+1. Sign in to `https://langfuse.<baseDomain>`, create an organization
+   and a project.
+
+2. Navigate to **Project Settings → API Keys → Create**. Copy the
+   public key (`pk-lf-...`) and secret key (`sk-lf-...`).
+
+3. Compute the Base64 Basic-auth header value:
+
+   ```bash
+   printf 'pk-lf-...:sk-lf-...' | base64
+   ```
+
+4. Create the credentials Secret using the template in
+   `examples/langfuse-otlp-credentials.example.yaml`. Set:
+
+   - `endpoint: https://langfuse.<baseDomain>/api/public/otel`
+   - `otlp-headers: Authorization=Basic <base64-value>`
+
+   Apply it to the `envoy-ai-gateway-system` namespace.
+
+5. Roll the extProc so it picks up the new env vars:
+
+   ```bash
+   kubectl rollout restart deploy \
+     -n envoy-ai-gateway-system \
+     -l app.kubernetes.io/name=ai-gateway-extproc
+   ```
+
+> **Note:** Automating project and API-key creation (e.g. via the
+> Langfuse management API) is a deliberate follow-up and is not part
+> of v1.
+
+### 14.4 Verification
+
+1. Mint an API key via the key-manager UI (see section 11.2).
+2. Call the external endpoint with that key (see section 11.3).
+3. Open `https://langfuse.<baseDomain>`, navigate to **Traces**, and
+   confirm a trace appears with non-zero input/output token usage
+   attributed to the clientID (`user-<name>-N`) of the key you minted.

From 03e1ee5d310404ee06aa6a4eac809ee532fa1083 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:23:51 -0500
Subject: [PATCH 07/20] fix: provide MinIO root-user secret key for Langfuse S3

---
 examples/langfuse.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/langfuse.yaml b/examples/langfuse.yaml
index 1c8c6d1..0de2d7c 100644
--- a/examples/langfuse.yaml
+++ b/examples/langfuse.yaml
@@ -19,6 +19,7 @@
 #     --from-literal=postgres-password="$(openssl rand -hex 24)" \
 #     --from-literal=clickhouse-password="$(openssl rand -hex 24)" \
 #     --from-literal=redis-password="$(openssl rand -hex 24)" \
+#     --from-literal=minio-root-user=minio \
 #     --from-literal=minio-root-password="$(openssl rand -hex 24)"
 #
 # encryption-key MUST be 64 hex chars (openssl rand -hex 32). The ArgoCD app
@@ -85,6 +86,7 @@ spec:
           deploy: true
           auth:
             existingSecret: langfuse-app-secrets
+            rootUserSecretKey: minio-root-user
             rootPasswordSecretKey: minio-root-password
 
   destination:

From 6379c05d234679ceae394a6785bd3b9010f46bef Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 16:31:38 -0500
Subject: [PATCH 08/20] fix: correct Envoy Gateway floor to v1.5.1 (not v1.7),
 keep v1.6.2 pin
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Live-cluster testing showed the v1.6.2 SecurityPolicy CRD already has
apiKeyAuth.forwardClientIDHeader/sanitize; they were added in EG v1.5.1.
The earlier v1.7.0 bump was unnecessary — revert examples + dev/Makefile
to the pack's existing v1.6.2 pin and fix the code/docs claims.
---
 dev/Makefile                                          | 2 +-
 docs/install-production.md                            | 6 +++---
 examples/envoy-gateway.yaml                           | 2 +-
 operator/internal/controller/reconcilers/auth.go      | 3 ++-
 operator/internal/controller/reconcilers/auth_test.go | 6 +++---
 5 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/dev/Makefile b/dev/Makefile
index 54889f6..1726038 100644
--- a/dev/Makefile
+++ b/dev/Makefile
@@ -15,7 +15,7 @@ setup: ## Create kind cluster and install dependencies
 	# GIE CRDs (v1.4.0, includes graduated k8s.io group)
 	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.4.0/manifests.yaml
 	# Envoy Gateway
-	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.7.0 -n envoy-gateway-system --create-namespace
+	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.6.2 -n envoy-gateway-system --create-namespace
 	kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s
 	# Envoy AI Gateway
 	helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v0.5.0 -n envoy-ai-gateway-system --create-namespace
diff --git a/docs/install-production.md b/docs/install-production.md
index 2b128da..61a2811 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1759,9 +1759,9 @@ The pipeline has four stages:
    key-manager **clientID** (e.g. `user-chuck-1`) — downstream as the
    `x-client-id` header. `sanitize: true` strips the raw API key from
    the request before it is proxied upstream. **These fields require
-   Envoy Gateway v1.7+**; the `examples/envoy-gateway.yaml` pin and
-   `dev/Makefile` were bumped to v1.7.0. Older Envoy Gateway will
-   reject the SecurityPolicy.
+   Envoy Gateway v1.5.1+** (verified present in v1.6.2, the pack's
+   pinned version; absent in v1.3). Older Envoy Gateway will reject or
+   ignore the fields.
 
 2. **Header → span attribute mapping.** The AI Gateway controller value
    `controller.spanRequestHeaderAttributes: "x-client-id:user.id"`
diff --git a/examples/envoy-gateway.yaml b/examples/envoy-gateway.yaml
index 630989d..a64a8dd 100644
--- a/examples/envoy-gateway.yaml
+++ b/examples/envoy-gateway.yaml
@@ -42,7 +42,7 @@ spec:
   source:
     chart: gateway-helm
     repoURL: docker.io/envoyproxy
-    targetRevision: v1.7.0       # v1.7+ REQUIRED for apiKeyAuth.forwardClientIDHeader/sanitize (per-user tracing); pin to a tested patch
+    targetRevision: v1.6.2       # apiKeyAuth.forwardClientIDHeader/sanitize (per-user tracing) need v1.5.1+; v1.6.2 is the validated pin
     helm:
       releaseName: envoy-gateway
       values: |
diff --git a/operator/internal/controller/reconcilers/auth.go b/operator/internal/controller/reconcilers/auth.go
index 336a567..6c5d8fd 100644
--- a/operator/internal/controller/reconcilers/auth.go
+++ b/operator/internal/controller/reconcilers/auth.go
@@ -136,7 +136,8 @@ func buildAPIKeyAuthSecurityPolicy(name, namespace string, labels map[string]int
 					// e.g. "user-chuck-1") downstream so the AI Gateway tracer can stamp
 					// it onto the GenAI span as user.id for per-user usage in Langfuse.
 					// sanitize removes the extracted API key from the request before
-					// it is proxied upstream. Both require Envoy Gateway v1.7+.
+					// it is proxied upstream. Both require Envoy Gateway v1.5.1+
+					// (present in the pack's pinned v1.6.2; absent in v1.3).
 					"forwardClientIDHeader": "x-client-id",
 					"sanitize":              true,
 				},
diff --git a/operator/internal/controller/reconcilers/auth_test.go b/operator/internal/controller/reconcilers/auth_test.go
index fd9c75e..bc7c74e 100644
--- a/operator/internal/controller/reconcilers/auth_test.go
+++ b/operator/internal/controller/reconcilers/auth_test.go
@@ -236,9 +236,9 @@ func TestBuildAuthResources(t *testing.T) { //nolint:gocyclo // table-driven tes
 			},
 		},
 		// NOTE: forwardClientIDHeader and sanitize are now rendered on the external
-		// SecurityPolicy. Both fields require Envoy Gateway v1.7+. Operators must
-		// ensure their Envoy Gateway install meets this minimum; the SecurityPolicy
-		// will be rejected by older CRDs.
+		// SecurityPolicy. Both fields require Envoy Gateway v1.5.1+ (present in the
+		// pack's pinned v1.6.2; absent in v1.3). Operators on an older Envoy Gateway
+		// must upgrade or the fields are rejected/ignored by the SecurityPolicy CRD.
 		{
 			name:  "External SecurityPolicy: extractFrom headers includes Authorization",
 			model: defaultAuthModel(),

From 84fee5a1145a9eb4982a3207fc0844f7eaeebc82 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 17:10:01 -0500
Subject: [PATCH 09/20] fix: AI Gateway OTLP env must be literal values (chart
 ignores valueFrom)

Live-cluster testing showed ai-gateway-helm v0.5.0 flattens
extProc.extraEnvVars into a CLI arg and only honors literal value:;
secretKeyRef rendered as nil and no traces exported. Switch the OTLP
endpoint/headers to literals (with a security caveat about the Basic
header containing the Langfuse secret key), drop the now-unused
interface-Secret example, and update docs.
---
 .../langfuse-otlp-credentials.example.yaml    | 28 -------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 examples/langfuse-otlp-credentials.example.yaml

diff --git a/examples/langfuse-otlp-credentials.example.yaml b/examples/langfuse-otlp-credentials.example.yaml
deleted file mode 100644
index 05bb2d3..0000000
--- a/examples/langfuse-otlp-credentials.example.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-# Interface Secret consumed by the Envoy AI Gateway extProc (see
-# examples/envoy-ai-gateway.yaml) to export GenAI traces to Langfuse.
-# Create it in the AI Gateway's namespace (envoy-ai-gateway-system).
-#
-#   endpoint     - Langfuse OTLP ingest URL, e.g.
-#                  https://langfuse.<baseDomain>/api/public/otel
-#   otlp-headers - OTLP exporter headers in W3C key=value form. Langfuse
-#                  authenticates its OTLP endpoint with HTTP Basic using a
-#                  project's public:secret API-key pair. Build it with:
-#                      printf 'pk-lf-...:sk-lf-...' | base64
-#                  then set: Authorization=Basic <that-base64>
-#
-# The project keys only exist after you create an org + project in the Langfuse
-# UI; see the "Langfuse project-key bootstrap" runbook in
-# docs/install-production.md.
-#
-# Replace both values, then:
-#   kubectl apply -n envoy-ai-gateway-system -f langfuse-otlp-credentials.yaml
-#   kubectl rollout restart deploy -n envoy-ai-gateway-system -l app.kubernetes.io/name=ai-gateway-extproc
-apiVersion: v1
-kind: Secret
-metadata:
-  name: langfuse-otlp-credentials
-  namespace: envoy-ai-gateway-system
-type: Opaque
-stringData:
-  endpoint: "https://langfuse.REPLACE-baseDomain/api/public/otel"
-  otlp-headers: "Authorization=Basic REPLACE-base64-of-pk:sk"

From cb510f60e6f5ec1bb6efb27965ddc04b0f0aeed4 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Thu, 25 Jun 2026 17:10:33 -0500
Subject: [PATCH 10/20] fix: switch AI Gateway OTLP env to literal values +
 update docs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Companion to the prior commit: ai-gateway-helm v0.5.0 only honors literal
value: in extProc.extraEnvVars (secretKeyRef -> nil), so the OTLP endpoint
and Basic-auth header are literals (with a security caveat). Docs §14
updated; the chart-limitation is documented.
---
 docs/install-production.md     | 40 ++++++++++++++++++++--------------
 examples/envoy-ai-gateway.yaml | 30 +++++++++++++++----------
 2 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/docs/install-production.md b/docs/install-production.md
index 61a2811..4100804 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1771,17 +1771,22 @@ The pipeline has four stages:
    span.
 
 3. **OTLP export to Langfuse.** The extProc exports OTLP directly to
-   Langfuse — there is no OTel Collector in the path. The endpoint and
-   credentials are read from a Secret named
-   `langfuse-otlp-credentials` in the `envoy-ai-gateway-system`
-   namespace via these extProc env vars:
-
-   | Env var | Secret key |
-   |---|---|
-   | `OTEL_TRACES_EXPORTER=otlp` | (literal) |
-   | `OTEL_SERVICE_NAME` | (literal) |
-   | `OTEL_EXPORTER_OTLP_ENDPOINT` | `endpoint` |
-   | `OTEL_EXPORTER_OTLP_HEADERS` | `otlp-headers` |
+   Langfuse — there is no OTel Collector in the path. Tracing is
+   configured through `extProc.extraEnvVars` in
+   `examples/envoy-ai-gateway.yaml`: `OTEL_TRACES_EXPORTER=otlp`,
+   `OTEL_SERVICE_NAME`, `OTEL_EXPORTER_OTLP_ENDPOINT`
+   (`https://langfuse.<baseDomain>/api/public/otel`), and
+   `OTEL_EXPORTER_OTLP_HEADERS` (`Authorization=Basic <base64(pk:sk)>`).
+
+   > **Chart limitation (verified on a live cluster):** `ai-gateway-helm`
+   > v0.5.0 flattens `extProc.extraEnvVars` into a single
+   > `--extProcExtraEnvVars` CLI arg and only honors a literal `value:` —
+   > `valueFrom`/`secretKeyRef` renders as `nil`. So these must be literal
+   > values in the Helm values, **including the Basic-auth header that
+   > contains the Langfuse project secret key.** Keep your GitOps repo
+   > private and/or encrypt this Application with SOPS or sealed-secrets.
+   > Secret-ref injection here is a follow-up that needs an upstream chart
+   > change.
 
 4. **Langfuse ingestion.** Langfuse ingests OTLP at
    `/api/public/otel`, reads token usage from the span, and uses
@@ -1834,13 +1839,16 @@ export traces.
    printf 'pk-lf-...:sk-lf-...' | base64
    ```
 
-4. Create the credentials Secret using the template in
-   `examples/langfuse-otlp-credentials.example.yaml`. Set:
+4. Set the literal values in `examples/envoy-ai-gateway.yaml`
+   (`extProc.extraEnvVars`) and re-sync the app:
 
-   - `endpoint: https://langfuse.<baseDomain>/api/public/otel`
-   - `otlp-headers: Authorization=Basic <base64-value>`
+   - `OTEL_EXPORTER_OTLP_ENDPOINT: https://langfuse.<baseDomain>/api/public/otel`
+   - `OTEL_EXPORTER_OTLP_HEADERS: Authorization=Basic <base64-value>`
 
-   Apply it to the `envoy-ai-gateway-system` namespace.
+   (See the chart-limitation note in §14.1 — these cannot be a
+   `secretKeyRef`.) After updating, roll the extProc so it picks up the
+   new controller args:
+   `kubectl rollout restart deploy -n envoy-gateway-system <gateway-proxy-deploy>`.
 
 5. Roll the extProc so it picks up the new env vars:
 
diff --git a/examples/envoy-ai-gateway.yaml b/examples/envoy-ai-gateway.yaml
index 236b444..5374d67 100644
--- a/examples/envoy-ai-gateway.yaml
+++ b/examples/envoy-ai-gateway.yaml
@@ -87,10 +87,9 @@ spec:
         # SecurityPolicy. Langfuse then attributes token usage per API-key owner
         # (per-key granularity; see docs/install-production.md).
         #
-        # The extProc OpenInference tracer exports OTLP to Langfuse. The endpoint
-        # and Basic-auth header are read from the langfuse-otlp-credentials
-        # Secret, which MUST exist in this namespace (envoy-ai-gateway-system).
-        # See examples/langfuse-otlp-credentials.example.yaml.
+        # The extProc OpenInference tracer exports OTLP to Langfuse, configured
+        # via the extProc.extraEnvVars below (see the chart-limitation note
+        # there: the OTLP endpoint + auth header must be literal values).
         controller:
           spanRequestHeaderAttributes: "x-client-id:user.id"
         extProc:
@@ -99,16 +98,23 @@ spec:
               value: "otlp"
             - name: OTEL_SERVICE_NAME
               value: "nebari-llm-ai-gateway"
+            # IMPORTANT: ai-gateway-helm v0.5.0 serializes extProc.extraEnvVars
+            # into a single --extProcExtraEnvVars CLI arg and only honors a
+            # literal `value:` — `valueFrom`/secretKeyRef renders as nil
+            # (verified on a live cluster). So the endpoint and the Basic-auth
+            # header MUST be literals here.
+            #
+            # SECURITY: OTEL_EXPORTER_OTLP_HEADERS contains the Langfuse project
+            # secret key (base64 of "pk-...:sk-..."). With this chart there is
+            # no secretKeyRef option, so it lands in the Helm values (i.e. in
+            # your GitOps repo). Keep that repo private and/or encrypt this app
+            # with SOPS/sealed-secrets. Upstream support for secret refs here is
+            # a follow-up (ai-gateway-helm would need to stop flattening
+            # extraEnvVars into a CLI arg).
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              valueFrom:
-                secretKeyRef:
-                  name: langfuse-otlp-credentials
-                  key: endpoint
+              value: "https://langfuse.REPLACE-baseDomain/api/public/otel"
             - name: OTEL_EXPORTER_OTLP_HEADERS
-              valueFrom:
-                secretKeyRef:
-                  name: langfuse-otlp-credentials
-                  key: otlp-headers
+              value: "Authorization=Basic REPLACE-base64-of-pk:sk"
 
   destination:
     server: https://kubernetes.default.svc

From 9ff050eb3590827c09fc52e88ceaa568348bc4b8 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 14:47:54 -0500
Subject: [PATCH 11/20] docs(operator): clarify clientID forwarding feeds the
 usage metric, not a Langfuse span

---
 operator/internal/controller/reconcilers/auth.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/operator/internal/controller/reconcilers/auth.go b/operator/internal/controller/reconcilers/auth.go
index 6c5d8fd..581972d 100644
--- a/operator/internal/controller/reconcilers/auth.go
+++ b/operator/internal/controller/reconcilers/auth.go
@@ -133,11 +133,14 @@ func buildAPIKeyAuthSecurityPolicy(name, namespace string, labels map[string]int
 						},
 					},
 					// Forward the matched credential's name (the key-manager clientID,
-					// e.g. "user-chuck-1") downstream so the AI Gateway tracer can stamp
-					// it onto the GenAI span as user.id for per-user usage in Langfuse.
-					// sanitize removes the extracted API key from the request before
-					// it is proxied upstream. Both require Envoy Gateway v1.5.1+
-					// (present in the pack's pinned v1.6.2; absent in v1.3).
+					// e.g. "user-chuck-1") downstream as the x-client-id header. The
+					// AI Gateway controller copies this header onto the GenAI
+					// token-usage metric as the label user.id (via
+					// controller.metricsRequestHeaderAttributes), which surfaces in
+					// Grafana as per-user token usage. sanitize removes the extracted
+					// API key from the request before it is proxied upstream. Both
+					// require Envoy Gateway v1.5.1+ (present in the pack's pinned
+					// v1.6.2; absent in v1.3).
 					"forwardClientIDHeader": "x-client-id",
 					"sanitize":              true,
 				},

From f0f6a7a989ff6acca34058196abfcca428675b3f Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 14:48:46 -0500
Subject: [PATCH 12/20] feat: label AI Gateway token-usage metric with user.id
 instead of OTLP-to-Langfuse

---
 examples/envoy-ai-gateway.yaml | 46 ++++++++--------------------------
 1 file changed, 11 insertions(+), 35 deletions(-)

diff --git a/examples/envoy-ai-gateway.yaml b/examples/envoy-ai-gateway.yaml
index 5374d67..fbb4206 100644
--- a/examples/envoy-ai-gateway.yaml
+++ b/examples/envoy-ai-gateway.yaml
@@ -77,44 +77,20 @@ spec:
     targetRevision: v0.5.0
     helm:
       values: |
-        # GenAI distributed tracing -> self-hosted Langfuse (OTLP).
+        # Per-user token usage as a Grafana metric label (no Langfuse, no OTLP).
         #
-        # controller.spanRequestHeaderAttributes maps an HTTP request header to
-        # an OTel span attribute (format "header:attribute"). We copy the
-        # x-client-id header onto the GenAI span as user.id. That header is the
-        # matched API-key clientID (e.g. "user-chuck-1"), forwarded by
+        # controller.metricsRequestHeaderAttributes maps an HTTP request header
+        # onto a label on the GenAI token-usage metric (format "header:label").
+        # We copy the x-client-id header onto gen_ai_client_token_usage as
+        # user.id (Prometheus label `user_id`). That header is the matched
+        # API-key clientID (e.g. "user-chuck-1"), forwarded by
         # apiKeyAuth.forwardClientIDHeader on the operator-generated external
-        # SecurityPolicy. Langfuse then attributes token usage per API-key owner
-        # (per-key granularity; see docs/install-production.md).
-        #
-        # The extProc OpenInference tracer exports OTLP to Langfuse, configured
-        # via the extProc.extraEnvVars below (see the chart-limitation note
-        # there: the OTLP endpoint + auth header must be literal values).
+        # SecurityPolicy. The metric is already scraped into Mimir by the LGTM
+        # collector (the proxy pod carries prometheus.io/scrape), so a Grafana
+        # table can sum tokens per user. See docs/install-production.md §14 and
+        # the chart's observability.dashboard value.
         controller:
-          spanRequestHeaderAttributes: "x-client-id:user.id"
-        extProc:
-          extraEnvVars:
-            - name: OTEL_TRACES_EXPORTER
-              value: "otlp"
-            - name: OTEL_SERVICE_NAME
-              value: "nebari-llm-ai-gateway"
-            # IMPORTANT: ai-gateway-helm v0.5.0 serializes extProc.extraEnvVars
-            # into a single --extProcExtraEnvVars CLI arg and only honors a
-            # literal `value:` — `valueFrom`/secretKeyRef renders as nil
-            # (verified on a live cluster). So the endpoint and the Basic-auth
-            # header MUST be literals here.
-            #
-            # SECURITY: OTEL_EXPORTER_OTLP_HEADERS contains the Langfuse project
-            # secret key (base64 of "pk-...:sk-..."). With this chart there is
-            # no secretKeyRef option, so it lands in the Helm values (i.e. in
-            # your GitOps repo). Keep that repo private and/or encrypt this app
-            # with SOPS/sealed-secrets. Upstream support for secret refs here is
-            # a follow-up (ai-gateway-helm would need to stop flattening
-            # extraEnvVars into a CLI arg).
-            - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              value: "https://langfuse.REPLACE-baseDomain/api/public/otel"
-            - name: OTEL_EXPORTER_OTLP_HEADERS
-              value: "Authorization=Basic REPLACE-base64-of-pk:sk"
+          metricsRequestHeaderAttributes: "x-client-id:user.id"
 
   destination:
     server: https://kubernetes.default.svc

From 5f124fc75a72c6ecb3769705d300de044704df7c Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 14:48:50 -0500
Subject: [PATCH 13/20] chore: remove self-hosted Langfuse ArgoCD app (replaced
 by Grafana metric)

---
 examples/langfuse.yaml | 152 -----------------------------------------
 1 file changed, 152 deletions(-)
 delete mode 100644 examples/langfuse.yaml

diff --git a/examples/langfuse.yaml b/examples/langfuse.yaml
deleted file mode 100644
index 0de2d7c..0000000
--- a/examples/langfuse.yaml
+++ /dev/null
@@ -1,152 +0,0 @@
-# Self-hosted Langfuse for per-user LLM token-usage observability.
-#
-# Langfuse is the OTLP trace backend for this pack's GenAI tracing (see
-# examples/envoy-ai-gateway.yaml). This file deploys Langfuse v3 (chart
-# langfuse/langfuse 1.5.36, appVersion 3.x) via ArgoCD, with bundled Postgres,
-# ClickHouse, Redis, and MinIO (S3). PVCs use the cluster's default
-# StorageClass (Longhorn on a NIC deploy).
-#
-# v1 scope: single-replica datastores. HA, external managed datastores, and
-# backups are out of scope.
-#
-# ── PREREQUISITE: create the app-secrets Secret (do NOT commit real values) ──
-#
-#   kubectl create namespace langfuse
-#   kubectl -n langfuse create secret generic langfuse-app-secrets \
-#     --from-literal=nextauth-secret="$(openssl rand -base64 32)" \
-#     --from-literal=salt="$(openssl rand -base64 32)" \
-#     --from-literal=encryption-key="$(openssl rand -hex 32)" \
-#     --from-literal=postgres-password="$(openssl rand -hex 24)" \
-#     --from-literal=clickhouse-password="$(openssl rand -hex 24)" \
-#     --from-literal=redis-password="$(openssl rand -hex 24)" \
-#     --from-literal=minio-root-user=minio \
-#     --from-literal=minio-root-password="$(openssl rand -hex 24)"
-#
-# encryption-key MUST be 64 hex chars (openssl rand -hex 32). The ArgoCD app
-# below references this Secret; it never contains secret material itself.
-#
-# Replace every REPLACE-baseDomain before applying.
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: langfuse
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "5"
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-spec:
-  project: foundational
-
-  source:
-    chart: langfuse
-    repoURL: https://langfuse.github.io/langfuse-k8s
-    targetRevision: 1.5.36
-    helm:
-      releaseName: langfuse
-      values: |
-        # Stable resource names: web Service becomes "langfuse-web".
-        fullnameOverride: langfuse
-        langfuse:
-          # Canonical site URL — used by NextAuth for callbacks. Must match the
-          # NebariApp hostname below.
-          nextauth:
-            url: https://langfuse.REPLACE-baseDomain
-            secret:
-              secretKeyRef:
-                name: langfuse-app-secrets
-                key: nextauth-secret
-          salt:
-            secretKeyRef:
-              name: langfuse-app-secrets
-              key: salt
-          encryptionKey:
-            secretKeyRef:
-              name: langfuse-app-secrets
-              key: encryption-key
-        postgresql:
-          deploy: true
-          auth:
-            existingSecret: langfuse-app-secrets
-            secretKeys:
-              userPasswordKey: postgres-password
-              adminPasswordKey: postgres-password
-        clickhouse:
-          deploy: true
-          auth:
-            existingSecret: langfuse-app-secrets
-            existingSecretKey: clickhouse-password
-        redis:
-          deploy: true
-          auth:
-            existingSecret: langfuse-app-secrets
-            existingSecretPasswordKey: redis-password
-        s3:
-          deploy: true
-          auth:
-            existingSecret: langfuse-app-secrets
-            rootUserSecretKey: minio-root-user
-            rootPasswordSecretKey: minio-root-password
-
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: langfuse
-
-  syncPolicy:
-    # Opt the langfuse namespace into Nebari management so the NebariApp below
-    # (which exposes the UI) is reconciled by the nebari-operator rather than
-    # rejected with NamespaceNotOptedIn.
-    managedNamespaceMetadata:
-      labels:
-        nebari.dev/managed: "true"
-    automated:
-      prune: true
-      selfHeal: true
-      allowEmpty: false
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m
----
-# Exposes the Langfuse web UI via the shared gateway with Keycloak OIDC auth and
-# a Nebari landing-page tile. Mirrors the key-manager NebariApp pattern. Lives
-# in the langfuse namespace so its same-namespace service reference resolves;
-# the namespace is opted into Nebari management by the Application above.
-apiVersion: reconcilers.nebari.dev/v1
-kind: NebariApp
-metadata:
-  name: langfuse
-  namespace: langfuse
-spec:
-  hostname: langfuse.REPLACE-baseDomain
-  gateway: nebari-gateway
-  service:
-    name: langfuse-web
-    port: 3000
-  routing:
-    publicRoutes: []
-  auth:
-    enabled: true
-    provider: keycloak
-    provisionClient: true
-    scopes:
-      - openid
-      - profile
-      - email
-  landingPage:
-    enabled: true
-    displayName: Langfuse
-    description: LLM observability — per-user token usage and traces
-    category: Monitoring
-    healthCheck:
-      enabled: true
-      path: /api/public/health
-      port: 3000
-      intervalSeconds: 30
-      timeoutSeconds: 5

From 672f7ce8a4dbbc1ff60a35e38a6e512b5dcf85c5 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 14:49:06 -0500
Subject: [PATCH 14/20] feat: add per-user token-usage Grafana dashboard JSON

---
 .../dashboards/per-user-token-usage.json      | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 charts/nebari-llm-serving/dashboards/per-user-token-usage.json

diff --git a/charts/nebari-llm-serving/dashboards/per-user-token-usage.json b/charts/nebari-llm-serving/dashboards/per-user-token-usage.json
new file mode 100644
index 0000000..bd4e25d
--- /dev/null
+++ b/charts/nebari-llm-serving/dashboards/per-user-token-usage.json
@@ -0,0 +1,75 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "graphTooltip": 0,
+  "panels": [
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "description": "Total LLM tokens (input + output) attributed to each external API-key owner over the last 30 days. Source: gen_ai_client_token_usage, labeled with user.id via the AI Gateway.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "align": "auto", "cellOptions": { "type": "auto" } },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 18, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "showHeader": true,
+        "sortBy": [{ "displayName": "Tokens (30d)", "desc": true }]
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${datasource}" },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=\"\"}[30d]))",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Tokens used per user (last 30 days)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true },
+            "indexByName": {},
+            "renameByName": { "user_id": "User", "Value": "Tokens (30d)" }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 39,
+  "tags": ["nebari", "llm", "token-usage"],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Data source",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": { "from": "now-30d", "to": "now" },
+  "timepicker": {},
+  "title": "LLM Per-User Token Usage",
+  "uid": "nebari-llm-token-usage",
+  "version": 1
+}

From 254c396084c1a1e2ff2196a820167df33e8ce75a Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 14:49:44 -0500
Subject: [PATCH 15/20] feat: ship per-user token-usage dashboard as a Grafana
 ConfigMap

---
 .../templates/token-usage-dashboard.yaml      | 22 +++++++++++++++++++
 charts/nebari-llm-serving/values.yaml         | 17 ++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 charts/nebari-llm-serving/templates/token-usage-dashboard.yaml

diff --git a/charts/nebari-llm-serving/templates/token-usage-dashboard.yaml b/charts/nebari-llm-serving/templates/token-usage-dashboard.yaml
new file mode 100644
index 0000000..d1004e6
--- /dev/null
+++ b/charts/nebari-llm-serving/templates/token-usage-dashboard.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.observability.dashboard.enabled }}
+# Per-user LLM token-usage Grafana dashboard.
+#
+# Labeled `grafana_dashboard: "1"` so the LGTM pack's grafana-sc-dashboard
+# sidecar auto-loads it (no Grafana restart needed). Placed in the monitoring
+# namespace (observability.dashboard.namespace) where the sidecar watches,
+# which differs from this chart's release namespace, so the namespace is set
+# explicitly. The panel sums gen_ai_client_token_usage per user_id over 30d;
+# the user_id label is populated by the AI Gateway's
+# metricsRequestHeaderAttributes mapping of the forwarded x-client-id header.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nebari-llm-serving.fullname" . }}-token-usage-dashboard
+  namespace: {{ .Values.observability.dashboard.namespace }}
+  labels:
+    {{- include "nebari-llm-serving.labels" . | nindent 4 }}
+    grafana_dashboard: "1"
+data:
+  per-user-token-usage.json: |-
+    {{- .Files.Get "dashboards/per-user-token-usage.json" | nindent 4 }}
+{{- end }}
diff --git a/charts/nebari-llm-serving/values.yaml b/charts/nebari-llm-serving/values.yaml
index b7e596a..bdaf081 100644
--- a/charts/nebari-llm-serving/values.yaml
+++ b/charts/nebari-llm-serving/values.yaml
@@ -152,3 +152,20 @@ defaults:
     storageClassName: ""
   monitoring:
     enabled: true
+
+# observability.dashboard provisions a Grafana dashboard showing per-user
+# LLM token usage (one row per API-key owner, total tokens over 30 days),
+# built on the LGTM pack already running in the cluster. The dashboard is
+# shipped as a ConfigMap labeled `grafana_dashboard: "1"`, which the Grafana
+# sidecar (grafana-sc-dashboard) auto-loads. Requires the AI Gateway value
+# controller.metricsRequestHeaderAttributes: "x-client-id:user.id" (see
+# examples/envoy-ai-gateway.yaml) so the gen_ai_client_token_usage metric
+# carries the user_id label.
+observability:
+  dashboard:
+    # enabled: provision the per-user token-usage dashboard ConfigMap.
+    enabled: true
+    # namespace: where the LGTM Grafana sidecar watches for dashboard
+    # ConfigMaps. Must match the namespace Grafana runs in. Defaults to the
+    # standard Nebari LGTM namespace.
+    namespace: monitoring

From 019e19fa9f45231a15c6a85e9faa44bd79a48768 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 14:50:26 -0500
Subject: [PATCH 16/20] docs: document per-user token usage via Grafana metric,
 drop Langfuse runbook

---
 docs/install-production.md | 174 +++++++++++++++----------------------
 1 file changed, 68 insertions(+), 106 deletions(-)

diff --git a/docs/install-production.md b/docs/install-production.md
index 4100804..dd4a41e 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1740,13 +1740,16 @@ kubectl rollout restart deploy/ai-gateway-controller -n envoy-ai-gateway-system
 Confirm recovery: the controller logs should show only the live
 model's `InferencePool` being reconciled, with no panic.
 
-## 14. Per-user token usage in Langfuse
+## 14. Per-user token usage in Grafana
 
 This feature attributes LLM token usage to the API-key owner who made
-each request, visible per-user in a self-hosted Langfuse instance.
+each request and renders it as a Grafana table — one row per user, total
+tokens used in the last 30 days — built entirely on the LGTM pack
+(Grafana + Mimir) already running in the cluster. No Langfuse, no extra
+datastores, no secrets in your GitOps repo.
 
-**Scope:** This covers the **external (API-key) access path only.**
-The internal JWT path is not traced to Langfuse in this version.
+**Scope:** This covers the **external (API-key) access path only.** The
+internal JWT path is not attributed in this version.
 
 ### 14.1 How it works
 
@@ -1754,118 +1757,77 @@ The pipeline has four stages:
 
 1. **Envoy Gateway credential forwarding.** The operator's external
    `SecurityPolicy` sets `apiKeyAuth.forwardClientIDHeader: x-client-id`
-   and `apiKeyAuth.sanitize: true`. When Envoy Gateway matches a
-   presented API key it forwards the matched credential's name — the
-   key-manager **clientID** (e.g. `user-chuck-1`) — downstream as the
-   `x-client-id` header. `sanitize: true` strips the raw API key from
-   the request before it is proxied upstream. **These fields require
-   Envoy Gateway v1.5.1+** (verified present in v1.6.2, the pack's
-   pinned version; absent in v1.3). Older Envoy Gateway will reject or
-   ignore the fields.
-
-2. **Header → span attribute mapping.** The AI Gateway controller value
-   `controller.spanRequestHeaderAttributes: "x-client-id:user.id"`
+   and `apiKeyAuth.sanitize: true`. When Envoy Gateway matches a presented
+   API key it forwards the matched credential's name — the key-manager
+   **clientID** (e.g. `user-chuck-1`) — downstream as the `x-client-id`
+   header. `sanitize: true` strips the raw API key from the request before
+   it is proxied upstream. **These fields require Envoy Gateway v1.5.1+**
+   (verified present in v1.6.2, the pack's pinned version; absent in v1.3).
+
+2. **Header → metric label mapping.** The AI Gateway controller value
+   `controller.metricsRequestHeaderAttributes: "x-client-id:user.id"`
    (set in `examples/envoy-ai-gateway.yaml`) maps the forwarded header
-   onto the OpenInference GenAI span as the `user.id` attribute. The
-   extProc OpenInference tracer already records token usage on that
-   span.
-
-3. **OTLP export to Langfuse.** The extProc exports OTLP directly to
-   Langfuse — there is no OTel Collector in the path. Tracing is
-   configured through `extProc.extraEnvVars` in
-   `examples/envoy-ai-gateway.yaml`: `OTEL_TRACES_EXPORTER=otlp`,
-   `OTEL_SERVICE_NAME`, `OTEL_EXPORTER_OTLP_ENDPOINT`
-   (`https://langfuse.<baseDomain>/api/public/otel`), and
-   `OTEL_EXPORTER_OTLP_HEADERS` (`Authorization=Basic <base64(pk:sk)>`).
-
-   > **Chart limitation (verified on a live cluster):** `ai-gateway-helm`
-   > v0.5.0 flattens `extProc.extraEnvVars` into a single
-   > `--extProcExtraEnvVars` CLI arg and only honors a literal `value:` —
-   > `valueFrom`/`secretKeyRef` renders as `nil`. So these must be literal
-   > values in the Helm values, **including the Basic-auth header that
-   > contains the Langfuse project secret key.** Keep your GitOps repo
-   > private and/or encrypt this Application with SOPS or sealed-secrets.
-   > Secret-ref injection here is a follow-up that needs an upstream chart
-   > change.
-
-4. **Langfuse ingestion.** Langfuse ingests OTLP at
-   `/api/public/otel`, reads token usage from the span, and uses
-   `user.id` to populate its per-user view.
-
-**Granularity note:** `user.id` is the verbatim clientID
-(e.g. `user-chuck-1`), which means attribution is **per-key**, not
-per-human-aggregated. Because clientIDs embed the username, the data
-is still effectively per-user and groupable in Langfuse, but a user
-with multiple keys will appear as multiple distinct user IDs. True
-per-user rollup across keys is a deferred follow-up.
-
-### 14.2 Deploying self-hosted Langfuse
-
-Langfuse is deployed via ArgoCD using `examples/langfuse.yaml` (chart
-`langfuse/langfuse` 1.5.36). The chart bundles Postgres, ClickHouse,
-Redis, and MinIO; PVCs use the default StorageClass (Longhorn on
-standard Nebari clusters). This is a single-replica v1 deployment —
-HA, external datastores, and backup configuration are out of scope.
-
-**Prerequisites:**
-
-- A `langfuse-app-secrets` Kubernetes Secret must exist before ArgoCD
-  syncs the application. The exact `kubectl create secret generic`
-  command (with all required keys) is in the comment header of
-  `examples/langfuse.yaml` — follow that file rather than duplicating
-  the command here.
-
-**DNS:** The UI is exposed via a `NebariApp` at
-`langfuse.<baseDomain>` with Keycloak OIDC and a landing-page tile.
-A `langfuse.<baseDomain>` DNS record is required; a wildcard CNAME on
-the base domain covers it.
-
-### 14.3 Langfuse project-key bootstrap
-
-OTLP authentication uses a Langfuse **project's** public and secret
-API keys. These keys only exist after a project is created in the UI,
-so this one-time bootstrap must be performed before the extProc can
-export traces.
+   onto the GenAI token-usage metric as the `user.id` attribute, which
+   Prometheus exposes as the label `user_id`. The extProc already records
+   `gen_ai_client_token_usage` (input and output token counts) on every
+   request.
+
+3. **Scrape into Mimir.** The gateway proxy pod already carries the
+   `prometheus.io/scrape` annotation, so the LGTM OpenTelemetry Collector
+   scrapes `gen_ai_*` metrics from the proxy's `/stats/prometheus` endpoint
+   and writes them to Mimir. No gateway-side OTLP export is configured.
+
+4. **Grafana table.** The pack ships a dashboard ConfigMap (labeled
+   `grafana_dashboard: "1"`) that the Grafana sidecar auto-loads. Its
+   single Table panel runs:
+
+   ```promql
+   sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=""}[30d]))
+   ```
 
-1. Sign in to `https://langfuse.<baseDomain>`, create an organization
-   and a project.
+   Summing across `gen_ai_token_type` (input + output) yields total tokens
+   per user over the last 30 days; `user_id != ""` drops unattributed
+   (e.g. internal) traffic.
 
-2. Navigate to **Project Settings → API Keys → Create**. Copy the
-   public key (`pk-lf-...`) and secret key (`sk-lf-...`).
+**Granularity note:** `user_id` is the verbatim clientID (e.g.
+`user-chuck-1`), so attribution is **per-key**, not per-human-aggregated.
+Because clientIDs embed the username, the data is still effectively
+per-user, but a user with multiple keys appears as multiple rows. True
+per-user rollup across keys is a deferred follow-up.
 
-3. Compute the Base64 Basic-auth header value:
+### 14.2 What you deploy
 
-   ```bash
-   printf 'pk-lf-...:sk-lf-...' | base64
-   ```
+Nothing extra beyond the pack and a healthy AI Gateway:
 
-4. Set the literal values in `examples/envoy-ai-gateway.yaml`
-   (`extProc.extraEnvVars`) and re-sync the app:
+- The metric label comes from the one-line `controller.metricsRequestHeaderAttributes`
+  value in `examples/envoy-ai-gateway.yaml` (already set). After changing
+  it, roll the gateway proxy so the new controller args take effect:
+  `kubectl rollout restart deploy -n envoy-gateway-system <gateway-proxy-deploy>`.
+- The dashboard ships as a chart-managed ConfigMap, controlled by
+  `observability.dashboard.enabled` (default `true`) and placed in
+  `observability.dashboard.namespace` (default `monitoring`). Set the
+  namespace to wherever your LGTM Grafana sidecar watches for dashboards.
 
-   - `OTEL_EXPORTER_OTLP_ENDPOINT: https://langfuse.<baseDomain>/api/public/otel`
-   - `OTEL_EXPORTER_OTLP_HEADERS: Authorization=Basic <base64-value>`
+**Prerequisites:** the LGTM pack (Grafana + Mimir) installed with the
+collector scraping the gateway proxy, and Mimir retention ≥ 30 days
+(the LGTM default).
 
-   (See the chart-limitation note in §14.1 — these cannot be a
-   `secretKeyRef`.) After updating, roll the extProc so it picks up the
-   new controller args:
-   `kubectl rollout restart deploy -n envoy-gateway-system <gateway-proxy-deploy>`.
+### 14.3 Verification
 
-5. Roll the extProc so it picks up the new env vars:
+1. Mint an API key via the key-manager UI (see section 11.2).
+2. Call the external endpoint with that key a few times (see section 11.3).
+3. Query Mimir directly to confirm the labeled metric landed:
 
-   ```bash
-   kubectl rollout restart deploy \
-     -n envoy-ai-gateway-system \
-     -l app.kubernetes.io/name=ai-gateway-extproc
+   ```promql
+   sum by (user_id) (increase(gen_ai_client_token_usage_sum{user_id!=""}[30d]))
    ```
 
-> **Note:** Automating project and API-key creation (e.g. via the
-> Langfuse management API) is a deliberate follow-up and is not part
-> of v1.
-
-### 14.4 Verification
+   Expect a row whose `user_id` is the clientID of the key you minted
+   (`user-<name>-N`) with a non-zero token count.
+4. Open Grafana, find the **LLM Per-User Token Usage** dashboard, and
+   confirm the table renders the same row. The dashboard appears without a
+   Grafana restart (the sidecar loads the ConfigMap live).
 
-1. Mint an API key via the key-manager UI (see section 11.2).
-2. Call the external endpoint with that key (see section 11.3).
-3. Open `https://langfuse.<baseDomain>`, navigate to **Traces**, and
-   confirm a trace appears with non-zero input/output token usage
-   attributed to the clientID (`user-<name>-N`) of the key you minted.
+> **Note:** Per-model and input/output split breakdowns, time-series/rate
+> panels, alerting, and cost estimation are deliberate follow-ups and are
+> not part of v1.

From f6e587421a4b8ec52be6643bc311290e1ab25213 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 15:40:13 -0500
Subject: [PATCH 17/20] fix: collect gen_ai metrics from extProc :1064, not
 envoy :19001

Live testing on tyler-hetzner-dev proved the gen_ai_* metrics are exposed on
the extProc sidecar admin port (:1064 /metrics), not on envoy's :19001
/stats/prometheus that the default prometheus.io/scrape annotation targets.
A dedicated collector scrape job for :1064 is required. Document the
scrape_config and correct the spec; the metricsRequestHeaderAttributes label
(user_id) is confirmed working.
---
 docs/install-production.md     | 74 ++++++++++++++++++++++++++--------
 examples/envoy-ai-gateway.yaml | 13 ++++--
 2 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/docs/install-production.md b/docs/install-production.md
index dd4a41e..f841dc7 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1772,10 +1772,14 @@ The pipeline has four stages:
    `gen_ai_client_token_usage` (input and output token counts) on every
    request.
 
-3. **Scrape into Mimir.** The gateway proxy pod already carries the
-   `prometheus.io/scrape` annotation, so the LGTM OpenTelemetry Collector
-   scrapes `gen_ai_*` metrics from the proxy's `/stats/prometheus` endpoint
-   and writes them to Mimir. No gateway-side OTLP export is configured.
+3. **Scrape into Mimir.** The `gen_ai_*` metrics are exposed on the extProc
+   sidecar's own admin port — `:1064` at `/metrics` — **not** on envoy's
+   `:19001 /stats/prometheus`. The proxy pod's default `prometheus.io/scrape`
+   annotation only covers `:19001`, and annotation-based discovery scrapes a
+   single port, so a **dedicated scrape job** targeting the gateway proxy pods
+   on `:1064` must be added to the LGTM OpenTelemetry Collector (see §14.2).
+   That job writes the `gen_ai_*` series to Mimir. No gateway-side OTLP export
+   is configured.
 
 4. **Grafana table.** The pack ships a dashboard ConfigMap (labeled
    `grafana_dashboard: "1"`) that the Grafana sidecar auto-loads. Its
@@ -1797,20 +1801,58 @@ per-user rollup across keys is a deferred follow-up.
 
 ### 14.2 What you deploy
 
-Nothing extra beyond the pack and a healthy AI Gateway:
+Three pieces — the metric label, the scrape job, and the dashboard:
 
-- The metric label comes from the one-line `controller.metricsRequestHeaderAttributes`
-  value in `examples/envoy-ai-gateway.yaml` (already set). After changing
-  it, roll the gateway proxy so the new controller args take effect:
-  `kubectl rollout restart deploy -n envoy-gateway-system <gateway-proxy-deploy>`.
-- The dashboard ships as a chart-managed ConfigMap, controlled by
-  `observability.dashboard.enabled` (default `true`) and placed in
-  `observability.dashboard.namespace` (default `monitoring`). Set the
-  namespace to wherever your LGTM Grafana sidecar watches for dashboards.
+**1. Metric label** — the one-line `controller.metricsRequestHeaderAttributes`
+value in `examples/envoy-ai-gateway.yaml` (already set). The controller passes
+this to the extProc sidecar as `-metricsRequestHeaderAttributes`, so the
+sidecar must be (re)created after the value changes. Roll the gateway proxy
+(the extProc is injected into it), which re-injects the sidecar with the new
+flag:
+`kubectl rollout restart deploy -n envoy-gateway-system <gateway-proxy-deploy>`.
 
-**Prerequisites:** the LGTM pack (Grafana + Mimir) installed with the
-collector scraping the gateway proxy, and Mimir retention ≥ 30 days
-(the LGTM default).
+**2. Scrape job for `:1064`** — add a scrape job to the LGTM OpenTelemetry
+Collector so the `gen_ai_*` metrics on the extProc admin port reach Mimir. The
+collector config is single-owner (see ADR-0005 in nebari-infrastructure-core);
+add this to the collector-override ConfigMap rather than editing the base. The
+job discovers the gateway proxy pods and rewrites the scrape target to `:1064`:
+
+```yaml
+# OpenTelemetry Collector prometheus receiver — scrape extProc gen_ai metrics.
+receivers:
+  prometheus/ai-gateway-genai:
+    config:
+      scrape_configs:
+        - job_name: ai-gateway-genai
+          metrics_path: /metrics
+          kubernetes_sd_configs:
+            - role: pod
+              namespaces:
+                names: [envoy-gateway-system]
+          relabel_configs:
+            # keep only the Envoy AI Gateway proxy pods
+            - source_labels:
+                - __meta_kubernetes_pod_label_gateway_envoyproxy_io_owning_gateway_name
+              regex: nebari-gateway
+              action: keep
+            # scrape the extProc admin port, not envoy's :19001
+            - source_labels: [__address__]
+              regex: (.+?)(?::\d+)?
+              target_label: __address__
+              replacement: ${1}:1064
+```
+
+Then add `prometheus/ai-gateway-genai` to the collector's `metrics` pipeline
+`receivers:` list so it flows through the existing `otlphttp/mimir` exporter.
+
+**3. Dashboard** — ships as a chart-managed ConfigMap, controlled by
+`observability.dashboard.enabled` (default `true`) and placed in
+`observability.dashboard.namespace` (default `monitoring`). Set the namespace
+to wherever your LGTM Grafana sidecar watches for dashboards.
+
+**Prerequisites:** the LGTM pack (Grafana + Mimir) installed with the OTel
+collector forwarding to Mimir, and Mimir retention ≥ 30 days (the LGTM
+default).
 
 ### 14.3 Verification
 
diff --git a/examples/envoy-ai-gateway.yaml b/examples/envoy-ai-gateway.yaml
index fbb4206..d23e80f 100644
--- a/examples/envoy-ai-gateway.yaml
+++ b/examples/envoy-ai-gateway.yaml
@@ -85,10 +85,15 @@ spec:
         # user.id (Prometheus label `user_id`). That header is the matched
         # API-key clientID (e.g. "user-chuck-1"), forwarded by
         # apiKeyAuth.forwardClientIDHeader on the operator-generated external
-        # SecurityPolicy. The metric is already scraped into Mimir by the LGTM
-        # collector (the proxy pod carries prometheus.io/scrape), so a Grafana
-        # table can sum tokens per user. See docs/install-production.md §14 and
-        # the chart's observability.dashboard value.
+        # SecurityPolicy.
+        #
+        # COLLECTION: the gen_ai_* metrics are exposed on the extproc sidecar's
+        # admin port (:1064 /metrics), NOT on envoy's :19001 /stats/prometheus.
+        # The proxy pod's default prometheus.io/scrape annotation targets :19001
+        # only, so a DEDICATED scrape job for :1064 must be added to the LGTM
+        # OTel collector (see docs/install-production.md §14.2 for the
+        # scrape_config). Then a Grafana table sums tokens per user.
+        # See also the chart's observability.dashboard value.
         controller:
           metricsRequestHeaderAttributes: "x-client-id:user.id"
 

From 644ef6f1b204c0fd24164da14f11bfe39a831180 Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 16:15:42 -0500
Subject: [PATCH 18/20] docs: escape OTel relabel replacement as $${1} for the
 :1064 scrape job

Verified on a rebuilt cluster: an unescaped ${1} crashes the OpenTelemetry
Collector (env-var expansion), so the documented scrape_config must use
$${1}. End-to-end scrape :1064 -> Mimir -> per-user token query confirmed.
---
 docs/install-production.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/install-production.md b/docs/install-production.md
index f841dc7..881b7de 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1839,12 +1839,20 @@ receivers:
             - source_labels: [__address__]
               regex: (.+?)(?::\d+)?
               target_label: __address__
-              replacement: ${1}:1064
+              replacement: $${1}:1064
 ```
 
 Then add `prometheus/ai-gateway-genai` to the collector's `metrics` pipeline
 `receivers:` list so it flows through the existing `otlphttp/mimir` exporter.
 
+> **OTel escaping gotcha (verified):** the relabel `replacement` is `$${1}`,
+> not `${1}`. The OpenTelemetry Collector expands `${...}` as an environment
+> variable at config-load time, so an unescaped `${1}` crashes the collector
+> with `environment variable "1" has invalid name`. Doubling the dollar sign
+> (`$${1}`) passes a literal `${1}` through to the Prometheus relabel engine.
+> (In a *raw* Prometheus `scrape_config` — not via the OTel collector — use the
+> normal `${1}`.)
+
 **3. Dashboard** — ships as a chart-managed ConfigMap, controlled by
 `observability.dashboard.enabled` (default `true`) and placed in
 `observability.dashboard.namespace` (default `monitoring`). Set the namespace

From f4bac664d1d8b0e70c586256f2a455b82bed945f Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Fri, 26 Jun 2026 17:00:59 -0500
Subject: [PATCH 19/20] docs: note the :1064 scrape gateway-name regex must
 match the shared gateway

---
 docs/install-production.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/install-production.md b/docs/install-production.md
index 881b7de..b4f9787 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1830,7 +1830,10 @@ receivers:
               namespaces:
                 names: [envoy-gateway-system]
           relabel_configs:
-            # keep only the Envoy AI Gateway proxy pods
+            # keep only the Envoy AI Gateway proxy pods. The regex must match
+            # your shared gateway's name (platform.gateway.external.name in the
+            # chart values; "nebari-gateway" by default). Use ".+" to scrape the
+            # extProc on every Envoy Gateway proxy regardless of name.
             - source_labels:
                 - __meta_kubernetes_pod_label_gateway_envoyproxy_io_owning_gateway_name
               regex: nebari-gateway

From 4a6516ae28d0b6c5e2d2b6c7bcb79220281441ed Mon Sep 17 00:00:00 2001
From: Tyler Potts <49161327+tylerpotts@users.noreply.github.com>
Date: Mon, 29 Jun 2026 10:43:59 -0500
Subject: [PATCH 20/20] docs: document streaming usage caveat (requires
 stream_options.include_usage)

Verified on a live cluster with a streaming mock: streaming requests are
attributed only when the client sends stream_options.include_usage=true
(10 with it -> exact usage recorded; 5 without -> zero). Non-streaming
always counted.
---
 docs/install-production.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/install-production.md b/docs/install-production.md
index b4f9787..2c1da45 100644
--- a/docs/install-production.md
+++ b/docs/install-production.md
@@ -1799,6 +1799,18 @@ Because clientIDs embed the username, the data is still effectively
 per-user, but a user with multiple keys appears as multiple rows. True
 per-user rollup across keys is a deferred follow-up.
 
+> **Streaming caveat (verified on a live cluster):** for **streaming**
+> requests (`"stream": true`), the token-usage metric is recorded **only if
+> the client sends `stream_options: {"include_usage": true}`** — that is what
+> makes the model emit the final usage chunk the extProc reads from. Streaming
+> requests without it complete normally (HTTP 200) but contribute **zero** to
+> the per-user totals. Non-streaming requests always carry `usage` and are
+> always counted. There is no gateway-side way to force `include_usage`, so
+> document this for users/SDKs, or treat streaming-without-usage traffic as a
+> known undercount. (Tested: 10 streaming requests with `include_usage`
+> incremented the metric by exactly their usage; 5 without it incremented by
+> nothing.)
+
 ### 14.2 What you deploy
 
 Three pieces — the metric label, the scrape job, and the dashboard: