feat(dev): align local dev stack with AI Gateway v0.5 + key-manager UI dev mode

.gitignore

@@ -1,3 +1,6 @@
 CLAUDE.md
 docs/superpowers/
 .claude/
+
+# Local dev secrets (OPENROUTER_API_KEY, etc.)
+.env

charts/nebari-llm-serving/templates/key-manager-deployment.yaml

@@ -48,6 +48,17 @@ spec:
             - name: LLM_OIDC_USERINFO_URL
               value: {{ .Values.keyManager.oidcUserinfoURL | quote }}
             {{- end }}
+            {{- if .Values.keyManager.devMode.enabled }}
+            # Dev mode: bypasses auth and injects a fixed identity so the UI
+            # works on a local cluster with no Keycloak. Never enable in a real
+            # deployment. See nebari-dev/llm-serving-pack#114.
+            - name: LLM_DEV_MODE
+              value: "true"
+            - name: LLM_DEV_USER
+              value: {{ .Values.keyManager.devMode.user | quote }}
+            - name: LLM_DEV_GROUPS
+              value: {{ .Values.keyManager.devMode.groups | join "," | quote }}
+            {{- end }}
           resources:
             limits:
               cpu: 200m

charts/nebari-llm-serving/values.yaml

@@ -62,6 +62,16 @@ envoyAIGateway:
 
 keyManager:
   enabled: true
+  # devMode bypasses the key-manager's auth and injects a fixed identity so the
+  # UI can run on a local cluster with no Keycloak / gateway OIDC layer. Off by
+  # default; never enable it in a real deployment. See
+  # nebari-dev/llm-serving-pack#114. Pair it with a direct port-forward to the
+  # key-manager Service so the gateway OIDC enforcement is bypassed too.
+  devMode:
+    enabled: false
+    user: dev
+    groups:
+      - llm
   image:
     repository: ghcr.io/nebari-dev/nebari-llm-serving-pack/key-manager
     # tag defaults to .Chart.AppVersion when empty so the chart version and

dev/.env.example

@@ -0,0 +1,5 @@
+# Copy to dev/.env and fill in. dev/.env is gitignored.
+#
+# OpenRouter API key (https://openrouter.ai/keys). Used as the upstream
+# provider credential for the PassthroughModels that `make run-dev` deploys.
+OPENROUTER_API_KEY=

dev/Makefile

@@ -1,28 +1,58 @@
 CLUSTER_NAME ?= llm-serving-test
 
-.PHONY: setup teardown build-images load-images deploy deploy-operator deploy-key-manager apply-test-model logs-operator logs-key-manager clean help
+# Dependency versions. These move together: Envoy AI Gateway v0.5.x requires
+# Envoy Gateway v1.6.x and Gateway API v1.4.0 (see
+# https://aigateway.envoyproxy.io/docs/compatibility/). Bump them as a set
+# when upgrading the AI Gateway.
+CERT_MANAGER_VERSION ?= v1.17.2
+GATEWAY_API_VERSION ?= v1.4.0
+GIE_VERSION ?= v1.4.0
+ENVOY_GATEWAY_VERSION ?= v1.6.7
+AI_GATEWAY_VERSION ?= v0.5.0
+
+# Helm 4 applies the eg chart's bundled Gateway API CRDs (crds/) server-side,
+# which conflicts with the standalone install in `setup` (owned by a different
+# field manager). Force ownership so the chart wins. Helm 3 has no server-side
+# apply and skips already-present crds/, so it neither needs nor accepts this
+# flag - keep it empty there. Empty when helm is missing (setup fails later).
+HELM_MAJOR := $(shell helm version --template '{{.Version}}' 2>/dev/null | sed -E 's/^v?([0-9]+).*/\1/')
+HELM_FORCE_CONFLICTS := $(if $(filter-out 1 2 3,$(HELM_MAJOR)),--force-conflicts,)
+
+.PHONY: setup teardown build-images load-images deploy deploy-operator deploy-key-manager apply-test-model apply-passthrough-model create-openrouter-secret run-dev ui logs-operator logs-key-manager clean help
 
 help: ## Show this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
+run-dev: ## One-command UI dev environment: cluster + models + port-forward + hot-reload UI server (needs dev/.env with OPENROUTER_API_KEY)
+	./run-dev.sh
+
 setup: ## Create kind cluster and install dependencies
-	kind create cluster --name $(CLUSTER_NAME)
+	# Idempotent: skip creation if the cluster already exists so a setup that
+	# died midway can be re-run without `make teardown` first.
+	@kind get clusters | grep -qx $(CLUSTER_NAME) || kind create cluster --name $(CLUSTER_NAME)
 	# cert-manager
-	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
 	kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
-	# Gateway API CRDs
-	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml
-	# GIE CRDs (v1.4.0, includes graduated k8s.io group)
-	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.4.0/manifests.yaml
-	# Envoy Gateway
-	helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.3.0 -n envoy-gateway-system --create-namespace
-	kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s
-	# Envoy AI Gateway
-	helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v0.5.0 -n envoy-ai-gateway-system --create-namespace
-	helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm --version v0.5.0 -n envoy-ai-gateway-system
+	# Gateway API CRDs. Server-side apply so the field ownership matches the
+	# eg chart's own SSA on Helm 4, letting it cleanly take co-ownership with
+	# --force-conflicts (see HELM_FORCE_CONFLICTS above) instead of erroring.
+	kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEWAY_API_VERSION)/standard-install.yaml
+	# GIE CRDs (includes the graduated inference.networking.k8s.io group)
+	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/$(GIE_VERSION)/manifests.yaml
+	# Envoy AI Gateway first: envoy-gateway's extensionManager below points at
+	# the ai-gateway-controller XDS service, so bring it up before reconfiguring
+	# envoy-gateway to avoid noisy connection-refused logs during translation.
+	helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version $(AI_GATEWAY_VERSION) -n envoy-ai-gateway-system --create-namespace
+	helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm --version $(AI_GATEWAY_VERSION) -n envoy-ai-gateway-system
 	kubectl -n envoy-ai-gateway-system wait --timeout=120s deployment/ai-gateway-controller --for=condition=Available
-	# LLMModel CRD
+	# Envoy Gateway, wired with the AI Gateway ext_proc extension (enableBackend,
+	# extensionManager, backendResources). Without this the per-model routing
+	# layer 404s and passthrough upstreams never get a TLS transport socket.
+	helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm --version $(ENVOY_GATEWAY_VERSION) -n envoy-gateway-system --create-namespace $(HELM_FORCE_CONFLICTS) -f eg-extension-values.yaml
+	kubectl -n envoy-gateway-system rollout status deployment/envoy-gateway --timeout=120s
+	# CRDs (served LLMModels and external-provider PassthroughModels)
 	kubectl apply -f ../charts/nebari-llm-serving/crds/llmmodel-crd.yaml
+	kubectl apply -f ../charts/nebari-llm-serving/crds/passthroughmodel-crd.yaml
 	# Test Gateways
 	kubectl apply -f gateways.yaml
 
@@ -45,13 +75,26 @@ deploy-operator: ## Deploy operator
 	kubectl apply -f manifests/webhook.yaml
 	kubectl -n llm-operator-system rollout status deployment/llm-operator --timeout=60s
 
-deploy-key-manager: ## Deploy key manager
+deploy-key-manager: ## Deploy key manager (dev mode: auth bypassed, dev identity injected)
 	kubectl apply -f manifests/key-manager.yaml
 	kubectl -n llm-operator-system rollout status deployment/llm-key-manager --timeout=60s
 
-apply-test-model: ## Apply test model
+apply-test-model: ## Apply served test model (mock vLLM)
 	kubectl apply -f manifests/test-model.yaml
 
+create-openrouter-secret: ## Create the OpenRouter provider credential (OPENROUTER_API_KEY env var required)
+	@test -n "$(OPENROUTER_API_KEY)" || { echo "set OPENROUTER_API_KEY"; exit 1; }
+	kubectl -n llm-operator-system create secret generic openrouter-api-key \
+		--from-literal=apiKey="$(OPENROUTER_API_KEY)" \
+		--dry-run=client -o yaml | kubectl apply -f -
+
+apply-passthrough-model: ## Apply the OpenRouter PassthroughModel (run create-openrouter-secret first)
+	kubectl apply -f manifests/passthrough-test-model.yaml
+
+ui: ## Port-forward the key-manager UI to http://localhost:8080 (dev mode, no login)
+	@echo "key-manager UI at http://localhost:8080 (dev mode injects user 'dev')"
+	kubectl -n llm-operator-system port-forward svc/llm-key-manager 8080:8080
+
 logs-operator: ## Tail operator logs
 	kubectl -n llm-operator-system logs -f deployment/llm-operator
 

dev/eg-extension-values.yaml

@@ -0,0 +1,35 @@
+# Envoy Gateway Helm values that wire in the Envoy AI Gateway ext_proc
+# extension. Mirrors docs/install-production.md section 6.1, which sources
+# these from envoyproxy/ai-gateway's manifests/envoy-gateway-values.yaml.
+#
+# - extensionApis.enableBackend lets HTTPRoutes reference InferencePool.
+# - extensionManager points envoy-gateway at the AI Gateway controller's XDS
+#   extension server (port 1063) so it inserts the ext_proc filter.
+# - backendResources lists the non-builtin backend kinds the extension handles.
+config:
+  envoyGateway:
+    gateway:
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+    extensionApis:
+      enableEnvoyPatchPolicy: true
+      enableBackend: true
+    extensionManager:
+      hooks:
+        xdsTranslator:
+          translation:
+            listener: { includeAll: true }
+            route: { includeAll: true }
+            cluster: { includeAll: true }
+            secret: { includeAll: true }
+          post:
+            - Translation
+            - Cluster
+            - Route
+      service:
+        fqdn:
+          hostname: ai-gateway-controller.envoy-ai-gateway-system.svc.cluster.local
+          port: 1063
+      backendResources:
+        - group: inference.networking.k8s.io
+          kind: InferencePool
+          version: v1

dev/manifests/dev-models.yaml

@@ -0,0 +1,60 @@
+# Three OpenRouter passthrough models for the local dev environment, applied by
+# `make run-dev` so the key-manager UI has a populated model list to work with.
+# Each is its own PassthroughModel so the UI lists three distinct models.
+#
+# Note: API keys are currently gateway-scoped, not model-scoped (see
+# nebari-dev/llm-serving-pack#116), so a key minted for one of these works for
+# all of them. That does not affect UI development.
+apiVersion: llm.nebari.dev/v1alpha1
+kind: PassthroughModel
+metadata:
+  name: claude-sonnet-45
+  namespace: llm-operator-system
+spec:
+  provider:
+    hostname: openrouter.ai
+    schemaVersion: api/v1
+    credentialSecretName: openrouter-api-key
+  models:
+    catchAll: false
+    declared:
+      - anthropic/claude-sonnet-4.5
+  access:
+    groups:
+      - llm
+---
+apiVersion: llm.nebari.dev/v1alpha1
+kind: PassthroughModel
+metadata:
+  name: gemini-25-flash
+  namespace: llm-operator-system
+spec:
+  provider:
+    hostname: openrouter.ai
+    schemaVersion: api/v1
+    credentialSecretName: openrouter-api-key
+  models:
+    catchAll: false
+    declared:
+      - google/gemini-2.5-flash
+  access:
+    groups:
+      - llm
+---
+apiVersion: llm.nebari.dev/v1alpha1
+kind: PassthroughModel
+metadata:
+  name: llama-33-70b
+  namespace: llm-operator-system
+spec:
+  provider:
+    hostname: openrouter.ai
+    schemaVersion: api/v1
+    credentialSecretName: openrouter-api-key
+  models:
+    catchAll: false
+    declared:
+      - meta-llama/llama-3.3-70b-instruct
+  access:
+    groups:
+      - llm

dev/manifests/key-manager.yaml

@@ -11,7 +11,7 @@ metadata:
   name: llm-key-manager-models
 rules:
   - apiGroups: ["llm.nebari.dev"]
-    resources: ["llmmodels"]
+    resources: ["llmmodels", "passthroughmodels"]
     verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -91,6 +91,16 @@ spec:
               value: "IdToken"
             - name: LLM_LISTEN_ADDR
               value: ":8080"
+            # Dev mode: there is no Keycloak or gateway OIDC layer on the kind
+            # cluster, so bypass auth and inject a fixed identity. The UI and
+            # /api/* then work behind a plain `make ui` port-forward. Never set
+            # this in a real deployment. See nebari-dev/llm-serving-pack#114.
+            - name: LLM_DEV_MODE
+              value: "true"
+            - name: LLM_DEV_USER
+              value: "dev"
+            - name: LLM_DEV_GROUPS
+              value: "llm"
           resources:
             limits:
               cpu: 200m

dev/manifests/operator.yaml

@@ -29,6 +29,15 @@ rules:
   - apiGroups: ["llm.nebari.dev"]
     resources: ["llmmodels/finalizers"]
     verbs: ["update"]
+  - apiGroups: ["llm.nebari.dev"]
+    resources: ["passthroughmodels"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["llm.nebari.dev"]
+    resources: ["passthroughmodels/status"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["llm.nebari.dev"]
+    resources: ["passthroughmodels/finalizers"]
+    verbs: ["update"]
   - apiGroups: ["apps"]
     resources: ["deployments"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
@@ -48,17 +57,30 @@ rules:
     resources: ["inferencepools"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiGroups: ["aigateway.envoyproxy.io"]
-    resources: ["aigatewayroutes"]
+    resources: ["aigatewayroutes", "aiservicebackends", "backendsecuritypolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiGroups: ["gateway.envoyproxy.io"]
-    resources: ["securitypolicies"]
+    resources: ["securitypolicies", "backends"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # PassthroughModel provider TLS validation toward the external provider.
+  - apiGroups: ["gateway.networking.k8s.io"]
+    resources: ["backendtlspolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiGroups: ["gateway.networking.k8s.io"]
     resources: ["referencegrants"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiGroups: ["monitoring.coreos.com"]
     resources: ["podmonitors"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # Shared-TLS reconciler: issues the cert covering llm/llm-internal via the
+  # selfsigned-issuer ClusterIssuer (see cert-manager-config.yaml) and patches
+  # HTTPS listeners onto the shared Gateways.
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["gateway.networking.k8s.io"]
+    resources: ["gateways"]
+    verbs: ["get", "list", "watch", "update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -95,7 +117,7 @@ spec:
           imagePullPolicy: Never
           env:
             - name: LLM_BASE_DOMAIN
-              value: "llm.local"
+              value: "local"
             - name: LLM_EXTERNAL_GATEWAY_NAME
               value: "nebari-gateway"
             - name: LLM_EXTERNAL_GATEWAY_NAMESPACE
@@ -104,8 +126,13 @@ spec:
               value: "nebari-internal-gateway"
             - name: LLM_INTERNAL_GATEWAY_NAMESPACE
               value: "envoy-gateway-system"
+            # Issue the shared llm.local / llm-internal.local cert via the
+            # local self-signed ClusterIssuer (cert-manager-config.yaml) so the
+            # operator can patch HTTPS listeners onto the gateways with no ACME.
+            - name: LLM_CLUSTER_ISSUER_NAME
+              value: "selfsigned-issuer"
             - name: LLM_OIDC_ISSUER_URL
-              value: "https://keycloak.llm.local/realms/nebari"
+              value: "https://keycloak.local/realms/nebari"
             - name: LLM_OIDC_GROUPS_CLAIM
               value: "groups"
             - name: ENABLE_WEBHOOKS
@@ -120,6 +147,16 @@ spec:
             - containerPort: 9443
               name: webhook-server
               protocol: TCP
+          # The validating webhook binds 9443 only after controller-runtime
+          # loads the mounted serving cert. Gate readiness on that port so
+          # `kubectl rollout status` in `make deploy` does not return before the
+          # webhook can accept PassthroughModel creates (the manager's /readyz
+          # is only a ping and goes green before the webhook is serving).
+          readinessProbe:
+            tcpSocket:
+              port: 9443
+            initialDelaySeconds: 2
+            periodSeconds: 2
           volumeMounts:
             - name: cert
               mountPath: /tmp/k8s-webhook-server/serving-certs

dev/manifests/passthrough-test-model.yaml

@@ -0,0 +1,33 @@
+# OpenRouter PassthroughModel for the local dev cluster.
+#
+# Prereq: the provider credential Secret. Create it with:
+#   make create-openrouter-secret OPENROUTER_API_KEY=sk-or-v1-...
+#
+# Once reconciled (kubectl -n llm-operator-system get passthroughmodel), reach
+# it through the gateway. The external endpoint uses API-key auth, so inject a
+# client key into the api-keys Secret to skip the key-manager:
+#   kubectl -n llm-operator-system patch secret openrouter-api-keys --type merge \
+#     -p '{"stringData":{"localtester":"sk-localtest-abc123"}}'
+#   kubectl -n envoy-gateway-system port-forward svc/envoy-...-nebari-gateway 8443:443 &
+#   curl -k https://llm.local:8443/v1/chat/completions \
+#     --resolve llm.local:8443:127.0.0.1 \
+#     -H "Authorization: Bearer sk-localtest-abc123" -H "Content-Type: application/json" \
+#     -d '{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"hi"}]}'
+apiVersion: llm.nebari.dev/v1alpha1
+kind: PassthroughModel
+metadata:
+  name: openrouter
+  namespace: llm-operator-system # per #59 lives in the operator namespace
+spec:
+  provider:
+    hostname: openrouter.ai
+    # OpenRouter serves the OpenAI API under /api/v1.
+    schemaVersion: api/v1
+    credentialSecretName: openrouter-api-key
+  models:
+    catchAll: true
+    declared:
+      - openai/gpt-4o-mini
+  access:
+    groups:
+      - llm