fix(operator): make per-model API-key auth work end-to-end on AI Gateway v0.5 (#116)

charts/nebari-llm-serving/crds/llmmodel-crd.yaml

@@ -1336,6 +1336,17 @@ spec:
               serving:
                 description: serving configures the vLLM serving layer
                 properties:
+                  command:
+                    description: |-
+                      command overrides the container command for the vLLM container.
+                      Defaults to the standard vLLM OpenAI-server entrypoint
+                      (python3 -m vllm.entrypoints.openai.api_server), which the default
+                      serving image requires because its entrypoint is the NVIDIA CUDA
+                      wrapper with no default CMD. Set this when a custom serving image
+                      needs a different launcher; vllmArgs are appended as arguments.
+                    items:
+                      type: string
+                    type: array
                   dataParallelism:
                     default: 1
                     description: dataParallelism controls data parallelism

charts/nebari-llm-serving/templates/operator-clusterrole.yaml

@@ -41,6 +41,13 @@ rules:
   - apiGroups: ["inference.networking.k8s.io"]
     resources: ["inferencepools"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # The operator creates a per-model EPP Role granting these
+  # inference.networking.x-k8s.io permissions (see reconcilers/inferencepool.go).
+  # RBAC privilege-escalation prevention requires the operator to already hold
+  # any permission it grants, so it must carry them here too.
+  - apiGroups: ["inference.networking.x-k8s.io"]
+    resources: ["inferenceobjectives", "inferencemodelrewrites", "inferencepools", "inferencepoolimports"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["aigateway.envoyproxy.io"]
     resources: ["aigatewayroutes", "aiservicebackends", "backendsecuritypolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

dev/manifests/operator.yaml

@@ -56,6 +56,14 @@ rules:
   - apiGroups: ["inference.networking.k8s.io"]
     resources: ["inferencepools"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # The operator creates a per-model EPP Role granting these
+  # inference.networking.x-k8s.io permissions (see reconcilers/inferencepool.go).
+  # RBAC privilege-escalation prevention requires the operator to already hold
+  # any permission it grants, so it must carry them here too (mirrors the Helm
+  # chart ClusterRole; #121).
+  - apiGroups: ["inference.networking.x-k8s.io"]
+    resources: ["inferenceobjectives", "inferencemodelrewrites", "inferencepools", "inferencepoolimports"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["aigateway.envoyproxy.io"]
     resources: ["aigatewayroutes", "aiservicebackends", "backendsecuritypolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

dev/manifests/passthrough-test-model.yaml

@@ -5,7 +5,9 @@
 #
 # Once reconciled (kubectl -n llm-operator-system get passthroughmodel), reach
 # it through the gateway. The external endpoint uses API-key auth, so inject a
-# client key into the api-keys Secret to skip the key-manager:
+# client key into the api-keys Secret to skip the key-manager. A curl right
+# after the patch can 403 until the operator re-renders the SecurityPolicy
+# allow-list from the Secret (a few seconds) - retry on 403:
 #   kubectl -n llm-operator-system patch secret openrouter-api-keys --type merge \
 #     -p '{"stringData":{"localtester":"sk-localtest-abc123"}}'
 #   kubectl -n envoy-gateway-system port-forward svc/envoy-...-nebari-gateway 8443:443 &

dev/manifests/test-model.yaml

@@ -23,6 +23,10 @@ spec:
       memory: "256Mi"
   serving:
     image: mock-vllm:dev
+    # The operator defaults the container command to the real vLLM entrypoint
+    # (required by the llm-d-cuda serving image); the mock image runs a plain
+    # Python HTTP server instead, so override it here.
+    command: ["python", "/server.py"]
     replicas: 1
     monitoring:
       enabled: false

dev/mock-vllm/server.py

@@ -1,4 +1,4 @@
-"""Mock vLLM server that responds to health check and model list endpoints."""
+"""Mock vLLM server that responds to health check, model list, and chat endpoints."""
 import json
 from http.server import HTTPServer, BaseHTTPRequestHandler
 
@@ -19,6 +19,32 @@ def do_GET(self):
             self.send_response(404)
             self.end_headers()
 
+    def do_POST(self):
+        # Minimal OpenAI-compatible chat-completion response so the dev stack can
+        # exercise the full request path (gateway auth/routing -> backend -> 200)
+        # without a real model.
+        if self.path in ("/v1/chat/completions", "/v1/completions"):
+            length = int(self.headers.get("Content-Length", 0) or 0)
+            if length:
+                self.rfile.read(length)
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            body = json.dumps({
+                "id": "chatcmpl-mock",
+                "object": "chat.completion",
+                "model": "test/tiny-model",
+                "choices": [{
+                    "index": 0,
+                    "message": {"role": "assistant", "content": "ok"},
+                    "finish_reason": "stop",
+                }],
+            })
+            self.wfile.write(body.encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
     def log_message(self, format, *args):
         pass  # suppress logs