test(kubernetes): scrape both metrics endpoints on a real cluster

test/e2e/kubernetes/run.sh

@@ -12,11 +12,13 @@ NAMESPACE="${NAMESPACE:-agent-platform}"
 RELEASE="${RELEASE:-agent-platform}"
 KIND_CLUSTER="${KIND_CLUSTER-agent-platform}"
 LOCAL_PORT="${LOCAL_PORT:-18080}"
+LOCAL_METRICS_PORT="${LOCAL_METRICS_PORT:-19090}"
+LOCAL_WORKER_METRICS_PORT="${LOCAL_WORKER_METRICS_PORT:-19091}"
 TAG="${TAG:-0.2.0}"
 
 # Docker-only by nature: these restart Compose services or detach container
 # networks. Everything else runs unchanged.
-FLOWS="${FLOWS:-seed session_flow mcp_flow mcp_sse_flow mcp_stdio_flow management_flow runtime_policy_flow}"
+FLOWS="${FLOWS:-seed session_flow mcp_flow mcp_sse_flow mcp_stdio_flow management_flow runtime_policy_flow metrics_flow}"
 
 cd "$REPO_ROOT"
 
@@ -40,13 +42,39 @@ if [ -n "$KIND_CLUSTER" ]; then
 fi
 
 echo "==> Installing the chart"
+# Generating fresh secrets on every run would rotate the Secret Store
+# encryption key, leaving every credential written by an earlier run
+# undecryptable, and would set a PostgreSQL password the retained volume never
+# accepted. Both fail well away from their cause, so an existing install keeps
+# the Secret it already has.
+#
+# The values are read back rather than pointed at with secrets.existingSecret,
+# which would stop the chart rendering the Secret at all and so delete it.
+SECRET_NAME="${RELEASE}-agent-platform-secrets"
+secret_value() {
+  kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" -o "jsonpath={.data.$1}" | base64 -d
+}
+if kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; then
+  echo "    reusing the secrets of the existing install"
+  SECRET_ARGS=(
+    --set secrets.secretEncryptionKey="$(secret_value SECRET_ENCRYPTION_KEY)"
+    --set secrets.workerToken="$(secret_value WORKER_TOKEN)"
+    --set postgresql.password="$(secret_value POSTGRES_PASSWORD)"
+    --set objectStore.secretKey="$(secret_value OBJECT_STORE_SECRET_KEY)"
+  )
+else
+  SECRET_ARGS=(
+    --set secrets.secretEncryptionKey="$(openssl rand -base64 32 | tr -d '=')"
+    --set secrets.workerToken="$(openssl rand -hex 24)"
+    --set postgresql.password="$(openssl rand -hex 16)"
+    --set objectStore.secretKey="$(openssl rand -hex 16)"
+  )
+fi
+
 helm upgrade --install "$RELEASE" deploy/helm/agent-platform \
   -n "$NAMESPACE" --create-namespace \
   --set image.tag="$TAG" \
-  --set secrets.secretEncryptionKey="$(openssl rand -base64 32 | tr -d '=')" \
-  --set secrets.workerToken="$(openssl rand -hex 24)" \
-  --set postgresql.password="$(openssl rand -hex 16)" \
-  --set objectStore.secretKey="$(openssl rand -hex 16)" \
+  "${SECRET_ARGS[@]}" \
   --wait --timeout 10m
 
 echo "==> Deploying the fixture Provider"
@@ -57,13 +85,57 @@ kubectl -n "$NAMESPACE" apply -f test/e2e/kubernetes/mock-provider.yaml
 kubectl -n "$NAMESPACE" rollout restart deploy/e2e-mock-provider
 kubectl -n "$NAMESPACE" rollout status deploy/e2e-mock-provider --timeout=180s
 
-echo "==> Port-forwarding the console"
+echo "==> Port-forwarding the console and both metrics endpoints"
+FORWARD_PIDS=()
+trap 'kill "${FORWARD_PIDS[@]}" 2>/dev/null || true' EXIT
+
 kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-web" \
   "${LOCAL_PORT}:8080" >/dev/null 2>&1 &
-FORWARD_PID=$!
-trap 'kill "$FORWARD_PID" 2>/dev/null || true' EXIT
-sleep 5
+FORWARD_PIDS+=($!)
+# The chart puts metrics on a port of their own on both components. Rendering
+# the templates proves the wiring exists; only a scrape proves it answers, so
+# the flow runs here as well as under Compose.
+kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-control-plane" \
+  "${LOCAL_METRICS_PORT}:9090" >/dev/null 2>&1 &
+FORWARD_PIDS+=($!)
+# The Worker's is headless: it exists only to give a scraper a stable target
+# for a Pod that takes no inbound traffic otherwise.
+kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-runtime-worker-metrics" \
+  "${LOCAL_WORKER_METRICS_PORT}:9090" >/dev/null 2>&1 &
+FORWARD_PIDS+=($!)
+
+# A port-forward whose local port is already taken dies immediately, and the
+# suite then talks to whatever else is listening — which looks like a platform
+# failure and is not one. Catch it here instead.
+sleep 2
+for pid in "${FORWARD_PIDS[@]}"; do
+  if ! kill -0 "$pid" 2>/dev/null; then
+    echo "a port-forward exited; are ${LOCAL_PORT}, ${LOCAL_METRICS_PORT}, or" \
+         "${LOCAL_WORKER_METRICS_PORT} already in use?" >&2
+    exit 1
+  fi
+done
+
+# A port-forward is accepted before it can carry a request, and the Control
+# Plane restarts until NATS accepts connections, so a fixed sleep turns a slow
+# start into a failed first flow. Wait for each endpoint to actually answer.
+wait_for() {
+  local url="$1" name="$2" attempt
+  for attempt in $(seq 60); do
+    if curl -fsS -o /dev/null --max-time 3 "$url"; then
+      return 0
+    fi
+    sleep 2
+  done
+  echo "$name did not answer at $url" >&2
+  return 1
+}
+wait_for "http://localhost:${LOCAL_PORT}/api/v1/auth/config" "the console"
+wait_for "http://localhost:${LOCAL_METRICS_PORT}/metrics" "control plane metrics"
+wait_for "http://localhost:${LOCAL_WORKER_METRICS_PORT}/metrics" "worker metrics"
 
 echo "==> Running flows"
 AGENT_PLATFORM_URL="http://localhost:${LOCAL_PORT}" \
+AGENT_PLATFORM_METRICS_URL="http://localhost:${LOCAL_METRICS_PORT}" \
+AGENT_PLATFORM_WORKER_METRICS_URL="http://localhost:${LOCAL_WORKER_METRICS_PORT}" \
   python3 test/e2e/run_all.py $FLOWS