From 5ce6bdb99f4570c783d68a03913b738193930c78 Mon Sep 17 00:00:00 2001 From: Oren Fromberg Date: Fri, 1 May 2026 07:04:50 -0400 Subject: [PATCH] Decouple K8s pod readiness from Ray Serve application state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Override KubeRay's default worker readiness/liveness probes which chain a raylet healthz check with `wget http://localhost:8000/-/healthz | grep success`. The Serve check requires a deployed Serve application AND a Serve HTTP proxy on the local pod — neither holds on a fresh cluster install (default `serveApplications: []`, default Ray Serve `proxy_location: HeadOnly` in Ray ≥2.10), so the worker stays 0/1 Ready indefinitely and KubeRay can't complete its rollout. Kubernetes pod readiness should reflect whether the Ray node itself is healthy (raylet up, resources allocated). Serve application health is tracked separately by the Serve controller and need not gate K8s readiness. The chart's serve-svc Service already targets only the head pod, so worker readiness state has no effect on user-visible HTTP routing. Changes: - chart/values.yaml: add `serve.proxyLocation` (default `EveryNode`), `worker.readinessProbe`, `worker.livenessProbe`, `head.readinessProbe`, `head.livenessProbe`. All probe fields are tunable; setting either worker probe to null (~) suppresses it and lets KubeRay's default apply. Head probes default to {} (KubeRay default applies); users can set explicit probes to override. - chart/templates/rayservice.yaml: template proxy_location from values; conditionally render head/worker probes from values via `with`. - chart/Chart.yaml: bump version 0.2.0 → 0.3.0. - .github/workflows/test.yaml: bump worker.replicas from 0 to 1 and add a wait for worker Ready condition. The previous test only ran with zero workers, so it didn't exercise the failure mode this PR fixes. Closes #7 --- .github/workflows/test.yaml | 24 ++++++++++++- chart/Chart.yaml | 2 +- chart/templates/rayservice.yaml | 17 ++++++++++ chart/values.yaml | 60 ++++++++++++++++++++++++++++++++- 4 files changed, 100 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9099d8..bc84c94 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,7 +38,11 @@ jobs: --set head.resources.requests.memory=512Mi \ --set head.resources.limits.cpu=1 \ --set head.resources.limits.memory=1Gi \ - --set worker.replicas=0 \ + --set worker.replicas=1 \ + --set worker.resources.requests.cpu=250m \ + --set worker.resources.requests.memory=512Mi \ + --set worker.resources.limits.cpu=500m \ + --set worker.resources.limits.memory=1Gi \ --timeout 5m - name: Wait for Ray head pod @@ -54,6 +58,24 @@ jobs: done kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=180s + # Exercises the fix from #7: with default values (no Serve apps deployed), + # the worker pod must reach Ready solely on raylet healthz. Before this + # chart override, KubeRay's default probe also required a Serve healthz + # response on localhost:8000, which never became available without an + # explicitly deployed Serve application. + - name: Wait for Ray worker pod + run: | + echo "Waiting for worker pod..." + for i in $(seq 1 60); do + if kubectl get pod -l ray.io/node-type=worker -o name 2>/dev/null | grep -q pod; then + echo "Worker pod found" + break + fi + echo " attempt $i/60..." + sleep 5 + done + kubectl wait --for=condition=ready pod -l ray.io/node-type=worker --timeout=240s + - name: Health check Ray Dashboard run: | kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 & diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 482e3ec..c074c9c 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nebari-rayserve description: A Nebari Software Pack for Ray Serve type: application -version: 0.2.0 +version: 0.3.0 appVersion: "2.43.0" dependencies: - name: kuberay-operator diff --git a/chart/templates/rayservice.yaml b/chart/templates/rayservice.yaml index 24e6ecb..c05ce9f 100644 --- a/chart/templates/rayservice.yaml +++ b/chart/templates/rayservice.yaml @@ -11,6 +11,7 @@ spec: # Empty by default; users add applications via values or deploy at runtime # via the Ray Dashboard REST API. serveConfigV2: | + proxy_location: {{ .Values.serve.proxyLocation }} http_options: host: "0.0.0.0" port: 8000 @@ -44,6 +45,14 @@ spec: name: client - containerPort: 8000 name: serve + {{- with .Values.head.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 16 }} + {{- end }} + {{- with .Values.head.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 16 }} + {{- end }} workerGroupSpecs: - replicas: {{ .Values.worker.replicas }} minReplicas: {{ .Values.worker.minReplicas | default .Values.worker.replicas }} @@ -64,3 +73,11 @@ spec: {{- end }} resources: {{- toYaml .Values.worker.resources | nindent 18 }} + {{- with .Values.worker.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 18 }} + {{- end }} + {{- with .Values.worker.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 18 }} + {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index bb95190..2a4c92f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -46,8 +46,18 @@ kuberay-operator: enabled: true # ============================================================================= -# Ray Serve Applications +# Ray Serve # ============================================================================= +serve: + # proxy_location for Ray Serve (https://docs.ray.io/en/latest/serve/architecture.html#http-proxies). + # EveryNode places an HTTP proxy on every Ray pod, which is typically what + # you want with KubeRay (each pod is its own scheduling unit, can ingress + # traffic, and pod-local probes can hit localhost:8000). HeadOnly runs a + # single proxy on the head pod — saves resources at the cost of a single + # ingress point, useful for very large clusters. Disabled turns HTTP off + # entirely (programmatic Serve handles only). + proxyLocation: EveryNode + # Declarative Serve applications deployed via RayService serveConfigV2. # Each application must specify an import_path to a module:app baked into # the Ray image. For production, build a custom image with your model code. @@ -85,6 +95,13 @@ head: cpu: "1" memory: "2Gi" + # Head pod readiness/liveness probes. Defaults to {} which lets KubeRay + # apply its built-in probes. Override here to set explicit probes; see + # the worker.readinessProbe / worker.livenessProbe block below for the + # rationale and a concrete example. + readinessProbe: {} + livenessProbe: {} + # ============================================================================= # Ray Workers # ============================================================================= @@ -102,6 +119,47 @@ worker: cpu: "1" memory: "2Gi" + # Worker pod readiness/liveness probes. + # + # KubeRay's default worker probes chain a raylet healthz check with + # `wget http://localhost:8000/-/healthz | grep success`. The Serve check + # requires both a deployed Serve application and a local Serve HTTP proxy, + # neither of which holds on a fresh cluster (default serveApplications is + # empty, default Ray Serve proxy_location varies by Ray version). When the + # Serve check fails the worker pod stays 0/1 Ready forever. + # + # The defaults below check raylet healthz alone — the Ray node is Ready + # when its raylet is healthy. Serve application health is tracked by the + # Serve controller and need not gate K8s pod readiness; the chart's + # serve-svc Service targets only the head pod, so worker readiness has no + # effect on user-visible HTTP routing. + # + # Set either probe to null (~ in YAML) to suppress it and let KubeRay's + # default apply. Setting to {} won't suppress — Helm's deep merge keeps + # existing keys when overlaying with an empty map. + # See https://github.com/nebari-dev/nebari-rayserve-pack/issues/7 for the + # full diagnosis. + readinessProbe: + exec: + command: + - bash + - -c + - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success" + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 1 + livenessProbe: + exec: + command: + - bash + - -c + - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success" + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 120 + # ============================================================================= # Overrides # =============================================================================