Decouple K8s pod readiness from Ray Serve application state

.github/workflows/test.yaml

@@ -38,7 +38,11 @@ jobs:
             --set head.resources.requests.memory=512Mi \
             --set head.resources.limits.cpu=1 \
             --set head.resources.limits.memory=1Gi \
-            --set worker.replicas=0 \
+            --set worker.replicas=1 \
+            --set worker.resources.requests.cpu=250m \
+            --set worker.resources.requests.memory=512Mi \
+            --set worker.resources.limits.cpu=500m \
+            --set worker.resources.limits.memory=1Gi \
             --timeout 5m
 
       - name: Wait for Ray head pod
@@ -54,6 +58,24 @@ jobs:
           done
           kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=180s
 
+      # Exercises the fix from #7: with default values (no Serve apps deployed),
+      # the worker pod must reach Ready solely on raylet healthz. Before this
+      # chart override, KubeRay's default probe also required a Serve healthz
+      # response on localhost:8000, which never became available without an
+      # explicitly deployed Serve application.
+      - name: Wait for Ray worker pod
+        run: |
+          echo "Waiting for worker pod..."
+          for i in $(seq 1 60); do
+            if kubectl get pod -l ray.io/node-type=worker -o name 2>/dev/null | grep -q pod; then
+              echo "Worker pod found"
+              break
+            fi
+            echo "  attempt $i/60..."
+            sleep 5
+          done
+          kubectl wait --for=condition=ready pod -l ray.io/node-type=worker --timeout=240s
+
       - name: Health check Ray Dashboard
         run: |
           kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 &

chart/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: nebari-rayserve
 description: A Nebari Software Pack for Ray Serve
 type: application
-version: 0.2.0
+version: 0.3.0
 appVersion: "2.43.0"
 dependencies:
   - name: kuberay-operator

chart/templates/rayservice.yaml

@@ -11,6 +11,7 @@ spec:
   # Empty by default; users add applications via values or deploy at runtime
   # via the Ray Dashboard REST API.
   serveConfigV2: |
+    proxy_location: {{ .Values.serve.proxyLocation }}
     http_options:
       host: "0.0.0.0"
       port: 8000
@@ -44,6 +45,14 @@ spec:
                   name: client
                 - containerPort: 8000
                   name: serve
+              {{- with .Values.head.readinessProbe }}
+              readinessProbe:
+                {{- toYaml . | nindent 16 }}
+              {{- end }}
+              {{- with .Values.head.livenessProbe }}
+              livenessProbe:
+                {{- toYaml . | nindent 16 }}
+              {{- end }}
     workerGroupSpecs:
       - replicas: {{ .Values.worker.replicas }}
         minReplicas: {{ .Values.worker.minReplicas | default .Values.worker.replicas }}
@@ -64,3 +73,11 @@ spec:
                 {{- end }}
                 resources:
                   {{- toYaml .Values.worker.resources | nindent 18 }}
+                {{- with .Values.worker.readinessProbe }}
+                readinessProbe:
+                  {{- toYaml . | nindent 18 }}
+                {{- end }}
+                {{- with .Values.worker.livenessProbe }}
+                livenessProbe:
+                  {{- toYaml . | nindent 18 }}
+                {{- end }}

chart/values.yaml

@@ -46,8 +46,18 @@ kuberay-operator:
   enabled: true
 
 # =============================================================================
-# Ray Serve Applications
+# Ray Serve
 # =============================================================================
+serve:
+  # proxy_location for Ray Serve (https://docs.ray.io/en/latest/serve/architecture.html#http-proxies).
+  # EveryNode places an HTTP proxy on every Ray pod, which is typically what
+  # you want with KubeRay (each pod is its own scheduling unit, can ingress
+  # traffic, and pod-local probes can hit localhost:8000). HeadOnly runs a
+  # single proxy on the head pod — saves resources at the cost of a single
+  # ingress point, useful for very large clusters. Disabled turns HTTP off
+  # entirely (programmatic Serve handles only).
+  proxyLocation: EveryNode
+
 # Declarative Serve applications deployed via RayService serveConfigV2.
 # Each application must specify an import_path to a module:app baked into
 # the Ray image. For production, build a custom image with your model code.
@@ -85,6 +95,13 @@ head:
       cpu: "1"
       memory: "2Gi"
 
+  # Head pod readiness/liveness probes. Defaults to {} which lets KubeRay
+  # apply its built-in probes. Override here to set explicit probes; see
+  # the worker.readinessProbe / worker.livenessProbe block below for the
+  # rationale and a concrete example.
+  readinessProbe: {}
+  livenessProbe: {}
+
 # =============================================================================
 # Ray Workers
 # =============================================================================
@@ -102,6 +119,47 @@ worker:
       cpu: "1"
       memory: "2Gi"
 
+  # Worker pod readiness/liveness probes.
+  #
+  # KubeRay's default worker probes chain a raylet healthz check with
+  # `wget http://localhost:8000/-/healthz | grep success`. The Serve check
+  # requires both a deployed Serve application and a local Serve HTTP proxy,
+  # neither of which holds on a fresh cluster (default serveApplications is
+  # empty, default Ray Serve proxy_location varies by Ray version). When the
+  # Serve check fails the worker pod stays 0/1 Ready forever.
+  #
+  # The defaults below check raylet healthz alone — the Ray node is Ready
+  # when its raylet is healthy. Serve application health is tracked by the
+  # Serve controller and need not gate K8s pod readiness; the chart's
+  # serve-svc Service targets only the head pod, so worker readiness has no
+  # effect on user-visible HTTP routing.
+  #
+  # Set either probe to null (~ in YAML) to suppress it and let KubeRay's
+  # default apply. Setting to {} won't suppress — Helm's deep merge keeps
+  # existing keys when overlaying with an empty map.
+  # See https://github.com/nebari-dev/nebari-rayserve-pack/issues/7 for the
+  # full diagnosis.
+  readinessProbe:
+    exec:
+      command:
+        - bash
+        - -c
+        - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success"
+    initialDelaySeconds: 10
+    periodSeconds: 5
+    timeoutSeconds: 2
+    failureThreshold: 1
+  livenessProbe:
+    exec:
+      command:
+        - bash
+        - -c
+        - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success"
+    initialDelaySeconds: 30
+    periodSeconds: 5
+    timeoutSeconds: 2
+    failureThreshold: 120
+
 # =============================================================================
 # Overrides
 # =============================================================================