diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9099d8..bc84c94 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,7 +38,11 @@ jobs: --set head.resources.requests.memory=512Mi \ --set head.resources.limits.cpu=1 \ --set head.resources.limits.memory=1Gi \ - --set worker.replicas=0 \ + --set worker.replicas=1 \ + --set worker.resources.requests.cpu=250m \ + --set worker.resources.requests.memory=512Mi \ + --set worker.resources.limits.cpu=500m \ + --set worker.resources.limits.memory=1Gi \ --timeout 5m - name: Wait for Ray head pod @@ -54,6 +58,24 @@ jobs: done kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=180s + # Exercises the fix from #7: with default values (no Serve apps deployed), + # the worker pod must reach Ready solely on raylet healthz. Before this + # chart override, KubeRay's default probe also required a Serve healthz + # response on localhost:8000, which never became available without an + # explicitly deployed Serve application. + - name: Wait for Ray worker pod + run: | + echo "Waiting for worker pod..." + for i in $(seq 1 60); do + if kubectl get pod -l ray.io/node-type=worker -o name 2>/dev/null | grep -q pod; then + echo "Worker pod found" + break + fi + echo " attempt $i/60..." + sleep 5 + done + kubectl wait --for=condition=ready pod -l ray.io/node-type=worker --timeout=240s + - name: Health check Ray Dashboard run: | kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 & diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 482e3ec..c074c9c 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nebari-rayserve description: A Nebari Software Pack for Ray Serve type: application -version: 0.2.0 +version: 0.3.0 appVersion: "2.43.0" dependencies: - name: kuberay-operator diff --git a/chart/templates/rayservice.yaml b/chart/templates/rayservice.yaml index 24e6ecb..c05ce9f 100644 --- a/chart/templates/rayservice.yaml +++ b/chart/templates/rayservice.yaml @@ -11,6 +11,7 @@ spec: # Empty by default; users add applications via values or deploy at runtime # via the Ray Dashboard REST API. serveConfigV2: | + proxy_location: {{ .Values.serve.proxyLocation }} http_options: host: "0.0.0.0" port: 8000 @@ -44,6 +45,14 @@ spec: name: client - containerPort: 8000 name: serve + {{- with .Values.head.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 16 }} + {{- end }} + {{- with .Values.head.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 16 }} + {{- end }} workerGroupSpecs: - replicas: {{ .Values.worker.replicas }} minReplicas: {{ .Values.worker.minReplicas | default .Values.worker.replicas }} @@ -64,3 +73,11 @@ spec: {{- end }} resources: {{- toYaml .Values.worker.resources | nindent 18 }} + {{- with .Values.worker.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 18 }} + {{- end }} + {{- with .Values.worker.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 18 }} + {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index bb95190..2a4c92f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -46,8 +46,18 @@ kuberay-operator: enabled: true # ============================================================================= -# Ray Serve Applications +# Ray Serve # ============================================================================= +serve: + # proxy_location for Ray Serve (https://docs.ray.io/en/latest/serve/architecture.html#http-proxies). + # EveryNode places an HTTP proxy on every Ray pod, which is typically what + # you want with KubeRay (each pod is its own scheduling unit, can ingress + # traffic, and pod-local probes can hit localhost:8000). HeadOnly runs a + # single proxy on the head pod — saves resources at the cost of a single + # ingress point, useful for very large clusters. Disabled turns HTTP off + # entirely (programmatic Serve handles only). + proxyLocation: EveryNode + # Declarative Serve applications deployed via RayService serveConfigV2. # Each application must specify an import_path to a module:app baked into # the Ray image. For production, build a custom image with your model code. @@ -85,6 +95,13 @@ head: cpu: "1" memory: "2Gi" + # Head pod readiness/liveness probes. Defaults to {} which lets KubeRay + # apply its built-in probes. Override here to set explicit probes; see + # the worker.readinessProbe / worker.livenessProbe block below for the + # rationale and a concrete example. + readinessProbe: {} + livenessProbe: {} + # ============================================================================= # Ray Workers # ============================================================================= @@ -102,6 +119,47 @@ worker: cpu: "1" memory: "2Gi" + # Worker pod readiness/liveness probes. + # + # KubeRay's default worker probes chain a raylet healthz check with + # `wget http://localhost:8000/-/healthz | grep success`. The Serve check + # requires both a deployed Serve application and a local Serve HTTP proxy, + # neither of which holds on a fresh cluster (default serveApplications is + # empty, default Ray Serve proxy_location varies by Ray version). When the + # Serve check fails the worker pod stays 0/1 Ready forever. + # + # The defaults below check raylet healthz alone — the Ray node is Ready + # when its raylet is healthy. Serve application health is tracked by the + # Serve controller and need not gate K8s pod readiness; the chart's + # serve-svc Service targets only the head pod, so worker readiness has no + # effect on user-visible HTTP routing. + # + # Set either probe to null (~ in YAML) to suppress it and let KubeRay's + # default apply. Setting to {} won't suppress — Helm's deep merge keeps + # existing keys when overlaying with an empty map. + # See https://github.com/nebari-dev/nebari-rayserve-pack/issues/7 for the + # full diagnosis. + readinessProbe: + exec: + command: + - bash + - -c + - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success" + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 1 + livenessProbe: + exec: + command: + - bash + - -c + - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success" + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 120 + # ============================================================================= # Overrides # =============================================================================