diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py index 0140863d91..356c2c9329 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py @@ -138,24 +138,9 @@ def _delivery_script(path: str, mode: int) -> str: # readinessProbe uses its own timeout_seconds instead. _DEFAULT_READINESS_TIMEOUT_SECONDS = 3 -# Headroom added to a network probe's own timeout when bounding the ExecSandbox RPC, so -# the RPC outlives the in-sandbox python timeout and captures its (non-zero) exit rather -# than being cut off first. _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS = 5 -# Loopback readiness probe programs, run by the sandbox's python. urlopen raises on a -# refused connection or an HTTP status >= 400 (so a still-starting 503 reads as not -# ready); create_connection raises until the socket is actually bound. The HTTP probe -# uses an unverified TLS context so an https readinessProbe against a loopback/self-signed -# cert is not rejected on verification (matching Kubernetes httpGet HTTPS probe semantics); -# the context is ignored for plain http, so one program covers both schemes. -_HTTP_PROBE_PROGRAM = ( - "import sys, ssl, urllib.request; " - "urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[2]), context=ssl._create_unverified_context())" -) -_TCP_PROBE_PROGRAM = ( - "import sys, socket; socket.create_connection((sys.argv[1], int(sys.argv[2])), timeout=float(sys.argv[3])).close()" -) +_CURL_ABS = "/usr/bin/curl" # Cached on first status read; needs the proto enums so it cannot be built at import # time (see _ensure_openshell). None until built. @@ -842,21 +827,16 @@ def _default_probe_port(container: Container) -> int | None: return None -def _loopback_probe_script(program: str, *args: str) -> str: - """Wrap a python probe *program* so it runs against the sandbox's own python. - - Selects ``python3`` then ``python`` off PATH and runs *program* with *args*, exiting - 0 when the probe connects and nonzero when it does not. If neither interpreter is on - PATH the probe cannot run, so it exits 0 to preserve the prior expose-on-alive - behaviour rather than wedging a workload in STARTING until the progress deadline. - """ - quoted_args = " ".join(shlex.quote(arg) for arg in args) - return ( - "if command -v python3 >/dev/null 2>&1; then _py=python3; " - "elif command -v python >/dev/null 2>&1; then _py=python; " - "else exit 0; fi; " - f'"$_py" -c {shlex.quote(program)} {quoted_args}' +def _curl_probe_script(curl_args: str, *, tcp_connect: bool = False) -> str: + preamble = ( + f"if [ -x {_CURL_ABS} ]; then _curl={_CURL_ABS}; " + "elif command -v curl >/dev/null 2>&1; then _curl=curl; " + "else exit 1; fi; " ) + invocation = f'"$_curl" {curl_args}' + if tcp_connect: + return f'{preamble}{invocation}; _rc=$?; [ "$_rc" -eq 7 ] && exit 1; exit 0' + return preamble + invocation def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] | None: @@ -870,7 +850,7 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] ``exec_timeout_seconds`` bounds the ExecSandbox RPC so a hung probe cannot stall the serial reconcile loop: an exec probe is bounded by its own ``timeoutSeconds``; a - network probe self-times in python, so the RPC is given that timeout plus headroom. + network probe self-times in curl, so the RPC is given that timeout plus headroom. """ probe = container.readiness_probe timeout = probe.timeout_seconds if probe is not None else _DEFAULT_READINESS_TIMEOUT_SECONDS @@ -888,8 +868,11 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] return None path = probe.http_get.path if probe.http_get.path.startswith("/") else f"/{probe.http_get.path}" url = f"{probe.http_get.scheme.lower()}://127.0.0.1:{port}{path}" - script = _loopback_probe_script(_HTTP_PROBE_PROGRAM, url, str(timeout)) - return ["/bin/sh", "-c", script], f"httpGet {url}", network_exec_timeout + args = ( + f"--fail --silent --show-error --insecure --noproxy '*' " + f"--max-time {shlex.quote(str(timeout))} --output /dev/null {shlex.quote(url)}" + ) + return ["/bin/sh", "-c", _curl_probe_script(args)], f"httpGet {url}", network_exec_timeout if probe is not None and probe.tcp_socket is not None: port = _resolve_probe_port(probe.tcp_socket.port, container) or _default_probe_port(container) @@ -897,7 +880,9 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] port = _default_probe_port(container) if port is None: return None - script = _loopback_probe_script(_TCP_PROBE_PROGRAM, "127.0.0.1", str(port), str(timeout)) + url = f"http://127.0.0.1:{port}/" + args = f"--silent --output /dev/null --noproxy '*' --max-time {shlex.quote(str(timeout))} {shlex.quote(url)}" + script = _curl_probe_script(args, tcp_connect=True) return ["/bin/sh", "-c", script], f"tcp 127.0.0.1:{port}", network_exec_timeout diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md b/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md index 01b0f01a7a..70736e087d 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md @@ -272,7 +272,7 @@ URL=$(curl -sf "$BASE/deployments/igw-agent" | jq -r '.endpoints[0].url') echo "$URL" # e.g. http://--.openshell.localhost:17670/ ``` -If the status reaches `FAILED` or never leaves `PENDING`/`STARTING`, jump to the recovery table below. Do not proceed to invoke until `READY`. `READY` gates on the sandbox being up, the serve launcher exiting cleanly, and the serve process still being alive on the poll, but not on `nat serve` having finished binding its port, so give serve a moment after `READY`. A serve process that exits (bad config, unresolvable model) flips the deployment to `FAILED` on the next poll, with the tail of its log in the status message. +If the status reaches `FAILED` or never leaves `PENDING`/`STARTING`, jump to the recovery table below. Do not proceed to invoke until `READY`. `READY` gates on the sandbox being up, the serve launcher exiting cleanly, the serve process still being alive on the poll, and `nat serve` answering a reachability probe on its port, so the endpoint is serving as soon as it reports `READY`. A serve process that exits (bad config, unresolvable model) flips the deployment to `FAILED` on the next poll, with the tail of its log in the status message. ## Step 6: Invoke the agent @@ -286,7 +286,7 @@ curl -sf -X POST "${URL}generate" -H 'content-type: application/json' \ Other routes the served workflow exposes: `POST /v1/chat/completions`, `/chat`, `/v1/workflow`, and `/generate/stream`. -Verification: the response must contain a non-empty `value`. A `502 Service endpoint is not reachable` means the gateway reached the sandbox but `nat serve` (port 9000) is not answering, because it crashed or has not come up. Give it 30-60s and retry; if it persists, read the serve log inside the sandbox: +Verification: the response must contain a non-empty `value`. A `502 Service endpoint is not reachable` means `nat serve` (port 9000) is not answering. Read the serve log inside the sandbox: ```bash SBX=$(curl -sf "$BASE/deployments/igw-agent" | jq -r '.endpoints[0].url' | sed -E 's#^https?://##; s#--.*##') @@ -345,7 +345,7 @@ docker compose -f plugins/nemo-deployments/examples/openshell/docker-compose.yml | `openshell gateway list` shows the endpoint on `:8080` | Gateway collides with the platform port | Recreate the gateway on `:17670` (docker driver, plaintext) per Pre-flight step 1; `:8080` belongs to the platform | | Deploy stuck `PENDING`/`STARTING` | Image missing sandbox user, or serve command wrong | `openshell sandbox list`; inspect the sandbox; confirm Step 2 used `--sandbox-runtime openshell` | | Deploy `FAILED` | Policy or gateway rejected the sandbox, the serve launcher exited non-zero, or the serve process died after launch | Check platform logs (`nemo services logs -n 100`) and the deployments reconciler output; the FAILED status message carries the launcher exit detail or the serve log tail. `nemo deployments logs ` returns the workload's own log | -| Invoke returns `502 Service endpoint is not reachable` | `nat serve` is alive but has not bound `:9000` yet | Retry after 30-60s. If it persists while the deployment stays `READY`, read the serve log with `nemo deployments logs `, or in the sandbox: `openshell sandbox exec --name -- cat /tmp/nemo-serve.log`. Common causes: baked config path does not match `--config_file`; a leftover `general.telemetry` block; or a `model_name` the Inference Gateway cannot resolve. Confirm the config is where serve looks (`openshell sandbox exec --name -- ls /workspace` -> `config.yaml`) and that `MODEL` exists (`nemo models list --all-pages`) | +| Invoke returns `502 Service endpoint is not reachable` | The deployment is not `READY`, or `nat serve` died after coming up | Wait for `READY` before invoking; if a `READY` deployment still 502s, read the serve log (`nemo deployments logs `, or `openshell sandbox exec --name -- cat /tmp/nemo-serve.log`). A bad config path, a leftover `general.telemetry` block, or an unresolvable `model_name` surfaces as `FAILED`, not a 502 | | Invoke returns 503 `inference service unavailable` (in the response or the serve log) | The `inference.local` route resolved, but the gateway's upstream hop to the platform failed | Almost always a platform bound to `127.0.0.1`; restart it with `--host 0.0.0.0`. Confirm the address the sandbox actually dials with `openshell sandbox exec --name -- cat /etc/hosts` (look for `host.openshell.internal`) and check the platform answers there (`curl -sf http://:8080/health/ready`). A clean `openshell inference set` does NOT rule this out: it validates from the host | | Invoke returns empty/error `value`, or serve log shows connection refused to `inference.local` | `inference.local` route not wired, or model misbehaved | Confirm `openshell inference get` shows the `nemo-igw` provider + your model; re-run Step 1's `provider create` / `inference set`; try a gpt-4o-mini-class model | | `example.com` reachable in Step 7 | Egress policy not applied | Confirm executor is `openshell-local` and the generated default-deny policy is attached to the sandbox | diff --git a/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py b/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py index 11f0f606a0..1875a07b03 100644 --- a/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py +++ b/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py @@ -117,3 +117,54 @@ async def test_openshell_roundtrip_serves_http() -> None: finally: await backend.delete_deployment("itest", "rt") backend.shutdown() + + +async def test_openshell_readiness_holds_until_bind() -> None: + # The workload sleeps before it binds, so READY must not be published until the curl + # readiness probe can actually reach :8000. Regression guard for AIRCORE-998 (READY + # published ahead of bind) that also exercises the real in-sandbox curl probe. + backend, mock_entities = _make_backend() + bind_delay = 15 + mock_entities.get.return_value = DeploymentConfig( + name="slow-cfg", + workspace="itest", + containers=[ + Container( + name="web", + image=IMAGE, + command=["/bin/sh", "-c"], + args=[f"sleep {bind_delay}; exec python3 -m http.server 8000 --bind 0.0.0.0"], + ports=[ContainerPort(containerPort=8000, name="http")], + ) + ], + ) + + try: + created = await backend.create_deployment( + workspace="itest", + name="slow", + config_name="slow-cfg", + labels={"managed-by": MANAGED_BY_LABEL}, + backend_config={}, + ) + assert created.status == "STARTING", created.status_message + + start = time.monotonic() + saw_starting = False + status = None + for _ in range(90): + status = await backend.read_status(workspace="itest", name="slow") + if status.status != "STARTING": + break + saw_starting = True + await asyncio.sleep(1.0) + + assert status is not None and status.status == "READY", status.status_message if status else "no status" + assert saw_starting, "expected STARTING before READY" + # Serve only starts once provisioning completes, so time-to-READY strictly exceeds the + # sleep. A probe that failed open would publish READY during the sleep window. + assert time.monotonic() - start >= bind_delay - 3, "READY published before the workload bound its port" + assert status.endpoints, "expected an exposed endpoint" + finally: + await backend.delete_deployment("itest", "slow") + backend.shutdown() diff --git a/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py index 06f8d9dc29..705c1099da 100644 --- a/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py +++ b/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py @@ -22,6 +22,7 @@ ) from nemo_deployments_plugin.backends.openshell.backend import ( _CONFIG_DELIVERED_MARKER, + _CURL_ABS, _DEFAULT_READINESS_TIMEOUT_SECONDS, _LAUNCH_MARKER, _LIVENESS_PROBE, @@ -769,7 +770,11 @@ def test_readiness_probe_command_defaults_to_tcp_on_the_first_port() -> None: assert command[:2] == ["/bin/sh", "-c"] assert "127.0.0.1" in command[2] assert description == "tcp 127.0.0.1:8000" - # A network probe self-times in python; the RPC gets that timeout plus headroom. + assert "127.0.0.1:8000" in command[2] + assert _CURL_ABS in command[2] + assert "--noproxy '*'" in command[2] + assert '[ "$_rc" -eq 7 ] && exit 1' in command[2] + assert "python" not in command[2] assert timeout == _DEFAULT_READINESS_TIMEOUT_SECONDS + _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS @@ -777,9 +782,22 @@ def test_readiness_probe_command_uses_declared_httpget() -> None: container = _config_with_readiness(Probe(httpGet=HTTPGetAction(path="/ready", port=8000))).containers[0] command, description, _timeout = _require_probe_command(container) assert "http://127.0.0.1:8000/ready" in command[2] + assert "--fail" in command[2] + assert "--noproxy '*'" in command[2] + assert _CURL_ABS in command[2] assert description == "httpGet http://127.0.0.1:8000/ready" +def test_readiness_probe_command_fails_closed_when_curl_is_missing() -> None: + tcp = _require_probe_command(_config().containers[0])[0][2] + http = _require_probe_command( + _config_with_readiness(Probe(httpGet=HTTPGetAction(path="/health", port=8000))).containers[0] + )[0][2] + for script in (tcp, http): + assert "else exit 1; fi" in script + assert "else exit 0" not in script + + def test_readiness_probe_command_runs_declared_exec_directly() -> None: probe = Probe(exec=ExecAction(command=["/bin/true"]), timeoutSeconds=4) container = _config_with_readiness(probe).containers[0] @@ -807,8 +825,7 @@ def test_readiness_probe_command_https_uses_unverified_context() -> None: ).containers[0] command, _description, _timeout = _require_probe_command(container) assert "https://127.0.0.1:8000/health" in command[2] - # An https probe against a loopback/self-signed cert must not fail verification. - assert "_create_unverified_context" in command[2] + assert "--insecure" in command[2] def test_readiness_probe_command_normalizes_httpget_path_without_leading_slash() -> None: