From adba8eead3de65e81e75e1731c92f94dedf9448b Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Wed, 12 Aug 2026 13:55:55 -0400 Subject: [PATCH 1/2] fix(nemo-deployments): gate OpenShell READY on a curl reachability probe The readiness gate added in #1139 never ran. It located an interpreter with `command -v python3 || command -v python`, but packaged agent images ship Python only in /workspace/.venv, which is not on the exec PATH. With no interpreter found, the probe took its `else exit 0` branch, which the gate reads as reachable, so READY was published about 14s before nat serve bound its port. Probe with curl instead. curl is present in these images (the NAT and Fabric Dockerfiles install it, and the sandbox policy pins /usr/bin/curl) and is resolved at that absolute path with a PATH fallback. A missing curl now exits nonzero, so the gate fails closed. - httpGet: curl --fail --insecure (5xx not-ready, self-signed loopback ok) - tcpSocket and default: loopback connect classified by curl exit code (7 = refused = not-ready, any other exit = ready) - deploy-sandbox SKILL: drop the "retry a 502 for 30-60s" guidance, since READY now means the port is reachable - integration: add a slow-bind regression test that asserts READY waits for bind Fixes AIRCORE-998. Signed-off-by: Max Dubrinsky --- .../backends/openshell/backend.py | 53 +++++++------------ .../skills/deploy-sandbox/SKILL.md | 4 +- .../openshell/test_openshell_backend.py | 51 ++++++++++++++++++ .../unit/backends/openshell/test_backend.py | 21 ++++++-- 4 files changed, 90 insertions(+), 39 deletions(-) diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py index 0140863d91..bf47f3afb2 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py @@ -138,24 +138,9 @@ def _delivery_script(path: str, mode: int) -> str: # readinessProbe uses its own timeout_seconds instead. _DEFAULT_READINESS_TIMEOUT_SECONDS = 3 -# Headroom added to a network probe's own timeout when bounding the ExecSandbox RPC, so -# the RPC outlives the in-sandbox python timeout and captures its (non-zero) exit rather -# than being cut off first. _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS = 5 -# Loopback readiness probe programs, run by the sandbox's python. urlopen raises on a -# refused connection or an HTTP status >= 400 (so a still-starting 503 reads as not -# ready); create_connection raises until the socket is actually bound. The HTTP probe -# uses an unverified TLS context so an https readinessProbe against a loopback/self-signed -# cert is not rejected on verification (matching Kubernetes httpGet HTTPS probe semantics); -# the context is ignored for plain http, so one program covers both schemes. -_HTTP_PROBE_PROGRAM = ( - "import sys, ssl, urllib.request; " - "urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[2]), context=ssl._create_unverified_context())" -) -_TCP_PROBE_PROGRAM = ( - "import sys, socket; socket.create_connection((sys.argv[1], int(sys.argv[2])), timeout=float(sys.argv[3])).close()" -) +_CURL_ABS = "/usr/bin/curl" # Cached on first status read; needs the proto enums so it cannot be built at import # time (see _ensure_openshell). None until built. @@ -842,21 +827,16 @@ def _default_probe_port(container: Container) -> int | None: return None -def _loopback_probe_script(program: str, *args: str) -> str: - """Wrap a python probe *program* so it runs against the sandbox's own python. - - Selects ``python3`` then ``python`` off PATH and runs *program* with *args*, exiting - 0 when the probe connects and nonzero when it does not. If neither interpreter is on - PATH the probe cannot run, so it exits 0 to preserve the prior expose-on-alive - behaviour rather than wedging a workload in STARTING until the progress deadline. - """ - quoted_args = " ".join(shlex.quote(arg) for arg in args) - return ( - "if command -v python3 >/dev/null 2>&1; then _py=python3; " - "elif command -v python >/dev/null 2>&1; then _py=python; " - "else exit 0; fi; " - f'"$_py" -c {shlex.quote(program)} {quoted_args}' +def _curl_probe_script(curl_args: str, *, tcp_connect: bool = False) -> str: + preamble = ( + f"if [ -x {_CURL_ABS} ]; then _curl={_CURL_ABS}; " + "elif command -v curl >/dev/null 2>&1; then _curl=curl; " + "else exit 1; fi; " ) + invocation = f'"$_curl" {curl_args}' + if tcp_connect: + return f'{preamble}{invocation}; _rc=$?; [ "$_rc" -eq 7 ] && exit 1; exit 0' + return preamble + invocation def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] | None: @@ -870,7 +850,7 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] ``exec_timeout_seconds`` bounds the ExecSandbox RPC so a hung probe cannot stall the serial reconcile loop: an exec probe is bounded by its own ``timeoutSeconds``; a - network probe self-times in python, so the RPC is given that timeout plus headroom. + network probe self-times in curl, so the RPC is given that timeout plus headroom. """ probe = container.readiness_probe timeout = probe.timeout_seconds if probe is not None else _DEFAULT_READINESS_TIMEOUT_SECONDS @@ -888,8 +868,11 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] return None path = probe.http_get.path if probe.http_get.path.startswith("/") else f"/{probe.http_get.path}" url = f"{probe.http_get.scheme.lower()}://127.0.0.1:{port}{path}" - script = _loopback_probe_script(_HTTP_PROBE_PROGRAM, url, str(timeout)) - return ["/bin/sh", "-c", script], f"httpGet {url}", network_exec_timeout + args = ( + f"--fail --silent --show-error --insecure --max-time {shlex.quote(str(timeout))} " + f"--output /dev/null {shlex.quote(url)}" + ) + return ["/bin/sh", "-c", _curl_probe_script(args)], f"httpGet {url}", network_exec_timeout if probe is not None and probe.tcp_socket is not None: port = _resolve_probe_port(probe.tcp_socket.port, container) or _default_probe_port(container) @@ -897,7 +880,9 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] port = _default_probe_port(container) if port is None: return None - script = _loopback_probe_script(_TCP_PROBE_PROGRAM, "127.0.0.1", str(port), str(timeout)) + url = f"http://127.0.0.1:{port}/" + args = f"--silent --output /dev/null --max-time {shlex.quote(str(timeout))} {shlex.quote(url)}" + script = _curl_probe_script(args, tcp_connect=True) return ["/bin/sh", "-c", script], f"tcp 127.0.0.1:{port}", network_exec_timeout diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md b/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md index 01b0f01a7a..6f9b171e18 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md @@ -286,7 +286,7 @@ curl -sf -X POST "${URL}generate" -H 'content-type: application/json' \ Other routes the served workflow exposes: `POST /v1/chat/completions`, `/chat`, `/v1/workflow`, and `/generate/stream`. -Verification: the response must contain a non-empty `value`. A `502 Service endpoint is not reachable` means the gateway reached the sandbox but `nat serve` (port 9000) is not answering, because it crashed or has not come up. Give it 30-60s and retry; if it persists, read the serve log inside the sandbox: +Verification: the response must contain a non-empty `value`. A `502 Service endpoint is not reachable` means `nat serve` (port 9000) is not answering. Read the serve log inside the sandbox: ```bash SBX=$(curl -sf "$BASE/deployments/igw-agent" | jq -r '.endpoints[0].url' | sed -E 's#^https?://##; s#--.*##') @@ -345,7 +345,7 @@ docker compose -f plugins/nemo-deployments/examples/openshell/docker-compose.yml | `openshell gateway list` shows the endpoint on `:8080` | Gateway collides with the platform port | Recreate the gateway on `:17670` (docker driver, plaintext) per Pre-flight step 1; `:8080` belongs to the platform | | Deploy stuck `PENDING`/`STARTING` | Image missing sandbox user, or serve command wrong | `openshell sandbox list`; inspect the sandbox; confirm Step 2 used `--sandbox-runtime openshell` | | Deploy `FAILED` | Policy or gateway rejected the sandbox, the serve launcher exited non-zero, or the serve process died after launch | Check platform logs (`nemo services logs -n 100`) and the deployments reconciler output; the FAILED status message carries the launcher exit detail or the serve log tail. `nemo deployments logs ` returns the workload's own log | -| Invoke returns `502 Service endpoint is not reachable` | `nat serve` is alive but has not bound `:9000` yet | Retry after 30-60s. If it persists while the deployment stays `READY`, read the serve log with `nemo deployments logs `, or in the sandbox: `openshell sandbox exec --name -- cat /tmp/nemo-serve.log`. Common causes: baked config path does not match `--config_file`; a leftover `general.telemetry` block; or a `model_name` the Inference Gateway cannot resolve. Confirm the config is where serve looks (`openshell sandbox exec --name -- ls /workspace` -> `config.yaml`) and that `MODEL` exists (`nemo models list --all-pages`) | +| Invoke returns `502 Service endpoint is not reachable` | The deployment is not `READY`, or `nat serve` died after coming up | Wait for `READY` before invoking; if a `READY` deployment still 502s, read the serve log (`nemo deployments logs `, or `openshell sandbox exec --name -- cat /tmp/nemo-serve.log`). A bad config path, a leftover `general.telemetry` block, or an unresolvable `model_name` surfaces as `FAILED`, not a 502 | | Invoke returns 503 `inference service unavailable` (in the response or the serve log) | The `inference.local` route resolved, but the gateway's upstream hop to the platform failed | Almost always a platform bound to `127.0.0.1`; restart it with `--host 0.0.0.0`. Confirm the address the sandbox actually dials with `openshell sandbox exec --name -- cat /etc/hosts` (look for `host.openshell.internal`) and check the platform answers there (`curl -sf http://:8080/health/ready`). A clean `openshell inference set` does NOT rule this out: it validates from the host | | Invoke returns empty/error `value`, or serve log shows connection refused to `inference.local` | `inference.local` route not wired, or model misbehaved | Confirm `openshell inference get` shows the `nemo-igw` provider + your model; re-run Step 1's `provider create` / `inference set`; try a gpt-4o-mini-class model | | `example.com` reachable in Step 7 | Egress policy not applied | Confirm executor is `openshell-local` and the generated default-deny policy is attached to the sandbox | diff --git a/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py b/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py index 11f0f606a0..1875a07b03 100644 --- a/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py +++ b/plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py @@ -117,3 +117,54 @@ async def test_openshell_roundtrip_serves_http() -> None: finally: await backend.delete_deployment("itest", "rt") backend.shutdown() + + +async def test_openshell_readiness_holds_until_bind() -> None: + # The workload sleeps before it binds, so READY must not be published until the curl + # readiness probe can actually reach :8000. Regression guard for AIRCORE-998 (READY + # published ahead of bind) that also exercises the real in-sandbox curl probe. + backend, mock_entities = _make_backend() + bind_delay = 15 + mock_entities.get.return_value = DeploymentConfig( + name="slow-cfg", + workspace="itest", + containers=[ + Container( + name="web", + image=IMAGE, + command=["/bin/sh", "-c"], + args=[f"sleep {bind_delay}; exec python3 -m http.server 8000 --bind 0.0.0.0"], + ports=[ContainerPort(containerPort=8000, name="http")], + ) + ], + ) + + try: + created = await backend.create_deployment( + workspace="itest", + name="slow", + config_name="slow-cfg", + labels={"managed-by": MANAGED_BY_LABEL}, + backend_config={}, + ) + assert created.status == "STARTING", created.status_message + + start = time.monotonic() + saw_starting = False + status = None + for _ in range(90): + status = await backend.read_status(workspace="itest", name="slow") + if status.status != "STARTING": + break + saw_starting = True + await asyncio.sleep(1.0) + + assert status is not None and status.status == "READY", status.status_message if status else "no status" + assert saw_starting, "expected STARTING before READY" + # Serve only starts once provisioning completes, so time-to-READY strictly exceeds the + # sleep. A probe that failed open would publish READY during the sleep window. + assert time.monotonic() - start >= bind_delay - 3, "READY published before the workload bound its port" + assert status.endpoints, "expected an exposed endpoint" + finally: + await backend.delete_deployment("itest", "slow") + backend.shutdown() diff --git a/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py index 06f8d9dc29..809296bd13 100644 --- a/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py +++ b/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py @@ -22,6 +22,7 @@ ) from nemo_deployments_plugin.backends.openshell.backend import ( _CONFIG_DELIVERED_MARKER, + _CURL_ABS, _DEFAULT_READINESS_TIMEOUT_SECONDS, _LAUNCH_MARKER, _LIVENESS_PROBE, @@ -769,7 +770,10 @@ def test_readiness_probe_command_defaults_to_tcp_on_the_first_port() -> None: assert command[:2] == ["/bin/sh", "-c"] assert "127.0.0.1" in command[2] assert description == "tcp 127.0.0.1:8000" - # A network probe self-times in python; the RPC gets that timeout plus headroom. + assert "127.0.0.1:8000" in command[2] + assert _CURL_ABS in command[2] + assert '[ "$_rc" -eq 7 ] && exit 1' in command[2] + assert "python" not in command[2] assert timeout == _DEFAULT_READINESS_TIMEOUT_SECONDS + _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS @@ -777,9 +781,21 @@ def test_readiness_probe_command_uses_declared_httpget() -> None: container = _config_with_readiness(Probe(httpGet=HTTPGetAction(path="/ready", port=8000))).containers[0] command, description, _timeout = _require_probe_command(container) assert "http://127.0.0.1:8000/ready" in command[2] + assert "--fail" in command[2] + assert _CURL_ABS in command[2] assert description == "httpGet http://127.0.0.1:8000/ready" +def test_readiness_probe_command_fails_closed_when_curl_is_missing() -> None: + tcp = _require_probe_command(_config().containers[0])[0][2] + http = _require_probe_command( + _config_with_readiness(Probe(httpGet=HTTPGetAction(path="/health", port=8000))).containers[0] + )[0][2] + for script in (tcp, http): + assert "else exit 1; fi" in script + assert "else exit 0" not in script + + def test_readiness_probe_command_runs_declared_exec_directly() -> None: probe = Probe(exec=ExecAction(command=["/bin/true"]), timeoutSeconds=4) container = _config_with_readiness(probe).containers[0] @@ -807,8 +823,7 @@ def test_readiness_probe_command_https_uses_unverified_context() -> None: ).containers[0] command, _description, _timeout = _require_probe_command(container) assert "https://127.0.0.1:8000/health" in command[2] - # An https probe against a loopback/self-signed cert must not fail verification. - assert "_create_unverified_context" in command[2] + assert "--insecure" in command[2] def test_readiness_probe_command_normalizes_httpget_path_without_leading_slash() -> None: From 350b9a6c9df19d89237b1e34ca38331450f5a13d Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Wed, 12 Aug 2026 15:21:42 -0400 Subject: [PATCH 2/2] fix(nemo-deployments): bypass proxies in curl readiness probes Loopback readiness probes ran plain curl, so http_proxy/https_proxy in the sandbox image env would route the 127.0.0.1 request through a proxy. For the TCP probe (exit 7 = not-ready, anything else = ready) a reachable-but-erroring proxy returns a non-7 code, flipping the deployment to READY before nat serve binds its port -- the exact race this gate closes. Add --noproxy '*' to both the httpGet and TCP probe curls, and assert it in the unit tests. Also drop the stale 'give serve a moment after READY' note in the deploy skill: READY now gates on a reachability probe, so the endpoint serves as soon as it reports READY. Signed-off-by: Max Dubrinsky --- .../nemo_deployments_plugin/backends/openshell/backend.py | 6 +++--- .../nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md | 2 +- .../tests/unit/backends/openshell/test_backend.py | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py index bf47f3afb2..356c2c9329 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py @@ -869,8 +869,8 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] path = probe.http_get.path if probe.http_get.path.startswith("/") else f"/{probe.http_get.path}" url = f"{probe.http_get.scheme.lower()}://127.0.0.1:{port}{path}" args = ( - f"--fail --silent --show-error --insecure --max-time {shlex.quote(str(timeout))} " - f"--output /dev/null {shlex.quote(url)}" + f"--fail --silent --show-error --insecure --noproxy '*' " + f"--max-time {shlex.quote(str(timeout))} --output /dev/null {shlex.quote(url)}" ) return ["/bin/sh", "-c", _curl_probe_script(args)], f"httpGet {url}", network_exec_timeout @@ -881,7 +881,7 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] if port is None: return None url = f"http://127.0.0.1:{port}/" - args = f"--silent --output /dev/null --max-time {shlex.quote(str(timeout))} {shlex.quote(url)}" + args = f"--silent --output /dev/null --noproxy '*' --max-time {shlex.quote(str(timeout))} {shlex.quote(url)}" script = _curl_probe_script(args, tcp_connect=True) return ["/bin/sh", "-c", script], f"tcp 127.0.0.1:{port}", network_exec_timeout diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md b/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md index 6f9b171e18..70736e087d 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md @@ -272,7 +272,7 @@ URL=$(curl -sf "$BASE/deployments/igw-agent" | jq -r '.endpoints[0].url') echo "$URL" # e.g. http://--.openshell.localhost:17670/ ``` -If the status reaches `FAILED` or never leaves `PENDING`/`STARTING`, jump to the recovery table below. Do not proceed to invoke until `READY`. `READY` gates on the sandbox being up, the serve launcher exiting cleanly, and the serve process still being alive on the poll, but not on `nat serve` having finished binding its port, so give serve a moment after `READY`. A serve process that exits (bad config, unresolvable model) flips the deployment to `FAILED` on the next poll, with the tail of its log in the status message. +If the status reaches `FAILED` or never leaves `PENDING`/`STARTING`, jump to the recovery table below. Do not proceed to invoke until `READY`. `READY` gates on the sandbox being up, the serve launcher exiting cleanly, the serve process still being alive on the poll, and `nat serve` answering a reachability probe on its port, so the endpoint is serving as soon as it reports `READY`. A serve process that exits (bad config, unresolvable model) flips the deployment to `FAILED` on the next poll, with the tail of its log in the status message. ## Step 6: Invoke the agent diff --git a/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py index 809296bd13..705c1099da 100644 --- a/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py +++ b/plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py @@ -772,6 +772,7 @@ def test_readiness_probe_command_defaults_to_tcp_on_the_first_port() -> None: assert description == "tcp 127.0.0.1:8000" assert "127.0.0.1:8000" in command[2] assert _CURL_ABS in command[2] + assert "--noproxy '*'" in command[2] assert '[ "$_rc" -eq 7 ] && exit 1' in command[2] assert "python" not in command[2] assert timeout == _DEFAULT_READINESS_TIMEOUT_SECONDS + _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS @@ -782,6 +783,7 @@ def test_readiness_probe_command_uses_declared_httpget() -> None: command, description, _timeout = _require_probe_command(container) assert "http://127.0.0.1:8000/ready" in command[2] assert "--fail" in command[2] + assert "--noproxy '*'" in command[2] assert _CURL_ABS in command[2] assert description == "httpGet http://127.0.0.1:8000/ready"