fix(nemo-deployments): gate OpenShell READY on a curl reachability probe

plugins/nemo-deployments/src/nemo_deployments_plugin/backends/openshell/backend.py

@@ -138,24 +138,9 @@ def _delivery_script(path: str, mode: int) -> str:
 # readinessProbe uses its own timeout_seconds instead.
 _DEFAULT_READINESS_TIMEOUT_SECONDS = 3
 
-# Headroom added to a network probe's own timeout when bounding the ExecSandbox RPC, so
-# the RPC outlives the in-sandbox python timeout and captures its (non-zero) exit rather
-# than being cut off first.
 _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS = 5
 
-# Loopback readiness probe programs, run by the sandbox's python. urlopen raises on a
-# refused connection or an HTTP status >= 400 (so a still-starting 503 reads as not
-# ready); create_connection raises until the socket is actually bound. The HTTP probe
-# uses an unverified TLS context so an https readinessProbe against a loopback/self-signed
-# cert is not rejected on verification (matching Kubernetes httpGet HTTPS probe semantics);
-# the context is ignored for plain http, so one program covers both schemes.
-_HTTP_PROBE_PROGRAM = (
-    "import sys, ssl, urllib.request; "
-    "urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[2]), context=ssl._create_unverified_context())"
-)
-_TCP_PROBE_PROGRAM = (
-    "import sys, socket; socket.create_connection((sys.argv[1], int(sys.argv[2])), timeout=float(sys.argv[3])).close()"
-)
+_CURL_ABS = "/usr/bin/curl"
 
 # Cached on first status read; needs the proto enums so it cannot be built at import
 # time (see _ensure_openshell). None until built.
@@ -842,21 +827,16 @@ def _default_probe_port(container: Container) -> int | None:
     return None
 
 
-def _loopback_probe_script(program: str, *args: str) -> str:
-    """Wrap a python probe *program* so it runs against the sandbox's own python.
-
-    Selects ``python3`` then ``python`` off PATH and runs *program* with *args*, exiting
-    0 when the probe connects and nonzero when it does not. If neither interpreter is on
-    PATH the probe cannot run, so it exits 0 to preserve the prior expose-on-alive
-    behaviour rather than wedging a workload in STARTING until the progress deadline.
-    """
-    quoted_args = " ".join(shlex.quote(arg) for arg in args)
-    return (
-        "if command -v python3 >/dev/null 2>&1; then _py=python3; "
-        "elif command -v python >/dev/null 2>&1; then _py=python; "
-        "else exit 0; fi; "
-        f'"$_py" -c {shlex.quote(program)} {quoted_args}'
+def _curl_probe_script(curl_args: str, *, tcp_connect: bool = False) -> str:
+    preamble = (
+        f"if [ -x {_CURL_ABS} ]; then _curl={_CURL_ABS}; "
+        "elif command -v curl >/dev/null 2>&1; then _curl=curl; "
+        "else exit 1; fi; "
     )
+    invocation = f'"$_curl" {curl_args}'
+    if tcp_connect:
+        return f'{preamble}{invocation}; _rc=$?; [ "$_rc" -eq 7 ] && exit 1; exit 0'
+    return preamble + invocation
 
 
 def _readiness_probe_command(container: Container) -> tuple[list[str], str, int] | None:
@@ -870,7 +850,7 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int]
 
     ``exec_timeout_seconds`` bounds the ExecSandbox RPC so a hung probe cannot stall the
     serial reconcile loop: an exec probe is bounded by its own ``timeoutSeconds``; a
-    network probe self-times in python, so the RPC is given that timeout plus headroom.
+    network probe self-times in curl, so the RPC is given that timeout plus headroom.
     """
     probe = container.readiness_probe
     timeout = probe.timeout_seconds if probe is not None else _DEFAULT_READINESS_TIMEOUT_SECONDS
@@ -888,16 +868,21 @@ def _readiness_probe_command(container: Container) -> tuple[list[str], str, int]
             return None
         path = probe.http_get.path if probe.http_get.path.startswith("/") else f"/{probe.http_get.path}"
         url = f"{probe.http_get.scheme.lower()}://127.0.0.1:{port}{path}"
-        script = _loopback_probe_script(_HTTP_PROBE_PROGRAM, url, str(timeout))
-        return ["/bin/sh", "-c", script], f"httpGet {url}", network_exec_timeout
+        args = (
+            f"--fail --silent --show-error --insecure --noproxy '*' "
+            f"--max-time {shlex.quote(str(timeout))} --output /dev/null {shlex.quote(url)}"
+        )
+        return ["/bin/sh", "-c", _curl_probe_script(args)], f"httpGet {url}", network_exec_timeout
 
     if probe is not None and probe.tcp_socket is not None:
         port = _resolve_probe_port(probe.tcp_socket.port, container) or _default_probe_port(container)
     else:
         port = _default_probe_port(container)
     if port is None:
         return None
-    script = _loopback_probe_script(_TCP_PROBE_PROGRAM, "127.0.0.1", str(port), str(timeout))
+    url = f"http://127.0.0.1:{port}/"
+    args = f"--silent --output /dev/null --noproxy '*' --max-time {shlex.quote(str(timeout))} {shlex.quote(url)}"
+    script = _curl_probe_script(args, tcp_connect=True)
     return ["/bin/sh", "-c", script], f"tcp 127.0.0.1:{port}", network_exec_timeout
 
 

plugins/nemo-deployments/src/nemo_deployments_plugin/skills/deploy-sandbox/SKILL.md

@@ -272,7 +272,7 @@ URL=$(curl -sf "$BASE/deployments/igw-agent" | jq -r '.endpoints[0].url')
 echo "$URL"      # e.g. http://<sandbox>--<svc>.openshell.localhost:17670/
 ```
 
-If the status reaches `FAILED` or never leaves `PENDING`/`STARTING`, jump to the recovery table below. Do not proceed to invoke until `READY`. `READY` gates on the sandbox being up, the serve launcher exiting cleanly, and the serve process still being alive on the poll, but not on `nat serve` having finished binding its port, so give serve a moment after `READY`. A serve process that exits (bad config, unresolvable model) flips the deployment to `FAILED` on the next poll, with the tail of its log in the status message.
+If the status reaches `FAILED` or never leaves `PENDING`/`STARTING`, jump to the recovery table below. Do not proceed to invoke until `READY`. `READY` gates on the sandbox being up, the serve launcher exiting cleanly, the serve process still being alive on the poll, and `nat serve` answering a reachability probe on its port, so the endpoint is serving as soon as it reports `READY`. A serve process that exits (bad config, unresolvable model) flips the deployment to `FAILED` on the next poll, with the tail of its log in the status message.
 
 ## Step 6: Invoke the agent
 
@@ -286,7 +286,7 @@ curl -sf -X POST "${URL}generate" -H 'content-type: application/json' \
 
 Other routes the served workflow exposes: `POST /v1/chat/completions`, `/chat`, `/v1/workflow`, and `/generate/stream`.
 
-Verification: the response must contain a non-empty `value`. A `502 Service endpoint is not reachable` means the gateway reached the sandbox but `nat serve` (port 9000) is not answering, because it crashed or has not come up. Give it 30-60s and retry; if it persists, read the serve log inside the sandbox:
+Verification: the response must contain a non-empty `value`. A `502 Service endpoint is not reachable` means `nat serve` (port 9000) is not answering. Read the serve log inside the sandbox:
 
 ```bash
 SBX=$(curl -sf "$BASE/deployments/igw-agent" | jq -r '.endpoints[0].url' | sed -E 's#^https?://##; s#--.*##')
@@ -345,7 +345,7 @@ docker compose -f plugins/nemo-deployments/examples/openshell/docker-compose.yml
 | `openshell gateway list` shows the endpoint on `:8080` | Gateway collides with the platform port | Recreate the gateway on `:17670` (docker driver, plaintext) per Pre-flight step 1; `:8080` belongs to the platform |
 | Deploy stuck `PENDING`/`STARTING` | Image missing sandbox user, or serve command wrong | `openshell sandbox list`; inspect the sandbox; confirm Step 2 used `--sandbox-runtime openshell` |
 | Deploy `FAILED` | Policy or gateway rejected the sandbox, the serve launcher exited non-zero, or the serve process died after launch | Check platform logs (`nemo services logs -n 100`) and the deployments reconciler output; the FAILED status message carries the launcher exit detail or the serve log tail. `nemo deployments logs <name>` returns the workload's own log |
-| Invoke returns `502 Service endpoint is not reachable` | `nat serve` is alive but has not bound `:9000` yet | Retry after 30-60s. If it persists while the deployment stays `READY`, read the serve log with `nemo deployments logs <name>`, or in the sandbox: `openshell sandbox exec --name <nmp-hash> -- cat /tmp/nemo-serve.log`. Common causes: baked config path does not match `--config_file`; a leftover `general.telemetry` block; or a `model_name` the Inference Gateway cannot resolve. Confirm the config is where serve looks (`openshell sandbox exec --name <nmp-hash> -- ls /workspace` -> `config.yaml`) and that `MODEL` exists (`nemo models list --all-pages`) |
+| Invoke returns `502 Service endpoint is not reachable` | The deployment is not `READY`, or `nat serve` died after coming up | Wait for `READY` before invoking; if a `READY` deployment still 502s, read the serve log (`nemo deployments logs <name>`, or `openshell sandbox exec --name <nmp-hash> -- cat /tmp/nemo-serve.log`). A bad config path, a leftover `general.telemetry` block, or an unresolvable `model_name` surfaces as `FAILED`, not a 502 |
 | Invoke returns 503 `inference service unavailable` (in the response or the serve log) | The `inference.local` route resolved, but the gateway's upstream hop to the platform failed | Almost always a platform bound to `127.0.0.1`; restart it with `--host 0.0.0.0`. Confirm the address the sandbox actually dials with `openshell sandbox exec --name <nmp-hash> -- cat /etc/hosts` (look for `host.openshell.internal`) and check the platform answers there (`curl -sf http://<that-ip>:8080/health/ready`). A clean `openshell inference set` does NOT rule this out: it validates from the host |
 | Invoke returns empty/error `value`, or serve log shows connection refused to `inference.local` | `inference.local` route not wired, or model misbehaved | Confirm `openshell inference get` shows the `nemo-igw` provider + your model; re-run Step 1's `provider create` / `inference set`; try a gpt-4o-mini-class model |
 | `example.com` reachable in Step 7 | Egress policy not applied | Confirm executor is `openshell-local` and the generated default-deny policy is attached to the sandbox |

plugins/nemo-deployments/tests/integration/backends/openshell/test_openshell_backend.py

@@ -117,3 +117,54 @@ async def test_openshell_roundtrip_serves_http() -> None:
     finally:
         await backend.delete_deployment("itest", "rt")
         backend.shutdown()
+
+
+async def test_openshell_readiness_holds_until_bind() -> None:
+    # The workload sleeps before it binds, so READY must not be published until the curl
+    # readiness probe can actually reach :8000. Regression guard for AIRCORE-998 (READY
+    # published ahead of bind) that also exercises the real in-sandbox curl probe.
+    backend, mock_entities = _make_backend()
+    bind_delay = 15
+    mock_entities.get.return_value = DeploymentConfig(
+        name="slow-cfg",
+        workspace="itest",
+        containers=[
+            Container(
+                name="web",
+                image=IMAGE,
+                command=["/bin/sh", "-c"],
+                args=[f"sleep {bind_delay}; exec python3 -m http.server 8000 --bind 0.0.0.0"],
+                ports=[ContainerPort(containerPort=8000, name="http")],
+            )
+        ],
+    )
+
+    try:
+        created = await backend.create_deployment(
+            workspace="itest",
+            name="slow",
+            config_name="slow-cfg",
+            labels={"managed-by": MANAGED_BY_LABEL},
+            backend_config={},
+        )
+        assert created.status == "STARTING", created.status_message
+
+        start = time.monotonic()
+        saw_starting = False
+        status = None
+        for _ in range(90):
+            status = await backend.read_status(workspace="itest", name="slow")
+            if status.status != "STARTING":
+                break
+            saw_starting = True
+            await asyncio.sleep(1.0)
+
+        assert status is not None and status.status == "READY", status.status_message if status else "no status"
+        assert saw_starting, "expected STARTING before READY"
+        # Serve only starts once provisioning completes, so time-to-READY strictly exceeds the
+        # sleep. A probe that failed open would publish READY during the sleep window.
+        assert time.monotonic() - start >= bind_delay - 3, "READY published before the workload bound its port"
+        assert status.endpoints, "expected an exposed endpoint"
+    finally:
+        await backend.delete_deployment("itest", "slow")
+        backend.shutdown()

plugins/nemo-deployments/tests/unit/backends/openshell/test_backend.py

@@ -22,6 +22,7 @@
 )
 from nemo_deployments_plugin.backends.openshell.backend import (
     _CONFIG_DELIVERED_MARKER,
+    _CURL_ABS,
     _DEFAULT_READINESS_TIMEOUT_SECONDS,
     _LAUNCH_MARKER,
     _LIVENESS_PROBE,
@@ -769,17 +770,34 @@ def test_readiness_probe_command_defaults_to_tcp_on_the_first_port() -> None:
     assert command[:2] == ["/bin/sh", "-c"]
     assert "127.0.0.1" in command[2]
     assert description == "tcp 127.0.0.1:8000"
-    # A network probe self-times in python; the RPC gets that timeout plus headroom.
+    assert "127.0.0.1:8000" in command[2]
+    assert _CURL_ABS in command[2]
+    assert "--noproxy '*'" in command[2]
+    assert '[ "$_rc" -eq 7 ] && exit 1' in command[2]
+    assert "python" not in command[2]
     assert timeout == _DEFAULT_READINESS_TIMEOUT_SECONDS + _READINESS_EXEC_TIMEOUT_MARGIN_SECONDS
 
 
 def test_readiness_probe_command_uses_declared_httpget() -> None:
     container = _config_with_readiness(Probe(httpGet=HTTPGetAction(path="/ready", port=8000))).containers[0]
     command, description, _timeout = _require_probe_command(container)
     assert "http://127.0.0.1:8000/ready" in command[2]
+    assert "--fail" in command[2]
+    assert "--noproxy '*'" in command[2]
+    assert _CURL_ABS in command[2]
     assert description == "httpGet http://127.0.0.1:8000/ready"
 
 
+def test_readiness_probe_command_fails_closed_when_curl_is_missing() -> None:
+    tcp = _require_probe_command(_config().containers[0])[0][2]
+    http = _require_probe_command(
+        _config_with_readiness(Probe(httpGet=HTTPGetAction(path="/health", port=8000))).containers[0]
+    )[0][2]
+    for script in (tcp, http):
+        assert "else exit 1; fi" in script
+        assert "else exit 0" not in script
+
+
 def test_readiness_probe_command_runs_declared_exec_directly() -> None:
     probe = Probe(exec=ExecAction(command=["/bin/true"]), timeoutSeconds=4)
     container = _config_with_readiness(probe).containers[0]
@@ -807,8 +825,7 @@ def test_readiness_probe_command_https_uses_unverified_context() -> None:
     ).containers[0]
     command, _description, _timeout = _require_probe_command(container)
     assert "https://127.0.0.1:8000/health" in command[2]
-    # An https probe against a loopback/self-signed cert must not fail verification.
-    assert "_create_unverified_context" in command[2]
+    assert "--insecure" in command[2]
 
 
 def test_readiness_probe_command_normalizes_httpget_path_without_leading_slash() -> None: