From ef0d7f2220b790885571287cc5c8149258a9d9ef Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Mon, 15 Jun 2026 17:41:57 +0000 Subject: [PATCH 01/23] feat(sandbox): enforce network_mode allowlist on docker (ENG-219) network_mode is now the authoritative network control across the launch path; the deprecated allow_internet boolean is a derived shim. Previously every backend keyed off allow_internet, so allowed_hosts never reached the sandbox and network_mode: allowlist was validated but unenforceable - rejected at preflight on every backend. - allowlist on the docker sandbox is now ENFORCED: the agent container joins an internal (no-egress) network and its HTTP(S) traffic routes through a stdlib egress-proxy sidecar that forwards only to allowed_hosts. Other hosts, raw-IP connections, and proxy-ignoring tools have no route off-box (default deny). New: sandbox/_egress.py, _egress_proxy.py. - sandbox/network_policy.py resolves the effective policy (open/block-all/ allowlist) and gates allowlist to backends that can enforce it; daytona/ modal fail closed (never silently open). - runtime_capabilities: allowlist supported on docker, rejected at preflight elsewhere with a clear message. - setup.py: preserve_agent_network lifts ANY restrictive policy (no-network OR allowlist) to public for web-disabled agent runs (they need the model API; no-web is enforced at the agent layer). - docs/task-authoring-task-md.md network-policy section + CHANGELOG. Verified on real docker (egress e2e): allowed host -> 200, non-allowed -> blocked, direct socket with proxy stripped -> no route. Full suite green. --- CHANGELOG.md | 13 ++ docs/task-authoring-task-md.md | 37 +++++ src/benchflow/sandbox/_egress.py | 79 +++++++++++ src/benchflow/sandbox/_egress_proxy.py | 150 +++++++++++++++++++++ src/benchflow/sandbox/docker.py | 30 ++++- src/benchflow/sandbox/network_policy.py | 100 ++++++++++++++ src/benchflow/sandbox/setup.py | 19 ++- src/benchflow/task/runtime_capabilities.py | 19 ++- tests/test_network_policy.py | 134 ++++++++++++++++++ tests/test_runtime_capabilities.py | 21 ++- 10 files changed, 585 insertions(+), 17 deletions(-) create mode 100644 src/benchflow/sandbox/_egress.py create mode 100644 src/benchflow/sandbox/_egress_proxy.py create mode 100644 src/benchflow/sandbox/network_policy.py create mode 100644 tests/test_network_policy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc5ecd39..21c0f5cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## [Unreleased] +### Added +- **`network_mode: allowlist` is now enforced on the `docker` sandbox.** A task + that sets `environment.network_mode: allowlist` + `allowed_hosts` runs behind + an internal network and an egress-proxy sidecar that forwards only to those + hosts; every other destination (other hosts, raw-IP connections, tools that + ignore the proxy) has no route off-box. `network_mode` is now the + authoritative network field across the launch path and `allow_internet` is a + derived back-compat shim. On sandboxes without per-host egress control + (`daytona`, `modal`) an `allowlist` task fails closed at preflight with a + clear message instead of running unrestricted (previously `allowlist` was + validated but unenforceable everywhere). (ENG-219) + + ### Changed - **`task.md` is now the sole task authoring format.** `bench tasks init` scaffolds a native `task.md` package (`task.md` + `environment/` + `oracle/` + diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index a50b6d5cd..abcc9219c 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -113,6 +113,43 @@ a minimal authored file and its canonical form never drift apart. --- +## Network policy + +`network_mode` (on `[environment]`, and optionally per-`agent`/`verifier`) +controls outbound network access. It is the authoritative field; the legacy +`allow_internet` boolean is deprecated and derived from it. + +| `network_mode` | Behavior | `allowed_hosts` | +|---|---|---| +| `public` *(default)* | Unrestricted egress | must be omitted | +| `no-network` | Container fully detached from the network | must be omitted | +| `allowlist` | Egress confined to `allowed_hosts` | required, non-empty | + +```yaml +environment: + network_mode: allowlist + allowed_hosts: [pypi.org, files.pythonhosted.org] +``` + +**Enforcement** + +- `no-network` and `public` are enforced on every sandbox. +- `allowlist` is enforced on the **`docker`** sandbox: the container joins an + internal (no-egress) network and its HTTP(S) traffic is routed through a proxy + sidecar that forwards only to `allowed_hosts`. Any other host, a raw-IP + connection, or a tool that ignores the proxy has no route off-box (default + deny). On sandboxes without per-host egress control (`daytona`, `modal`) an + `allowlist` task is **rejected at preflight** rather than run unrestricted — + use `docker`, `no-network`, or `public` there. Wider per-sandbox allowlist + support is tracked in ENG-219. +- For **agent runs with web tools disabled**, the agent still needs the model + API, so a restrictive `network_mode` (`no-network` or `allowlist`) is lifted + to `public` and the no-web policy is enforced at the agent layer instead. + `allowlist` therefore governs oracle/verifier and non-web-disabled runs. + +`allowed_hosts` entries are hostnames only (no scheme, port, or path) and match +the host exactly or as a parent domain (`example.com` matches `api.example.com`). + ## Prompt body and prompts/ sidecars The body below the frontmatter is the base prompt — free-form markdown, no diff --git a/src/benchflow/sandbox/_egress.py b/src/benchflow/sandbox/_egress.py new file mode 100644 index 000000000..26716634d --- /dev/null +++ b/src/benchflow/sandbox/_egress.py @@ -0,0 +1,79 @@ +"""Docker/compose allowlist egress enforcement. + +Generates a compose override that confines the ``main`` service to an +``internal: true`` network (no direct route off-box) and routes its HTTP(S) +traffic through a sidecar (``bf-egress``) running ``_egress_proxy.py``, which +forwards only to ``allowed_hosts``. Used by the docker and daytona-dind +backends when the resolved policy is ``ALLOWLIST`` (see ``network_policy.py``). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +#: Minimal image used for the proxy sidecar. Pinned; only needs stdlib python3. +DEFAULT_EGRESS_PROXY_IMAGE = "python:3.12-alpine" +_PROXY_SCRIPT = Path(__file__).parent / "_egress_proxy.py" +_EGRESS_INTERNAL_NET = "bf_egress_internal" +_EGRESS_EXTERNAL_NET = "bf_egress_external" +_EGRESS_SERVICE = "bf-egress" +_EGRESS_PORT = 8080 + + +def build_egress_override( + allowed_hosts: list[str] | tuple[str, ...], + *, + out_dir: Path, + proxy_image: str = DEFAULT_EGRESS_PROXY_IMAGE, +) -> Path: + """Write the allowlist egress compose override into *out_dir*; return its path. + + Copies the proxy script next to the override so both live on a host path the + daemon can bind-mount. The agent service ``main`` is detached from the + default bridge (sequence ``networks`` is replaced on merge) and pointed at + the proxy via ``HTTP(S)_PROXY``; the proxy alone bridges to the outside. + """ + out_dir.mkdir(parents=True, exist_ok=True) + script_dst = out_dir / "egress_proxy.py" + script_dst.write_text(_PROXY_SCRIPT.read_text()) + proxy_url = f"http://{_EGRESS_SERVICE}:{_EGRESS_PORT}" + override = { + "services": { + "main": { + # Sequence networks are REPLACED on compose merge → drops `default`. + "networks": [_EGRESS_INTERNAL_NET], + "environment": { + "HTTP_PROXY": proxy_url, + "HTTPS_PROXY": proxy_url, + "http_proxy": proxy_url, + "https_proxy": proxy_url, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + }, + "depends_on": [_EGRESS_SERVICE], + }, + _EGRESS_SERVICE: { + "image": proxy_image, + "networks": [_EGRESS_INTERNAL_NET, _EGRESS_EXTERNAL_NET], + "command": ["python3", "/egress_proxy.py"], + "environment": { + "ALLOWED_HOSTS": ",".join(allowed_hosts), + "PORT": str(_EGRESS_PORT), + }, + "volumes": [f"{script_dst.resolve()}:/egress_proxy.py:ro"], + "labels": {"benchflow.owned": "true"}, + "restart": "on-failure", + }, + }, + "networks": { + _EGRESS_INTERNAL_NET: { + "internal": True, + "labels": {"benchflow.owned": "true"}, + }, + _EGRESS_EXTERNAL_NET: {"labels": {"benchflow.owned": "true"}}, + }, + } + path = out_dir / "docker-compose-egress.json" + path.write_text(json.dumps(override, indent=2)) + return path diff --git a/src/benchflow/sandbox/_egress_proxy.py b/src/benchflow/sandbox/_egress_proxy.py new file mode 100644 index 000000000..785048faa --- /dev/null +++ b/src/benchflow/sandbox/_egress_proxy.py @@ -0,0 +1,150 @@ +"""Allowlist egress proxy — stdlib-only, default-deny. + +Runs inside the ``bf-egress`` sidecar (a minimal ``python`` image). The agent +container is attached to an ``internal: true`` Docker network with no route to +the outside world and is pointed at this proxy via ``HTTP(S)_PROXY``; the proxy +is the *only* path off-box, and it forwards a request only when the target host +matches the allowlist. Everything else — non-allowlisted hosts, raw-IP CONNECTs, +and any tool that ignores the proxy env and tries a direct socket — is refused +or has no route, so egress is confined to ``ALLOWED_HOSTS``. + +Config via env: ``ALLOWED_HOSTS`` (comma-separated), ``PORT`` (default 8080). +Matching is exact or proper-subdomain (``api.example.com`` matches the entry +``example.com``); never substring. No request body, header, or host is logged +beyond the host being allowed/denied. +""" + +from __future__ import annotations + +import contextlib +import os +import select +import socket +import sys +import threading + +_ALLOWED: tuple[str, ...] = tuple( + h.strip().lower().rstrip(".") + for h in os.environ.get("ALLOWED_HOSTS", "").split(",") + if h.strip() +) +_PORT = int(os.environ.get("PORT", "8080")) +_BUF = 65536 + + +def _host_allowed(host: str) -> bool: + host = host.strip().lower().rstrip(".") + if not host or not _ALLOWED: + return False + return any(host == a or host.endswith("." + a) for a in _ALLOWED) + + +def _recv_headers(sock: socket.socket) -> bytes: + data = b"" + while b"\r\n\r\n" not in data and len(data) < _BUF: + chunk = sock.recv(_BUF) + if not chunk: + break + data += chunk + return data + + +def _pipe(a: socket.socket, b: socket.socket) -> None: + socks = [a, b] + try: + while True: + r, _, x = select.select(socks, [], socks, 60) + if x or not r: + break + for s in r: + data = s.recv(_BUF) + if not data: + return + (b if s is a else a).sendall(data) + except OSError: + return + + +def _deny(client: socket.socket, host: str) -> None: + client.sendall( + b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n" + b"X-Benchflow-Egress: denied\r\nConnection: close\r\n\r\n" + ) + sys.stderr.write(f"egress-proxy: DENY {host or '?'}\n") + sys.stderr.flush() + + +def _handle(client: socket.socket) -> None: + try: + client.settimeout(30) + header = _recv_headers(client) + if not header: + return + line = header.split(b"\r\n", 1)[0].decode("latin-1", "replace") + parts = line.split(" ") + if len(parts) < 2: + return + method, target = parts[0], parts[1] + + if method.upper() == "CONNECT": + host = target.rsplit(":", 1)[0].strip("[]") + port = int(target.rsplit(":", 1)[1]) if ":" in target else 443 + if not _host_allowed(host): + _deny(client, host) + return + try: + upstream = socket.create_connection((host, port), timeout=30) + except OSError: + client.sendall(b"HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n") + return + client.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n") + _pipe(client, upstream) + upstream.close() + return + + # Plain HTTP: target is an absolute URI (http://host/path) in proxy form. + host = "" + if "://" in target: + host = target.split("://", 1)[1].split("/", 1)[0] + if not host: + for hl in header.split(b"\r\n"): + if hl.lower().startswith(b"host:"): + host = hl.split(b":", 1)[1].decode("latin-1").strip() + break + hostname = host.rsplit(":", 1)[0].strip("[]") + port = int(host.rsplit(":", 1)[1]) if ":" in host and "]" not in host else 80 + if not _host_allowed(hostname): + _deny(client, hostname) + return + try: + upstream = socket.create_connection((hostname, port), timeout=30) + except OSError: + client.sendall(b"HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n") + return + upstream.sendall(header) + _pipe(client, upstream) + upstream.close() + except Exception: + pass + finally: + with contextlib.suppress(OSError): + client.close() + + +def main() -> None: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("0.0.0.0", _PORT)) + srv.listen(128) + sys.stderr.write(f"egress-proxy: listening on :{_PORT}; allow={list(_ALLOWED)}\n") + sys.stderr.flush() + while True: + try: + client, _ = srv.accept() + except OSError: + continue + threading.Thread(target=_handle, args=(client,), daemon=True).start() + + +if __name__ == "__main__": + main() diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index de4bf1f57..3c951d1d0 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -256,11 +256,35 @@ def _docker_compose_paths(self) -> list[Path]: if self._mounts_compose_path: paths.append(self._mounts_compose_path) - if not self.task_env_config.allow_internet: - paths.append(self._DOCKER_COMPOSE_NO_NETWORK_PATH) - + paths.extend(self._network_policy_compose_paths()) return paths + def _network_policy_compose_paths(self) -> list[Path]: + """Compose overrides enforcing the task's resolved network policy. + + ``no-network`` detaches the container network; ``allowlist`` confines + egress to ``allowed_hosts`` via an internal network + proxy sidecar + (see ``_egress.build_egress_override``); ``public`` adds nothing. An + allowlist with no writable rollout dir fails closed to no-network. + """ + from benchflow.sandbox._egress import build_egress_override + from benchflow.sandbox.network_policy import ( + EffectivePolicy, + resolve_network_decision, + ) + + decision = resolve_network_decision(self.task_env_config, "docker") + if decision.policy is EffectivePolicy.OPEN: + return [] + if decision.policy is EffectivePolicy.ALLOWLIST and self.rollout_paths: + override = build_egress_override( + decision.allowed_hosts, + out_dir=self.rollout_paths.rollout_dir, + ) + return [override] + # BLOCK_ALL, or allowlist with nowhere to stage the proxy → fail closed. + return [self._DOCKER_COMPOSE_NO_NETWORK_PATH] + def _write_mounts_compose_file(self) -> Path: compose = {"services": {"main": {"volumes": self._mounts_json}}} assert self.rollout_paths is not None diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py new file mode 100644 index 000000000..8172ae48b --- /dev/null +++ b/src/benchflow/sandbox/network_policy.py @@ -0,0 +1,100 @@ +"""Central resolution of a task's effective network policy. + +Backends historically keyed network enforcement off the deprecated +``allow_internet`` boolean, so ``network_mode`` (the documented authority) and +``allowed_hosts`` never reached the sandbox — ``allowlist`` was validated but +unenforced. This module makes ``network_mode`` authoritative while keeping +``allow_internet`` as a derived back-compat input, and decides per backend +whether an ``allowlist`` can be enforced (compose backends, via an egress +proxy) or must fail closed (backends with only a binary block-all control). + +See ``_egress.py`` for the Docker/compose allowlist mechanism and Linear +ENG-219 for the roadmap context. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from benchflow.task.config import NetworkMode, SandboxConfig + +#: Sandboxes that can enforce a per-host allowlist (docker-compose egress proxy). +#: Others expose only a binary block-all control, so ``allowlist`` is rejected +#: at preflight by ``runtime_capabilities`` and fails closed here as defense in +#: depth. (daytona-dind shares the compose mechanism and is a natural follow-up, +#: but the ``daytona`` preflight sandbox string can't distinguish dind from the +#: direct strategy, so it is intentionally excluded from this first cut.) +ALLOWLIST_CAPABLE_SANDBOXES: frozenset[str] = frozenset({"docker"}) + + +def _normalize_sandbox(sandbox: str | None) -> str: + return (sandbox or "").replace("_", "-").strip().lower() + + +def sandbox_supports_allowlist(sandbox: str | None) -> bool: + """Whether *sandbox* can enforce a per-host ``allowlist``.""" + return _normalize_sandbox(sandbox) in ALLOWLIST_CAPABLE_SANDBOXES + + +class EffectivePolicy(StrEnum): + """The concrete network posture a backend must apply.""" + + OPEN = "open" + BLOCK_ALL = "block-all" + ALLOWLIST = "allowlist" + + +@dataclass(frozen=True) +class NetworkDecision: + policy: EffectivePolicy + allowed_hosts: tuple[str, ...] = () + downgraded_from: NetworkMode | None = None # set when allowlist failed closed + note: str = "" + + +def resolve_network_mode(env_config: SandboxConfig) -> NetworkMode: + """Return the authoritative ``NetworkMode``. + + ``network_mode`` wins. ``allow_internet`` is deprecated; an explicit + ``False`` still tightens a ``public`` policy to ``no-network`` so legacy + callers that mutate ``allow_internet`` after validation (e.g. the + ``preserve_agent_network`` lift in ``sandbox/setup.py``) keep working. + """ + mode = env_config.network_mode + if env_config.allow_internet is False and mode is NetworkMode.PUBLIC: + return NetworkMode.NO_NETWORK + return mode + + +def resolve_network_decision( + env_config: SandboxConfig, sandbox: str +) -> NetworkDecision: + """Resolve the effective policy a *sandbox* should enforce for *env_config*.""" + mode = resolve_network_mode(env_config) + if mode is NetworkMode.NO_NETWORK: + return NetworkDecision(EffectivePolicy.BLOCK_ALL) + if mode is NetworkMode.ALLOWLIST: + hosts = tuple(env_config.allowed_hosts or ()) + if sandbox_supports_allowlist(sandbox): + return NetworkDecision(EffectivePolicy.ALLOWLIST, allowed_hosts=hosts) + # Defense in depth: runtime_capabilities rejects allowlist at preflight on + # these sandboxes, but if that gate is bypassed we fail CLOSED (never open). + return NetworkDecision( + EffectivePolicy.BLOCK_ALL, + allowed_hosts=hosts, + downgraded_from=NetworkMode.ALLOWLIST, + note=( + f"network_mode='allowlist' is not enforceable on sandbox " + f"'{sandbox}' — failing closed to no-network" + ), + ) + return NetworkDecision(EffectivePolicy.OPEN) + + +def network_blocks_all(env_config: SandboxConfig, sandbox: str) -> bool: + """Back-compat shim for the historic ``not allow_internet`` block-all gate.""" + return ( + resolve_network_decision(env_config, sandbox).policy + is EffectivePolicy.BLOCK_ALL + ) diff --git a/src/benchflow/sandbox/setup.py b/src/benchflow/sandbox/setup.py index 15bba8a2e..206b8e875 100644 --- a/src/benchflow/sandbox/setup.py +++ b/src/benchflow/sandbox/setup.py @@ -649,13 +649,18 @@ def _create_sandbox_environment( sandbox_type=sandbox_type, task_path=task_path, ) - if preserve_agent_network and env_config.allow_internet is False: + if preserve_agent_network and _network_policy_is_restrictive(env_config): # LLM agents run inside the sandbox and need outbound network for model # APIs and first-run agent installation. BenchFlow enforces the task's # no-web policy at the agent layer instead of applying the container - # network block for these runs. + # network block for these runs, so lift ANY restrictive policy + # (no-network or allowlist) to public for them. + from benchflow.task.config import NetworkMode + env_config = env_config.model_copy(deep=True) env_config.allow_internet = True + env_config.network_mode = NetworkMode.PUBLIC + env_config.allowed_hosts = None manifest_env: dict[str, str] = {} if environment_manifest is not None: @@ -758,6 +763,16 @@ def _create_sandbox_environment( ) +def _network_policy_is_restrictive(env_config: object) -> bool: + """True when the resolved network policy is no-network or allowlist.""" + from benchflow.sandbox.network_policy import resolve_network_mode + from benchflow.task.config import NetworkMode, SandboxConfig + + if not isinstance(env_config, SandboxConfig): + return getattr(env_config, "allow_internet", True) is False + return resolve_network_mode(env_config) is not NetworkMode.PUBLIC + + def _validate_task_runtime_for_launch( task: Task, *, diff --git a/src/benchflow/task/runtime_capabilities.py b/src/benchflow/task/runtime_capabilities.py index 3d58f4d54..b9ef62b38 100644 --- a/src/benchflow/task/runtime_capabilities.py +++ b/src/benchflow/task/runtime_capabilities.py @@ -269,12 +269,19 @@ def _append_network_issue( sandbox: str, ) -> None: if mode == NetworkMode.ALLOWLIST: - _issue( - unsupported, - path=path, - reason="network allowlists are parsed but not enforced per sandbox", - sandbox=sandbox, - ) + from benchflow.sandbox.network_policy import sandbox_supports_allowlist + + if not sandbox_supports_allowlist(sandbox): + _issue( + unsupported, + path=path, + reason=( + "network_mode='allowlist' is enforced only on the 'docker' " + "sandbox (egress proxy); not available on this sandbox — use " + "'docker', 'no-network', or 'public' (ENG-219)" + ), + sandbox=sandbox, + ) def _append_document_issues( diff --git a/tests/test_network_policy.py b/tests/test_network_policy.py new file mode 100644 index 000000000..2d0d28469 --- /dev/null +++ b/tests/test_network_policy.py @@ -0,0 +1,134 @@ +"""network_mode enforcement: policy resolution, egress override, capability gate. + +The end-to-end egress proof (allowed host reachable, others blocked, direct +egress blocked) runs as a Docker integration test on a host with docker; these +unit tests pin the resolution logic, the generated compose override shape, the +proxy host-matching, and the preflight capability gate. +""" + +import json + +from benchflow.sandbox import _egress_proxy +from benchflow.sandbox._egress import build_egress_override +from benchflow.sandbox.network_policy import ( + EffectivePolicy, + resolve_network_decision, + resolve_network_mode, + sandbox_supports_allowlist, +) +from benchflow.task.config import NetworkMode, SandboxConfig + + +def _cfg(**kw) -> SandboxConfig: + return SandboxConfig(**kw) + + +class TestResolveMode: + def test_public_default(self): + assert resolve_network_mode(_cfg()) is NetworkMode.PUBLIC + + def test_no_network_explicit(self): + assert ( + resolve_network_mode(_cfg(network_mode="no-network")) + is NetworkMode.NO_NETWORK + ) + + def test_allowlist(self): + c = _cfg(network_mode="allowlist", allowed_hosts=["example.com"]) + assert resolve_network_mode(c) is NetworkMode.ALLOWLIST + + def test_deprecated_allow_internet_false_forces_no_network(self): + # config reconciliation already does this for the public default; the + # resolver must agree even if a caller mutates allow_internet post-init. + c = _cfg() + c.allow_internet = False + assert resolve_network_mode(c) is NetworkMode.NO_NETWORK + + +class TestResolveDecision: + def test_docker_allowlist_enforced(self): + c = _cfg(network_mode="allowlist", allowed_hosts=["a.com", "b.com"]) + d = resolve_network_decision(c, "docker") + assert d.policy is EffectivePolicy.ALLOWLIST + assert d.allowed_hosts == ("a.com", "b.com") + assert d.downgraded_from is None + + def test_non_docker_allowlist_fails_closed(self): + c = _cfg(network_mode="allowlist", allowed_hosts=["a.com"]) + for sb in ("daytona", "modal"): + d = resolve_network_decision(c, sb) + assert d.policy is EffectivePolicy.BLOCK_ALL + assert d.downgraded_from is NetworkMode.ALLOWLIST # never silently open + + def test_no_network(self): + assert ( + resolve_network_decision(_cfg(network_mode="no-network"), "docker").policy + is EffectivePolicy.BLOCK_ALL + ) + + def test_public_open(self): + assert resolve_network_decision(_cfg(), "docker").policy is EffectivePolicy.OPEN + + def test_capability_predicate(self): + assert sandbox_supports_allowlist("docker") + assert not sandbox_supports_allowlist("daytona") + assert not sandbox_supports_allowlist("modal") + assert not sandbox_supports_allowlist(None) + + +class TestEgressOverride: + def test_structure(self, tmp_path): + path = build_egress_override(["example.com", "api.test.org"], out_dir=tmp_path) + doc = json.loads(path.read_text()) + main = doc["services"]["main"] + proxy = doc["services"]["bf-egress"] + # main detached from default bridge → only the internal network + assert main["networks"] == ["bf_egress_internal"] + assert main["environment"]["HTTPS_PROXY"].startswith("http://bf-egress:") + assert main["depends_on"] == ["bf-egress"] + # proxy bridges internal + external and carries the allowlist + assert set(proxy["networks"]) == {"bf_egress_internal", "bf_egress_external"} + assert proxy["environment"]["ALLOWED_HOSTS"] == "example.com,api.test.org" + assert doc["networks"]["bf_egress_internal"]["internal"] is True + assert "internal" not in doc["networks"]["bf_egress_external"] + # the proxy script is staged next to the override for bind-mounting + assert (tmp_path / "egress_proxy.py").exists() + + +class TestProxyHostMatching: + def test_match(self, monkeypatch): + monkeypatch.setattr(_egress_proxy, "_ALLOWED", ("example.com", "test.org")) + assert _egress_proxy._host_allowed("example.com") + assert _egress_proxy._host_allowed("api.example.com") # subdomain + assert _egress_proxy._host_allowed("EXAMPLE.COM") # case-insensitive + assert _egress_proxy._host_allowed("example.com.") # trailing dot + assert not _egress_proxy._host_allowed("notexample.com") # not a subdomain + assert not _egress_proxy._host_allowed("evil.com") + assert not _egress_proxy._host_allowed("example.com.evil.com") + + def test_empty_allowlist_denies_all(self, monkeypatch): + monkeypatch.setattr(_egress_proxy, "_ALLOWED", ()) + assert not _egress_proxy._host_allowed("example.com") + + +class TestCapabilityGate: + def test_allowlist_rejected_off_docker_accepted_on_docker(self): + from benchflow.task.config import TaskConfig + from benchflow.task.runtime_capabilities import validate_task_runtime_support + + cfg = TaskConfig( + schema_version="1.3", + environment={"network_mode": "allowlist", "allowed_hosts": ["example.com"]}, + ) + docker_issues = [ + i + for i in validate_task_runtime_support(cfg, sandbox="docker") + if "network_mode" in i.path + ] + daytona_issues = [ + i + for i in validate_task_runtime_support(cfg, sandbox="daytona") + if "network_mode" in i.path + ] + assert docker_issues == [] # enforced on docker + assert daytona_issues # rejected at preflight elsewhere diff --git a/tests/test_runtime_capabilities.py b/tests/test_runtime_capabilities.py index c6c826559..c3db983c8 100644 --- a/tests/test_runtime_capabilities.py +++ b/tests/test_runtime_capabilities.py @@ -128,7 +128,8 @@ def test_runtime_view_preserves_legacy_split_packages(tmp_path: Path) -> None: def test_validator_reports_allowlist_as_runtime_gap() -> None: - """Parsed allowlists are invalid for launch until a sandbox enforces them.""" + """Allowlist is enforced on docker (egress proxy) but rejected at preflight + on sandboxes without per-host egress control (ENG-219).""" config = TaskConfig.model_validate( { "agent": { @@ -142,12 +143,20 @@ def test_validator_reports_allowlist_as_runtime_gap() -> None: } ) - issues = validate_task_runtime_support(config, sandbox="docker") - - assert [issue.path for issue in issues] == [ - "agent.network_mode", - "environment.network_mode", + # docker enforces allowlist via the egress proxy -> no network_mode gap + docker_net = [ + i.path + for i in validate_task_runtime_support(config, sandbox="docker") + if "network_mode" in i.path + ] + assert docker_net == [] + # sandboxes without per-host egress control reject allowlist at preflight + daytona_net = [ + i.path + for i in validate_task_runtime_support(config, sandbox="daytona") + if "network_mode" in i.path ] + assert daytona_net == ["agent.network_mode", "environment.network_mode"] def test_validator_reports_unknown_sandbox_backend() -> None: From 2b58aa62021d8a21323d13ccf15895d574a3ac9c Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Mon, 15 Jun 2026 22:36:37 +0000 Subject: [PATCH 02/23] feat: wildcard allowed_hosts + model lane for restrictive docker egress Wildcard allowed_hosts: a single leading `*.` label (e.g. *.example.com) matches subdomains at any depth (Harbor/nginx semantics) but not the apex; mid/trailing wildcards are rejected at parse time. Validator in task/config.py, matcher in sandbox/_egress_proxy.py. Model lane: a restrictive network_mode (no-network or allowlist) on docker now keeps one always-allow lane to the host-side benchflow model proxy open, so an agent run reaches the model without opening the sandbox to the public internet (no-network becomes model-only egress). The egress sidecar permits the docker host (_docker_host_address()) and routes to it via the bridge gateway / host-gateway. Replaces the blanket lift-to-public for web-disabled docker runs (extracted to _lift_agent_network_to_public, now docker-excluded); other sandboxes keep the lift. allow_model_endpoint:false (default true) closes the lane for a hermetic, no-model run. Live-verified through the real egress proxy: lane host + allowed host reachable, non-allowed host blocked (403); wildcard subdomain reachable, apex blocked. ENG-219 --- CHANGELOG.md | 13 +++ docs/task-authoring-task-md.md | 17 +++- src/benchflow/sandbox/_egress.py | 40 +++++--- src/benchflow/sandbox/_egress_proxy.py | 18 +++- src/benchflow/sandbox/docker.py | 22 +++- src/benchflow/sandbox/network_policy.py | 11 +- src/benchflow/sandbox/setup.py | 39 +++++--- src/benchflow/task/config.py | 15 ++- tests/test_internet_policy.py | 12 +-- tests/test_network_modes.py | 127 ++++++++++++++++++++++++ 10 files changed, 271 insertions(+), 43 deletions(-) create mode 100644 tests/test_network_modes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 21c0f5cae..e74c5c95c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,19 @@ (`daytona`, `modal`) an `allowlist` task fails closed at preflight with a clear message instead of running unrestricted (previously `allowlist` was validated but unenforceable everywhere). (ENG-219) +- **Wildcard `allowed_hosts`.** An `allowed_hosts` entry may use a single leading + `*.` label (e.g. `*.example.com`) to match subdomains at any depth (Harbor / + nginx semantics); the bare apex (`example.com`) is **not** matched by the + wildcard. Mid- or trailing-wildcard entries (`a.*.com`, `*example.com`, + `ex*mple.com`) are rejected at config parse time. (ENG-219) +- **Dedicated model lane under a restrictive `network_mode` on `docker`.** A + restrictive task (`no-network` or `allowlist`) now keeps a single always-allow + lane open to the host-side benchflow model proxy, so an agent run reaches the + model without opening the sandbox to the public internet (a `no-network` run + becomes model-only egress). This replaces the previous blanket lift-to-`public` + for web-disabled `docker` runs; the other sandboxes keep the lift. Set + `environment.allow_model_endpoint: false` to close the lane for a fully + hermetic, no-model run. (ENG-219) ### Changed diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index abcc9219c..4aa8badc9 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -142,13 +142,22 @@ environment: `allowlist` task is **rejected at preflight** rather than run unrestricted — use `docker`, `no-network`, or `public` there. Wider per-sandbox allowlist support is tracked in ENG-219. -- For **agent runs with web tools disabled**, the agent still needs the model - API, so a restrictive `network_mode` (`no-network` or `allowlist`) is lifted - to `public` and the no-web policy is enforced at the agent layer instead. - `allowlist` therefore governs oracle/verifier and non-web-disabled runs. +- **Model access under a restrictive policy.** An agent run still needs the + model API. On **`docker`**, a restrictive `network_mode` (`no-network` or + `allowlist`) is kept intact and a single always-allow lane to the host-side + model proxy is added, so the agent reaches the model without opening the + sandbox to the public internet (a `no-network` run becomes model-only egress). + Set `allow_model_endpoint: false` on `[environment]` (default `true`) to close + that lane for a fully hermetic, no-model run. On the other sandboxes the + restrictive policy is instead lifted to `public` for web-disabled agent runs, + with the no-web policy enforced at the agent layer. `allowed_hosts` entries are hostnames only (no scheme, port, or path) and match the host exactly or as a parent domain (`example.com` matches `api.example.com`). +A single leading `*.` label is also allowed: `*.example.com` matches subdomains +at any depth (`api.example.com`, `a.b.example.com`) but **not** the bare apex +`example.com`. The wildcard must be the leading label only — `a.*.com`, +`*example.com`, and `ex*mple.com` are rejected at parse time. ## Prompt body and prompts/ sidecars diff --git a/src/benchflow/sandbox/_egress.py b/src/benchflow/sandbox/_egress.py index 26716634d..7f63447b0 100644 --- a/src/benchflow/sandbox/_egress.py +++ b/src/benchflow/sandbox/_egress.py @@ -9,8 +9,10 @@ from __future__ import annotations +import ipaddress import json from pathlib import Path +from typing import Any #: Minimal image used for the proxy sidecar. Pinned; only needs stdlib python3. DEFAULT_EGRESS_PROXY_IMAGE = "python:3.12-alpine" @@ -26,6 +28,7 @@ def build_egress_override( *, out_dir: Path, proxy_image: str = DEFAULT_EGRESS_PROXY_IMAGE, + model_lane: str | None = None, ) -> Path: """Write the allowlist egress compose override into *out_dir*; return its path. @@ -38,6 +41,30 @@ def build_egress_override( script_dst = out_dir / "egress_proxy.py" script_dst.write_text(_PROXY_SCRIPT.read_text()) proxy_url = f"http://{_EGRESS_SERVICE}:{_EGRESS_PORT}" + egress_env: dict[str, str] = { + "ALLOWED_HOSTS": ",".join(allowed_hosts), + "PORT": str(_EGRESS_PORT), + } + egress_service: dict[str, Any] = { + "image": proxy_image, + "networks": [_EGRESS_INTERNAL_NET, _EGRESS_EXTERNAL_NET], + "command": ["python3", "/egress_proxy.py"], + "environment": egress_env, + "volumes": [f"{script_dst.resolve()}:/egress_proxy.py:ro"], + "labels": {"benchflow.owned": "true"}, + "restart": "on-failure", + } + if model_lane: + # Always-allow lane to the host-side model proxy. The agent's base_url + # already targets the docker host (_docker_host_address():port), so we only + # need the sidecar to (a) permit that host and (b) be able to route to it. + egress_env["BENCHFLOW_EGRESS_LANE_HOST"] = model_lane + try: + ipaddress.ip_address(model_lane) + except ValueError: + # Hostname (e.g. host.docker.internal on macOS): give the sidecar the + # blessed container->host route. IP literals are already routable. + egress_service["extra_hosts"] = [f"{model_lane}:host-gateway"] override = { "services": { "main": { @@ -53,18 +80,7 @@ def build_egress_override( }, "depends_on": [_EGRESS_SERVICE], }, - _EGRESS_SERVICE: { - "image": proxy_image, - "networks": [_EGRESS_INTERNAL_NET, _EGRESS_EXTERNAL_NET], - "command": ["python3", "/egress_proxy.py"], - "environment": { - "ALLOWED_HOSTS": ",".join(allowed_hosts), - "PORT": str(_EGRESS_PORT), - }, - "volumes": [f"{script_dst.resolve()}:/egress_proxy.py:ro"], - "labels": {"benchflow.owned": "true"}, - "restart": "on-failure", - }, + _EGRESS_SERVICE: egress_service, }, "networks": { _EGRESS_INTERNAL_NET: { diff --git a/src/benchflow/sandbox/_egress_proxy.py b/src/benchflow/sandbox/_egress_proxy.py index 785048faa..732ec3ad0 100644 --- a/src/benchflow/sandbox/_egress_proxy.py +++ b/src/benchflow/sandbox/_egress_proxy.py @@ -30,13 +30,27 @@ ) _PORT = int(os.environ.get("PORT", "8080")) _BUF = 65536 +#: Dedicated always-allow target (the benchflow model proxy) — permitted even +#: when the task allowlist is empty (no-network + model-only egress). +_LANE = os.environ.get("BENCHFLOW_EGRESS_LANE_HOST", "").strip().lower().rstrip(".") def _host_allowed(host: str) -> bool: host = host.strip().lower().rstrip(".") - if not host or not _ALLOWED: + if not host: return False - return any(host == a or host.endswith("." + a) for a in _ALLOWED) + if _LANE and host == _LANE: # the model lane, allowed regardless of _ALLOWED + return True + if not _ALLOWED: + return False + for a in _ALLOWED: + if a.startswith("*."): + # leading-label wildcard: any subdomain at any depth, never the apex + if host.endswith("." + a[2:]): + return True + elif host == a or host.endswith("." + a): + return True + return False def _recv_headers(sock: socket.socket) -> bytes: diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index 3c951d1d0..ca2ac2169 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -276,13 +276,29 @@ def _network_policy_compose_paths(self) -> list[Path]: decision = resolve_network_decision(self.task_env_config, "docker") if decision.policy is EffectivePolicy.OPEN: return [] - if decision.policy is EffectivePolicy.ALLOWLIST and self.rollout_paths: + lane = None + if decision.model_lane: + from benchflow.providers.litellm_runtime import _docker_host_address + + lane = _docker_host_address() + # An allowlist, or a no-network run that keeps only the model lane open, is + # enforced by the egress sidecar; both need a writable rollout dir to stage + # the proxy compose override. + if self.rollout_paths and ( + decision.policy is EffectivePolicy.ALLOWLIST or lane + ): + hosts = ( + decision.allowed_hosts + if decision.policy is EffectivePolicy.ALLOWLIST + else () + ) override = build_egress_override( - decision.allowed_hosts, + hosts, out_dir=self.rollout_paths.rollout_dir, + model_lane=lane, ) return [override] - # BLOCK_ALL, or allowlist with nowhere to stage the proxy → fail closed. + # BLOCK_ALL with no lane, or nowhere to stage the proxy → fail closed. return [self._DOCKER_COMPOSE_NO_NETWORK_PATH] def _write_mounts_compose_file(self) -> Path: diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py index 8172ae48b..778a83e14 100644 --- a/src/benchflow/sandbox/network_policy.py +++ b/src/benchflow/sandbox/network_policy.py @@ -51,6 +51,9 @@ class NetworkDecision: allowed_hosts: tuple[str, ...] = () downgraded_from: NetworkMode | None = None # set when allowlist failed closed note: str = "" + model_lane: bool = ( + False # keep a lane to the model proxy under a restrictive policy + ) def resolve_network_mode(env_config: SandboxConfig) -> NetworkMode: @@ -72,12 +75,15 @@ def resolve_network_decision( ) -> NetworkDecision: """Resolve the effective policy a *sandbox* should enforce for *env_config*.""" mode = resolve_network_mode(env_config) + lane = bool(getattr(env_config, "allow_model_endpoint", True)) if mode is NetworkMode.NO_NETWORK: - return NetworkDecision(EffectivePolicy.BLOCK_ALL) + return NetworkDecision(EffectivePolicy.BLOCK_ALL, model_lane=lane) if mode is NetworkMode.ALLOWLIST: hosts = tuple(env_config.allowed_hosts or ()) if sandbox_supports_allowlist(sandbox): - return NetworkDecision(EffectivePolicy.ALLOWLIST, allowed_hosts=hosts) + return NetworkDecision( + EffectivePolicy.ALLOWLIST, allowed_hosts=hosts, model_lane=lane + ) # Defense in depth: runtime_capabilities rejects allowlist at preflight on # these sandboxes, but if that gate is bypassed we fail CLOSED (never open). return NetworkDecision( @@ -88,6 +94,7 @@ def resolve_network_decision( f"network_mode='allowlist' is not enforceable on sandbox " f"'{sandbox}' — failing closed to no-network" ), + model_lane=lane, ) return NetworkDecision(EffectivePolicy.OPEN) diff --git a/src/benchflow/sandbox/setup.py b/src/benchflow/sandbox/setup.py index 206b8e875..de2d8ff41 100644 --- a/src/benchflow/sandbox/setup.py +++ b/src/benchflow/sandbox/setup.py @@ -621,6 +621,31 @@ def _patched(self, include_os_env=True): # type: ignore[override] _DIND_PATCH_APPLIED = True +def _lift_agent_network_to_public(env_config: Any, sandbox_type: str) -> Any: + """Lift a restrictive network policy to public for in-sandbox LLM agents. + + LLM agents run inside the sandbox and need outbound network for model APIs + and first-run agent installation. For most sandboxes BenchFlow enforces the + task's no-web policy at the agent layer and lifts ANY restrictive policy + (no-network or allowlist) to public here. + + Docker is excluded: it preserves the restrictive policy and instead carves + out a dedicated lane to the model proxy (see ``network_policy.model_lane`` + + ``_egress``), so the agent reaches the model without opening the sandbox to + the public internet. Returns a possibly-copied ``env_config`` (the original + is never mutated). + """ + if sandbox_type == "docker" or not _network_policy_is_restrictive(env_config): + return env_config + from benchflow.task.config import NetworkMode + + env_config = env_config.model_copy(deep=True) + env_config.allow_internet = True + env_config.network_mode = NetworkMode.PUBLIC + env_config.allowed_hosts = None + return env_config + + def _create_sandbox_environment( sandbox_type: str, task: Task, @@ -649,18 +674,8 @@ def _create_sandbox_environment( sandbox_type=sandbox_type, task_path=task_path, ) - if preserve_agent_network and _network_policy_is_restrictive(env_config): - # LLM agents run inside the sandbox and need outbound network for model - # APIs and first-run agent installation. BenchFlow enforces the task's - # no-web policy at the agent layer instead of applying the container - # network block for these runs, so lift ANY restrictive policy - # (no-network or allowlist) to public for them. - from benchflow.task.config import NetworkMode - - env_config = env_config.model_copy(deep=True) - env_config.allow_internet = True - env_config.network_mode = NetworkMode.PUBLIC - env_config.allowed_hosts = None + if preserve_agent_network: + env_config = _lift_agent_network_to_public(env_config, sandbox_type) manifest_env: dict[str, str] = {} if environment_manifest is not None: diff --git a/src/benchflow/task/config.py b/src/benchflow/task/config.py index e0c52917f..f39eb7c27 100644 --- a/src/benchflow/task/config.py +++ b/src/benchflow/task/config.py @@ -79,6 +79,11 @@ def _validate_allowed_hosts(hosts: list[str] | None) -> list[str] | None: host = raw_host.strip().lower().rstrip(".") if not host: raise ValueError("allowed_hosts entries must be non-empty hostnames") + wildcard = "" + if host.startswith("*."): + # Leading-label wildcard (Harbor #1854 / nginx semantics): matches + # subdomains at any depth; the remainder is validated as a hostname. + wildcard, host = "*.", host[2:] if "://" in host or "/" in host or ":" in host: raise ValueError( "allowed_hosts entries must be hostnames, not URLs, ports, or paths" @@ -89,7 +94,7 @@ def _validate_allowed_hosts(hosts: list[str] | None) -> list[str] | None: "allowed_hosts entries must be valid hostnames containing only " "letters, digits, hyphens, and dots" ) - normalized.append(host) + normalized.append(wildcard + host) return normalized @@ -488,6 +493,14 @@ class SandboxConfig(TaskConfigModel): default=True, description="Deprecated compatibility field; use network_mode instead.", ) + allow_model_endpoint: bool = Field( + default=True, + description=( + "Keep a dedicated lane to the benchflow model proxy open even under " + "a restrictive network_mode (model-only egress), so the agent can " + "reach its model. Set false for a fully-hermetic, no-model run." + ), + ) memory: str | None = Field( default=None, deprecated="Use 'memory_mb' instead.", diff --git a/tests/test_internet_policy.py b/tests/test_internet_policy.py index 32a6915fd..761916c26 100644 --- a/tests/test_internet_policy.py +++ b/tests/test_internet_policy.py @@ -123,14 +123,11 @@ def test_host_skill_nudge_reads_task_skills_when_explicitly_enabled( assert prompts[0].endswith("Do the thing.") -def test_create_environment_preserves_agent_network_for_llm_runs(tmp_path): +def test_create_environment_docker_preserves_policy_for_llm_runs(tmp_path): from benchflow.sandbox.setup import _create_environment original_env = MagicMock() original_env.allow_internet = False - copied_env = MagicMock() - copied_env.allow_internet = False - original_env.model_copy.return_value = copied_env task = SimpleNamespace( paths=SimpleNamespace(environment_dir=tmp_path / "environment"), config=SimpleNamespace(environment=original_env), @@ -146,9 +143,10 @@ def test_create_environment_preserves_agent_network_for_llm_runs(tmp_path): preserve_agent_network=True, ) - assert copied_env.allow_internet is True - assert original_env.allow_internet is False - assert docker_env.call_args.kwargs["task_env_config"] is copied_env + # Docker preserves the restrictive policy: the model lane (not a public lift) + # carries model access, so no copy is made and the original env passes through. + original_env.model_copy.assert_not_called() + assert docker_env.call_args.kwargs["task_env_config"] is original_env def test_create_environment_keeps_oracle_network_policy(tmp_path): diff --git a/tests/test_network_modes.py b/tests/test_network_modes.py new file mode 100644 index 000000000..c1fdea6c6 --- /dev/null +++ b/tests/test_network_modes.py @@ -0,0 +1,127 @@ +"""Behavior tests for the #785 follow-up: wildcard allowlist + LLM-lane. + +Driven out one behavior at a time (TDD). Tests exercise public interfaces: +SandboxConfig construction (validation), the egress proxy's host-match, the +egress override builder, and the network-policy resolver. +""" + +import pytest +from pydantic import ValidationError + +from benchflow.sandbox import _egress_proxy +from benchflow.task.config import SandboxConfig + + +def test_allowlist_accepts_leading_wildcard(): + cfg = SandboxConfig(network_mode="allowlist", allowed_hosts=["*.example.com"]) + assert "*.example.com" in cfg.allowed_hosts + + +def test_allowlist_rejects_non_leading_wildcard(): + for bad in ["a.*.com", "*example.com", "example.*", "ex*mple.com"]: + with pytest.raises(ValidationError): + SandboxConfig(network_mode="allowlist", allowed_hosts=[bad]) + + +def test_proxy_wildcard_matches_subdomains_not_apex(monkeypatch): + monkeypatch.setattr(_egress_proxy, "_ALLOWED", ("*.example.com",)) + assert _egress_proxy._host_allowed("a.example.com") + assert _egress_proxy._host_allowed("x.y.example.com") # multi-level + assert not _egress_proxy._host_allowed("example.com") # apex excluded + assert not _egress_proxy._host_allowed("evil.com") + + +def test_proxy_bare_host_matches_apex_and_subdomain(monkeypatch): + monkeypatch.setattr(_egress_proxy, "_ALLOWED", ("example.com",)) + assert _egress_proxy._host_allowed("example.com") # apex + assert _egress_proxy._host_allowed("api.example.com") # subdomain + assert not _egress_proxy._host_allowed("notexample.com") + + +def test_allow_model_endpoint_defaults_true_and_settable(): + assert SandboxConfig().allow_model_endpoint is True + assert SandboxConfig(allow_model_endpoint=False).allow_model_endpoint is False + + +def test_proxy_lane_host_allowed_even_with_empty_allowlist(monkeypatch): + monkeypatch.setattr(_egress_proxy, "_ALLOWED", ()) + monkeypatch.setattr(_egress_proxy, "_LANE", "host.docker.internal", raising=False) + assert _egress_proxy._host_allowed("host.docker.internal") # the model lane + assert not _egress_proxy._host_allowed("evil.com") + + +def test_egress_override_wires_hostname_model_lane(tmp_path): + import json + + from benchflow.sandbox._egress import build_egress_override + + # A hostname lane (macOS host.docker.internal) needs the host-gateway route + # mapped so the sidecar can reach the host-side model proxy. + path = build_egress_override( + [], out_dir=tmp_path, model_lane="host.docker.internal" + ) + proxy = json.loads(path.read_text())["services"]["bf-egress"] + assert proxy["environment"]["BENCHFLOW_EGRESS_LANE_HOST"] == "host.docker.internal" + assert "host.docker.internal:host-gateway" in proxy["extra_hosts"] + + +def test_egress_override_ip_model_lane_needs_no_extra_hosts(tmp_path): + import json + + from benchflow.sandbox._egress import build_egress_override + + # A bridge-gateway IP lane (Linux) is already routable from the sidecar — no + # extra_hosts mapping, just the always-allow lane host. + path = build_egress_override([], out_dir=tmp_path, model_lane="172.17.0.1") + proxy = json.loads(path.read_text())["services"]["bf-egress"] + assert proxy["environment"]["BENCHFLOW_EGRESS_LANE_HOST"] == "172.17.0.1" + assert "extra_hosts" not in proxy + + +def test_resolve_sets_model_lane_for_restrictive(): + from benchflow.sandbox.network_policy import resolve_network_decision + + assert ( + resolve_network_decision( + SandboxConfig(network_mode="no-network"), "docker" + ).model_lane + is True + ) + assert ( + resolve_network_decision( + SandboxConfig(network_mode="allowlist", allowed_hosts=["x.com"]), "docker" + ).model_lane + is True + ) + assert ( + resolve_network_decision(SandboxConfig(), "docker").model_lane is False + ) # public + + +def test_lift_agent_network_skips_docker(): + # Docker preserves the restrictive policy (the model lane handles model access). + from benchflow.sandbox.setup import _lift_agent_network_to_public + + cfg = SandboxConfig(network_mode="no-network") + out = _lift_agent_network_to_public(cfg, "docker") + assert out is cfg # untouched, not copied + assert out.network_mode == "no-network" + + +def test_lift_agent_network_public_for_non_docker(): + from benchflow.sandbox.setup import _lift_agent_network_to_public + + cfg = SandboxConfig(network_mode="no-network") + out = _lift_agent_network_to_public(cfg, "daytona") + assert out is not cfg # copied, original never mutated + assert out.network_mode == "public" + assert cfg.network_mode == "no-network" + + +def test_resolve_model_lane_opt_out(): + from benchflow.sandbox.network_policy import resolve_network_decision + + d = resolve_network_decision( + SandboxConfig(network_mode="no-network", allow_model_endpoint=False), "docker" + ) + assert d.model_lane is False From ca4ff27ccd61117feae4ccacd72acb69ef0d2686 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Mon, 15 Jun 2026 23:00:28 +0000 Subject: [PATCH 03/23] docs: add network-access prior-art/credits page + cross-links New docs/network-mode-prior-art.md records the network-mode design model (no-network/public/allowlist + wildcard + model lane), a mode/mechanism taxonomy, and credits to the prior-art platforms the design draws on (Harbor, Modal, AISI Inspect, WAREX, SWE-bench, SWE-bench-Live, WebArena, Cybench, tau2-bench, browser-use) with primary-source links. Registered in the Mintlify nav and cross-linked from the task-authoring guide and sandbox-hardening. Citations verified against primary sources; corrects the Harbor PR#1854 framing (wildcard-depth clarification, not the feature origin) and notes the "N0/N1/N2" shorthand is ours, not AISI nomenclature. ENG-219 --- docs/docs.json | 1 + docs/network-mode-prior-art.md | 132 +++++++++++++++++++++++++++++++++ docs/sandbox-hardening.md | 1 + docs/task-authoring-task-md.md | 2 + 4 files changed, 136 insertions(+) create mode 100644 docs/network-mode-prior-art.md diff --git a/docs/docs.json b/docs/docs.json index 4dc5078d2..fa4852da9 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -29,6 +29,7 @@ "skill-eval", "llm-judge", "sandbox-hardening", + "network-mode-prior-art", "architecture", "environment-plane" ] diff --git a/docs/network-mode-prior-art.md b/docs/network-mode-prior-art.md new file mode 100644 index 000000000..6a7d6427e --- /dev/null +++ b/docs/network-mode-prior-art.md @@ -0,0 +1,132 @@ +# Network access: design and prior art + +BenchFlow controls a task's outbound network with `network_mode` +(`no-network` / `public` / `allowlist`) plus `allowed_hosts` and +`allow_model_endpoint`. This page records the **design model**, what is +enforced today, and **credits the prior-art platforms** whose network-access +designs informed it. For the authoring reference see +[Authoring native task.md tasks](./task-authoring-task-md.md#network-policy); for +enforcement internals see [Sandbox hardening](./sandbox-hardening.md). + +The enforcing substrate is the owned egress proxy from +PR [#785](https://github.com/benchflow-ai/benchflow/pull/785) +(`feat/network-mode-enforcement`, ENG-219): an `internal: true` Docker network +plus a stdlib CONNECT/forward proxy sidecar. Everything below distinguishes +**what BenchFlow owns** from **patterns credited to other platforms**. + +## What BenchFlow enforces today + +- **`no-network` / `public`** — every sandbox. +- **`allowlist`** — the `docker` sandbox only. The container joins an + `internal: true` (no-egress) network and all HTTP(S) traffic is forced through + the proxy sidecar, which forwards only to `allowed_hosts`. A host matches + **exactly or as a parent domain** (`example.com` matches `api.example.com`). A + single **leading-label wildcard** `*.example.com` matches subdomains at any + depth but **not** the bare apex (Harbor / nginx semantics); mid- or + trailing-wildcards are rejected at parse time. Non-allowlisted hosts, raw-IP + connections, and proxy-ignoring tools have no route off-box (default deny). On + sandboxes without per-host egress control (`daytona`, `modal`) an `allowlist` + task is **rejected at preflight** rather than run unrestricted. +- **Model lane** — under a restrictive policy on `docker`, a single always-allow + lane to the host-side model proxy stays open, so an agent run reaches the model + without opening the sandbox to the public internet (a `no-network` run becomes + model-only egress). Governed by `allow_model_endpoint` (default `true`; set + `false` for a fully hermetic, no-model run). This replaced an earlier + blanket lift-to-`public` for web-disabled docker runs. + +## Behavioral modes + +Four behavioral postures, plus orthogonal modifiers. "own" = BenchFlow's own +design/mechanism; "credit" = pattern adopted from the named platform. + +| Mode | How it is implemented | BenchFlow status | Credit / source | +|---|---|---|---| +| **no-network** | Docker `network_mode: none` compose overlay | have | own (universal pattern) | +| **public** (default) | No restriction | have | own (field default everywhere) | +| **hostname allowlist** (exact / subdomain) | Internal net + CONNECT egress-proxy sidecar, host-matched | have (#785, docker) | own proxy; taxonomy ≡ Harbor, AISI Inspect | +| **wildcard host** (`*.example.com`, multi-level) | Glob match in proxy + validator accepts leading `*.` | have (#785) | credit: Harbor, Modal | +| **CIDR allowlist** | IP-prefix egress rule (needs IP-routing, not pure CONNECT) | gap | credit: Modal | +| **port-scoped egress** (`host:port`) | Rule type carries port; proxy already tunnels any CONNECT port | mechanism ok, rule type missing | credit: Modal | +| **offline mirror** (pkg / content) | Pre-baked layered images / pinned mirror | gap | credit: SWE-bench | +| **temporal package pin** | pip-index proxy filtering by release date | gap | credit: SWE-bench-Live | +| **record / replay (+ fault inject)** | MITM proxy + CA injected into sandbox | gap (proxy is non-MITM by design) | credit: WAREX | +| **mocked tool layer** | Tools served from a local DB; no real net | task-authored, not a mode | credit: τ²-bench | +| **ingress / published ports** | Host-publish + PREROUTING, or internal shared net | no inbound axis today | credit: WebArena, Cybench | +| **always-on model lane** | Auto-allow the host model proxy; default-on, settable | have (#785, docker) | own | +| **per-scene / per-turn policy** | network field on Role/Turn → `Step.data` → switch at connect/execute | design (cheap add) | own (Harbor has per-phase/step) | +| **separate-verifier-env** | Distinct policy for the grading env | field exists, enforcement is sandbox-wide | credit: Harbor | + +## Enforcement mechanisms + +| Mechanism | Enforces | Who uses it | BenchFlow | +|---|---|---|---| +| Docker `network_mode: none` / internal network | no-network; default-deny base for allowlist | everyone (AISI, SWE-bench, …) | have | +| **Owned forward/CONNECT egress proxy sidecar** | hostname allowlist, vendor-neutral | **benchflow #785** | have / own | +| Cloud-platform egress allowlist (domain + CIDR) | hostname & CIDR allowlist | Modal, Daytona, Harbor backends | credit | +| CoreDNS `allowDomains` (k8s) | hostname allowlist via DNS | UK AISI Inspect (k8s sandbox) | credit | +| MITM proxy + CA injection | record/replay, fault injection, body inspection | WAREX | credit (deliberately not done) | +| Pre-baked layered images / offline mirror | hermetic — no runtime egress needed | SWE-bench, DeepSWE | credit | +| Temporal pip-index proxy | version pinning by date (not connectivity) | SWE-bench-Live | credit | +| Host-published ports + PREROUTING | inbound to self-hosted target apps | WebArena / VisualWebArena | credit (no inbound axis today) | +| Shared internal Docker network | agent ↔ target topology (CTF) | Cybench | credit | +| Browser-SDK `allowed_domains` | app-level (browser) egress scope | browser-use / BenchFlow CUA | have (browser app) | +| Mocked tools over a local DB | no network in the tool layer | τ²-bench | credit (task-authored) | + +## Credits + +Primary sources for each credited platform. Mechanism descriptions are stated as +verified against these sources; see the caveats inline. + +- **Harbor** — agent-evaluation / RL framework (the `harbor-framework` org, not + the container registry). Per-host `network_mode = "allowlist"` + `allowed_hosts` + with leading-wildcard subdomain matching (nginx-style, multi-level), per-phase + overrides (`[environment]`/`[agent]`/`[verifier]`/`[steps.*]`), and a + shared-by-default vs. separate verifier environment. BenchFlow's `allowlist` + taxonomy and wildcard semantics follow Harbor's. + · docs . + (Harbor PR [#1854](https://github.com/harbor-framework/harbor/pull/1854) + *clarifies* that wildcards match multi-level subdomains — it documents the + depth rule rather than introducing the feature.) +- **Modal** — Sandbox outbound controls: `block_network`, + `outbound_cidr_allowlist` (CIDR ranges), and `outbound_domain_allowlist` + (domains). Source of the CIDR-allowlist and port-scoping patterns BenchFlow + does not yet implement. . +- **UK AISI Inspect** (k8s sandbox) — hostname allowlisting via `allowDomains` + (FQDN list) enforced by a per-Pod CoreDNS sidecar, plus `allowCIDR`. + · docs + . *(The "N0/N1/N2" + network-level shorthand sometimes used in BenchFlow design notes is **our own** + label, not Inspect nomenclature.)* +- **WAREX** — routes web-agent traffic through mitmproxy with split-TLS (its CA + is trusted inside the sandbox) to decrypt and **inject faults** (delays, + 4xx/5xx, injected JS/overlays). BenchFlow deliberately keeps its proxy + **non-MITM**; WAREX is the reference for the record/replay + fault-injection + axis we do not implement. . +- **SWE-bench** — per-task hermetic execution via three layered Docker images + (base → environment → instance), removing runtime network dependence. + · . +- **SWE-bench-Live** — "time-machine" pip-index proxy that serves only package + versions released no later than the instance's base-commit timestamp. + · . +- **WebArena / VisualWebArena** — self-host target web apps as Docker containers + with host-published ports, resettable to an initial state; VisualWebArena adds + a self-hosted Wikipedia knowledge base. The reference for an inbound/ingress + axis BenchFlow does not model today. + · + . +- **Cybench** — CTF benchmark where a Kali agent container reaches one or more + task-server containers over a shared Docker network. + · . +- **τ²-bench** — tool-calling benchmark whose tools operate over a local JSON + database (`db.json` per domain); grading compares final DB state to a goal + state, with no real network in the tool layer. + · τ-bench paper + . +- **browser-use** — browser-automation SDK whose `BrowserProfile.allowed_domains` + scopes which domains the browser may visit (supports `*.example.com` patterns). + BenchFlow's CUA browser-app egress scope reuses this. + . + +> **Scope note.** Per-tool network scoping and a 3-way +> agent / verifier / environment egress split are *not* standard in the field — +> treat them as BenchFlow-specific design choices, not parity gaps. diff --git a/docs/sandbox-hardening.md b/docs/sandbox-hardening.md index 29eada1b5..e36c0d582 100644 --- a/docs/sandbox-hardening.md +++ b/docs/sandbox-hardening.md @@ -79,3 +79,4 @@ Known residual risk: - [`docs/labs/reward-hack-matrix/README.md`](https://github.com/benchflow-ai/benchflow/tree/main/docs/labs/reward-hack-matrix) — methodology, exploit taxonomy, sweep results (historical, 0.2.x-era). - [`progressive-disclosure.md`](./progressive-disclosure.md) — soft-verify (the relaxed hardening used between rounds in multi-round trials). - [`task-authoring.md`](./task-authoring.md) — the `task.toml` schema including `[verifier.hardening]` opt-outs. +- [`network-mode-prior-art.md`](./network-mode-prior-art.md) — network-access design model, mode/mechanism taxonomy, and credits to prior-art platforms. diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index 4aa8badc9..5ee49b642 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -159,6 +159,8 @@ at any depth (`api.example.com`, `a.b.example.com`) but **not** the bare apex `example.com`. The wildcard must be the leading label only — `a.*.com`, `*example.com`, and `ex*mple.com` are rejected at parse time. +For the full mode/mechanism taxonomy and credits to the platforms this design draws on, see [Network access: design and prior art](./network-mode-prior-art.md). + ## Prompt body and prompts/ sidecars The body below the frontmatter is the base prompt — free-form markdown, no From accab1b7cde14e102c7af06e1081ab32d7f263eb Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 03:34:09 +0000 Subject: [PATCH 04/23] fix(daytona): fail closed when block-all egress leaks (ENG-263) Derive daytona block-all from resolve_network_decision (not the deprecated allow_internet bool) and verify it after start with a raw-TCP egress canary; abort with SandboxStartupError if a block-all policy can still reach the network (platform did not honor network_block_all). --- src/benchflow/sandbox/daytona.py | 53 +++++++++++++++++++++++-- src/benchflow/sandbox/network_policy.py | 11 +++++ tests/test_network_runtime.py | 38 ++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 tests/test_network_runtime.py diff --git a/src/benchflow/sandbox/daytona.py b/src/benchflow/sandbox/daytona.py index 1bcbb2bed..cfc04a038 100644 --- a/src/benchflow/sandbox/daytona.py +++ b/src/benchflow/sandbox/daytona.py @@ -77,6 +77,10 @@ def wait_exponential(*_args: Any, **_kwargs: Any) -> Any: ) from benchflow.sandbox.daytona_strategies import _DaytonaDirect, _DaytonaStrategy from benchflow.sandbox.metadata import persist_sandbox_info +from benchflow.sandbox.network_policy import ( + blockall_enforcement_violation, + network_blocks_all, +) from benchflow.sandbox.protocol import ( SandboxImage, SandboxStartupError, @@ -97,6 +101,11 @@ def wait_exponential(*_args: Any, **_kwargs: Any) -> Any: # ``benchflow.sandbox.daytona`` keeps resolving from this path unchanged. __all__ = ["DaytonaSandbox", "SandboxStartupError"] +#: Raw-TCP egress canary (no DNS) used to verify a block-all policy is actually +#: enforced on daytona; reachable under block-all => the platform leaked. +_EGRESS_CANARY_HOST = "1.1.1.1" +_EGRESS_CANARY_PORT = 53 + def _ensure_daytona_anyio_compat() -> None: """Patch the anyio symbol that Daytona 0.176 imports on newer anyio.""" @@ -347,14 +356,14 @@ def __init__( self._snapshot_template_name = snapshot_template_name if network_block_all is not None: self._network_block_all = network_block_all - expected = not task_env_config.allow_internet + expected = network_blocks_all(task_env_config, "daytona") if network_block_all != expected: self.logger.warning( f"network_block_all={network_block_all} overrides task config " f"allow_internet={task_env_config.allow_internet}" ) else: - self._network_block_all = not task_env_config.allow_internet + self._network_block_all = network_blocks_all(task_env_config, "daytona") self._sandbox: AsyncSandbox | None = None # pyright: ignore[reportInvalidTypeForm] self._client_manager: DaytonaClientManager | None = None @@ -769,7 +778,45 @@ async def _download_files_individually(self, file_downloads: list[Any]) -> None: # Public interface — delegates to strategy async def start(self, force_build: bool) -> None: - return await self._strategy.start(force_build) + await self._strategy.start(force_build) + await self._verify_network_enforcement() + + async def _verify_network_enforcement(self) -> None: + """Fail closed if a block-all policy resolves but egress still works. + + Daytona delegates the block to a platform ``network_block_all`` flag with + no in-guest fallback or verification; if the platform ignores it, a + ``no-network`` task would silently run with full internet and produce a + falsely-rewarded result. Probe a raw-TCP canary and abort if reachable. + """ + if not self._network_block_all: + return + if blockall_enforcement_violation( + block_all=True, canary_reachable=await self._egress_reachable() + ): + raise SandboxStartupError( + "network resolves to block-all but the sandbox reached " + f"{_EGRESS_CANARY_HOST}:{_EGRESS_CANARY_PORT} — the daytona " + "platform did not enforce the egress block; failing closed instead " + "of running with leaked network" + ) + + async def _egress_reachable(self) -> bool: + probe = ( + "python3 - <<'PYEOF' 2>/dev/null || true\n" + "import socket\n" + "s = socket.socket(); s.settimeout(6)\n" + "try:\n" + f" s.connect(({_EGRESS_CANARY_HOST!r}, {_EGRESS_CANARY_PORT})); print('REACH')\n" + "except Exception:\n" + " pass\n" + "PYEOF" + ) + try: + res = await self.exec(probe, timeout_sec=20, user="root") + except Exception: + return False + return "REACH" in (res.stdout or "") async def stop(self, delete: bool) -> None: return await self._strategy.stop(delete) diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py index 778a83e14..6537c8518 100644 --- a/src/benchflow/sandbox/network_policy.py +++ b/src/benchflow/sandbox/network_policy.py @@ -105,3 +105,14 @@ def network_blocks_all(env_config: SandboxConfig, sandbox: str) -> bool: resolve_network_decision(env_config, sandbox).policy is EffectivePolicy.BLOCK_ALL ) + + +def blockall_enforcement_violation(*, block_all: bool, canary_reachable: bool) -> bool: + """Fail-closed check for a block-all policy. + + A sandbox that resolves to ``BLOCK_ALL`` must have no off-box route. If an + external canary is still reachable from inside, the platform did not honor the + block (e.g. daytona's ``network_block_all`` flag was ignored) — the run should + abort rather than produce a falsely-rewarded "offline" result. + """ + return block_all and canary_reachable diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py new file mode 100644 index 000000000..a70b93876 --- /dev/null +++ b/tests/test_network_runtime.py @@ -0,0 +1,38 @@ +"""ENG-263: runtime network-control behavior. + +TDD for the three runtime fixes: daytona fail-closed enforcement, forcing the +host-side usage proxy under a restrictive policy, and the docker relock plan. +Tests exercise pure decision functions through public interfaces. +""" + +from benchflow.task.config import SandboxConfig + + +# ---- Fix #2: daytona fail-closed enforcement ------------------------------- + + +def test_network_blocks_all_for_daytona(): + from benchflow.sandbox.network_policy import network_blocks_all + + assert ( + network_blocks_all(SandboxConfig(network_mode="no-network"), "daytona") is True + ) + assert network_blocks_all(SandboxConfig(), "daytona") is False # public default + # allowlist is unenforceable on daytona → resolve fails closed to block-all + assert ( + network_blocks_all( + SandboxConfig(network_mode="allowlist", allowed_hosts=["x.com"]), "daytona" + ) + is True + ) + + +def test_blockall_enforcement_violation(): + from benchflow.sandbox.network_policy import blockall_enforcement_violation + + # a block-all policy is VIOLATED when the sandbox can still reach the canary + assert blockall_enforcement_violation(block_all=True, canary_reachable=True) + # block-all and correctly unreachable → no violation + assert not blockall_enforcement_violation(block_all=True, canary_reachable=False) + # not block-all → never a violation + assert not blockall_enforcement_violation(block_all=False, canary_reachable=True) From 813985936ed11bea21402a947a1fa86868a56c94 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 03:39:22 +0000 Subject: [PATCH 05/23] fix(litellm): fail closed when usage proxy unavailable under a restrictive policy (ENG-263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under no-network/allowlist the agent cannot silently fall back to the direct provider (the egress allowlist blocks it) — so a skipped/unavailable LiteLLM usage proxy must abort the run instead of leaving the agent pointed at a blocked, untracked provider endpoint. Adds network_is_restrictive + proxy_unavailable_is_fatal and threads it through ensure_litellm_runtime. --- src/benchflow/providers/litellm_runtime.py | 20 ++++++++++++- src/benchflow/sandbox/network_policy.py | 19 +++++++++++++ tests/test_network_runtime.py | 33 +++++++++++++++++++--- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index be88f84ab..f25642677 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -47,6 +47,10 @@ extract_usage_from_trajectory, trajectory_from_litellm_callback_log, ) +from benchflow.sandbox.network_policy import ( + network_is_restrictive, + proxy_unavailable_is_fatal, +) from benchflow.sandbox.providers import OFF_BOX_MODEL_PROVIDERS from benchflow.trajectories.types import Trajectory from benchflow.usage_tracking import UsageTrackingConfig, usage_unavailable @@ -960,12 +964,17 @@ async def _skip_litellm_runtime( async def _fallback_or_raise_for_unavailable_litellm( *, usage_cfg: UsageTrackingConfig, + network_restrictive: bool = False, agent_env: dict[str, str], runtime: Any | None, required_error: str, fallback_reason: str, ) -> tuple[dict[str, str], Any | None]: - if usage_cfg.mode == "required": + if proxy_unavailable_is_fatal( + usage_mode=usage_cfg.mode, network_restrictive=network_restrictive + ): + # A restrictive network policy can't silently fall back to the direct + # provider (the egress allowlist would block it) — fail closed. raise RuntimeError(required_error) return await _skip_litellm_runtime( agent_env, @@ -1010,6 +1019,12 @@ async def ensure_litellm_runtime( return await _skip_litellm_runtime(agent_env, runtime) assert model is not None + network_restrictive = bool( + sandbox is not None + and getattr(sandbox, "task_env_config", None) is not None + and network_is_restrictive(sandbox.task_env_config, environment) + ) + if environment in _SANDBOX_LOCAL_ENVIRONMENTS and sandbox is None: raise RuntimeError("sandbox-local LiteLLM requires a sandbox handle") @@ -1018,6 +1033,7 @@ async def ensure_litellm_runtime( except ValueError as exc: return await _fallback_or_raise_for_unavailable_litellm( usage_cfg=usage_cfg, + network_restrictive=network_restrictive, agent_env=agent_env, runtime=runtime, required_error=( @@ -1037,6 +1053,7 @@ async def ensure_litellm_runtime( ) return await _fallback_or_raise_for_unavailable_litellm( usage_cfg=usage_cfg, + network_restrictive=network_restrictive, agent_env=agent_env, runtime=runtime, required_error=required_error, @@ -1092,6 +1109,7 @@ async def ensure_litellm_runtime( except Exception as exc: return await _fallback_or_raise_for_unavailable_litellm( usage_cfg=usage_cfg, + network_restrictive=network_restrictive, agent_env=agent_env, runtime=None, required_error=( diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py index 6537c8518..b2935d60d 100644 --- a/src/benchflow/sandbox/network_policy.py +++ b/src/benchflow/sandbox/network_policy.py @@ -116,3 +116,22 @@ def blockall_enforcement_violation(*, block_all: bool, canary_reachable: bool) - abort rather than produce a falsely-rewarded "offline" result. """ return block_all and canary_reachable + + +def network_is_restrictive(env_config: SandboxConfig, sandbox: str) -> bool: + """True iff the resolved policy is anything other than fully-open egress.""" + return ( + resolve_network_decision(env_config, sandbox).policy is not EffectivePolicy.OPEN + ) + + +def proxy_unavailable_is_fatal(*, usage_mode: str, network_restrictive: bool) -> bool: + """Whether an unavailable LiteLLM usage proxy must abort the run instead of + silently falling back to the direct provider. + + Fatal when usage tracking is ``required``, or when the network policy is + restrictive (the direct provider would be blocked by the egress allowlist, so + skipping the proxy leaves the model unreachable). ``off`` is an explicit + opt-out and is never forced fatal here. + """ + return usage_mode == "required" or (network_restrictive and usage_mode != "off") diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index a70b93876..daa8d018f 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -10,13 +10,10 @@ # ---- Fix #2: daytona fail-closed enforcement ------------------------------- - def test_network_blocks_all_for_daytona(): from benchflow.sandbox.network_policy import network_blocks_all - assert ( - network_blocks_all(SandboxConfig(network_mode="no-network"), "daytona") is True - ) + assert network_blocks_all(SandboxConfig(network_mode="no-network"), "daytona") is True assert network_blocks_all(SandboxConfig(), "daytona") is False # public default # allowlist is unenforceable on daytona → resolve fails closed to block-all assert ( @@ -36,3 +33,31 @@ def test_blockall_enforcement_violation(): assert not blockall_enforcement_violation(block_all=True, canary_reachable=False) # not block-all → never a violation assert not blockall_enforcement_violation(block_all=False, canary_reachable=True) + + +# ---- Fix #3: force host-side usage proxy under a restrictive policy ---------- + +def test_network_is_restrictive(): + from benchflow.sandbox.network_policy import network_is_restrictive + + assert network_is_restrictive(SandboxConfig(network_mode="no-network"), "docker") is True + assert ( + network_is_restrictive( + SandboxConfig(network_mode="allowlist", allowed_hosts=["x.com"]), "docker" + ) + is True + ) + assert network_is_restrictive(SandboxConfig(), "docker") is False # public + + +def test_proxy_unavailable_is_fatal(): + from benchflow.sandbox.network_policy import proxy_unavailable_is_fatal + + # 'required' is always fatal (existing contract) + assert proxy_unavailable_is_fatal(usage_mode="required", network_restrictive=False) + # a restrictive policy can't silently fall back to the (blocked) direct provider + assert proxy_unavailable_is_fatal(usage_mode="auto", network_restrictive=True) + # public 'auto' may skip the proxy and use the provider directly + assert not proxy_unavailable_is_fatal(usage_mode="auto", network_restrictive=False) + # 'off' is an explicit opt-out — the caller owns provider reachability + assert not proxy_unavailable_is_fatal(usage_mode="off", network_restrictive=True) From 058ecf17d748fe94bd9bb9ed687e2cbf8ad9c761 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 03:57:55 +0000 Subject: [PATCH 06/23] fix(docker): install-before-lockdown via non-destructive relock (ENG-263) The container now comes up open so the agent can install; relock_network() then drops it off the public bridge (and, for allowlist/model-lane, starts the bf-egress sidecar + moves it onto the internal-only net, returning HTTP_PROXY for the agent). The main container is never recreated, so the install survives. Called from install_agent() after lockdown_paths. Fixes rc=127 agent-install failures under no-network/tight-allowlist on docker. --- src/benchflow/rollout/__init__.py | 15 +++++++ src/benchflow/sandbox/docker.py | 71 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index 0a1291fe9..acae72cc0 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -684,6 +684,7 @@ async def install_agent(self) -> None: self._env, cfg.generated_skills_root, cfg.sandbox_user ) await self._planes.lockdown_paths(self._env, self._effective_locked) + await self._lock_down_network() self._phase = "installed" return @@ -744,8 +745,22 @@ async def install_agent(self) -> None: ) await self._planes.lockdown_paths(self._env, self._effective_locked) + await self._lock_down_network() self._phase = "installed" + async def _lock_down_network(self) -> None: + """Apply the task's restrictive network policy after agent install + (install-before-lockdown). Docker swaps the container onto an internal + network + egress sidecar and returns the proxy env the agent must use; + other sandboxes no-op. Merges the returned proxy env into the agent env. + """ + relock = getattr(self._env, "relock_network", None) + if relock is None: + return + proxy_env = await relock() + if proxy_env: + self._agent_env = {**self._agent_env, **proxy_env} + # Phase 3b: CONNECT (ACP session — re-entrant) async def connect(self) -> None: diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index ca2ac2169..db9cdbab1 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -203,6 +203,9 @@ def __init__( self._use_prebuilt = False self._compose_task_env: dict[str, str] = {} + # Install-before-lockdown: the restrictive network policy is applied by + # relock_network() AFTER the agent installs, not at sandbox start. + self._network_locked = False if task_env_config.env and self._uses_compose: self._compose_task_env = resolve_env_vars(task_env_config.env) @@ -273,6 +276,10 @@ def _network_policy_compose_paths(self) -> list[Path]: resolve_network_decision, ) + if not self._network_locked: + # Stay open during the install phase; relock_network() applies the + # restrictive policy once the agent has been installed. + return [] decision = resolve_network_decision(self.task_env_config, "docker") if decision.policy is EffectivePolicy.OPEN: return [] @@ -301,6 +308,70 @@ def _network_policy_compose_paths(self) -> list[Path]: # BLOCK_ALL with no lane, or nowhere to stage the proxy → fail closed. return [self._DOCKER_COMPOSE_NO_NETWORK_PATH] + async def relock_network(self) -> dict[str, str]: + """Apply the task's restrictive network policy to the running container. + + The container came up open so the agent could install (install-before- + lockdown); now drop it off the public bridge. For allowlist / model-lane + runs, start the egress sidecar and move the container onto the internal- + only network, returning the HTTP(S)_PROXY env the agent must use. ``public`` + is a no-op. The ``main`` container is never recreated, so the install + survives. Returns the proxy env to merge into the agent launch env. + """ + from benchflow.sandbox._egress import ( + _EGRESS_INTERNAL_NET, + _EGRESS_PORT, + _EGRESS_SERVICE, + ) + from benchflow.sandbox.network_policy import ( + EffectivePolicy, + resolve_network_decision, + ) + + decision = resolve_network_decision(self.task_env_config, "docker") + if decision.policy is EffectivePolicy.OPEN: + return {} + + # Gate _network_policy_compose_paths to emit the real override now. + self._network_locked = True + cid = await self._main_container_id() + if not cid: + self.logger.warning("relock_network: no 'main' container; skipping") + return {} + + project = _sanitize_docker_compose_project_name(self.session_id) + paths = self._network_policy_compose_paths() + use_sidecar = bool(paths and paths[0] != self._DOCKER_COMPOSE_NO_NETWORK_PATH) + + if use_sidecar: + # Bring up ONLY the egress sidecar (creates the bf_egress_* networks); + # --no-deps leaves the already-running 'main' container in place. + await self._run_docker_compose_command( + ["up", "--detach", "--no-deps", _EGRESS_SERVICE] + ) + await self._docker_cli( + ["network", "connect", f"{project}_{_EGRESS_INTERNAL_NET}", cid], + check=False, + ) + # Lockdown: detach the container from the public bridge. + await self._docker_cli( + ["network", "disconnect", f"{project}_default", cid], check=False + ) + self.logger.info( + "relock_network: %s applied (sidecar=%s)", decision.policy.name, use_sidecar + ) + if use_sidecar: + proxy = f"http://{_EGRESS_SERVICE}:{_EGRESS_PORT}" + return { + "HTTP_PROXY": proxy, + "HTTPS_PROXY": proxy, + "http_proxy": proxy, + "https_proxy": proxy, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + } + return {} + def _write_mounts_compose_file(self) -> Path: compose = {"services": {"main": {"volumes": self._mounts_json}}} assert self.rollout_paths is not None From cbeb5c19f31843a188f2b779a9614a294a5908ca Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 04:42:53 +0000 Subject: [PATCH 07/23] fix(daytona): actually enforce no-network (gate lift + reliable egress probe) (ENG-263) The daytona no-network leak was the _lift_agent_network_to_public blanket lift to public (the same P0 the model lane replaced for docker), still firing on non-docker. Gate the lift for daytona/daytona-dind too (no lane there -> the honest behavior is to keep the policy and fail closed). Also fix the egress canary probe to a single-line python -c (the heredoc form did not execute through daytona exec) so block-all enforcement is actually verified at start. Verified live: daytona no-network now resolves block_all=True and the canary (1.1.1.1:443) is unreachable -- genuinely offline, no longer silently public. --- src/benchflow/sandbox/daytona.py | 30 ++++++++++++++++++++---------- src/benchflow/sandbox/setup.py | 7 ++++++- tests/test_network_modes.py | 15 ++++++++------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/benchflow/sandbox/daytona.py b/src/benchflow/sandbox/daytona.py index cfc04a038..7a42e8a51 100644 --- a/src/benchflow/sandbox/daytona.py +++ b/src/benchflow/sandbox/daytona.py @@ -104,7 +104,7 @@ def wait_exponential(*_args: Any, **_kwargs: Any) -> Any: #: Raw-TCP egress canary (no DNS) used to verify a block-all policy is actually #: enforced on daytona; reachable under block-all => the platform leaked. _EGRESS_CANARY_HOST = "1.1.1.1" -_EGRESS_CANARY_PORT = 53 +_EGRESS_CANARY_PORT = 443 def _ensure_daytona_anyio_compat() -> None: @@ -789,6 +789,9 @@ async def _verify_network_enforcement(self) -> None: ``no-network`` task would silently run with full internet and produce a falsely-rewarded result. Probe a raw-TCP canary and abort if reachable. """ + self.logger.info( + "verify_network_enforcement: block_all=%s", self._network_block_all + ) if not self._network_block_all: return if blockall_enforcement_violation( @@ -802,21 +805,28 @@ async def _verify_network_enforcement(self) -> None: ) async def _egress_reachable(self) -> bool: + py = ( + "import socket;socket.setdefaulttimeout(6);" + f"socket.create_connection(({_EGRESS_CANARY_HOST!r},{_EGRESS_CANARY_PORT}))" + ".close();print('REACH')" + ) probe = ( - "python3 - <<'PYEOF' 2>/dev/null || true\n" - "import socket\n" - "s = socket.socket(); s.settimeout(6)\n" - "try:\n" - f" s.connect(({_EGRESS_CANARY_HOST!r}, {_EGRESS_CANARY_PORT})); print('REACH')\n" - "except Exception:\n" - " pass\n" - "PYEOF" + f"(python3 -c {shlex.quote(py)} || python -c {shlex.quote(py)}) " + "2>/dev/null || true" ) try: res = await self.exec(probe, timeout_sec=20, user="root") except Exception: + self.logger.warning("egress canary probe failed to run", exc_info=True) return False - return "REACH" in (res.stdout or "") + reachable = "REACH" in (res.stdout or "") + self.logger.info( + "egress canary %s:%s reachable=%s", + _EGRESS_CANARY_HOST, + _EGRESS_CANARY_PORT, + reachable, + ) + return reachable async def stop(self, delete: bool) -> None: return await self._strategy.stop(delete) diff --git a/src/benchflow/sandbox/setup.py b/src/benchflow/sandbox/setup.py index de2d8ff41..beea53f28 100644 --- a/src/benchflow/sandbox/setup.py +++ b/src/benchflow/sandbox/setup.py @@ -621,6 +621,9 @@ def _patched(self, include_os_env=True): # type: ignore[override] _DIND_PATCH_APPLIED = True +_NO_LIFT_SANDBOXES = frozenset({"docker", "daytona", "daytona-dind"}) + + def _lift_agent_network_to_public(env_config: Any, sandbox_type: str) -> Any: """Lift a restrictive network policy to public for in-sandbox LLM agents. @@ -635,7 +638,9 @@ def _lift_agent_network_to_public(env_config: Any, sandbox_type: str) -> Any: the public internet. Returns a possibly-copied ``env_config`` (the original is never mutated). """ - if sandbox_type == "docker" or not _network_policy_is_restrictive(env_config): + if sandbox_type in _NO_LIFT_SANDBOXES or not _network_policy_is_restrictive( + env_config + ): return env_config from benchflow.task.config import NetworkMode diff --git a/tests/test_network_modes.py b/tests/test_network_modes.py index c1fdea6c6..8c213489f 100644 --- a/tests/test_network_modes.py +++ b/tests/test_network_modes.py @@ -98,21 +98,22 @@ def test_resolve_sets_model_lane_for_restrictive(): ) # public -def test_lift_agent_network_skips_docker(): - # Docker preserves the restrictive policy (the model lane handles model access). +def test_lift_agent_network_skips_docker_and_daytona(): + # Docker (model lane) and daytona (fail-closed, no lane) preserve the policy. from benchflow.sandbox.setup import _lift_agent_network_to_public cfg = SandboxConfig(network_mode="no-network") - out = _lift_agent_network_to_public(cfg, "docker") - assert out is cfg # untouched, not copied - assert out.network_mode == "no-network" + for sb in ("docker", "daytona", "daytona-dind"): + out = _lift_agent_network_to_public(cfg, sb) + assert out is cfg # untouched, not copied + assert out.network_mode == "no-network" -def test_lift_agent_network_public_for_non_docker(): +def test_lift_agent_network_public_for_modal(): from benchflow.sandbox.setup import _lift_agent_network_to_public cfg = SandboxConfig(network_mode="no-network") - out = _lift_agent_network_to_public(cfg, "daytona") + out = _lift_agent_network_to_public(cfg, "modal") assert out is not cfg # copied, original never mutated assert out.network_mode == "public" assert cfg.network_mode == "no-network" From 1010b2191af2848a024dd6d6b6304d865daa4863 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 16:34:11 +0000 Subject: [PATCH 08/23] fix: sort imports in test_network_runtime (ruff I001, ENG-263) --- tests/test_network_runtime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index daa8d018f..59be71a76 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -7,7 +7,6 @@ from benchflow.task.config import SandboxConfig - # ---- Fix #2: daytona fail-closed enforcement ------------------------------- def test_network_blocks_all_for_daytona(): From f6d7ffdb463cbdf0788da8180460721103181b1b Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 16:35:26 +0000 Subject: [PATCH 09/23] style: ruff format test_network_runtime (ENG-263) --- tests/test_network_runtime.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index 59be71a76..c5ca3ecf7 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -9,10 +9,13 @@ # ---- Fix #2: daytona fail-closed enforcement ------------------------------- + def test_network_blocks_all_for_daytona(): from benchflow.sandbox.network_policy import network_blocks_all - assert network_blocks_all(SandboxConfig(network_mode="no-network"), "daytona") is True + assert ( + network_blocks_all(SandboxConfig(network_mode="no-network"), "daytona") is True + ) assert network_blocks_all(SandboxConfig(), "daytona") is False # public default # allowlist is unenforceable on daytona → resolve fails closed to block-all assert ( @@ -36,10 +39,14 @@ def test_blockall_enforcement_violation(): # ---- Fix #3: force host-side usage proxy under a restrictive policy ---------- + def test_network_is_restrictive(): from benchflow.sandbox.network_policy import network_is_restrictive - assert network_is_restrictive(SandboxConfig(network_mode="no-network"), "docker") is True + assert ( + network_is_restrictive(SandboxConfig(network_mode="no-network"), "docker") + is True + ) assert ( network_is_restrictive( SandboxConfig(network_mode="allowlist", allowed_hosts=["x.com"]), "docker" From 10582d1012e531b1969bd05429aac86e63762ddd Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 16:43:20 +0000 Subject: [PATCH 10/23] fix: _lock_down_network tolerates mock/non-async sandboxes (ENG-263) The install-before-lockdown relock hook awaited self._env.relock_network() unconditionally; with a MagicMock/AsyncMock sandbox in tests that raised (MagicMock not awaitable / **mock dict). Only await when isawaitable and only merge a real dict result. Fixes 8 CI test failures in test_trial_install_agent_timeout and test_verify. --- src/benchflow/rollout/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index acae72cc0..941ff6579 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -39,6 +39,7 @@ import asyncio import contextlib +import inspect import logging import shlex import shutil @@ -757,8 +758,11 @@ async def _lock_down_network(self) -> None: relock = getattr(self._env, "relock_network", None) if relock is None: return - proxy_env = await relock() - if proxy_env: + proxy_env = relock() + if inspect.isawaitable(proxy_env): + proxy_env = await proxy_env + # Tolerate mock sandboxes in tests: only merge a real dict result. + if isinstance(proxy_env, dict) and proxy_env: self._agent_env = {**self._agent_env, **proxy_env} # Phase 3b: CONNECT (ACP session — re-entrant) From 7ff71eb25c1c7145e6a9128d8b68d9a44c369b0d Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 17:05:12 +0000 Subject: [PATCH 11/23] fix(docker): fail closed if relock_network lockdown didn't take effect (greptile P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit relock_network ran `docker network connect/disconnect` with check=False and discarded the results, then logged "applied" + returned the proxy env unconditionally. Root cause: a silently-failed disconnect (after a successful connect) left the container on BOTH {project}_default and the internal net — routing around the egress proxy, so allowlist/no-network was silently UNENFORCED; a failed connect stranded it with no network. Both were reported as success. Fix: after the swap, `docker inspect` the container's actual network attachments and raise SandboxStartupError via lockdown_complete() unless it is detached from {project}_default (and, for sidecar runs, attached to {project}_bf_egress_internal). Verifying the resulting state is strictly stronger than per-op check=True (an op can return 0 with the wrong end state, or non-zero on a benign race). Adds lockdown_complete() with 5 unit tests; live-verified a real allowlist relock still completes (no false positive). Refs ENG-263, greptile P1 on PR #785. --- src/benchflow/sandbox/docker.py | 24 ++++++++++++++++++++++++ src/benchflow/sandbox/network_policy.py | 19 +++++++++++++++++++ tests/test_network_runtime.py | 20 ++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index db9cdbab1..6459d0671 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -357,6 +357,30 @@ async def relock_network(self) -> dict[str, str]: await self._docker_cli( ["network", "disconnect", f"{project}_default", cid], check=False ) + # Fail closed: confirm the swap actually took effect. A silently-failed + # connect/disconnect could leave the container bypassing the proxy (still + # on the public bridge) or stranded (on no network); inspect the real + # attachments and refuse to run an unenforced policy. + from benchflow.sandbox.network_policy import lockdown_complete + from benchflow.sandbox.protocol import SandboxStartupError + + inspect_res = await self._docker_cli( + [ + "inspect", + cid, + "--format", + "{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}", + ], + check=False, + ) + attached = set(inspect_res.stdout.split()) + internal_net = f"{project}_{_EGRESS_INTERNAL_NET}" if use_sidecar else None + if not lockdown_complete(attached, f"{project}_default", internal_net): + raise SandboxStartupError( + f"relock_network: {decision.policy.name} lockdown did not take " + f"effect (container networks={sorted(attached)}); refusing to run " + "with an unenforced network policy" + ) self.logger.info( "relock_network: %s applied (sidecar=%s)", decision.policy.name, use_sidecar ) diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py index b2935d60d..64dc85d57 100644 --- a/src/benchflow/sandbox/network_policy.py +++ b/src/benchflow/sandbox/network_policy.py @@ -135,3 +135,22 @@ def proxy_unavailable_is_fatal(*, usage_mode: str, network_restrictive: bool) -> opt-out and is never forced fatal here. """ return usage_mode == "required" or (network_restrictive and usage_mode != "off") + + +def lockdown_complete( + attached_networks: set[str], default_net: str, internal_net: str | None +) -> bool: + """True iff a docker relock actually took effect. + + After install-before-lockdown swaps the container's networks, it must be + detached from the public bridge (*default_net*) and, when an egress sidecar + is in use, attached to *internal_net*. If a ``network connect``/``disconnect`` + silently failed the container could sit on BOTH nets (egress proxy bypassed) + or on NONE (stranded) — callers fail closed when this returns ``False`` + rather than running with an unenforced policy. + """ + on_public_bridge = default_net in attached_networks + missing_internal = ( + internal_net is not None and internal_net not in attached_networks + ) + return not (on_public_bridge or missing_internal) diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index c5ca3ecf7..cf45f98bf 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -67,3 +67,23 @@ def test_proxy_unavailable_is_fatal(): assert not proxy_unavailable_is_fatal(usage_mode="auto", network_restrictive=False) # 'off' is an explicit opt-out — the caller owns provider reachability assert not proxy_unavailable_is_fatal(usage_mode="off", network_restrictive=True) + + +# ---- Fix #1b: relock fail-closed verification (greptile P1) ------------------ + + +def test_lockdown_complete_docker_relock(): + from benchflow.sandbox.network_policy import lockdown_complete + + default = "proj_default" + internal = "proj_bf_egress_internal" + # allowlist / model-lane: detached from the public bridge, on the internal net + assert lockdown_complete({internal}, default, internal) is True + # disconnect silently failed -> still on default + internal -> egress BYPASS + assert lockdown_complete({default, internal}, default, internal) is False + # connect silently failed -> sidecar expected but not on internal -> stranded + assert lockdown_complete(set(), default, internal) is False + # hermetic (no sidecar): detached, attached to nothing -> fully dark, ok + assert lockdown_complete(set(), default, None) is True + # hermetic but disconnect failed -> still on default -> not locked down + assert lockdown_complete({default}, default, None) is False From 3228ec7e9facb6ff78ef5a40069bf1cfc92ec106 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 19:12:44 +0000 Subject: [PATCH 12/23] fix(egress): rewrite plain-HTTP requests to origin-form (greptile P2 #1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stdlib egress proxy forwarded plain-HTTP requests with the absolute-URI request line (POST http://host/path HTTP/1.1), which FastAPI/uvicorn upstreams 404 ({detail: Not Found}) — verified via repro. This is the model-lane blocker: the host litellm proxy is plain-HTTP, so the agent model call traversed bf-egress in absolute form and 404d. Add _to_origin_form() (rewrites the request line to origin-form; 3 unit tests) and apply it before forwarding. Refs ENG-262/263. NOTE: removes the 404 but plain-HTTP forwarding still has response/keep-alive gaps (litellm 500/timeout) — model-lane reachability under restrictive policy needs the follow-up below. --- src/benchflow/sandbox/_egress_proxy.py | 21 +++++++++++++++++- tests/test_network_runtime.py | 30 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/benchflow/sandbox/_egress_proxy.py b/src/benchflow/sandbox/_egress_proxy.py index 732ec3ad0..2dbf0d14c 100644 --- a/src/benchflow/sandbox/_egress_proxy.py +++ b/src/benchflow/sandbox/_egress_proxy.py @@ -135,7 +135,7 @@ def _handle(client: socket.socket) -> None: except OSError: client.sendall(b"HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n") return - upstream.sendall(header) + upstream.sendall(_to_origin_form(header)) _pipe(client, upstream) upstream.close() except Exception: @@ -145,6 +145,25 @@ def _handle(client: socket.socket) -> None: client.close() +def _to_origin_form(header: bytes) -> bytes: + """Rewrite a proxy-style absolute-URI request line to origin-form. + + ``POST http://host:port/path?q HTTP/1.1`` -> ``POST /path?q HTTP/1.1`` so the + upstream (e.g. a FastAPI/uvicorn server such as the host litellm proxy on the + model lane) routes it instead of returning 404/400 for the absolute form. + Already-origin-form request lines are returned unchanged. + """ + line, sep, rest = header.partition(b"\r\n") + parts = line.split(b" ") + if len(parts) != 3 or b"://" not in parts[1]: + return header + method, target, version = parts + after = target.split(b"://", 1)[1] + slash = after.find(b"/") + path = after[slash:] if slash != -1 else b"/" + return method + b" " + path + b" " + version + sep + rest + + def main() -> None: srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index cf45f98bf..a59dc889c 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -87,3 +87,33 @@ def test_lockdown_complete_docker_relock(): assert lockdown_complete(set(), default, None) is True # hermetic but disconnect failed -> still on default -> not locked down assert lockdown_complete({default}, default, None) is False + + +# ---- Fix #1c: plain-HTTP origin-form rewrite (greptile P2 #1; lane blocker) --- + +def test_to_origin_form_rewrites_absolute_uri(): + from benchflow.sandbox._egress_proxy import _to_origin_form + + # proxy absolute-URI request line -> origin-form; headers + body preserved + h = b"POST http://172.17.0.1:8080/chat/completions HTTP/1.1\r\nHost: x\r\nContent-Length: 2\r\n\r\n{}" + assert ( + _to_origin_form(h) + == b"POST /chat/completions HTTP/1.1\r\nHost: x\r\nContent-Length: 2\r\n\r\n{}" + ) + + +def test_to_origin_form_query_and_root_default(): + from benchflow.sandbox._egress_proxy import _to_origin_form + + assert ( + _to_origin_form(b"GET http://h:80/p?a=1&b=2 HTTP/1.1\r\n\r\n") + == b"GET /p?a=1&b=2 HTTP/1.1\r\n\r\n" + ) + assert _to_origin_form(b"GET http://h HTTP/1.1\r\n\r\n") == b"GET / HTTP/1.1\r\n\r\n" + + +def test_to_origin_form_passthrough_already_origin(): + from benchflow.sandbox._egress_proxy import _to_origin_form + + h = b"GET /already HTTP/1.1\r\nHost: h\r\n\r\n" + assert _to_origin_form(h) == h From ce0858d4cc2580e2cc35dd2dc69bb1d33f931059 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 19:13:02 +0000 Subject: [PATCH 13/23] style: ruff format after origin-form rewrite (ENG-263) --- tests/test_network_runtime.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index a59dc889c..82008e227 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -91,6 +91,7 @@ def test_lockdown_complete_docker_relock(): # ---- Fix #1c: plain-HTTP origin-form rewrite (greptile P2 #1; lane blocker) --- + def test_to_origin_form_rewrites_absolute_uri(): from benchflow.sandbox._egress_proxy import _to_origin_form @@ -109,7 +110,9 @@ def test_to_origin_form_query_and_root_default(): _to_origin_form(b"GET http://h:80/p?a=1&b=2 HTTP/1.1\r\n\r\n") == b"GET /p?a=1&b=2 HTTP/1.1\r\n\r\n" ) - assert _to_origin_form(b"GET http://h HTTP/1.1\r\n\r\n") == b"GET / HTTP/1.1\r\n\r\n" + assert ( + _to_origin_form(b"GET http://h HTTP/1.1\r\n\r\n") == b"GET / HTTP/1.1\r\n\r\n" + ) def test_to_origin_form_passthrough_already_origin(): From 92b7c8c23f6a62fc00eae4b8cdcd7fba091b705a Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Tue, 16 Jun 2026 20:20:51 +0000 Subject: [PATCH 14/23] fix(network): make restrictive-docker allowlist reach the provider over HTTPS Under a restrictive docker policy the agent runs on an internal-only network and reaches the outside world only through the bf-egress CONNECT proxy. Two gaps kept the agent from ever completing an LLM call (ACP -32603 "Connection error"): 1. The host LiteLLM proxy forwarding path (plain-HTTP via the egress proxy) is fragile. Pivot: under a restrictive docker policy, skip the host proxy and let the agent reach the provider directly over HTTPS. The provider host is added to the egress allowlist (provider_host_for_model + relock_network extra_allowed_hosts) so the CONNECT tunnel is permitted; token usage falls back to native-ACP telemetry. 2. ROOT CAUSE of the -32603: the scene/role path (connect_as) rebuilds agent_env from config via resolve_agent_env, discarding the HTTP(S)_PROXY env that _lock_down_network() had merged into self._agent_env. The agent then launched on the internal-only net with no proxy -> litellm "Connection error". _lock_down_network now persists the proxy env (self._lockdown_proxy_env) and connect_as re-merges it into the rebuilt agent_env. connect() already carried it via self._agent_env. Also harden relock_network's lockdown verification against a None docker-inspect stdout (fail-closed). Verified live: citation-check-allow on docker (deepseek/deepseek-v4-flash via api.deepseek.com) now runs a full session (12 tool calls) reaching both the model and the task allowlist hosts (crossref/doi/arxiv) through the proxy, with 0 connection errors and a clean verifier run. Reward is a model-correctness outcome, separate from network enforcement. Tests: connect_as lockdown-proxy regression (test_connect_as_env.py), provider_host_for_model (test_network_runtime.py). 68 relevant tests pass; ruff + ty clean. --- src/benchflow/agents/providers.py | 24 +++++++++++++++++++++ src/benchflow/providers/litellm_runtime.py | 11 ++++++++++ src/benchflow/rollout/__init__.py | 25 +++++++++++++++++++++- src/benchflow/sandbox/docker.py | 21 +++++++++++++----- tests/test_connect_as_env.py | 25 ++++++++++++++++++++++ tests/test_network_runtime.py | 18 ++++++++++++++++ 6 files changed, 118 insertions(+), 6 deletions(-) diff --git a/src/benchflow/agents/providers.py b/src/benchflow/agents/providers.py index 0f711ef24..ba370c92f 100644 --- a/src/benchflow/agents/providers.py +++ b/src/benchflow/agents/providers.py @@ -465,6 +465,30 @@ def resolve_base_url( return url.format_map(replacements) +def provider_host_for_model(model: str, env: dict[str, str]) -> str | None: + """Hostname of a model's resolved provider base_url, or ``None``. + + Used to allowlist the model provider's host under a restrictive network + policy so the agent can reach it directly over HTTPS (a clean CONNECT + tunnel). Returns ``None`` when the model has no registered provider prefix + or the provider's base_url env is unset — don't guess; leave the allowlist + unchanged. + """ + from urllib.parse import urlparse + + found = find_provider(model) + if found is None: + return None + _, cfg = found + try: + url = resolve_base_url(cfg, env) + except KeyError: + return None + if not url: + return None + return urlparse(url if "://" in url else f"https://{url}").hostname + + def strip_provider_prefix(model: str) -> str: """Strip a *registered* provider prefix. Unregistered inputs pass through. diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index f25642677..8f69af590 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -1025,6 +1025,17 @@ async def ensure_litellm_runtime( and network_is_restrictive(sandbox.task_env_config, environment) ) + if network_restrictive and environment == "docker": + # The host litellm proxy is plain-HTTP and reached via the egress + # proxy; that forwarding path is fragile. Under a restrictive docker + # policy the agent instead reaches the provider directly over HTTPS + # (its host is egress-allowlisted); usage is captured by native-ACP. + return await _skip_litellm_runtime( + agent_env, + runtime, + reason="restrictive docker policy: direct-provider over HTTPS (egress-allowlisted)", + ) + if environment in _SANDBOX_LOCAL_ENVIRONMENTS and sandbox is None: raise RuntimeError("sandbox-local LiteLLM requires a sandbox handle") diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index 941ff6579..e449a13dd 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -327,6 +327,12 @@ def __init__(self, config: RolloutConfig) -> None: self._ask_user_handler_set: bool = False self._agent_name: str = "" self._active_role: Role | None = None + #: Egress proxy env (HTTP(S)_PROXY -> bf-egress sidecar) produced by + #: the network lockdown. Empty unless a restrictive docker policy is + #: active. The scene/role path (connect_as) rebuilds agent_env from + #: config, so it must re-merge this or the agent can't reach the + #: allowlisted provider (the internal-only net has no direct route). + self._lockdown_proxy_env: dict[str, str] = {} # Cursors into the live ACP session's cumulative trajectory and # tool-call totals so execute() and the partial-capture path extend # only the delta since the last read. Reset per session in @@ -758,11 +764,20 @@ async def _lock_down_network(self) -> None: relock = getattr(self._env, "relock_network", None) if relock is None: return - proxy_env = relock() + # Allowlist the model provider host so the agent can reach it directly + # over HTTPS under a restrictive policy (the host-proxy path is skipped). + from benchflow.agents.providers import provider_host_for_model + + host = provider_host_for_model( + self._config.primary_model or "", self._agent_env + ) + extra = (host,) if host else () + proxy_env = relock(extra_allowed_hosts=extra) if inspect.isawaitable(proxy_env): proxy_env = await proxy_env # Tolerate mock sandboxes in tests: only merge a real dict result. if isinstance(proxy_env, dict) and proxy_env: + self._lockdown_proxy_env = dict(proxy_env) self._agent_env = {**self._agent_env, **proxy_env} # Phase 3b: CONNECT (ACP session — re-entrant) @@ -1655,6 +1670,14 @@ async def connect_as(self, role: Role) -> None: usage_tracking=cfg.usage_tracking, sandbox=self._env, ) + # The network lockdown (install-before-lockdown) put the container on + # an internal-only net reachable off-box only through the bf-egress + # proxy. resolve_agent_env above rebuilt agent_env from config and + # dropped that proxy env, so re-merge it here or the agent's LLM + # client hits a connection error reaching the allowlisted provider. + lockdown_proxy_env = getattr(self, "_lockdown_proxy_env", None) + if lockdown_proxy_env: + agent_env = {**agent_env, **lockdown_proxy_env} role_agent_differs = role.agent != cfg.primary_agent needs_role_credentials = ( diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index 6459d0671..a5b75e7bd 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -206,6 +206,9 @@ def __init__( # Install-before-lockdown: the restrictive network policy is applied by # relock_network() AFTER the agent installs, not at sandbox start. self._network_locked = False + #: Extra hosts unioned into the egress allowlist at relock (the model + #: provider host, so the agent reaches it directly over HTTPS). + self._extra_allowed_hosts: tuple[str, ...] = () if task_env_config.env and self._uses_compose: self._compose_task_env = resolve_env_vars(task_env_config.env) @@ -295,9 +298,12 @@ def _network_policy_compose_paths(self) -> list[Path]: decision.policy is EffectivePolicy.ALLOWLIST or lane ): hosts = ( - decision.allowed_hosts - if decision.policy is EffectivePolicy.ALLOWLIST - else () + tuple( + decision.allowed_hosts + if decision.policy is EffectivePolicy.ALLOWLIST + else () + ) + + self._extra_allowed_hosts ) override = build_egress_override( hosts, @@ -308,7 +314,9 @@ def _network_policy_compose_paths(self) -> list[Path]: # BLOCK_ALL with no lane, or nowhere to stage the proxy → fail closed. return [self._DOCKER_COMPOSE_NO_NETWORK_PATH] - async def relock_network(self) -> dict[str, str]: + async def relock_network( + self, extra_allowed_hosts: tuple[str, ...] = () + ) -> dict[str, str]: """Apply the task's restrictive network policy to the running container. The container came up open so the agent could install (install-before- @@ -334,6 +342,7 @@ async def relock_network(self) -> dict[str, str]: # Gate _network_policy_compose_paths to emit the real override now. self._network_locked = True + self._extra_allowed_hosts = tuple(extra_allowed_hosts) cid = await self._main_container_id() if not cid: self.logger.warning("relock_network: no 'main' container; skipping") @@ -373,7 +382,9 @@ async def relock_network(self) -> dict[str, str]: ], check=False, ) - attached = set(inspect_res.stdout.split()) + # stdout is str|None (check=False); None -> empty set -> lockdown_complete + # fails closed below (treated as not-detached), which is the safe default. + attached: set[str] = set((inspect_res.stdout or "").split()) internal_net = f"{project}_{_EGRESS_INTERNAL_NET}" if use_sidecar else None if not lockdown_complete(attached, f"{project}_default", internal_net): raise SandboxStartupError( diff --git a/tests/test_connect_as_env.py b/tests/test_connect_as_env.py index 129e1d86b..ca8348d45 100644 --- a/tests/test_connect_as_env.py +++ b/tests/test_connect_as_env.py @@ -140,6 +140,31 @@ def fake_resolve(agent, model, env): assert captured["env"] == {} + @pytest.mark.asyncio + async def test_lockdown_proxy_env_reaches_agent(self, _mock_trial): + """Egress proxy env from network lockdown must reach the agent launched + via connect_as. Under a restrictive docker policy the container is on an + internal-only network and reaches the allowlisted provider only through + the bf-egress sidecar (HTTPS_PROXY). The scene/role path rebuilds + agent_env from config, so it must re-merge the lockdown proxy env or the + agent's LLM client gets "Connection error". Regression for that path.""" + proxy = { + "HTTP_PROXY": "http://bf-egress:8080", + "HTTPS_PROXY": "http://bf-egress:8080", + "http_proxy": "http://bf-egress:8080", + "https_proxy": "http://bf-egress:8080", + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + } + _mock_trial._lockdown_proxy_env = dict(proxy) + role = _mock_trial._config.scenes[0].roles[0] + await _mock_trial.connect_as(role) + + sent = _mock_trial._planes.connect_acp.await_args.kwargs["agent_env"] + assert sent.get("HTTPS_PROXY") == "http://bf-egress:8080" + assert sent.get("https_proxy") == "http://bf-egress:8080" + assert sent.get("NO_PROXY") == "localhost,127.0.0.1" + @pytest.mark.asyncio async def test_same_agent_different_model_refreshes_credentials(self, _mock_trial): """Guards ENG-91 P0 same-agent role credential refresh regression.""" diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index 82008e227..fd210ffb8 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -120,3 +120,21 @@ def test_to_origin_form_passthrough_already_origin(): h = b"GET /already HTTP/1.1\r\nHost: h\r\n\r\n" assert _to_origin_form(h) == h + + +# ---- Fix #3 pivot: resolve provider host to allowlist under restrictive policy --- + +def test_provider_host_for_model(): + from benchflow.agents.providers import provider_host_for_model + + assert ( + provider_host_for_model( + "deepseek/deepseek-v4-flash", {"DEEPSEEK_BASE_URL": "https://api.deepseek.com"} + ) + == "api.deepseek.com" + ) + assert provider_host_for_model("openai/gpt-4o", {}) == "api.openai.com" + # no provider prefix -> unknown -> None (caller leaves allowlist unchanged) + assert provider_host_for_model("deepseek-v4-flash", {}) is None + # provider prefix but its base_url env is missing -> None (don't guess) + assert provider_host_for_model("deepseek/x", {}) is None From 80d3601274783a26a102cc2f891397d86f097302 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Wed, 17 Jun 2026 00:36:56 +0000 Subject: [PATCH 15/23] feat(network): enforce allowlist on daytona (enforce-when-faithful), parity with docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daytona has a native allowlist (`network_allow_list`) + runtime `update_network_settings` (verified in pinned SDK 0.184.0), so the ENG-219 "allowlist -> reject at preflight on daytona" was conservative, not a hard limit. Close the docker<->daytona gap with an enforce-when-faithful policy: daytona now ENFORCES a per-host allowlist when it can be faithfully expressed as an IPv4-CIDR list, and fails closed with a precise reason otherwise. The mechanisms differ (docker = hostname/SNI egress proxy; daytona = IPv4 CIDR), so a hostname allowlist is mapped to /32 CIDRs at lockdown. This is only done when faithful; we never silently weaken a policy the user wrote. - network_policy: `plan_daytona_allowlist(allowed_hosts, model_host, resolve)` -> enforce(CIDRs) | reject(reason). Rejects wildcards (inexpressible as IPs), unresolvable hosts, and >10 IPs (daytona's cap). `resolve_ipv4` helper. daytona added to ALLOWLIST_CAPABLE_SANDBOXES; IP_BASED_ALLOWLIST_SANDBOXES. - runtime_capabilities: daytona allowlist no longer blanket-rejected. The only model-independent preflight reject is wildcards (offline/deterministic); the >10-IP / unresolvable rejects need DNS + the model host and fail closed at lockdown. - daytona.relock_network(extra_allowed_hosts): mirrors docker install-before- lockdown — sandbox comes up open, agent installs, then resolve hosts (+ the model host) to /32 CIDRs and apply via `update_network_settings`; fail closed if the plan is unfaithful OR if a non-allowlisted canary is still reachable after applying (same posture as the block-all canary). Returns {} (daytona enforces at the network layer; no HTTP(S)_PROXY needed). The existing rollout `_lock_down_network` already calls relock_network with the model host. - /etc/hosts pinning: an IPv4-only allow list blocks DNS (the sandbox resolvers 1.1.1.1/8.8.8.8 are not allowlisted), so a hostname connection would hang. relock_network pins each allowlisted host -> its resolved IP in the sandbox's /etc/hosts, so the agent resolves without DNS egress and connects to exactly the allowlisted IP (this also removes IP-rotation drift). TLS SNI/cert still use the hostname, so HTTPS stays valid. - litellm skip broadened to daytona: under a restrictive daytona policy the in-sandbox usage proxy can't pip-install post-lockdown (pypi blocked), so we skip it (like docker) and reach the provider directly; usage via native-ACP. Behavior parity table (deepseek-v4-flash): public : identical (open) on both no-network : identical (block-all + canary) on both allowlist : docker enforces by hostname; daytona enforces by IPv4 CIDR when faithful, else fails closed at preflight (wildcards) or lockdown (>10 IPs / unresolvable) with a precise reason wildcard : docker enforces; daytona rejects at preflight (IPs can't express wildcards) -> use docker Caveats (documented in net/docs/daytona-allowlist-parity.md): daytona's IPv4 allowlist is coarser than the hostname proxy (CDN IP rotation can stale-block or over-permit via shared IPs; max 10 CIDRs; IPv4 only). Verified live on daytona: citation-check-deny (allowlist=example.com, 3 CIDRs) -> ALLOWLIST applied, canary blocked, agent runs; citation-check-allow (4 CDN hosts -> 13 IPs) -> fails closed "exceeding daytona's 10-CIDR limit; use docker"; wildcard -> rejected at preflight. New tests: plan_daytona_allowlist (6), relock_network orchestration (4), updated daytona-capability contracts. ruff + relevant suite green. --- src/benchflow/providers/litellm_runtime.py | 18 +- src/benchflow/sandbox/daytona.py | 61 +++++ src/benchflow/sandbox/network_policy.py | 130 +++++++++- src/benchflow/task/runtime_capabilities.py | 28 ++- tests/test_network_policy.py | 24 +- tests/test_network_runtime.py | 277 ++++++++++++++++++++- tests/test_runtime_capabilities.py | 22 +- 7 files changed, 540 insertions(+), 20 deletions(-) diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index 8f69af590..db362c0bd 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -1025,15 +1025,21 @@ async def ensure_litellm_runtime( and network_is_restrictive(sandbox.task_env_config, environment) ) - if network_restrictive and environment == "docker": - # The host litellm proxy is plain-HTTP and reached via the egress - # proxy; that forwarding path is fragile. Under a restrictive docker - # policy the agent instead reaches the provider directly over HTTPS - # (its host is egress-allowlisted); usage is captured by native-ACP. + if network_restrictive and environment in ("docker", "daytona"): + # The benchflow LiteLLM usage proxy is fragile under a restrictive + # policy: on docker it is reached over the plain-HTTP egress-proxy + # forwarding path; on daytona it must pip-install in-sandbox AFTER the + # allowlist is applied (pypi is not allowlisted -> install fails). In + # both cases the agent instead reaches the provider DIRECTLY over + # HTTPS — the provider host is allowlisted (docker egress proxy / + # daytona IPv4 CIDR) — and usage is captured by native-ACP telemetry. return await _skip_litellm_runtime( agent_env, runtime, - reason="restrictive docker policy: direct-provider over HTTPS (egress-allowlisted)", + reason=( + f"restrictive {environment} policy: direct-provider over HTTPS " + "(provider host allowlisted)" + ), ) if environment in _SANDBOX_LOCAL_ENVIRONMENTS and sandbox is None: diff --git a/src/benchflow/sandbox/daytona.py b/src/benchflow/sandbox/daytona.py index 6c3ab59fb..39bc629e0 100644 --- a/src/benchflow/sandbox/daytona.py +++ b/src/benchflow/sandbox/daytona.py @@ -788,6 +788,67 @@ async def start(self, force_build: bool) -> None: await self._strategy.start(force_build) await self._verify_network_enforcement() + async def relock_network( + self, extra_allowed_hosts: tuple[str, ...] = () + ) -> dict[str, str]: + """Apply the task's allowlist as a daytona IPv4 CIDR list, post-install. + + Mirrors docker install-before-lockdown: the sandbox came up open so the + agent could install; now resolve the hostname allowlist (+ the model + host) to /32 CIDRs and push them via ``update_network_settings``. Fails + closed if the policy can't be faithfully expressed as <=10 IPv4 CIDRs + (wildcards, unresolvable, or too many IPs) or if a non-allowlisted + canary is still reachable after applying it. ``block-all``/``open`` are + handled at creation, so this is a no-op for them. Returns ``{}`` — the + daytona allowlist is enforced at the network layer, so (unlike docker) + the agent needs no HTTP(S)_PROXY env. + """ + from benchflow.sandbox.network_policy import ( + EffectivePolicy, + plan_daytona_allowlist, + resolve_network_decision, + ) + + decision = resolve_network_decision(self.task_env_config, "daytona") + if decision.policy is not EffectivePolicy.ALLOWLIST: + return {} + model_host = extra_allowed_hosts[0] if extra_allowed_hosts else None + plan = plan_daytona_allowlist(decision.allowed_hosts, model_host=model_host) + if not plan.enforceable: + raise SandboxStartupError( + f"daytona cannot enforce network_mode='allowlist': {plan.reject_reason}" + ) + # Pin allowlisted hosts in /etc/hosts so the agent resolves them WITHOUT + # DNS egress (the sandbox resolvers are not in the allow list) and without + # IP-rotation drift — it connects to exactly the IP we allowlisted. TLS + # SNI/cert still use the hostname, so HTTPS stays valid. + if plan.host_ips: + lines = "".join(f"{ip}\t{host}\n" for host, ip in plan.host_ips) + await self.exec( + f"printf %s {shlex.quote(lines)} >> /etc/hosts", + user="root", + timeout_sec=20, + ) + sandbox = self._require_sandbox() + await sandbox.update_network_settings(network_allow_list=",".join(plan.cidrs)) + # Fail closed: a non-allowlisted canary must now be UNREACHABLE. If it + # still resolves, the platform didn't apply the allow list — refuse to + # run an unenforced policy (same posture as the block-all canary). + if blockall_enforcement_violation( + block_all=True, canary_reachable=await self._egress_reachable() + ): + raise SandboxStartupError( + f"daytona applied a {len(plan.cidrs)}-CIDR allow list but the " + f"sandbox still reached {_EGRESS_CANARY_HOST}:{_EGRESS_CANARY_PORT} " + "(a non-allowlisted host) — the platform did not enforce the " + "allow list; failing closed" + ) + self.logger.info( + "relock_network: ALLOWLIST applied (daytona, %d cidrs)", + len(plan.cidrs), + ) + return {} + async def _verify_network_enforcement(self) -> None: """Fail closed if a block-all policy resolves but egress still works. diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py index 64dc85d57..369287c05 100644 --- a/src/benchflow/sandbox/network_policy.py +++ b/src/benchflow/sandbox/network_policy.py @@ -14,6 +14,8 @@ from __future__ import annotations +import socket +from collections.abc import Callable from dataclasses import dataclass from enum import StrEnum @@ -25,13 +27,22 @@ #: depth. (daytona-dind shares the compose mechanism and is a natural follow-up, #: but the ``daytona`` preflight sandbox string can't distinguish dind from the #: direct strategy, so it is intentionally excluded from this first cut.) -ALLOWLIST_CAPABLE_SANDBOXES: frozenset[str] = frozenset({"docker"}) +ALLOWLIST_CAPABLE_SANDBOXES: frozenset[str] = frozenset({"docker", "daytona"}) + +#: Sandboxes whose allowlist is enforced by IPv4 CIDR (not hostname/SNI). They +#: need hostname->IP resolution at lockdown and cannot express wildcards. +IP_BASED_ALLOWLIST_SANDBOXES: frozenset[str] = frozenset({"daytona"}) def _normalize_sandbox(sandbox: str | None) -> str: return (sandbox or "").replace("_", "-").strip().lower() +def allowlist_is_ip_based(sandbox: str | None) -> bool: + """True if the sandbox enforces allowlists by IPv4 CIDR (daytona).""" + return _normalize_sandbox(sandbox) in IP_BASED_ALLOWLIST_SANDBOXES + + def sandbox_supports_allowlist(sandbox: str | None) -> bool: """Whether *sandbox* can enforce a per-host ``allowlist``.""" return _normalize_sandbox(sandbox) in ALLOWLIST_CAPABLE_SANDBOXES @@ -154,3 +165,120 @@ def lockdown_complete( internal_net is not None and internal_net not in attached_networks ) return not (on_public_bridge or missing_internal) + + +# --- Daytona allowlist parity (enforce-when-faithful) ----------------------- +# +# Daytona has a native allowlist (`network_allow_list`) but it is **IPv4-CIDR** +# based (max 10 entries, no hostnames/wildcards), whereas the docker `bf-egress` +# proxy matches on hostname/SNI. To make daytona enforce an allowlist (instead +# of failing closed at preflight) we resolve the hostname allowlist to /32 CIDRs +# at lockdown. We only do this when the policy is *faithfully* expressible as an +# IPv4 list; otherwise we fail closed with a precise reason (the policy a user +# wrote can't be honored on this sandbox, so don't silently weaken it). + +#: Daytona caps `network_allow_list` at 10 IPv4 CIDR entries. +DAYTONA_MAX_ALLOWLIST_CIDRS = 10 + + +@dataclass(frozen=True) +class DaytonaAllowlistPlan: + """Outcome of mapping a hostname allowlist onto daytona's IPv4-CIDR control. + + Exactly one state holds: ``cidrs`` non-empty (enforce these), or + ``reject_reason`` set (fail closed — the allowlist can't be faithfully + represented as an IPv4 list on daytona). + """ + + cidrs: tuple[str, ...] = () + reject_reason: str | None = None + #: (hostname, primary IPv4) pairs to pin in the sandbox's /etc/hosts so the + #: agent resolves allowlisted hosts WITHOUT DNS egress (the resolvers are not + #: allowlisted) and without IP-rotation drift (pinned to the allowlisted IP). + host_ips: tuple[tuple[str, str], ...] = () + + @property + def enforceable(self) -> bool: + return self.reject_reason is None + + +def resolve_ipv4(host: str) -> tuple[str, ...]: + """Resolve *host* to its IPv4 addresses (empty tuple if none / on failure).""" + try: + infos = socket.getaddrinfo(host, None, family=socket.AF_INET) + except OSError: + return () + seen: list[str] = [] + for info in infos: + ip = str(info[4][0]) + if ip not in seen: + seen.append(ip) + return tuple(seen) + + +def plan_daytona_allowlist( + allowed_hosts: tuple[str, ...], + *, + model_host: str | None, + resolve: Callable[[str], tuple[str, ...]] = resolve_ipv4, +) -> DaytonaAllowlistPlan: + """Map a hostname allowlist (+ the model host) onto daytona IPv4 CIDRs. + + Enforce-when-faithful: returns CIDRs when the policy is expressible as a + <=10 IPv4 list with every host resolving; otherwise returns a precise + ``reject_reason`` so the caller fails closed (never silently downgrades a + policy the user explicitly requested). + """ + wild = [h for h in allowed_hosts if h.startswith("*.")] + if wild: + return DaytonaAllowlistPlan( + reject_reason=( + "daytona's allowlist is IPv4-CIDR based and cannot express " + f"wildcard host(s) {sorted(wild)}; use the 'docker' sandbox for " + "wildcard allowlists or list exact hostnames" + ) + ) + + hosts = list(allowed_hosts) + if model_host: + hosts.append(model_host) + + cidrs: list[str] = [] + host_ips: list[tuple[str, str]] = [] + seen: set[str] = set() + unresolved: list[str] = [] + for host in hosts: + ips = resolve(host) + if not ips: + if host not in unresolved: + unresolved.append(host) + continue + # pin host -> first resolved IP (which we also allowlist below) + host_ips.append((host, ips[0])) + for ip in ips: + cidr = f"{ip}/32" + if cidr not in seen: + seen.add(cidr) + cidrs.append(cidr) + + if unresolved: + return DaytonaAllowlistPlan( + reject_reason=( + "could not resolve an IPv4 address for allowlist host(s) " + f"{sorted(unresolved)} (required to build the daytona network " + "allow list); failing closed" + ) + ) + if not cidrs: + return DaytonaAllowlistPlan( + reject_reason="allowlist resolved to zero hosts; failing closed" + ) + if len(cidrs) > DAYTONA_MAX_ALLOWLIST_CIDRS: + return DaytonaAllowlistPlan( + reject_reason=( + f"allowlist resolves to {len(cidrs)} IPv4 addresses, exceeding " + f"daytona's {DAYTONA_MAX_ALLOWLIST_CIDRS}-CIDR limit; reduce the " + "host list or use the 'docker' sandbox" + ) + ) + return DaytonaAllowlistPlan(cidrs=tuple(cidrs), host_ips=tuple(host_ips)) diff --git a/src/benchflow/task/runtime_capabilities.py b/src/benchflow/task/runtime_capabilities.py index b9ef62b38..bf06173c6 100644 --- a/src/benchflow/task/runtime_capabilities.py +++ b/src/benchflow/task/runtime_capabilities.py @@ -162,18 +162,21 @@ def _append_config_issues( unsupported, path="agent.network_mode", mode=config.agent.network_mode, + allowed_hosts=tuple(getattr(config.agent, "allowed_hosts", None) or ()), sandbox=sandbox, ) _append_network_issue( unsupported, path="environment.network_mode", mode=config.environment.network_mode, + allowed_hosts=tuple(getattr(config.environment, "allowed_hosts", None) or ()), sandbox=sandbox, ) _append_network_issue( unsupported, path="verifier.network_mode", mode=config.verifier.network_mode, + allowed_hosts=tuple(getattr(config.verifier, "allowed_hosts", None) or ()), sandbox=sandbox, ) @@ -267,9 +270,13 @@ def _append_network_issue( path: str, mode: NetworkMode | None, sandbox: str, + allowed_hosts: tuple[str, ...] = (), ) -> None: if mode == NetworkMode.ALLOWLIST: - from benchflow.sandbox.network_policy import sandbox_supports_allowlist + from benchflow.sandbox.network_policy import ( + allowlist_is_ip_based, + sandbox_supports_allowlist, + ) if not sandbox_supports_allowlist(sandbox): _issue( @@ -282,6 +289,25 @@ def _append_network_issue( ), sandbox=sandbox, ) + return + # IP-CIDR allowlist sandboxes (daytona) can't express wildcards. This + # is the only model-independent reject we can make at preflight; the + # >10-IP / unresolvable checks need DNS + the model host and fail + # closed at lockdown (relock_network). + if allowlist_is_ip_based(sandbox): + wild = sorted(h for h in allowed_hosts if h.startswith("*.")) + if wild: + _issue( + unsupported, + path=path, + reason=( + f"network_mode='allowlist' on the '{sandbox}' sandbox uses " + f"an IPv4 allow list and cannot express wildcard host(s) " + f"{wild}; use the 'docker' sandbox for wildcard allowlists " + "or list exact hostnames" + ), + sandbox=sandbox, + ) def _append_document_issues( diff --git a/tests/test_network_policy.py b/tests/test_network_policy.py index 2d0d28469..0d1242b13 100644 --- a/tests/test_network_policy.py +++ b/tests/test_network_policy.py @@ -55,10 +55,14 @@ def test_docker_allowlist_enforced(self): def test_non_docker_allowlist_fails_closed(self): c = _cfg(network_mode="allowlist", allowed_hosts=["a.com"]) - for sb in ("daytona", "modal"): - d = resolve_network_decision(c, sb) - assert d.policy is EffectivePolicy.BLOCK_ALL - assert d.downgraded_from is NetworkMode.ALLOWLIST # never silently open + # modal has no per-host egress control -> still fails closed to block-all + d = resolve_network_decision(c, "modal") + assert d.policy is EffectivePolicy.BLOCK_ALL + assert d.downgraded_from is NetworkMode.ALLOWLIST # never silently open + # daytona enforces via its native IPv4-CIDR allow list -> ALLOWLIST + d_day = resolve_network_decision(c, "daytona") + assert d_day.policy is EffectivePolicy.ALLOWLIST + assert d_day.allowed_hosts == ("a.com",) def test_no_network(self): assert ( @@ -71,7 +75,7 @@ def test_public_open(self): def test_capability_predicate(self): assert sandbox_supports_allowlist("docker") - assert not sandbox_supports_allowlist("daytona") + assert sandbox_supports_allowlist("daytona") # native IPv4-CIDR allow list assert not sandbox_supports_allowlist("modal") assert not sandbox_supports_allowlist(None) @@ -130,5 +134,11 @@ def test_allowlist_rejected_off_docker_accepted_on_docker(self): for i in validate_task_runtime_support(cfg, sandbox="daytona") if "network_mode" in i.path ] - assert docker_issues == [] # enforced on docker - assert daytona_issues # rejected at preflight elsewhere + modal_issues = [ + i + for i in validate_task_runtime_support(cfg, sandbox="modal") + if "network_mode" in i.path + ] + assert docker_issues == [] # enforced on docker (hostname proxy) + assert daytona_issues == [] # enforced on daytona (IPv4-CIDR allow list) + assert modal_issues # no per-host egress control -> rejected at preflight diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index fd210ffb8..d2040f8d7 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -17,12 +17,14 @@ def test_network_blocks_all_for_daytona(): network_blocks_all(SandboxConfig(network_mode="no-network"), "daytona") is True ) assert network_blocks_all(SandboxConfig(), "daytona") is False # public default - # allowlist is unenforceable on daytona → resolve fails closed to block-all + # allowlist is now ENFORCEABLE on daytona (native IPv4-CIDR allow list), so it + # resolves to ALLOWLIST — NOT block-all. Faithfulness (wildcards / >10 IPs) + # is decided by plan_daytona_allowlist at lockdown, not here. assert ( network_blocks_all( SandboxConfig(network_mode="allowlist", allowed_hosts=["x.com"]), "daytona" ) - is True + is False ) @@ -124,12 +126,14 @@ def test_to_origin_form_passthrough_already_origin(): # ---- Fix #3 pivot: resolve provider host to allowlist under restrictive policy --- + def test_provider_host_for_model(): from benchflow.agents.providers import provider_host_for_model assert ( provider_host_for_model( - "deepseek/deepseek-v4-flash", {"DEEPSEEK_BASE_URL": "https://api.deepseek.com"} + "deepseek/deepseek-v4-flash", + {"DEEPSEEK_BASE_URL": "https://api.deepseek.com"}, ) == "api.deepseek.com" ) @@ -138,3 +142,270 @@ def test_provider_host_for_model(): assert provider_host_for_model("deepseek-v4-flash", {}) is None # provider prefix but its base_url env is missing -> None (don't guess) assert provider_host_for_model("deepseek/x", {}) is None + + +# ---- Daytona allowlist parity: enforce-when-faithful plan (ENG-219 follow-up) ---- + + +def _fake_resolver(table): + def resolve(host): + return tuple(table.get(host, ())) + + return resolve + + +def test_daytona_plan_resolves_hosts_and_model_to_cidrs(): + from benchflow.sandbox.network_policy import plan_daytona_allowlist + + plan = plan_daytona_allowlist( + ("api.crossref.org", "doi.org"), + model_host="api.deepseek.com", + resolve=_fake_resolver( + { + "api.crossref.org": ("1.1.1.1",), + "doi.org": ("2.2.2.2",), + "api.deepseek.com": ("3.3.3.3",), + } + ), + ) + assert plan.enforceable + assert plan.reject_reason is None + assert plan.cidrs == ("1.1.1.1/32", "2.2.2.2/32", "3.3.3.3/32") + + +def test_daytona_plan_dedups_shared_ips(): + from benchflow.sandbox.network_policy import plan_daytona_allowlist + + plan = plan_daytona_allowlist( + ("a.example.com", "b.example.com"), + model_host=None, + resolve=_fake_resolver( + {"a.example.com": ("9.9.9.9", "8.8.8.8"), "b.example.com": ("9.9.9.9",)} + ), + ) + assert plan.enforceable + assert plan.cidrs == ("9.9.9.9/32", "8.8.8.8/32") + + +def test_daytona_plan_rejects_wildcard(): + from benchflow.sandbox.network_policy import plan_daytona_allowlist + + plan = plan_daytona_allowlist( + ("*.crossref.org", "doi.org"), + model_host="api.deepseek.com", + resolve=_fake_resolver( + {"doi.org": ("2.2.2.2",), "api.deepseek.com": ("3.3.3.3",)} + ), + ) + assert not plan.enforceable + assert plan.cidrs == () + assert "wildcard" in plan.reject_reason.lower() + assert "*.crossref.org" in plan.reject_reason + + +def test_daytona_plan_rejects_unresolvable_host(): + from benchflow.sandbox.network_policy import plan_daytona_allowlist + + plan = plan_daytona_allowlist( + ("api.crossref.org", "nope.invalid"), + model_host=None, + resolve=_fake_resolver({"api.crossref.org": ("1.1.1.1",)}), + ) + assert not plan.enforceable + assert "nope.invalid" in plan.reject_reason + assert "resolve" in plan.reject_reason.lower() + + +def test_daytona_plan_rejects_over_ten_ips(): + from benchflow.sandbox.network_policy import plan_daytona_allowlist + + table = {f"h{i}.example.com": (f"10.0.0.{i}",) for i in range(11)} + plan = plan_daytona_allowlist( + tuple(table), model_host=None, resolve=_fake_resolver(table) + ) + assert not plan.enforceable + assert "10" in plan.reject_reason + + +def test_daytona_plan_rejects_empty_resolution(): + from benchflow.sandbox.network_policy import plan_daytona_allowlist + + plan = plan_daytona_allowlist((), model_host=None, resolve=_fake_resolver({})) + assert not plan.enforceable + + +# ---- Daytona relock_network: enforce-or-fail-closed orchestration ---- + +import logging # noqa: E402 + +import pytest # noqa: E402 + + +def _make_daytona_stub(task_env_config): + from benchflow.sandbox.daytona import DaytonaSandbox + + sb = DaytonaSandbox.__new__(DaytonaSandbox) + sb.task_env_config = task_env_config + sb.logger = logging.getLogger("test-daytona") + applied = {} + + class _Inner: + async def update_network_settings( + self, *, network_allow_list=None, network_block_all=None + ): + applied["allow_list"] = network_allow_list + + sb._sandbox = _Inner() + return sb, applied + + +@pytest.mark.asyncio +async def test_daytona_relock_applies_faithful_allowlist(monkeypatch): + from benchflow.sandbox import network_policy + from benchflow.task.config import SandboxConfig + + sb, applied = _make_daytona_stub( + SandboxConfig(network_mode="allowlist", allowed_hosts=["a.com"]) + ) + monkeypatch.setattr( + network_policy, + "plan_daytona_allowlist", + lambda hosts, *, model_host, resolve=None: network_policy.DaytonaAllowlistPlan( + cidrs=("9.9.9.9/32", "3.3.3.3/32") + ), + ) + + async def _unreachable(): + return False # non-allowlisted canary correctly blocked + + sb._egress_reachable = _unreachable + out = await sb.relock_network(extra_allowed_hosts=("api.model.test",)) + assert out == {} + assert applied["allow_list"] == "9.9.9.9/32,3.3.3.3/32" + + +@pytest.mark.asyncio +async def test_daytona_relock_fails_closed_on_unfaithful_plan(monkeypatch): + from benchflow.sandbox import network_policy + from benchflow.sandbox.protocol import SandboxStartupError + from benchflow.task.config import SandboxConfig + + sb, _ = _make_daytona_stub( + SandboxConfig(network_mode="allowlist", allowed_hosts=["a.com"]) + ) + monkeypatch.setattr( + network_policy, + "plan_daytona_allowlist", + lambda hosts, *, model_host, resolve=None: network_policy.DaytonaAllowlistPlan( + reject_reason="resolves to 13 IPv4 addresses, exceeding daytona's 10-CIDR limit" + ), + ) + with pytest.raises(SandboxStartupError, match="cannot enforce"): + await sb.relock_network(extra_allowed_hosts=()) + + +@pytest.mark.asyncio +async def test_daytona_relock_fails_closed_if_canary_still_reachable(monkeypatch): + from benchflow.sandbox import network_policy + from benchflow.sandbox.protocol import SandboxStartupError + from benchflow.task.config import SandboxConfig + + sb, _ = _make_daytona_stub( + SandboxConfig(network_mode="allowlist", allowed_hosts=["a.com"]) + ) + monkeypatch.setattr( + network_policy, + "plan_daytona_allowlist", + lambda hosts, *, model_host, resolve=None: network_policy.DaytonaAllowlistPlan( + cidrs=("9.9.9.9/32",) + ), + ) + + async def _reachable(): + return True # platform did NOT enforce -> leaked egress + + sb._egress_reachable = _reachable + with pytest.raises(SandboxStartupError, match="did not enforce"): + await sb.relock_network(extra_allowed_hosts=()) + + +@pytest.mark.asyncio +async def test_daytona_relock_noop_for_non_allowlist(): + from benchflow.task.config import SandboxConfig + + sb, applied = _make_daytona_stub(SandboxConfig(network_mode="no-network")) + out = await sb.relock_network() + assert out == {} + assert applied == {} # update_network_settings never called + + +@pytest.mark.asyncio +async def test_ensure_litellm_skips_under_restrictive_daytona(): + """Under a restrictive daytona policy the in-sandbox usage proxy can't be + installed (pypi blocked post-lockdown), so ensure_litellm_runtime skips it and + lets the agent reach the (allowlisted) provider directly — same as docker.""" + from benchflow.providers import litellm_runtime + from benchflow.task.config import SandboxConfig + + class _FakeSandbox: + task_env_config = SandboxConfig( + network_mode="allowlist", allowed_hosts=["a.com"] + ) + + agent_env = { + "LLM_BASE_URL": "https://api.deepseek.com", + "LLM_MODEL": "openai/deepseek-v4-flash", + "DEEPSEEK_API_KEY": "sk-test", + } + out_env, runtime = await litellm_runtime.ensure_litellm_runtime( + agent="openhands", + agent_env=agent_env, + model="deepseek/deepseek-v4-flash", + runtime=None, + environment="daytona", + usage_tracking="auto", + sandbox=_FakeSandbox(), + ) + assert runtime is None + assert out_env is agent_env # skipped: env returned unchanged + + +@pytest.mark.asyncio +async def test_daytona_relock_pins_etc_hosts(monkeypatch): + from benchflow.sandbox import network_policy + from benchflow.task.config import SandboxConfig + + sb, applied = _make_daytona_stub( + SandboxConfig(network_mode="allowlist", allowed_hosts=["a.com"]) + ) + execs = [] + + async def _fake_exec(cmd, *a, **k): + execs.append(cmd) + + class _R: + stdout = "" + result = "" + + return _R() + + sb.exec = _fake_exec + monkeypatch.setattr( + network_policy, + "plan_daytona_allowlist", + lambda hosts, *, model_host, resolve=None: network_policy.DaytonaAllowlistPlan( + cidrs=("9.9.9.9/32", "3.3.3.3/32"), + host_ips=(("a.com", "9.9.9.9"), ("api.model.test", "3.3.3.3")), + ), + ) + + async def _unreachable(): + return False + + sb._egress_reachable = _unreachable + await sb.relock_network(extra_allowed_hosts=("api.model.test",)) + # /etc/hosts pinned for both hosts before the allow list was applied + hosts_writes = [c for c in execs if "/etc/hosts" in c] + assert hosts_writes, "expected an /etc/hosts pin write" + assert "9.9.9.9" in hosts_writes[0] and "a.com" in hosts_writes[0] + assert applied["allow_list"] == "9.9.9.9/32,3.3.3.3/32" diff --git a/tests/test_runtime_capabilities.py b/tests/test_runtime_capabilities.py index 8ccf6fe0f..d452707e1 100644 --- a/tests/test_runtime_capabilities.py +++ b/tests/test_runtime_capabilities.py @@ -150,13 +150,31 @@ def test_validator_reports_allowlist_as_runtime_gap() -> None: if "network_mode" in i.path ] assert docker_net == [] - # sandboxes without per-host egress control reject allowlist at preflight + # daytona enforces exact-host allowlists via its native IPv4-CIDR list -> + # no gap for these hosts. daytona_net = [ i.path for i in validate_task_runtime_support(config, sandbox="daytona") if "network_mode" in i.path ] - assert daytona_net == ["agent.network_mode", "environment.network_mode"] + assert daytona_net == [] + # but daytona CANNOT express wildcards -> those still reject at preflight. + wild = TaskConfig.model_validate( + {"agent": {"network_mode": "allowlist", "allowed_hosts": ["*.example.com"]}} + ) + daytona_wild = [ + i.path + for i in validate_task_runtime_support(wild, sandbox="daytona") + if "network_mode" in i.path + ] + assert daytona_wild == ["agent.network_mode"] + # modal has no per-host control -> rejects exact-host allowlist too. + modal_net = [ + i.path + for i in validate_task_runtime_support(config, sandbox="modal") + if "network_mode" in i.path + ] + assert modal_net == ["agent.network_mode", "environment.network_mode"] def test_validator_reports_unknown_sandbox_backend() -> None: From 0351b5b7a6cea574d6d0624fdda27e74a9163dc7 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Wed, 17 Jun 2026 01:57:03 +0000 Subject: [PATCH 16/23] docs(network): prior-art reflects daytona allowlist enforcement (ENG-264) --- docs/network-mode-prior-art.md | 55 +++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/docs/network-mode-prior-art.md b/docs/network-mode-prior-art.md index 6a7d6427e..693ece96d 100644 --- a/docs/network-mode-prior-art.md +++ b/docs/network-mode-prior-art.md @@ -11,22 +11,34 @@ enforcement internals see [Sandbox hardening](./sandbox-hardening.md). The enforcing substrate is the owned egress proxy from PR [#785](https://github.com/benchflow-ai/benchflow/pull/785) (`feat/network-mode-enforcement`, ENG-219): an `internal: true` Docker network -plus a stdlib CONNECT/forward proxy sidecar. Everything below distinguishes -**what BenchFlow owns** from **patterns credited to other platforms**. +plus a stdlib CONNECT/forward proxy sidecar. On the `daytona` sandbox the same +policy is enforced through the platform's native IPv4 `network_allow_list` +(ENG-264). Everything below distinguishes **what BenchFlow owns** from +**patterns credited to other platforms**. ## What BenchFlow enforces today - **`no-network` / `public`** — every sandbox. -- **`allowlist`** — the `docker` sandbox only. The container joins an - `internal: true` (no-egress) network and all HTTP(S) traffic is forced through - the proxy sidecar, which forwards only to `allowed_hosts`. A host matches - **exactly or as a parent domain** (`example.com` matches `api.example.com`). A - single **leading-label wildcard** `*.example.com` matches subdomains at any - depth but **not** the bare apex (Harbor / nginx semantics); mid- or - trailing-wildcards are rejected at parse time. Non-allowlisted hosts, raw-IP - connections, and proxy-ignoring tools have no route off-box (default deny). On - sandboxes without per-host egress control (`daytona`, `modal`) an `allowlist` - task is **rejected at preflight** rather than run unrestricted. +- **`allowlist`** — enforced on the `docker` and `daytona` sandboxes. On + `docker`, the container joins an `internal: true` (no-egress) network and all + HTTP(S) traffic is forced through the proxy sidecar, which forwards only to + `allowed_hosts`. A host matches **exactly or as a parent domain** (`example.com` + matches `api.example.com`). A single **leading-label wildcard** `*.example.com` + matches subdomains at any depth but **not** the bare apex (Harbor / nginx + semantics); mid- or trailing-wildcards are rejected at parse time. + Non-allowlisted hosts, raw-IP connections, and proxy-ignoring tools have no + route off-box (default deny). On `daytona` the same intent is enforced through + the platform's native IPv4 allow list (`network_allow_list`): the hostname + allowlist is resolved to `/32` CIDRs at lockdown and each host is pinned in the + sandbox `/etc/hosts` (so it resolves without DNS egress — the resolvers are not + allowlisted — and without IP-rotation drift). Daytona enforces **when the + policy is faithfully expressible** as IPv4 (exact hosts resolving to ≤10 IPs) + and otherwise **fails closed with a precise reason**: wildcard allowlists are + rejected at preflight (an IPv4 list can't express `*.x`) and over-cap + allowlists (>10 IPs, e.g. CDN-fronted hosts) fail closed at lockdown pointing + to `docker`. On sandboxes with no per-host egress control (`modal`) an + `allowlist` task is **rejected at preflight** rather than run unrestricted. + (ENG-264) - **Model lane** — under a restrictive policy on `docker`, a single always-allow lane to the host-side model proxy stays open, so an agent run reaches the model without opening the sandbox to the public internet (a `no-network` run becomes @@ -43,9 +55,9 @@ design/mechanism; "credit" = pattern adopted from the named platform. |---|---|---|---| | **no-network** | Docker `network_mode: none` compose overlay | have | own (universal pattern) | | **public** (default) | No restriction | have | own (field default everywhere) | -| **hostname allowlist** (exact / subdomain) | Internal net + CONNECT egress-proxy sidecar, host-matched | have (#785, docker) | own proxy; taxonomy ≡ Harbor, AISI Inspect | +| **hostname allowlist** (exact / subdomain) | docker: internal net + CONNECT egress-proxy sidecar; daytona: resolve to IPv4 `/32`s + `/etc/hosts` pin | have (#785, docker + daytona) | own proxy; taxonomy ≡ Harbor, AISI Inspect | | **wildcard host** (`*.example.com`, multi-level) | Glob match in proxy + validator accepts leading `*.` | have (#785) | credit: Harbor, Modal | -| **CIDR allowlist** | IP-prefix egress rule (needs IP-routing, not pure CONNECT) | gap | credit: Modal | +| **CIDR allowlist** | Daytona native `network_allow_list` (IPv4 `/32`s resolved from the host allowlist) | have (#785, daytona, ENG-264) | own resolver; mechanism credit: Modal, Daytona | | **port-scoped egress** (`host:port`) | Rule type carries port; proxy already tunnels any CONNECT port | mechanism ok, rule type missing | credit: Modal | | **offline mirror** (pkg / content) | Pre-baked layered images / pinned mirror | gap | credit: SWE-bench | | **temporal package pin** | pip-index proxy filtering by release date | gap | credit: SWE-bench-Live | @@ -62,7 +74,7 @@ design/mechanism; "credit" = pattern adopted from the named platform. |---|---|---|---| | Docker `network_mode: none` / internal network | no-network; default-deny base for allowlist | everyone (AISI, SWE-bench, …) | have | | **Owned forward/CONNECT egress proxy sidecar** | hostname allowlist, vendor-neutral | **benchflow #785** | have / own | -| Cloud-platform egress allowlist (domain + CIDR) | hostname & CIDR allowlist | Modal, Daytona, Harbor backends | credit | +| Cloud-platform egress allowlist (domain + CIDR) | hostname & CIDR allowlist | Modal, Daytona, Harbor backends | have (daytona, ENG-264) | | CoreDNS `allowDomains` (k8s) | hostname allowlist via DNS | UK AISI Inspect (k8s sandbox) | credit | | MITM proxy + CA injection | record/replay, fault injection, body inspection | WAREX | credit (deliberately not done) | | Pre-baked layered images / offline mirror | hermetic — no runtime egress needed | SWE-bench, DeepSWE | credit | @@ -89,8 +101,17 @@ verified against these sources; see the caveats inline. depth rule rather than introducing the feature.) - **Modal** — Sandbox outbound controls: `block_network`, `outbound_cidr_allowlist` (CIDR ranges), and `outbound_domain_allowlist` - (domains). Source of the CIDR-allowlist and port-scoping patterns BenchFlow - does not yet implement. . + (domains). Source of the port-scoping pattern BenchFlow does not yet implement; + the CIDR-allowlist pattern is now enforced on the `daytona` sandbox via its + native `network_allow_list` (see Daytona below, ENG-264). + . +- **Daytona** — Sandbox `network_block_all` and `network_allow_list` + (comma-separated IPv4 CIDRs, max 10) at creation, plus a runtime + `update_network_settings`. BenchFlow uses these directly to enforce + `allowlist` / `no-network` on the `daytona` sandbox — resolving the hostname + allowlist to `/32`s and pinning `/etc/hosts`, since the IPv4 list cannot carry + hostnames (verified against SDK `daytona==0.184.0`). + . - **UK AISI Inspect** (k8s sandbox) — hostname allowlisting via `allowDomains` (FQDN list) enforced by a per-Pod CoreDNS sidecar, plus `allowCIDR`. · docs From 337487df65bc35fa2b11f24ffee495f8c3d30e23 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Wed, 17 Jun 2026 02:10:17 +0000 Subject: [PATCH 17/23] docs(network): cite specific prior-art PRs (Harbor #1455/#1840, Daytona #4124/#4604, browser-use #364) --- docs/network-mode-prior-art.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/network-mode-prior-art.md b/docs/network-mode-prior-art.md index 693ece96d..a873d9ea0 100644 --- a/docs/network-mode-prior-art.md +++ b/docs/network-mode-prior-art.md @@ -96,9 +96,11 @@ verified against these sources; see the caveats inline. shared-by-default vs. separate verifier environment. BenchFlow's `allowlist` taxonomy and wildcard semantics follow Harbor's. · docs . - (Harbor PR [#1854](https://github.com/harbor-framework/harbor/pull/1854) - *clarifies* that wildcards match multi-level subdomains — it documents the - depth rule rather than introducing the feature.) + Introduced in [#1455](https://github.com/harbor-framework/harbor/pull/1455) + (network mode + optional allowlist) and + [#1840](https://github.com/harbor-framework/harbor/pull/1840) (wildcard host + support); [#1854](https://github.com/harbor-framework/harbor/pull/1854) later + *clarifies* that wildcards match multi-level subdomains. - **Modal** — Sandbox outbound controls: `block_network`, `outbound_cidr_allowlist` (CIDR ranges), and `outbound_domain_allowlist` (domains). Source of the port-scoping pattern BenchFlow does not yet implement; @@ -110,8 +112,10 @@ verified against these sources; see the caveats inline. `update_network_settings`. BenchFlow uses these directly to enforce `allowlist` / `no-network` on the `daytona` sandbox — resolving the hostname allowlist to `/32`s and pinning `/etc/hosts`, since the IPv4 list cannot carry - hostnames (verified against SDK `daytona==0.184.0`). - . + hostnames (verified against SDK `daytona==0.184.0`). Allow/block-list format + + validation: [#4124](https://github.com/daytonaio/daytona/pull/4124); runtime + `update_network_settings`: [#4604](https://github.com/daytonaio/daytona/pull/4604). + Docs: . - **UK AISI Inspect** (k8s sandbox) — hostname allowlisting via `allowDomains` (FQDN list) enforced by a per-Pod CoreDNS sidecar, plus `allowCIDR`. · docs @@ -145,7 +149,8 @@ verified against these sources; see the caveats inline. . - **browser-use** — browser-automation SDK whose `BrowserProfile.allowed_domains` scopes which domains the browser may visit (supports `*.example.com` patterns). - BenchFlow's CUA browser-app egress scope reuses this. + BenchFlow's CUA browser-app egress scope reuses this. Introduced in + [#364](https://github.com/browser-use/browser-use/pull/364). . > **Scope note.** Per-tool network scoping and a 3-way From 7d0fd7d9c9d7cdb65deb3337cdd8ab704017a2f9 Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Wed, 17 Jun 2026 03:07:19 +0000 Subject: [PATCH 18/23] fix(egress): always close upstream socket in _handle (greptile P2) The finally block only closed the client socket; a raise in upstream.sendall/ _pipe skipped upstream.close() in both the CONNECT and plain-HTTP paths, leaking the fd on non-refcounting runtimes (PyPy). Track upstream from the top and close it in finally. Live e2e: allowed->200, non-allowlisted->blocked, direct->no route. --- src/benchflow/sandbox/_egress_proxy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/benchflow/sandbox/_egress_proxy.py b/src/benchflow/sandbox/_egress_proxy.py index 2dbf0d14c..946d8bcb5 100644 --- a/src/benchflow/sandbox/_egress_proxy.py +++ b/src/benchflow/sandbox/_egress_proxy.py @@ -89,6 +89,7 @@ def _deny(client: socket.socket, host: str) -> None: def _handle(client: socket.socket) -> None: + upstream: socket.socket | None = None try: client.settimeout(30) header = _recv_headers(client) @@ -113,7 +114,6 @@ def _handle(client: socket.socket) -> None: return client.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n") _pipe(client, upstream) - upstream.close() return # Plain HTTP: target is an absolute URI (http://host/path) in proxy form. @@ -137,10 +137,14 @@ def _handle(client: socket.socket) -> None: return upstream.sendall(_to_origin_form(header)) _pipe(client, upstream) - upstream.close() except Exception: pass finally: + # Always release the upstream socket — a raise in sendall/_pipe would + # otherwise skip its close() and leak the fd on non-refcounted runtimes. + if upstream is not None: + with contextlib.suppress(OSError): + upstream.close() with contextlib.suppress(OSError): client.close() From dc85a03c6b192db3082b8fa2fe402e03f5b206dd Mon Sep 17 00:00:00 2001 From: Yiminnn Date: Wed, 17 Jun 2026 03:52:50 +0000 Subject: [PATCH 19/23] fix(network): address multi-agent audit findings (4 confirmed defects) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 20-agent adversarial review (audit/test/verify/thermo-nuclear x 5 fixes) of the PR's network-mode fixes found four real, reproduced defects — all of which shipped green because the changed orchestration surface had ~zero direct test coverage. Fixed + added the missing orchestration/unit tests. 1. Egress proxy origin-form rewrite dropped/corrupted query strings and mangled already-origin-form requests (P1, correctness). `_to_origin_form` split the authority on the first '/' only and detected absolute-form by the substring '://', so `http://h?x=1` -> `/` (query dropped), `http://h?a=b/c` -> `/c` (corrupted), and an origin-form `/cb?u=http://e.com` was rewritten to `/`. The same substring confusion made `_handle` key the allowlist off a `://` inside the query value instead of the Host header. Fix: cut the authority at the first of '/', '?' or '#'; detect absolute-form by the http(s):// scheme prefix; extract the host via a shared `_absolute_uri_host` helper. 2. Restrictive-docker still recreated the exact ACP -32603 it set out to kill, for BARE model ids (P1). `provider_host_for_model` resolved only via `find_provider` (prefix match), so a bare id like `deepseek-v4-flash` returned None -> the provider host was never allowlisted, yet the LiteLLM skip still claimed "provider host allowlisted" and skipped, leaving the agent to 403 on its model CONNECT. Fix: fall back to `find_provider_for_bare_model` for bare ids, and make the restrictive-policy skip fail closed (raise) when the provider host can't be resolved instead of skipping with a false claim. 3. Docker relock fail-closed gate allow-listed two nets instead of denying extras, and fail-OPENED on a silent inspect failure on the hermetic path (P1). `lockdown_complete` only checked "off default AND on internal", so a task-author compose attaching `main` to a custom net survived the swap (off-box route bypassing the proxy) yet reported "applied"; and a swallowed `docker inspect` yielded an empty set that read as "on no networks" on the no-sidecar path. Fix: deny-by-default (attached must be a subset of the permitted set) and raise on `docker inspect` rc != 0. 4. Daytona allowlist hardening (P2). (a) The hard-coded 1.1.1.1 enforcement canary collides with Cloudflare-fronted allowlisted hosts (a host can resolve to 1.1.1.1), causing a false abort of a correctly-enforced run — now pick a canary IP not in the allow list. (b) A compose/DinD allowlist task applied `update_network_settings` to the OUTER sandbox while the agent ran in ungoverned inner containers — now fails closed. (c) The egress probe fail-OPENED (any probe error read as "blocked/enforced") — now tri-state (REACH/NOREACH/unknown) and an unverifiable probe counts as a violation (`blockall_enforcement_violation` treats None as a violation). Tests: origin-form query/host, bare-id resolution + fail-closed skip, lockdown_complete deny-extras + DockerSandbox.relock_network orchestration (stray-net/inspect-rc/happy), daytona canary pick + compose fail-closed + tri-state probe. Full suite 4272 passed, 0 regressions; ruff + ty clean; live proxy e2e still allowed->200 / blocked / no-direct-route. --- src/benchflow/agents/providers.py | 7 +- src/benchflow/providers/litellm_runtime.py | 16 +- src/benchflow/sandbox/_egress_proxy.py | 39 +++- src/benchflow/sandbox/daytona.py | 68 ++++-- src/benchflow/sandbox/docker.py | 11 +- src/benchflow/sandbox/network_policy.py | 49 ++-- tests/test_network_runtime.py | 247 ++++++++++++++++++++- 7 files changed, 387 insertions(+), 50 deletions(-) diff --git a/src/benchflow/agents/providers.py b/src/benchflow/agents/providers.py index ba370c92f..6ee4060fd 100644 --- a/src/benchflow/agents/providers.py +++ b/src/benchflow/agents/providers.py @@ -476,7 +476,12 @@ def provider_host_for_model(model: str, env: dict[str, str]) -> str | None: """ from urllib.parse import urlparse - found = find_provider(model) + # A bare id (prefix already stripped, e.g. 'deepseek-v4-flash') no longer + # matches find_provider; consult the bare-model registry so the host still + # resolves and gets allowlisted (otherwise a restrictive run can't reach it). + found = find_provider(model) or find_provider_for_bare_model( + strip_provider_prefix(model) + ) if found is None: return None _, cfg = found diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index db362c0bd..fee78b401 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -1031,8 +1031,20 @@ async def ensure_litellm_runtime( # forwarding path; on daytona it must pip-install in-sandbox AFTER the # allowlist is applied (pypi is not allowlisted -> install fails). In # both cases the agent instead reaches the provider DIRECTLY over - # HTTPS — the provider host is allowlisted (docker egress proxy / - # daytona IPv4 CIDR) — and usage is captured by native-ACP telemetry. + # HTTPS — but only if the provider host was actually allowlisted. The + # lockdown allowlists exactly provider_host_for_model(model); if that is + # None the host can't have been allowlisted, so skipping here would + # launch an agent whose model CONNECT the egress proxy then denies + # (ACP -32603). Fail closed with an actionable message instead. + from benchflow.agents.providers import provider_host_for_model + + if provider_host_for_model(model, agent_env) is None: + raise RuntimeError( + f"Restrictive network_mode on {environment!r}: cannot resolve a " + f"provider host for model {model!r} to allowlist it, so the agent " + "could not reach the model directly. Use a registered provider " + "prefix (e.g. 'deepseek/') or relax the network policy." + ) return await _skip_litellm_runtime( agent_env, runtime, diff --git a/src/benchflow/sandbox/_egress_proxy.py b/src/benchflow/sandbox/_egress_proxy.py index 946d8bcb5..7470f8329 100644 --- a/src/benchflow/sandbox/_egress_proxy.py +++ b/src/benchflow/sandbox/_egress_proxy.py @@ -116,10 +116,10 @@ def _handle(client: socket.socket) -> None: _pipe(client, upstream) return - # Plain HTTP: target is an absolute URI (http://host/path) in proxy form. - host = "" - if "://" in target: - host = target.split("://", 1)[1].split("/", 1)[0] + # Plain HTTP: an absolute-URI proxy target (http://host/path). Only treat + # it as absolute when it actually starts with a scheme — a '://' inside a + # query value of an origin-form target must NOT be read as the authority. + host = _absolute_uri_host(target) or "" if not host: for hl in header.split(b"\r\n"): if hl.lower().startswith(b"host:"): @@ -159,15 +159,40 @@ def _to_origin_form(header: bytes) -> bytes: """ line, sep, rest = header.partition(b"\r\n") parts = line.split(b" ") - if len(parts) != 3 or b"://" not in parts[1]: + if len(parts) != 3 or not parts[1].lower().startswith((b"http://", b"https://")): return header method, target, version = parts after = target.split(b"://", 1)[1] - slash = after.find(b"/") - path = after[slash:] if slash != -1 else b"/" + # The authority ends at the first of '/', '?' or '#'; everything from there + # is the origin-form path. A query/fragment with no path is prefixed with + # '/' so the query survives (http://h?x=1 -> /?x=1, not / ). + cuts = [ + i for i in (after.find(b"/"), after.find(b"?"), after.find(b"#")) if i != -1 + ] + if not cuts: + path = b"/" + else: + path = after[min(cuts) :] + if not path.startswith(b"/"): + path = b"/" + path return method + b" " + path + b" " + version + sep + rest +def _absolute_uri_host(target: str) -> str | None: + """Authority of an absolute-form proxy target, or None for origin-form. + + Only an actual ``http://`` / ``https://`` prefix counts as absolute-form; + a ``://`` appearing inside the query of an origin-form target (e.g. + ``/cb?u=http://evil``) must not be mistaken for the authority. The + authority ends at the first of ``/``, ``?`` or ``#``. + """ + if not target.lower().startswith(("http://", "https://")): + return None + after = target.split("://", 1)[1] + cuts = [i for i in (after.find("/"), after.find("?"), after.find("#")) if i != -1] + return after[: min(cuts)] if cuts else after + + def main() -> None: srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/src/benchflow/sandbox/daytona.py b/src/benchflow/sandbox/daytona.py index 39bc629e0..0912fc5d9 100644 --- a/src/benchflow/sandbox/daytona.py +++ b/src/benchflow/sandbox/daytona.py @@ -104,9 +104,21 @@ def wait_exponential(*_args: Any, **_kwargs: Any) -> Any: #: Raw-TCP egress canary (no DNS) used to verify a block-all policy is actually #: enforced on daytona; reachable under block-all => the platform leaked. _EGRESS_CANARY_HOST = "1.1.1.1" +#: Public IPs used as the 'should-be-blocked' enforcement canary. Reachable on +#: an open network; pick one that is NOT in the task's allow list (a host can +#: resolve to 1.1.1.1, which would otherwise be allowlisted and falsely abort). +_CANARY_CANDIDATES = ("1.1.1.1", "8.8.8.8", "9.9.9.9") _EGRESS_CANARY_PORT = 443 +def _pick_canary(cidrs: tuple[str, ...]) -> str: + """First canary IP whose /32 is NOT already in the allow list.""" + for host in _CANARY_CANDIDATES: + if f"{host}/32" not in cidrs: + return host + return _CANARY_CANDIDATES[0] + + def _ensure_daytona_anyio_compat() -> None: """Patch the anyio symbol that Daytona 0.176 imports on newer anyio.""" try: @@ -812,6 +824,15 @@ async def relock_network( decision = resolve_network_decision(self.task_env_config, "daytona") if decision.policy is not EffectivePolicy.ALLOWLIST: return {} + if self._compose_mode: + # DinD: update_network_settings governs the OUTER sandbox, but the + # agent runs in inner containers whose egress is ungoverned — the + # canary would pass while the agent has open egress. Fail closed. + raise SandboxStartupError( + "daytona compose/DinD does not support network_mode='allowlist' " + "enforcement (settings apply to the outer sandbox only); use the " + "'docker' sandbox or 'no-network'" + ) model_host = extra_allowed_hosts[0] if extra_allowed_hosts else None plan = plan_daytona_allowlist(decision.allowed_hosts, model_host=model_host) if not plan.enforceable: @@ -834,14 +855,15 @@ async def relock_network( # Fail closed: a non-allowlisted canary must now be UNREACHABLE. If it # still resolves, the platform didn't apply the allow list — refuse to # run an unenforced policy (same posture as the block-all canary). + canary = _pick_canary(plan.cidrs) if blockall_enforcement_violation( - block_all=True, canary_reachable=await self._egress_reachable() + block_all=True, canary_reachable=await self._egress_reachable(canary) ): raise SandboxStartupError( f"daytona applied a {len(plan.cidrs)}-CIDR allow list but the " - f"sandbox still reached {_EGRESS_CANARY_HOST}:{_EGRESS_CANARY_PORT} " - "(a non-allowlisted host) — the platform did not enforce the " - "allow list; failing closed" + f"sandbox could not confirm {canary}:{_EGRESS_CANARY_PORT} is " + "blocked (a non-allowlisted host) — the platform did not enforce " + "the allow list, or the probe could not run; failing closed" ) self.logger.info( "relock_network: ALLOWLIST applied (daytona, %d cidrs)", @@ -872,29 +894,41 @@ async def _verify_network_enforcement(self) -> None: "of running with leaked network" ) - async def _egress_reachable(self) -> bool: + async def _egress_reachable( + self, canary_host: str = _EGRESS_CANARY_HOST + ) -> bool | None: + """Tri-state egress probe: True=reachable, False=blocked, None=unknown. + + Emits an explicit NOREACH sentinel on a blocked connection so a probe + that could not run (python missing, timeout) is distinguishable from a + confirmed-blocked host and does NOT read as 'enforced' (fail-open). + """ py = ( - "import socket;socket.setdefaulttimeout(6);" - f"socket.create_connection(({_EGRESS_CANARY_HOST!r},{_EGRESS_CANARY_PORT}))" - ".close();print('REACH')" + "import socket\n" + "try:\n" + " socket.setdefaulttimeout(6)\n" + f" socket.create_connection(({canary_host!r}, {_EGRESS_CANARY_PORT})).close()\n" + " print('REACH')\n" + "except Exception:\n" + " print('NOREACH')" ) probe = ( - f"(python3 -c {shlex.quote(py)} || python -c {shlex.quote(py)}) " - "2>/dev/null || true" + f"python3 -c {shlex.quote(py)} 2>/dev/null || " + f"python -c {shlex.quote(py)} 2>/dev/null || true" ) try: res = await self.exec(probe, timeout_sec=20, user="root") except Exception: self.logger.warning("egress canary probe failed to run", exc_info=True) - return False - reachable = "REACH" in (res.stdout or "") + return None + out = res.stdout or "" + result: bool | None = ( + False if "NOREACH" in out else (True if "REACH" in out else None) + ) self.logger.info( - "egress canary %s:%s reachable=%s", - _EGRESS_CANARY_HOST, - _EGRESS_CANARY_PORT, - reachable, + "egress canary %s:%s -> %s", canary_host, _EGRESS_CANARY_PORT, result ) - return reachable + return result async def stop(self, delete: bool) -> None: return await self._strategy.stop(delete) diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index a5b75e7bd..6402748c4 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -382,8 +382,15 @@ async def relock_network( ], check=False, ) - # stdout is str|None (check=False); None -> empty set -> lockdown_complete - # fails closed below (treated as not-detached), which is the safe default. + # Fail closed on an unrunnable inspect: an empty network set would + # otherwise read as 'correctly on no networks' on the hermetic path + # (internal_net is None), masking a swallowed inspect error. + if inspect_res.return_code != 0: + raise SandboxStartupError( + "relock_network: could not inspect container networks " + f"(docker inspect rc={inspect_res.return_code}); failing closed " + "rather than running with an unverified network policy" + ) attached: set[str] = set((inspect_res.stdout or "").split()) internal_net = f"{project}_{_EGRESS_INTERNAL_NET}" if use_sidecar else None if not lockdown_complete(attached, f"{project}_default", internal_net): diff --git a/src/benchflow/sandbox/network_policy.py b/src/benchflow/sandbox/network_policy.py index 369287c05..0cfe2ea94 100644 --- a/src/benchflow/sandbox/network_policy.py +++ b/src/benchflow/sandbox/network_policy.py @@ -118,15 +118,19 @@ def network_blocks_all(env_config: SandboxConfig, sandbox: str) -> bool: ) -def blockall_enforcement_violation(*, block_all: bool, canary_reachable: bool) -> bool: - """Fail-closed check for a block-all policy. - - A sandbox that resolves to ``BLOCK_ALL`` must have no off-box route. If an - external canary is still reachable from inside, the platform did not honor the - block (e.g. daytona's ``network_block_all`` flag was ignored) — the run should - abort rather than produce a falsely-rewarded "offline" result. +def blockall_enforcement_violation( + *, block_all: bool, canary_reachable: bool | None +) -> bool: + """Fail-closed check for a restrictive policy (block-all or allowlist canary). + + A restrictive policy must have no off-box route to a non-allowlisted host. + ``canary_reachable`` is tri-state: ``True`` = the canary was reachable (the + platform did not enforce), ``False`` = confirmed unreachable (enforced), and + ``None`` = the probe could not run (python missing, timeout, exec error). An + unverifiable probe must NOT be read as 'enforced' — only an explicit ``False`` + clears the policy, so ``True`` and ``None`` both count as a violation. """ - return block_all and canary_reachable + return block_all and canary_reachable is not False def network_is_restrictive(env_config: SandboxConfig, sandbox: str) -> bool: @@ -149,22 +153,31 @@ def proxy_unavailable_is_fatal(*, usage_mode: str, network_restrictive: bool) -> def lockdown_complete( - attached_networks: set[str], default_net: str, internal_net: str | None + attached_networks: set[str], + default_net: str, + internal_net: str | None, + extra_permitted: frozenset[str] = frozenset(), ) -> bool: - """True iff a docker relock actually took effect. - - After install-before-lockdown swaps the container's networks, it must be - detached from the public bridge (*default_net*) and, when an egress sidecar - is in use, attached to *internal_net*. If a ``network connect``/``disconnect`` - silently failed the container could sit on BOTH nets (egress proxy bypassed) - or on NONE (stranded) — callers fail closed when this returns ``False`` - rather than running with an unenforced policy. + """True iff a docker relock actually took effect (deny-by-default). + + After install-before-lockdown swaps the container's networks, the container + must be attached to EXACTLY the permitted set: *internal_net* when an egress + sidecar is in use, otherwise nothing (plus any explicitly *extra_permitted* + benchflow-owned nets). It must NOT be on the public bridge (*default_net*) + and must NOT retain any other network. A silently failed + ``connect``/``disconnect`` could leave it on BOTH nets (proxy bypassed), on + NONE (stranded), or still on a task-author custom net that routes off-box + around the proxy — all return ``False`` so callers fail closed. """ on_public_bridge = default_net in attached_networks missing_internal = ( internal_net is not None and internal_net not in attached_networks ) - return not (on_public_bridge or missing_internal) + permitted = set(extra_permitted) + if internal_net is not None: + permitted.add(internal_net) + has_extra_net = not set(attached_networks).issubset(permitted) + return not (on_public_bridge or missing_internal or has_extra_net) # --- Daytona allowlist parity (enforce-when-faithful) ----------------------- diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index d2040f8d7..f617e8809 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -247,6 +247,7 @@ def _make_daytona_stub(task_env_config): sb = DaytonaSandbox.__new__(DaytonaSandbox) sb.task_env_config = task_env_config sb.logger = logging.getLogger("test-daytona") + sb._compose_mode = False applied = {} class _Inner: @@ -275,7 +276,7 @@ async def test_daytona_relock_applies_faithful_allowlist(monkeypatch): ), ) - async def _unreachable(): + async def _unreachable(canary=None): return False # non-allowlisted canary correctly blocked sb._egress_reachable = _unreachable @@ -321,7 +322,7 @@ async def test_daytona_relock_fails_closed_if_canary_still_reachable(monkeypatch ), ) - async def _reachable(): + async def _reachable(canary=None): return True # platform did NOT enforce -> leaked egress sb._egress_reachable = _reachable @@ -356,6 +357,7 @@ class _FakeSandbox: "LLM_BASE_URL": "https://api.deepseek.com", "LLM_MODEL": "openai/deepseek-v4-flash", "DEEPSEEK_API_KEY": "sk-test", + "DEEPSEEK_BASE_URL": "https://api.deepseek.com", } out_env, runtime = await litellm_runtime.ensure_litellm_runtime( agent="openhands", @@ -399,7 +401,7 @@ class _R: ), ) - async def _unreachable(): + async def _unreachable(canary=None): return False sb._egress_reachable = _unreachable @@ -409,3 +411,242 @@ async def _unreachable(): assert hosts_writes, "expected an /etc/hosts pin write" assert "9.9.9.9" in hosts_writes[0] and "a.com" in hosts_writes[0] assert applied["allow_list"] == "9.9.9.9/32,3.3.3.3/32" + + +# ---- origin-form rewrite: query handling + scheme-prefix detection (audit P1) ---- + + +def test_to_origin_form_preserves_query(): + from benchflow.sandbox._egress_proxy import _to_origin_form + + assert ( + _to_origin_form(b"GET http://h?x=1 HTTP/1.1\r\n\r\n") + == b"GET /?x=1 HTTP/1.1\r\n\r\n" + ) + # a '/' inside the query value must not be taken as the path boundary + assert ( + _to_origin_form(b"GET http://h?a=b/c HTTP/1.1\r\n\r\n") + == b"GET /?a=b/c HTTP/1.1\r\n\r\n" + ) + # authority with a port, query-only + assert ( + _to_origin_form(b"GET http://h:80?a=1 HTTP/1.1\r\n\r\n") + == b"GET /?a=1 HTTP/1.1\r\n\r\n" + ) + # normal path+query is unchanged in shape + assert ( + _to_origin_form(b"GET http://h/p?q=1 HTTP/1.1\r\n\r\n") + == b"GET /p?q=1 HTTP/1.1\r\n\r\n" + ) + + +def test_to_origin_form_leaves_origin_form_with_url_query_untouched(): + from benchflow.sandbox._egress_proxy import _to_origin_form + + # already origin-form, but the query contains '://' — must be returned verbatim + h = b"GET /cb?u=http://e.com HTTP/1.1\r\nHost: api.example.com\r\n\r\n" + assert _to_origin_form(h) == h + + +def test_absolute_uri_host_ignores_scheme_in_query(): + from benchflow.sandbox._egress_proxy import _absolute_uri_host + + # absolute-form: authority is the real host, stopping at '/', '?' or '#' + assert _absolute_uri_host("http://api.example.com/p?x=1") == "api.example.com" + assert ( + _absolute_uri_host("https://api.example.com?u=http://e.com") + == "api.example.com" + ) + assert _absolute_uri_host("http://h:8080") == "h:8080" + # origin-form with '://' in the query is NOT absolute -> None (use Host header) + assert _absolute_uri_host("/cb?u=http://evil.com") is None + assert _absolute_uri_host("/plain") is None + + +# ---- bare model-id provider host resolution (audit P1: -32603 for bare ids) ---- + + +def test_provider_host_for_model_resolves_bare_id(): + from benchflow.agents.providers import provider_host_for_model + + # the documented dev model in BARE form (prefix already stripped) must still + # resolve its host so a restrictive run allowlists it + assert ( + provider_host_for_model( + "deepseek-v4-flash", {"DEEPSEEK_BASE_URL": "https://api.deepseek.com"} + ) + == "api.deepseek.com" + ) + # prefixed form keeps working + assert ( + provider_host_for_model( + "deepseek/deepseek-v4-flash", + {"DEEPSEEK_BASE_URL": "https://api.deepseek.com"}, + ) + == "api.deepseek.com" + ) + # genuinely unknown model -> None (caller fails closed) + assert provider_host_for_model("totally-unknown-model-xyz", {}) is None + + +@pytest.mark.asyncio +async def test_ensure_litellm_fails_closed_when_provider_host_unresolvable(): + """Under a restrictive policy, if the provider host can't be resolved (so it + was never allowlisted), skipping the proxy would launch an agent that 403s on + its model CONNECT. ensure_litellm_runtime must fail closed instead.""" + import pytest as _pytest + + from benchflow.providers import litellm_runtime + from benchflow.task.config import SandboxConfig + + class _FakeSandbox: + task_env_config = SandboxConfig( + network_mode="allowlist", allowed_hosts=["a.com"] + ) + + with _pytest.raises(RuntimeError, match="cannot resolve a provider host"): + await litellm_runtime.ensure_litellm_runtime( + agent="openhands", + agent_env={"LLM_BASE_URL": "https://x"}, + model="totally-unknown-model-xyz", + runtime=None, + environment="docker", + usage_tracking="auto", + sandbox=_FakeSandbox(), + ) + + +# ---- docker relock fail-closed gate: deny extras + inspect-rc (audit P1/P2) ---- + + +def test_lockdown_complete_denies_extra_network(): + from benchflow.sandbox.network_policy import lockdown_complete + + internal = "proj_bf_egress_internal" + default = "proj_default" + assert lockdown_complete({internal}, default, internal) is True + # a stray non-internal net survives the swap -> NOT complete (bypass risk) + assert lockdown_complete({internal, "proj_mynet"}, default, internal) is False + assert lockdown_complete(set(), default, None) is True + assert lockdown_complete({"proj_mynet"}, default, None) is False + assert lockdown_complete({internal, default}, default, internal) is False + # explicitly-permitted benign net is allowed + assert ( + lockdown_complete( + {internal, "proj_ok"}, default, internal, frozenset({"proj_ok"}) + ) + is True + ) + + +def _relock_project(): + from benchflow.sandbox.docker import _sanitize_docker_compose_project_name + + return _sanitize_docker_compose_project_name("relocktest") + + +def _make_docker_relock_stub(inspect_stdout, inspect_rc): + import logging + + from benchflow.sandbox._base import ExecResult + from benchflow.sandbox.docker import DockerSandbox + from benchflow.task.config import SandboxConfig + + sb = DockerSandbox.__new__(DockerSandbox) + sb.task_env_config = SandboxConfig( + network_mode="allowlist", allowed_hosts=["a.com"] + ) + sb.session_id = "relocktest" + sb._network_locked = False + sb._extra_allowed_hosts = () + sb.logger = logging.getLogger("relock-test") + + async def _cid(): + return "cid123" + + sb._main_container_id = _cid + sb._network_policy_compose_paths = lambda: ["/x/egress.json"] + + async def _compose(_args): + return None + + sb._run_docker_compose_command = _compose + + async def _cli(args, check=True): + if "inspect" in args: + return ExecResult(stdout=inspect_stdout, stderr="", return_code=inspect_rc) + return ExecResult(stdout="", stderr="", return_code=0) + + sb._docker_cli = _cli + return sb + + +@pytest.mark.asyncio +async def test_docker_relock_raises_on_stray_network(): + from benchflow.sandbox.protocol import SandboxStartupError + + proj = _relock_project() + sb = _make_docker_relock_stub(f"{proj}_bf_egress_internal {proj}_mynet", 0) + with pytest.raises(SandboxStartupError, match="did not take effect"): + await sb.relock_network() + + +@pytest.mark.asyncio +async def test_docker_relock_raises_on_inspect_error(): + from benchflow.sandbox.protocol import SandboxStartupError + + sb = _make_docker_relock_stub("", 1) + with pytest.raises(SandboxStartupError, match="could not inspect"): + await sb.relock_network() + + +@pytest.mark.asyncio +async def test_docker_relock_happy_sidecar_returns_proxy_env(): + proj = _relock_project() + sb = _make_docker_relock_stub(f"{proj}_bf_egress_internal", 0) + out = await sb.relock_network() + assert out.get("HTTPS_PROXY", "").startswith("http://bf-egress:") + + +# ---- daytona allowlist hardening (audit P2: canary / compose / probe) ---- + + +def test_blockall_violation_treats_unverifiable_probe_as_violation(): + from benchflow.sandbox.network_policy import blockall_enforcement_violation + + assert blockall_enforcement_violation(block_all=True, canary_reachable=True) is True + assert ( + blockall_enforcement_violation(block_all=True, canary_reachable=False) is False + ) + # probe could not run -> cannot confirm blocked -> fail closed + assert blockall_enforcement_violation(block_all=True, canary_reachable=None) is True + assert ( + blockall_enforcement_violation(block_all=False, canary_reachable=None) is False + ) + + +def test_pick_canary_avoids_allowlisted_ip(): + from benchflow.sandbox.daytona import _pick_canary + + assert _pick_canary(()) == "1.1.1.1" + # a host resolved to 1.1.1.1 is allowlisted -> canary must move off it + assert _pick_canary(("1.1.1.1/32",)) == "8.8.8.8" + assert _pick_canary(("1.1.1.1/32", "8.8.8.8/32")) == "9.9.9.9" + + +@pytest.mark.asyncio +async def test_daytona_relock_fails_closed_for_compose_mode(): + import logging + + from benchflow.sandbox.daytona import DaytonaSandbox + from benchflow.sandbox.protocol import SandboxStartupError + from benchflow.task.config import SandboxConfig + + sb = DaytonaSandbox.__new__(DaytonaSandbox) + sb.task_env_config = SandboxConfig( + network_mode="allowlist", allowed_hosts=["a.com"] + ) + sb._compose_mode = True + sb.logger = logging.getLogger("daytona-compose-test") + with pytest.raises(SandboxStartupError, match="compose/DinD"): + await sb.relock_network(extra_allowed_hosts=()) From 0c78eadd35647bc47fb7bb4536d64d8555696589 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 10 Jul 2026 09:05:45 -0700 Subject: [PATCH 20/23] fix(network): wait for egress sidecar readiness --- src/benchflow/sandbox/_egress.py | 21 +++++- .../sandbox/docker_network_lockdown.py | 65 ++++++++++++++++++- tests/test_network_policy.py | 3 +- tests/test_network_runtime.py | 54 ++++++++++++++- 4 files changed, 135 insertions(+), 8 deletions(-) diff --git a/src/benchflow/sandbox/_egress.py b/src/benchflow/sandbox/_egress.py index 7f63447b0..938887a53 100644 --- a/src/benchflow/sandbox/_egress.py +++ b/src/benchflow/sandbox/_egress.py @@ -53,6 +53,23 @@ def build_egress_override( "volumes": [f"{script_dst.resolve()}:/egress_proxy.py:ro"], "labels": {"benchflow.owned": "true"}, "restart": "on-failure", + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + ( + "import os, socket; " + "port = int(os.environ.get('PORT', '8080')); " + "sock = socket.create_connection(('127.0.0.1', port), timeout=1); " + "sock.close()" + ), + ], + "interval": "1s", + "timeout": "2s", + "retries": 30, + "start_period": "1s", + }, } if model_lane: # Always-allow lane to the host-side model proxy. The agent's base_url @@ -78,7 +95,9 @@ def build_egress_override( "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", }, - "depends_on": [_EGRESS_SERVICE], + "depends_on": { + _EGRESS_SERVICE: {"condition": "service_healthy"}, + }, }, _EGRESS_SERVICE: egress_service, }, diff --git a/src/benchflow/sandbox/docker_network_lockdown.py b/src/benchflow/sandbox/docker_network_lockdown.py index 3644d79ac..159df74da 100644 --- a/src/benchflow/sandbox/docker_network_lockdown.py +++ b/src/benchflow/sandbox/docker_network_lockdown.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from pathlib import Path from typing import Any @@ -18,6 +19,9 @@ ) from benchflow.sandbox.protocol import SandboxStartupError +_EGRESS_HEALTH_ATTEMPTS = 30 +_EGRESS_HEALTH_INTERVAL_SEC = 1.0 + def docker_network_policy_compose_paths(sandbox: Any) -> list[Path]: """Return compose overrides for the sandbox's active network policy.""" @@ -57,6 +61,55 @@ def docker_network_policy_compose_paths(sandbox: Any) -> list[Path]: return [sandbox._DOCKER_COMPOSE_NO_NETWORK_PATH] +async def _wait_for_egress_sidecar_ready(sandbox: Any) -> None: + ps_res = await sandbox._run_docker_compose_command( + ["ps", "--quiet", _EGRESS_SERVICE], check=False + ) + if ps_res.return_code != 0: + raise SandboxStartupError( + "relock_network: could not resolve egress sidecar container " + f"(docker compose ps rc={ps_res.return_code}); failing closed " + "rather than returning proxy env before bf-egress is ready" + ) + + cid = next((line.strip() for line in (ps_res.stdout or "").splitlines()), "") + if not cid: + raise SandboxStartupError( + "relock_network: egress sidecar container was not found; failing closed " + "rather than returning proxy env before bf-egress is ready" + ) + + last_status = "unknown" + for attempt in range(_EGRESS_HEALTH_ATTEMPTS): + inspect_res = await sandbox._docker_cli( + [ + "inspect", + cid, + "--format", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", + ], + check=False, + ) + if inspect_res.return_code == 0: + status = (inspect_res.stdout or "").strip() + if status == "healthy": + return + last_status = status or "empty" + if status == "unhealthy": + break + else: + last_status = f"inspect rc={inspect_res.return_code}" + + if attempt + 1 < _EGRESS_HEALTH_ATTEMPTS: + await asyncio.sleep(_EGRESS_HEALTH_INTERVAL_SEC) + + raise SandboxStartupError( + "relock_network: egress sidecar did not become ready " + f"(last status={last_status}); failing closed rather than returning " + "proxy env before bf-egress is ready" + ) + + async def relock_docker_network( sandbox: Any, *, @@ -129,12 +182,18 @@ async def relock_docker_network( "with an unenforced network policy" ) - sandbox.logger.info( - "relock_network: %s applied (sidecar=%s)", decision.policy.name, use_sidecar - ) if not use_sidecar: + sandbox.logger.info( + "relock_network: %s applied (sidecar=%s)", + decision.policy.name, + use_sidecar, + ) return {} + await _wait_for_egress_sidecar_ready(sandbox) + sandbox.logger.info( + "relock_network: %s applied (sidecar=%s)", decision.policy.name, use_sidecar + ) proxy = f"http://{_EGRESS_SERVICE}:{_EGRESS_PORT}" return { "HTTP_PROXY": proxy, diff --git a/tests/test_network_policy.py b/tests/test_network_policy.py index 0d1242b13..6a5b507ee 100644 --- a/tests/test_network_policy.py +++ b/tests/test_network_policy.py @@ -89,10 +89,11 @@ def test_structure(self, tmp_path): # main detached from default bridge → only the internal network assert main["networks"] == ["bf_egress_internal"] assert main["environment"]["HTTPS_PROXY"].startswith("http://bf-egress:") - assert main["depends_on"] == ["bf-egress"] + assert main["depends_on"] == {"bf-egress": {"condition": "service_healthy"}} # proxy bridges internal + external and carries the allowlist assert set(proxy["networks"]) == {"bf_egress_internal", "bf_egress_external"} assert proxy["environment"]["ALLOWED_HOSTS"] == "example.com,api.test.org" + assert proxy["healthcheck"]["test"][:3] == ["CMD", "python3", "-c"] assert doc["networks"]["bf_egress_internal"]["internal"] is True assert "internal" not in doc["networks"]["bf_egress_external"] # the proxy script is staged next to the override for bind-mounting diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index d2aa945e5..b20ea4426 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -567,14 +567,18 @@ async def _cid(): sb._main_container_id = _cid sb._network_policy_compose_paths = lambda: ["/x/egress.json"] - async def _compose(_args): - return None + async def _compose(args, check=True): + if args == ["ps", "--quiet", "bf-egress"]: + return ExecResult(stdout="egresscid\n", stderr="", return_code=0) + return ExecResult(stdout="", stderr="", return_code=0) sb._run_docker_compose_command = _compose async def _cli(args, check=True): - if "inspect" in args: + if args[:2] == ["inspect", "cid123"]: return ExecResult(stdout=inspect_stdout, stderr="", return_code=inspect_rc) + if args[:2] == ["inspect", "egresscid"]: + return ExecResult(stdout="healthy\n", stderr="", return_code=0) return ExecResult(stdout="", stderr="", return_code=0) sb._docker_cli = _cli @@ -639,6 +643,50 @@ async def test_docker_relock_happy_sidecar_returns_proxy_env(): assert out.get("HTTPS_PROXY", "").startswith("http://bf-egress:") +@pytest.mark.asyncio +async def test_docker_relock_waits_for_egress_health_before_proxy_env(monkeypatch): + """Guards PR #785 against returning proxy env before bf-egress is ready.""" + from benchflow.sandbox import docker_network_lockdown + from benchflow.sandbox._base import ExecResult + + monkeypatch.setattr(docker_network_lockdown, "_EGRESS_HEALTH_INTERVAL_SEC", 0) + proj = _relock_project() + sb = _make_docker_relock_stub(f"{proj}_bf_egress_internal", 0) + events: list[str] = [] + health_statuses = iter(["starting\n", "healthy\n"]) + + async def _compose(args, check=True): + events.append(f"compose:{' '.join(args)}") + if args == ["ps", "--quiet", "bf-egress"]: + return ExecResult(stdout="egresscid\n", stderr="", return_code=0) + return ExecResult(stdout="", stderr="", return_code=0) + + async def _cli(args, check=True): + if args[:2] == ["inspect", "cid123"]: + events.append("main-networks-inspected") + return ExecResult( + stdout=f"{proj}_bf_egress_internal", stderr="", return_code=0 + ) + if args[:2] == ["inspect", "egresscid"]: + status = next(health_statuses) + events.append(f"egress-health:{status.strip()}") + return ExecResult(stdout=status, stderr="", return_code=0) + return ExecResult(stdout="", stderr="", return_code=0) + + sb._run_docker_compose_command = _compose + sb._docker_cli = _cli + + out = await sb.relock_network() + + assert out["HTTP_PROXY"].startswith("http://bf-egress:") + assert events.index("main-networks-inspected") < events.index( + "compose:ps --quiet bf-egress" + ) + assert events.index("egress-health:starting") < events.index( + "egress-health:healthy" + ) + + @pytest.mark.asyncio async def test_docker_restore_rejects_restrictive_network_policy(): from benchflow.sandbox.docker import DockerSandbox From ddc6e93b9059ada6053a8d98ebb570f55ba78dcf Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 10 Jul 2026 10:24:26 -0700 Subject: [PATCH 21/23] fix(network): fail closed when model lane disabled --- src/benchflow/providers/litellm_runtime.py | 23 ++++++++++++++----- tests/test_network_runtime.py | 26 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index 6434e7abd..56ad155dd 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -50,6 +50,7 @@ ) from benchflow.sandbox.network_policy import ( network_is_restrictive, + resolve_network_decision, ) from benchflow.sandbox.providers import OFF_BOX_MODEL_PROVIDERS from benchflow.trajectories.types import Trajectory @@ -1236,13 +1237,25 @@ async def ensure_litellm_runtime( # forwarding path; on daytona it must pip-install in-sandbox AFTER the # allowlist is applied (pypi is not allowlisted -> install fails). In # both cases the agent instead reaches the provider DIRECTLY over - # HTTPS — but only if the provider host was actually allowlisted. The - # lockdown allowlists exactly provider_host_for_model(model); if that is - # None the host can't have been allowlisted, so skipping here would - # launch an agent whose model CONNECT the egress proxy then denies - # (ACP -32603). Fail closed with an actionable message instead. + # HTTPS — but only if the model lane was allowed and the provider host + # was actually allowlisted. A task with allow_model_endpoint=false + # intentionally has no model lane; skipping the proxy would launch an + # agent whose model CONNECT cannot ever succeed. Likewise, lockdown + # allowlists exactly provider_host_for_model(model); if that is None the + # host can't have been allowlisted. Fail closed with an actionable + # message in both cases instead of surfacing a later ACP -32603. from benchflow.agents.providers import provider_host_for_model + assert sandbox is not None + task_env_config = sandbox.task_env_config + decision = resolve_network_decision(task_env_config, environment) + if not decision.model_lane: + raise RuntimeError( + f"Restrictive network_mode on {environment!r} has " + "allow_model_endpoint=false, so the model endpoint is not " + "reachable. Enable allow_model_endpoint or relax the network " + "policy for model-backed agents." + ) if provider_host_for_model(model, agent_env) is None: raise RuntimeError( f"Restrictive network_mode on {environment!r}: cannot resolve a " diff --git a/tests/test_network_runtime.py b/tests/test_network_runtime.py index b20ea4426..4c6dc6918 100644 --- a/tests/test_network_runtime.py +++ b/tests/test_network_runtime.py @@ -516,6 +516,32 @@ class _FakeSandbox: ) +@pytest.mark.asyncio +async def test_ensure_litellm_fails_closed_when_model_lane_disabled(): + """Guards PR #785: restrictive tasks that opt out of the model lane must not + skip the proxy and launch an agent whose model endpoint is unreachable.""" + from benchflow.providers import litellm_runtime + from benchflow.task.config import SandboxConfig + + class _FakeSandbox: + task_env_config = SandboxConfig( + network_mode="allowlist", + allowed_hosts=["api.deepseek.com"], + allow_model_endpoint=False, + ) + + with pytest.raises(RuntimeError, match="allow_model_endpoint=false"): + await litellm_runtime.ensure_litellm_runtime( + agent="openhands", + agent_env={"DEEPSEEK_API_KEY": "sk"}, + model="deepseek/deepseek-v4-flash", + runtime=None, + environment="docker", + usage_tracking="auto", + sandbox=_FakeSandbox(), + ) + + # ---- docker relock fail-closed gate: deny extras + inspect-rc (audit P1/P2) ---- From 8164d0fde394e951427f868b83e7071bd1487caf Mon Sep 17 00:00:00 2001 From: Bingran You Date: Mon, 13 Jul 2026 06:10:51 -0700 Subject: [PATCH 22/23] Refresh lock for click advisory --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 4f6cfcfbf..e5806fd2a 100644 --- a/uv.lock +++ b/uv.lock @@ -621,14 +621,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] From 9f7599f638e9737cae3e51325fc21e321df645c8 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Mon, 13 Jul 2026 06:18:26 -0700 Subject: [PATCH 23/23] Allow Click 8.4 option error wording --- tests/test_trace_import_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_trace_import_cli.py b/tests/test_trace_import_cli.py index b80b0a719..efcb3956f 100644 --- a/tests/test_trace_import_cli.py +++ b/tests/test_trace_import_cli.py @@ -37,7 +37,7 @@ def test_tasks_generate_help_uses_long_options_only() -> None: def test_tasks_generate_rejects_removed_short_options( alias: str, value: str, tmp_path: Path ) -> None: - """Guards ENG-96: removed task-generation short options stay rejected.""" + """Guards ENG-96 and the Click 8.4.2 lock refresh from PR #788.""" result = CliRunner().invoke( app, [ @@ -52,7 +52,9 @@ def test_tasks_generate_rejects_removed_short_options( ) assert result.exit_code != 0 - assert f"No such option: {alias}" in click.unstyle(result.output) + output = click.unstyle(result.output) + assert "No such option" in output + assert alias in output def test_tasks_generate_dry_run_uses_generation_filters(