From c99f269809a9bbb13839c2262b84be9571f0483b Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 14 Jun 2026 15:07:29 +0300 Subject: [PATCH 1/5] feat(run): add --mode crane, materialize evals at run time Where --mode container runs the pre-built evals/-- combination image, --mode crane runs ONE generic core/crane-runner image that materializes the eval at run time. It crane-exports the per-axis benchmarks/ + agents/ images into a rootfs and fuses them in-container with bwrap/chroot: no Docker daemon, no DinD. So the per-(benchmark, agent) combination matrix becomes an optional pre-bake, and per-task benchmarks (SWE-bench) run from one image (the runner pulls the per-task rootfs). Additive: compose/container/job modes are untouched and nothing depends on the new image. Opt in with --mode crane. cli: Mode::Crane + run_crane (mirrors run_container, passes EVAL_BENCHMARK_ENV=per-task for per-task benchmarks) and naming::crane_runner_image with a unit test. containers/core/crane-runner: the runner image (Dockerfile + materialize entrypoint) plus a daemonless proof (crane-poc.sh). First cut: the fusion is proven daemonless (crane-poc.sh). Wiring the in-rootfs gateway/otel via process-compose and the root-only grader perms is the remaining build-out (see README); pin pulls by digest before shipping. Doctrine note: late- vs early-binding change; needs a rule for when crane applies plus the digest-pin requirement. Signed-off-by: Elron Bandel --- cli/src/naming.rs | 17 +++++ cli/src/run.rs | 83 +++++++++++++++++++++++ containers/core/crane-runner/Dockerfile | 26 +++++++ containers/core/crane-runner/README.md | 45 ++++++++++++ containers/core/crane-runner/crane-poc.sh | 42 ++++++++++++ containers/core/crane-runner/materialize | 42 ++++++++++++ 6 files changed, 255 insertions(+) create mode 100644 containers/core/crane-runner/Dockerfile create mode 100644 containers/core/crane-runner/README.md create mode 100755 containers/core/crane-runner/crane-poc.sh create mode 100755 containers/core/crane-runner/materialize diff --git a/cli/src/naming.rs b/cli/src/naming.rs index ba00ae54..78d538c2 100644 --- a/cli/src/naming.rs +++ b/cli/src/naming.rs @@ -62,6 +62,15 @@ pub fn eval_task_image( ) } +/// `{registry}/core/crane-runner:` — the generic runtime-fusion runner. +/// One image for every (benchmark, agent): `--mode crane` runs it and it +/// crane-pulls the per-axis `benchmarks/` + `agents/` images and fuses +/// them at run time, so the per-(benchmark, agent) `evals/` images above are not +/// required (they stay an optional pre-bake). +pub fn crane_runner_image(registry: &str, tag: &str) -> String { + format!("{registry}/core/crane-runner:{tag}") +} + /// `{registry}/evaluate` — the single published evaluation compose artifact. /// `run --mode compose` consumes it as `oci://{registry}/evaluate`; one generic, /// `EVAL_BENCHMARK`-parameterized artifact, not one per benchmark. @@ -148,6 +157,14 @@ mod tests { ); } + #[test] + fn crane_runner_is_one_generic_core_image() { + assert_eq!( + crane_runner_image(REG, "latest"), + "ghcr.io/exgentic/core/crane-runner:latest" + ); + } + #[test] fn category_images_are_namespaced() { assert_eq!( diff --git a/cli/src/run.rs b/cli/src/run.rs index abb8119f..8fa552b8 100644 --- a/cli/src/run.rs +++ b/cli/src/run.rs @@ -52,6 +52,12 @@ pub enum Mode { /// One k8s `Job` + one Pod + three containers (NetworkPolicy on runner). /// Invocation: `kubectl apply`. Production k8s surface. Job, + /// One generic runner container that materializes the eval at run time: it + /// crane-pulls the per-axis `benchmarks/` + `agents/` images and fuses + /// them in-container (bwrap/chroot, no daemon, no DinD), so no + /// `evals/--` combination image is needed. Invocation: `docker run` + /// the `core/crane-runner` image. Additive — the modes above are untouched. + Crane, } #[derive(Args)] @@ -203,6 +209,14 @@ pub fn execute(registry: &str, args: RunArgs) -> Result<(), String> { args.dry_run, ), Mode::Job => run_job(registry, &benchmark, &args, &envs), + Mode::Crane => run_crane( + registry, + &benchmark, + &args.agent, + &envs, + args.local, + args.dry_run, + ), } } @@ -372,6 +386,75 @@ fn run_container( Ok(()) } +/// `--mode crane` → docker run -e EVAL_* +/// +/// Unlike `--mode container` (which runs the pre-built `evals/--` image), +/// crane mode runs ONE generic runner that materializes the eval at run time: it +/// crane-pulls the per-axis `benchmarks/` + `agents/` images and fuses +/// them in-container — no combination image, no DinD. `EVAL_BENCHMARK`, +/// `EVAL_AGENT`, and `EVAL_TASK_ID` (already in `envs`) tell the runner what to +/// assemble; for a per-task benchmark we also pass `EVAL_BENCHMARK_ENV=per-task` +/// so it materializes the per-task rootfs. This is the single-container analog +/// of the generic compose file, one better: it composes the axes at run time, so +/// the combination matrix is optional. +fn run_crane( + registry: &str, + benchmark: &str, + agent: &Option, + envs: &[(&str, String)], + local: bool, + dry_run: bool, +) -> Result<(), String> { + if agent.is_none() { + return Err("--agent is required in crane mode".to_string()); + } + if local { + // The runner pulls published per-axis images at run time; there is no + // local combination image to build (that is the whole point). + return Err("--local is not supported in crane mode".to_string()); + } + let image = eval_containers::naming::crane_runner_image(registry, "latest"); + + // Per-task benchmarks materialize `benchmarks/-`; tell the runner. + let mut env_list: Vec<(&str, String)> = envs.to_vec(); + if eval_containers::benchmark::is_per_task_by_name(benchmark) { + env_list.push(("EVAL_BENCHMARK_ENV", "per-task".to_string())); + } + + let env_str = env_list + .iter() + .map(|(k, v)| format!("-e {k}={v}")) + .collect::>() + .join(" "); + eprintln!("$ docker run --rm {env_str} -v output:/output {image}"); + if dry_run { + eprintln!("(--dry-run: stopping before docker run)"); + return Ok(()); + } + + let mut cmd = Command::new("docker"); + cmd.arg("run").arg("--rm"); + for (k, v) in &env_list { + cmd.arg("-e").arg(format!("{k}={v}")); + } + // Same in-container gateway as `--mode container`, so forward the upstream + // credentials it proxies to (skipped when the caller didn't set them). + for var in GATEWAY_CRED_VARS { + if std::env::var_os(var).is_some() { + cmd.arg("-e").arg(var); + } + } + cmd.arg("-v").arg("output:/output"); + cmd.arg(&image); + let status = cmd + .status() + .map_err(|e| format!("failed to docker run: {e}"))?; + if !status.success() { + return Err(format!("docker run failed with {status}")); + } + Ok(()) +} + /// `--mode job` → `helm template oci:///charts/eval … | kubectl apply -f -` /// (or `./benchmarks/_chart` with `--local`). /// diff --git a/containers/core/crane-runner/Dockerfile b/containers/core/crane-runner/Dockerfile new file mode 100644 index 00000000..6ca5ec36 --- /dev/null +++ b/containers/core/crane-runner/Dockerfile @@ -0,0 +1,26 @@ +# The crane runner -- ONE generic image that materializes any (benchmark, agent) +# eval at run time, instead of pulling a pre-built evals/-- combination +# image. It crane-exports the per-axis images to a rootfs and fuses them with +# bwrap/chroot: no Docker daemon, no DinD (proven daemonless -- see crane-poc.sh). +# +# STATUS: first cut. The fusion ([1]/[2] in `materialize`) is proven; wiring the +# in-rootfs gateway/otel via process-compose and the root-only grader perms is +# the remaining build-out + validation gate (see README.md). Nothing else in the +# repo depends on this image -- `--mode crane` opts in. +FROM debian:stable-slim + +# Userspace image tool (no daemon) + the sandbox. crane exports an image's +# filesystem; bwrap/chroot runs a process inside it. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl bubblewrap jq tar \ + && rm -rf /var/lib/apt/lists/* + +# Pin crane for reproducibility (override with --build-arg CRANE_VERSION=...). +ARG CRANE_VERSION=v0.20.3 +RUN arch="$(dpkg --print-architecture)"; case "$arch" in amd64) arch=x86_64 ;; esac; \ + curl -fsSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${arch}.tar.gz" \ + | tar -xz -C /usr/local/bin crane + +COPY materialize /usr/local/bin/materialize +RUN chmod +x /usr/local/bin/materialize +ENTRYPOINT ["/usr/local/bin/materialize"] diff --git a/containers/core/crane-runner/README.md b/containers/core/crane-runner/README.md new file mode 100644 index 00000000..fd0af800 --- /dev/null +++ b/containers/core/crane-runner/README.md @@ -0,0 +1,45 @@ +# crane-runner (`--mode crane`) + +One generic image that materializes any `(benchmark, agent)` eval at **run time** +by fusing the per-axis images, instead of pulling a pre-built +`evals/--` combination image. + +## Why + +`compose` / `container` / `job` modes all run a pre-built combination image, so +the `evals/--` matrix (benchmarks × agents) must exist. Per-task benchmarks +(SWE-bench) make it worse — a different image per task, which a single +`images.benchmark` can't fan out. The crane runner pulls the per-axis +`benchmarks/` + `agents/` images at run time and fuses them in one +container, so: + +- the combination matrix becomes an **optional pre-bake**, and +- per-task benchmarks run from **one** image (it pulls the per-task rootfs). + +It is the single-container analog of the generic `compose` file — one better: it +composes the *axes* at run time instead of pulling a pre-fused product. + +## How (daemonless — no DinD) + +`materialize` does: **crane export** the benchmark rootfs (download + untar, no +daemon) → **overlay the agent** → **bwrap/chroot** in to run agent + grade. The +distinction from DinD: crane only *downloads a filesystem*; it never runs a +daemon. The core primitive (crane pull + run-in-rootfs + agent-edits-testbed) is +proven daemonless — run [`crane-poc.sh`](crane-poc.sh) in any Linux container. + +## Status — first cut + +- ✅ **Fusion** ([1]/[2] in `materialize`) — the proven primitive. +- ⛳ **Validation gate**: wiring the in-rootfs gateway/otel via `process-compose` + and the root-only grader perms ([3]) reuses the combination image's existing + `/usr/local/bin/run` mechanism; **end-to-end against real images is not yet + run**. Pin pulls by **digest** for reproducibility before shipping. +- This is a **doctrine-level** addition (late- vs early-binding): it needs a rule + for *when* crane applies (`per-task`, or opt-in) and the digest-pin requirement. + +## Use (additive, opt-in — nothing else depends on it) + +```bash +eval-containers run aime --agent codex --model openai/ --mode crane +# → docker run --rm -e EVAL_* -v output:/output /core/crane-runner:latest +``` diff --git a/containers/core/crane-runner/crane-poc.sh b/containers/core/crane-runner/crane-poc.sh new file mode 100755 index 00000000..1a16942d --- /dev/null +++ b/containers/core/crane-runner/crane-poc.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Daemonless proof for the crane runner. Run it inside a plain Linux container -- +# that container models the eval pod; nothing below talks to a Docker daemon: +# +# docker run --rm -i debian:stable-slim sh < crane-poc.sh +# +set -e + +echo "=== environment: a plain Linux box (the 'eval pod') ===" +if command -v docker >/dev/null 2>&1; then echo "docker daemon in here: PRESENT (unexpected!)"; else echo "docker daemon in here: NONE <-- so nothing below is DinD"; fi +echo + +echo "[setup] fetch crane + bwrap (userspace tools, no daemon)..." +apt-get update -qq >/dev/null 2>&1 +apt-get install -y -qq --no-install-recommends ca-certificates curl bubblewrap >/dev/null 2>&1 +curl -sSL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_arm64.tar.gz \ + | tar -xz -C /usr/local/bin crane +echo " crane=$(crane version 2>/dev/null || echo ok) bwrap=$(command -v bwrap)" +echo + +echo "[1] DAEMONLESS PULL (this is the part DinD would otherwise do)" +echo " crane export docker://alpine -> a root filesystem, just download+untar" +mkdir -p /bench +crane export --platform linux/arm64 alpine:latest - | tar -x -C /bench +echo " got rootfs: $(du -sh /bench 2>/dev/null | cut -f1), top: $(ls /bench | tr '\n' ' ')" +echo + +echo "[2] RUN a binary FROM the pulled rootfs (chroot, as root, no daemon)" +printf ' /etc/alpine-release inside the pulled fs: ' +chroot /bench /bin/sh -c 'cat /etc/alpine-release' +echo + +echo "[3] COMPOSE (the universal-runner trick): drop an 'agent' into the benchmark" +echo " rootfs and run it there -- it reads and edits the testbed filesystem" +printf '%s\n' \ + '#!/bin/sh' \ + 'echo " agent is running INSIDE the benchmark fs (alpine $(cat /etc/alpine-release))"' \ + ': > /patch.diff && echo " agent edited the testbed: $(ls -la /patch.diff)"' \ + > /bench/agent.sh +chroot /bench /bin/sh /agent.sh +echo +echo "=== PROVEN: pull rootfs (crane) + run in it (chroot/bwrap) = no daemon, no DinD ===" diff --git a/containers/core/crane-runner/materialize b/containers/core/crane-runner/materialize new file mode 100755 index 00000000..e6df21b6 --- /dev/null +++ b/containers/core/crane-runner/materialize @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Materialize one eval by fusing per-axis images at run time, then run it. +# Proven primitive: crane export (daemonless pull) + bwrap/chroot (run in rootfs). +set -euo pipefail +: "${EVAL_REGISTRY:?}" "${EVAL_BENCHMARK:?}" "${EVAL_AGENT:?}" +btag="${EVAL_BENCHMARK_TAG:-latest}" +atag="${EVAL_AGENT_TAG:-latest}" +root=/run/eval +mkdir -p "$root" + +echo "crane-runner: fusing ${EVAL_BENCHMARK} x ${EVAL_AGENT} at run time (no combination image)" + +# [1] Benchmark filesystem (testbed + tasks + grader + entrypoint). Per-task +# benchmarks resolve to benchmarks/-; shared-env to benchmarks/. +if [ "${EVAL_BENCHMARK_ENV:-shared-env}" = "per-task" ]; then + bench="${EVAL_REGISTRY}/benchmarks/${EVAL_BENCHMARK}-${EVAL_TASK_ID:?per-task needs EVAL_TASK_ID}:${btag}" +else + bench="${EVAL_REGISTRY}/benchmarks/${EVAL_BENCHMARK}:${btag}" +fi +echo " [1] crane export ${bench}" +crane export "$bench" - | tar -x -C "$root" + +# [2] Overlay the agent into the benchmark rootfs (mirrors the combination +# Dockerfile's `COPY --from=agent /opt/agent`). +agent="${EVAL_REGISTRY}/agents/${EVAL_AGENT}:${atag}" +echo " [2] crane export ${agent} -> overlay" +crane export "$agent" - | tar -x -C "$root" + +# Run inside the fused rootfs: bwrap if user namespaces are available +# (isolation), else chroot (root, still daemonless). +in_root() { bwrap --bind "$root" / --dev /dev --proc /proc "$@" 2>/dev/null || chroot "$root" "$@"; } + +# --------------------------------------------------------------------------- +# VALIDATION GATE (see README): wiring the in-rootfs gateway + otel via +# process-compose and the root-only grader perms is the remaining build-out. It +# reuses the EXACT mechanism the combination image already ships +# (/usr/local/bin/run + /etc/process-compose.yaml). This first cut proves the +# FUSION above; end-to-end against real benchmark/agent images is not yet run. +# --------------------------------------------------------------------------- +export EVAL_TASK_ID="${EVAL_TASK_ID:-0}" +echo " [3] fused rootfs ready at ${root} ($(du -sh "$root" 2>/dev/null | cut -f1))" +echo " next (build-out): in_root /entrypoint.sh /run.sh with gateway via process-compose" From 985341692cba59b88a370d97dfc5d541fc60290b Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 14 Jun 2026 16:08:17 +0300 Subject: [PATCH 2/5] docs(crane): record the statically-verified build-out spec Static verification confirmed: export preserves root-only perms (so the answer-isolation invariant holds by reusing gosu/process-compose, not by inventing isolation); the orchestration is the existing 5-process pipeline launched by /usr/local/bin/run; arch must match (swe-bench is x86_64). README status now states the precise [3] build-out (bake core, pull gateway axis, extract as root, exec entrypoint+run) and the remaining gates (digest-pin, conformance, bake target, doctrine). Signed-off-by: Elron Bandel --- containers/core/crane-runner/README.md | 32 ++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/containers/core/crane-runner/README.md b/containers/core/crane-runner/README.md index fd0af800..9270c852 100644 --- a/containers/core/crane-runner/README.md +++ b/containers/core/crane-runner/README.md @@ -27,15 +27,29 @@ distinction from DinD: crane only *downloads a filesystem*; it never runs a daemon. The core primitive (crane pull + run-in-rootfs + agent-edits-testbed) is proven daemonless — run [`crane-poc.sh`](crane-poc.sh) in any Linux container. -## Status — first cut - -- ✅ **Fusion** ([1]/[2] in `materialize`) — the proven primitive. -- ⛳ **Validation gate**: wiring the in-rootfs gateway/otel via `process-compose` - and the root-only grader perms ([3]) reuses the combination image's existing - `/usr/local/bin/run` mechanism; **end-to-end against real images is not yet - run**. Pin pulls by **digest** for reproducibility before shipping. -- This is a **doctrine-level** addition (late- vs early-binding): it needs a rule - for *when* crane applies (`per-task`, or opt-in) and the digest-pin requirement. +## Status — first cut (statically verified) + +Proven: +- **Fusion** ([1]/[2]) against **real** images — the real swe-bench `/testbed` (sympy + repo) ⊎ the real claude-code `/opt/agent` reproduce the combination image's layout. +- **Isolation survives the fusion**: `export` preserves root-only modes/owners + (`/tasks` `0600`, `/tests` & `/opt/gateway` `0700`, root), so *extract as root* and + run the agent through the existing `gosu agent` / `env -i` pipeline (which even hides + the task id from the model) and the answers stay unreadable. The invariant is + **reused, not reinvented**. + +Build-out ([3]) — wire the existing runtime, don't invent one: +1. **Bake the fixed core** into this image: `otelcol`, `process-compose`, `gosu`, + `/usr/local/bin/{run,write-result}` + configs. +2. **Pull the 3rd axis** (`models/` → `/opt/gateway`) alongside benchmark+agent; + run the agent image's `install.sh` (symlinks). +3. **Extract as root**, then `exec /entrypoint.sh /usr/local/bin/run` — the existing + 5-process pipeline (otelcol → gateway → agent → verifier → result). + +Hard constraint: **arch** — the runner, the node, and the pulled rootfs must match +(swe-bench is `x86_64`; the runner built here is `arm64`). Plus **digest-pinned** pulls, +a **conformance test** (`crane(X) == container(X)`), a **bake target**, and a **doctrine +rule** (when crane applies + the digest-pin + isolation invariants). ## Use (additive, opt-in — nothing else depends on it) From f6a81a89ae0b80ce7ead5d9eeb12e61a9bf4414b Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 14 Jun 2026 16:17:21 +0300 Subject: [PATCH 3/5] refactor(crane): share the docker-run helper, drop dead/unused code run.rs: --mode container and --mode crane duplicated ~30 lines of docker-run boilerplate; extract one docker_run_eval(image, envs, dry_run) both call. materialize: drop the unused in_root function (the [3] build-out is still a stub). Dockerfile: drop unused jq. crane-poc.sh: detect arch instead of hardcoding arm64 so it runs in any Linux container. Net -33 lines; behavior unchanged (dry-runs identical for container + crane, per-task env still injected, container mode unaffected). Signed-off-by: Elron Bandel --- cli/src/run.rs | 55 ++++++----------------- containers/core/crane-runner/Dockerfile | 2 +- containers/core/crane-runner/crane-poc.sh | 7 +-- containers/core/crane-runner/materialize | 17 +++---- 4 files changed, 24 insertions(+), 57 deletions(-) diff --git a/cli/src/run.rs b/cli/src/run.rs index 8fa552b8..5f8a6f60 100644 --- a/cli/src/run.rs +++ b/cli/src/run.rs @@ -349,41 +349,7 @@ fn run_container( eval_containers::naming::eval_image(registry, benchmark, &agent, "latest") }; - let env_str = envs - .iter() - .map(|(k, v)| format!("-e {k}={v}")) - .collect::>() - .join(" "); - eprintln!("$ docker run --rm {env_str} -v output:/output {image}"); - if dry_run { - eprintln!("(--dry-run: stopping before docker run)"); - return Ok(()); - } - - let mut cmd = Command::new("docker"); - cmd.arg("run").arg("--rm"); - for (k, v) in envs { - cmd.arg("-e").arg(format!("{k}={v}")); - } - // Single-image mode runs the gateway in-container, so it needs the upstream - // credentials the gateway service gets from `eval-secrets` (k8s) or the - // shell env (compose). Forward them from the caller's environment with - // docker's `-e NAME` passthrough (no value → not rendered into logs); unset - // vars are skipped, so this is a no-op when the caller didn't provide them. - for var in GATEWAY_CRED_VARS { - if std::env::var_os(var).is_some() { - cmd.arg("-e").arg(var); - } - } - cmd.arg("-v").arg("output:/output"); - cmd.arg(&image); - let status = cmd - .status() - .map_err(|e| format!("failed to docker run: {e}"))?; - if !status.success() { - return Err(format!("docker run failed with {status}")); - } - Ok(()) + docker_run_eval(&image, envs, dry_run) } /// `--mode crane` → docker run -e EVAL_* @@ -421,7 +387,16 @@ fn run_crane( env_list.push(("EVAL_BENCHMARK_ENV", "per-task".to_string())); } - let env_str = env_list + docker_run_eval(&image, &env_list, dry_run) +} + +/// `docker run --rm -e EVAL_* [-e …] -v output:/output ` — or print it +/// for `--dry-run`. Shared by the single-image surfaces (`--mode container` and +/// `--mode crane`), which run the gateway in-container and so forward the upstream +/// credentials (`OPENAI_API_*`) it proxies to — skipped when the caller didn't set +/// them, so it's a no-op otherwise. +fn docker_run_eval(image: &str, envs: &[(&str, String)], dry_run: bool) -> Result<(), String> { + let env_str = envs .iter() .map(|(k, v)| format!("-e {k}={v}")) .collect::>() @@ -431,21 +406,17 @@ fn run_crane( eprintln!("(--dry-run: stopping before docker run)"); return Ok(()); } - let mut cmd = Command::new("docker"); cmd.arg("run").arg("--rm"); - for (k, v) in &env_list { + for (k, v) in envs { cmd.arg("-e").arg(format!("{k}={v}")); } - // Same in-container gateway as `--mode container`, so forward the upstream - // credentials it proxies to (skipped when the caller didn't set them). for var in GATEWAY_CRED_VARS { if std::env::var_os(var).is_some() { cmd.arg("-e").arg(var); } } - cmd.arg("-v").arg("output:/output"); - cmd.arg(&image); + cmd.arg("-v").arg("output:/output").arg(image); let status = cmd .status() .map_err(|e| format!("failed to docker run: {e}"))?; diff --git a/containers/core/crane-runner/Dockerfile b/containers/core/crane-runner/Dockerfile index 6ca5ec36..f18d7b8d 100644 --- a/containers/core/crane-runner/Dockerfile +++ b/containers/core/crane-runner/Dockerfile @@ -12,7 +12,7 @@ FROM debian:stable-slim # Userspace image tool (no daemon) + the sandbox. crane exports an image's # filesystem; bwrap/chroot runs a process inside it. RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl bubblewrap jq tar \ + ca-certificates curl bubblewrap tar \ && rm -rf /var/lib/apt/lists/* # Pin crane for reproducibility (override with --build-arg CRANE_VERSION=...). diff --git a/containers/core/crane-runner/crane-poc.sh b/containers/core/crane-runner/crane-poc.sh index 1a16942d..d7acef82 100755 --- a/containers/core/crane-runner/crane-poc.sh +++ b/containers/core/crane-runner/crane-poc.sh @@ -13,15 +13,16 @@ echo echo "[setup] fetch crane + bwrap (userspace tools, no daemon)..." apt-get update -qq >/dev/null 2>&1 apt-get install -y -qq --no-install-recommends ca-certificates curl bubblewrap >/dev/null 2>&1 -curl -sSL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_arm64.tar.gz \ +case "$(uname -m)" in x86_64) ca=x86_64; pa=amd64 ;; aarch64|arm64) ca=arm64; pa=arm64 ;; *) ca="$(uname -m)"; pa="$ca" ;; esac +curl -sSL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_${ca}.tar.gz" \ | tar -xz -C /usr/local/bin crane -echo " crane=$(crane version 2>/dev/null || echo ok) bwrap=$(command -v bwrap)" +echo " crane=$(crane version 2>/dev/null || echo ok) bwrap=$(command -v bwrap) arch=${pa}" echo echo "[1] DAEMONLESS PULL (this is the part DinD would otherwise do)" echo " crane export docker://alpine -> a root filesystem, just download+untar" mkdir -p /bench -crane export --platform linux/arm64 alpine:latest - | tar -x -C /bench +crane export --platform "linux/${pa}" alpine:latest - | tar -x -C /bench echo " got rootfs: $(du -sh /bench 2>/dev/null | cut -f1), top: $(ls /bench | tr '\n' ' ')" echo diff --git a/containers/core/crane-runner/materialize b/containers/core/crane-runner/materialize index e6df21b6..0b4c4890 100755 --- a/containers/core/crane-runner/materialize +++ b/containers/core/crane-runner/materialize @@ -26,17 +26,12 @@ agent="${EVAL_REGISTRY}/agents/${EVAL_AGENT}:${atag}" echo " [2] crane export ${agent} -> overlay" crane export "$agent" - | tar -x -C "$root" -# Run inside the fused rootfs: bwrap if user namespaces are available -# (isolation), else chroot (root, still daemonless). -in_root() { bwrap --bind "$root" / --dev /dev --proc /proc "$@" 2>/dev/null || chroot "$root" "$@"; } - # --------------------------------------------------------------------------- -# VALIDATION GATE (see README): wiring the in-rootfs gateway + otel via -# process-compose and the root-only grader perms is the remaining build-out. It -# reuses the EXACT mechanism the combination image already ships -# (/usr/local/bin/run + /etc/process-compose.yaml). This first cut proves the -# FUSION above; end-to-end against real benchmark/agent images is not yet run. +# BUILD-OUT ([3], see README): wire the EXISTING runtime into the fused rootfs -- +# bake otelcol/process-compose/gosu/run, pull the gateway axis, extract as root, +# then `exec /entrypoint.sh /usr/local/bin/run` (the 5-process pipeline). `export` +# preserves root-only perms, so answer-isolation holds by reuse. This first cut +# proves the FUSION ([1]/[2]); end-to-end is the remaining work. # --------------------------------------------------------------------------- -export EVAL_TASK_ID="${EVAL_TASK_ID:-0}" echo " [3] fused rootfs ready at ${root} ($(du -sh "$root" 2>/dev/null | cut -f1))" -echo " next (build-out): in_root /entrypoint.sh /run.sh with gateway via process-compose" +echo " build-out: bake core + pull gateway + exec /entrypoint.sh /usr/local/bin/run" From 9a70c78ff8fbf84ecdf387191e9efc545d3d8ab2 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 14 Jun 2026 16:22:12 +0300 Subject: [PATCH 4/5] refactor(crane): drop now-unused bubblewrap/tar from the runner image Removing the dead in_root function left bubblewrap unused (the first cut only does crane export | tar -x); tar and chroot are already in debian:stable-slim. apt install is now just crane fetch deps (ca-certificates curl). bwrap lands with the [3] build-out, when first used. Signed-off-by: Elron Bandel --- containers/core/crane-runner/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/containers/core/crane-runner/Dockerfile b/containers/core/crane-runner/Dockerfile index f18d7b8d..51b165f4 100644 --- a/containers/core/crane-runner/Dockerfile +++ b/containers/core/crane-runner/Dockerfile @@ -9,10 +9,11 @@ # repo depends on this image -- `--mode crane` opts in. FROM debian:stable-slim -# Userspace image tool (no daemon) + the sandbox. crane exports an image's -# filesystem; bwrap/chroot runs a process inside it. +# crane (a daemonless registry client) downloads an image's filesystem; curl + +# CA certs fetch crane itself at build time. tar and chroot are already in the +# base image; bwrap (hardened isolation) lands with the [3] build-out when used. RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl bubblewrap tar \ + ca-certificates curl \ && rm -rf /var/lib/apt/lists/* # Pin crane for reproducibility (override with --build-arg CRANE_VERSION=...). From 7338d09c58ea7f9518d9014c9a387b38fefb3b52 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 14 Jun 2026 16:25:57 +0300 Subject: [PATCH 5/5] refactor(crane): drop crane-poc.sh from image source containers/core// holds image build sources (Dockerfile + COPYd files); crane-poc.sh was a standalone demo, COPYd nowhere and used by nothing. The daemonless mechanism is captured in the PR; dropped the file and its two references (Dockerfile header, README). Signed-off-by: Elron Bandel --- containers/core/crane-runner/Dockerfile | 2 +- containers/core/crane-runner/README.md | 6 ++-- containers/core/crane-runner/crane-poc.sh | 43 ----------------------- 3 files changed, 4 insertions(+), 47 deletions(-) delete mode 100755 containers/core/crane-runner/crane-poc.sh diff --git a/containers/core/crane-runner/Dockerfile b/containers/core/crane-runner/Dockerfile index 51b165f4..27ec8bee 100644 --- a/containers/core/crane-runner/Dockerfile +++ b/containers/core/crane-runner/Dockerfile @@ -1,7 +1,7 @@ # The crane runner -- ONE generic image that materializes any (benchmark, agent) # eval at run time, instead of pulling a pre-built evals/-- combination # image. It crane-exports the per-axis images to a rootfs and fuses them with -# bwrap/chroot: no Docker daemon, no DinD (proven daemonless -- see crane-poc.sh). +# chroot/bwrap: no Docker daemon, no DinD (crane just downloads a filesystem). # # STATUS: first cut. The fusion ([1]/[2] in `materialize`) is proven; wiring the # in-rootfs gateway/otel via process-compose and the root-only grader perms is diff --git a/containers/core/crane-runner/README.md b/containers/core/crane-runner/README.md index 9270c852..7c498190 100644 --- a/containers/core/crane-runner/README.md +++ b/containers/core/crane-runner/README.md @@ -22,10 +22,10 @@ composes the *axes* at run time instead of pulling a pre-fused product. ## How (daemonless — no DinD) `materialize` does: **crane export** the benchmark rootfs (download + untar, no -daemon) → **overlay the agent** → **bwrap/chroot** in to run agent + grade. The +daemon) → **overlay the agent** → **chroot/bwrap** in to run agent + grade. The distinction from DinD: crane only *downloads a filesystem*; it never runs a -daemon. The core primitive (crane pull + run-in-rootfs + agent-edits-testbed) is -proven daemonless — run [`crane-poc.sh`](crane-poc.sh) in any Linux container. +daemon — so nothing here needs privilege or a Docker socket. Verified daemonless +against the real swe-bench + claude-code images (see the PR). ## Status — first cut (statically verified) diff --git a/containers/core/crane-runner/crane-poc.sh b/containers/core/crane-runner/crane-poc.sh deleted file mode 100755 index d7acef82..00000000 --- a/containers/core/crane-runner/crane-poc.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# Daemonless proof for the crane runner. Run it inside a plain Linux container -- -# that container models the eval pod; nothing below talks to a Docker daemon: -# -# docker run --rm -i debian:stable-slim sh < crane-poc.sh -# -set -e - -echo "=== environment: a plain Linux box (the 'eval pod') ===" -if command -v docker >/dev/null 2>&1; then echo "docker daemon in here: PRESENT (unexpected!)"; else echo "docker daemon in here: NONE <-- so nothing below is DinD"; fi -echo - -echo "[setup] fetch crane + bwrap (userspace tools, no daemon)..." -apt-get update -qq >/dev/null 2>&1 -apt-get install -y -qq --no-install-recommends ca-certificates curl bubblewrap >/dev/null 2>&1 -case "$(uname -m)" in x86_64) ca=x86_64; pa=amd64 ;; aarch64|arm64) ca=arm64; pa=arm64 ;; *) ca="$(uname -m)"; pa="$ca" ;; esac -curl -sSL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_${ca}.tar.gz" \ - | tar -xz -C /usr/local/bin crane -echo " crane=$(crane version 2>/dev/null || echo ok) bwrap=$(command -v bwrap) arch=${pa}" -echo - -echo "[1] DAEMONLESS PULL (this is the part DinD would otherwise do)" -echo " crane export docker://alpine -> a root filesystem, just download+untar" -mkdir -p /bench -crane export --platform "linux/${pa}" alpine:latest - | tar -x -C /bench -echo " got rootfs: $(du -sh /bench 2>/dev/null | cut -f1), top: $(ls /bench | tr '\n' ' ')" -echo - -echo "[2] RUN a binary FROM the pulled rootfs (chroot, as root, no daemon)" -printf ' /etc/alpine-release inside the pulled fs: ' -chroot /bench /bin/sh -c 'cat /etc/alpine-release' -echo - -echo "[3] COMPOSE (the universal-runner trick): drop an 'agent' into the benchmark" -echo " rootfs and run it there -- it reads and edits the testbed filesystem" -printf '%s\n' \ - '#!/bin/sh' \ - 'echo " agent is running INSIDE the benchmark fs (alpine $(cat /etc/alpine-release))"' \ - ': > /patch.diff && echo " agent edited the testbed: $(ls -la /patch.diff)"' \ - > /bench/agent.sh -chroot /bench /bin/sh /agent.sh -echo -echo "=== PROVEN: pull rootfs (crane) + run in it (chroot/bwrap) = no daemon, no DinD ==="