benchflow-ai · Yiminnn · Jun 28, 2026 · Jun 28, 2026 · Jun 28, 2026 · Jun 28, 2026
diff --git a/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py b/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py
@@ -24,7 +24,6 @@
 from pathlib import Path
 from typing import Any
 
-
 TOKEN_KEYS = {
     "input_tokens",
     "output_tokens",
@@ -109,9 +108,12 @@ def has_token_usage(value: Any) -> bool:
     for obj in iter_dicts(value):
         for key in TOKEN_KEYS:
             token_value = obj.get(key)
-            if isinstance(token_value, (int, float)) and not isinstance(token_value, bool):
-                if token_value > 0:
-                    return True
+            if (
+                isinstance(token_value, (int, float))
+                and not isinstance(token_value, bool)
+                and token_value > 0
+            ):
+                return True
     return False
 
 
@@ -215,7 +217,9 @@ def validate_acp(rows: list[dict[str, Any]], path: Path) -> list[str]:
     return issues
 
 
-def validate_llm(rows: list[dict[str, Any]], path: Path) -> tuple[list[str], dict[str, Any]]:
+def validate_llm(
+    rows: list[dict[str, Any]], path: Path
+) -> tuple[list[str], dict[str, Any]]:
     issues: list[str] = []
     request_count = 0
     response_count = 0
@@ -270,7 +274,9 @@ def load_run_config(root: Path) -> dict[str, Any] | None:
     return None
 
 
-def validate_rollout(root: Path, *, allow_oracle_without_llm: bool = False) -> dict[str, Any]:
+def validate_rollout(
+    root: Path, *, allow_oracle_without_llm: bool = False
+) -> dict[str, Any]:
     result_path = root / "result.json"
     result, result_error = read_json(result_path)
     issues: list[str] = []
@@ -319,7 +325,12 @@ def validate_rollout(root: Path, *, allow_oracle_without_llm: bool = False) -> d
 
     error_text = " ".join(
         str(result.get(key) or "")
-        for key in ("error", "verifier_error", "error_category", "verifier_error_category")
+        for key in (
+            "error",
+            "verifier_error",
+            "error_category",
+            "verifier_error_category",
+        )
     ).lower()
     if any(marker in error_text for marker in INFRA_ERROR_MARKERS):
         issues.append("result carries infra/provider error markers")

diff --git a/benchmarks/casinobench/Dockerfile b/benchmarks/casinobench/Dockerfile
@@ -0,0 +1,7 @@
+# Run image for the casinobench benchmark = the prebuilt base (agent backends +
+# casino CLI + the in-sandbox casino-service). The manifest's `image` key
+# (ghcr.io/yiminnn/casinobench-base:2.0.6) selects it as a prebuilt, so this Dockerfile is not
+# rebuilt at run time — it exists so the sandbox's environment_dir validates and
+# so `docker build -t ghcr.io/yiminnn/casinobench-base:2.0.6 benchmarks/casinobench` works too.
+# Build the base first: see docker/casinobench-base.Dockerfile + README.md.
+FROM ghcr.io/yiminnn/casinobench-base:2.0.6
diff --git a/benchmarks/casinobench/README.md b/benchmarks/casinobench/README.md
@@ -0,0 +1,89 @@
+# CasinoBench — in-sandbox env-0 casino benchmark
+
+CasinoBench is a stateful, single-service BenchFlow benchmark. A casino
+**mock-service** (FastAPI/uvicorn) runs *inside the rollout's own sandbox* — the
+env-0 / ClawsBench pattern — and the agent plays through the `casino` seven-tool
+CLI over `localhost:9001`. Each game is one task; the game is selected per task
+by the `CASINOBENCH_GAME` env var. The service is the chip authority, so the
+verifier just reads the live final standing and scores net chips. There is **no
+host-subprocess wiring** — the service is declared in the manifest and started
+by BenchFlow's Environment plane in-sandbox.
+
+```
+benchmarks/casinobench/
+├── environment.toml                 # the in-sandbox manifest (the whole framework seam)
+├── docker/casinobench-base.Dockerfile  # casinobench-base: working service + ACP backends + casino CLI
+├── verifier/test.sh                 # shared scorer: curl /_admin/state -> net chips -> reward
+└── tasks/
+    └── blackjack/
+        ├── task.md                  # CASINOBENCH_GAME=six-deck-blackjack-s17, manifest -> ../../environment.toml
+        └── tests/test.sh            # symlink -> ../../../verifier/test.sh
+```
+
+## The base image
+
+`casino-service` (the env-0 mock-service) and the ACP seats live in **one**
+image so competing agents can share a sandbox. The image is built in two steps:
+
+```bash
+CB=~/casinobench   # your casinobench engine checkout (proprietary, separate repo)
+
+# 1. assemble the agent-seat build context (gitignored — see agent_env/.gitignore)
+cp src/benchflow/agents/deepagents_acp_shim.py examples/casino/agent_env/deepagents-acp-shim
+cp -r "$CB/packages/environments/casino"        examples/casino/agent_env/casino-pkg
+cp -r "$CB"                                      examples/casino/agent_env/casinobench-engine
+
+# 2. the seat image (ACP backends + casino CLI; service still a --no-deps stub)
+docker build -t casino-agent-seat:latest examples/casino/agent_env \
+  -f examples/casino/agent_env/Dockerfile
+
+# 3. casinobench-base: extends the seat image with the engine + service extra so
+#    `casino-service` actually runs. This is the manifest's run `image`.
+docker build -t env0acrdd8632.azurecr.io/casinobench-base:2.0.1 examples/casino/agent_env \
+  -f benchmarks/casinobench/docker/casinobench-base.Dockerfile
+```
+
+Verify the service is real (not the broken `--no-deps` stub):
+
+```bash
+docker run --rm env0acrdd8632.azurecr.io/casinobench-base:2.0.1 casino-service --help   # exits 0
+```
+
+## Run it
+
+```bash
+bench eval run \
+  --tasks-dir benchmarks/casinobench/tasks/blackjack \
+  --environment-manifest benchmarks/casinobench/environment.toml \
+  --agents roster.yaml
+```
+
+`roster.yaml` lists the seats (each an ACP backend baked into the base image):
+
+```yaml
+agents:
+  - { name: claude, agent: claude-agent-acp, model: claude-haiku-4-5 }
+  - { name: codex,  agent: codex-acp,        model: gpt-5.5 }
+```
+
+Single-agent runs work too — swap `--agents roster.yaml` for `--agent
+claude-agent-acp --model claude-haiku-4-5`.
+
+## How a trial flows
+
+1. The Environment plane reads `environment.toml`, runs `env0acrdd8632.azurecr.io/casinobench-base:2.0.1`,
+   forwards `CASINOBENCH_GAME` / `CASINOBENCH_HANDS` / `BENCHFLOW_SEED`, starts
+   `casino-service` on `:9001`, and health-gates it on `/health`.
+2. The agent plays via the `casino` CLI (`lobby` → `join` → `observe`/`act` …).
+   The CLI is a thin HTTP client of `$CASINO_URL` (default `http://localhost:9001`).
+3. `tests/test.sh` (the shared `verifier/test.sh`) curls `/_admin/state` for the
+   final chips and writes the net-chips reward to `/logs/verifier/reward.txt`
+   (and `reward.json`). A failed read aborts with no reward file so a verifier
+   error is recorded rather than a fabricated `0`.
+
+## Adding another game
+
+Copy `tasks/blackjack/` to `tasks/<game>/`, set `CASINOBENCH_GAME` to a
+registered game id (`casino lobby` lists them — e.g. `european-roulette`,
+`punto-banco-baccarat`, `jacks-or-better-video-poker`, `infinite-deck-blackjack`),
+and keep the `tests/test.sh` symlink to the shared verifier.
diff --git a/benchmarks/casinobench/docker/casinobench-base.Dockerfile b/benchmarks/casinobench/docker/casinobench-base.Dockerfile
@@ -0,0 +1,48 @@
+# casinobench-base — the in-sandbox CasinoBench image.
+#
+# Extends the agent-seat image (examples/casino/agent_env, tagged
+# `casino-agent-seat:latest`) so ONE shared sandbox carries both halves:
+#   - the ACP agent backends (codex-acp / claude-agent-acp / deepagents) + the
+#     `casino` seven-tool CLI  — inherited from casino-agent-seat, untouched;
+#   - a WORKING `casino-service` (the env-0 HTTP mock-service on :9001).
+#
+# Why this image exists: casino-agent-seat installs env_0_casino with
+# `pip install --no-deps`, so `casino-service` is a broken stub — it crashes on
+#   from env_0_casino.app import create_app   (fastapi + the casinobench engine
+# were never installed). Here we bake the casinobench engine + its `service`
+# extra (fastapi / uvicorn / click / httpx) so `casino-service --help` exits 0
+# and the service actually serves. env_0_casino itself is already installed in
+# the base, so its console scripts (`casino`, `casino-service`) just light up.
+#
+# Build context = examples/casino/agent_env (same as casino-agent-seat). Assemble
+# it first (see benchmarks/casinobench/README.md): the deepagents shim, the
+# env_0_casino package as `casino-pkg/`, AND the casinobench engine checkout as
+# `casinobench-engine/`. Then:
+#   docker build -t casino-agent-seat:latest examples/casino/agent_env \
+#     -f examples/casino/agent_env/Dockerfile
+#   docker build -t casinobench-base:latest examples/casino/agent_env \
+#     -f benchmarks/casinobench/docker/casinobench-base.Dockerfile
+FROM casino-agent-seat:latest
+
+# The casinobench engine (pure-Python, deterministic kernel — zero runtime deps)
+# plus its `service` extra. `[service]` == fastapi + uvicorn[standard] + click +
+# httpx, exactly the deps env_0_casino's server.py imports. The engine is
+# proprietary (github.com/benchflow-ai/casinobench), so it is vendored into the
+# build context rather than pulled from a public index.
+COPY casinobench-engine /opt/casinobench-engine
+RUN pip install --no-cache-dir "/opt/casinobench-engine[service]" && \
+    python -c "import fastapi, uvicorn, casinobench.catalog" && \
+    casino-service --help >/dev/null && \
+    casino --help >/dev/null && \
+    echo "casino-service + casino cli ok"
+
+# Reinstall env_0_casino from the (patched) build-context copy so the `casino` CLI
+# carries the cwd-based seat fallback: in the shared-sandbox floor each seat runs
+# in /work/<seat>, so cwd IS the seat id even when the agent runtime doesn't
+# propagate CASINOBENCH_SEAT_ID to its casino subprocess.
+COPY casino-pkg /opt/casino-pkg-patched
+RUN pip install --no-cache-dir --no-deps --force-reinstall /opt/casino-pkg-patched && \
+    python -c "import inspect, env_0_casino.cli as c; assert 'seat identity' in inspect.getsource(c._seat), 'cwd-seat patch missing'" && \
+    echo "casino cli cwd-seat fallback ok"
+
+WORKDIR /app
diff --git a/benchmarks/casinobench/environment.toml b/benchmarks/casinobench/environment.toml
@@ -0,0 +1,46 @@
+# CasinoBench — Environment-plane manifest (in-sandbox, ClawsBench-style).
+#
+# The casino mock-service runs INSIDE the rollout's own sandbox (env-0 style),
+# NOT as a host subprocess. BenchFlow's ManifestEnvironment starts it before the
+# agent, health-gates it on /health, and the agent reaches it on localhost:9001
+# via the `casino` CLI. The ACP backends are baked into the same base image so
+# competing seats can run in the same shared sandbox. Run a task with:
+#   bench eval run --tasks-dir benchmarks/casinobench/tasks/blackjack \
+#     --environment-manifest benchmarks/casinobench/environment.toml \
+#     --agents roster.yaml
+
+# One ready-to-run image serves every game (selected by CASINOBENCH_GAME), so
+# this is `image` (the run target), not `base_image` (which only names what
+# per-task images are built FROM and is NOT runnable on its own —
+# resolve_manifest_image returns it as None).
+[environment]
+name = "casinobench"
+# Published, versioned base image. The casinobench publish-base-image CI builds it
+# on each release tag and pushes to BOTH the env0 ACR (this tag — the registry
+# Daytona is authed to pull) and ghcr (for org-authed consumers). --sandbox daytona
+# pulls this ACR tag via Image.base(); --sandbox docker uses the same tag built
+# locally (docker build -t <this tag> …), so there's no drift between "built
+# locally" and "what Daytona pulls".
+image = "ghcr.io/yiminnn/casinobench-base:2.0.6"
+owns_lifecycle = false                    # the framework starts the service below
+isolation = "per_task"
+
+[environment.task_selection]
+mechanism = "env_var"
+key = "CASINOBENCH_GAME"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Forward the per-task game/seed into the service process.
+# BENCHFLOW_SEED is deliberately NOT forwarded: a seat with the world seed can
+# compute the deterministic roulette spin from (seed, table_id, hand) and bet the
+# winning number. The multiplayer World now draws a secret random seed of its own.
+[environment.forward_env]
+keys = ["CASINOBENCH_GAME", "CASINOBENCH_HANDS"]
+
+[[environment.services]]
+name = "casino"
+command = "casino-service"     # uvicorn on $CASINO_PORT (9001)
+port = 9001
+health_path = "/health"
diff --git a/benchmarks/casinobench/tasks/blackjack/environment/Dockerfile b/benchmarks/casinobench/tasks/blackjack/environment/Dockerfile
@@ -0,0 +1,7 @@
+# Task-package contract requires environment/Dockerfile. CasinoBench is
+# manifest-backed: when run with --environment-manifest, the manifest's run
+# `image` (ghcr.io/yiminnn/casinobench-base:2.0.6) is the sandbox target and THIS file is
+# bypassed. It is a trivial passthrough so the per-game task carries no per-task
+# build (the game is chosen at runtime by CASINOBENCH_GAME, not baked here), and
+# so the non-manifest path still gets the working casino-service image.
+FROM ghcr.io/yiminnn/casinobench-base:2.0.6
diff --git a/benchmarks/casinobench/tasks/blackjack/task.md b/benchmarks/casinobench/tasks/blackjack/task.md
@@ -0,0 +1,53 @@
+---
+schema_version: '1.0'
+metadata:
+  author_name: benchflow
+  category: games
+  tags:
+  - casino
+  - casinobench
+  - house
+  - blackjack
+# The in-sandbox env-0 service is provisioned by the Environment plane from the
+# manifest below; the game is selected by CASINOBENCH_GAME (env_var task_selection).
+benchflow:
+  environment:
+    manifest: ../../environment.toml
+agent:
+  timeout_sec: 1800.0
+verifier:
+  type: test-script
+  timeout_sec: 120.0
+  service: main
+  # reward = net chips (final - starting): did-nothing -> 0, loss < 0, profit > 0.
+  reward_range: [-1000.0, 5000.0]
+environment:
+  network_mode: public
+  cpus: 1
+  memory_mb: 2048
+  env:
+    CASINOBENCH_GAME: 'six-deck-blackjack-s17'
+    CASINOBENCH_HANDS: '10'
+oracle:
+  env: {}
+---
+
+## prompt
+
+You have a seat at the CasinoBench casino. Play **blackjack** (the
+`six-deck-blackjack-s17` table: six decks, dealer stands on all 17, double-down
+allowed) and finish with as many chips as you can.
+
+The casino runs as a mock service inside this sandbox; reach it with the
+`casino` CLI (the seven tools):
+
+    casino lobby                            # open games, tables, your bankroll
+    casino rules six-deck-blackjack-s17     # the rules
+    casino join six-deck-blackjack-s17      # take your seat
+    # then repeat until observe reports "done": true
+    casino observe                          # request_id + your view + legal actions
+    casino act <request_id> '<one legal action, copied from legal_actions>'
+    casino cashier                          # your bankroll and realized PnL
+
+Choose actions only from the `legal_actions` returned by `casino observe`. Your
+score is your net chips at the end, so play to win and manage risk.
diff --git a/benchmarks/casinobench/tasks/blackjack/tests/test.sh b/benchmarks/casinobench/tasks/blackjack/tests/test.sh
@@ -0,0 +1 @@
+../../../verifier/test.sh
diff --git a/benchmarks/casinobench/verifier/test.sh b/benchmarks/casinobench/verifier/test.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# CasinoBench verifier (env-0 style). The casino mock-service runs INSIDE this
+# same sandbox on localhost:9001 (started by the BenchFlow Environment plane from
+# benchmarks/casinobench/environment.toml). The service is the chip authority —
+# the agent plays only through the `casino` CLI and cannot fake the count — so we
+# read the live standing (final chips) from /_admin/state and score it. No replay.
+#
+# Reward = NET chips (final - starting): did-nothing -> 0, a loss is negative, a
+# profit positive. A failed READ must NOT score a fabricated 0 (indistinguishable
+# from a real break-even): abort nonzero with no reward file so BenchFlow records
+# a verifier error instead.
+#
+# This file is the single shared verifier for every casinobench game task; each
+# task points its `tests/test.sh` here (symlink). It writes the bare scalar to
+# /logs/verifier/reward.txt and the structured map to reward.json.
+set -euo pipefail
+BASE="${CASINO_URL:-http://localhost:9001}"
+LOGS_DIR="${LOGS_DIR:-/logs/verifier}"
+mkdir -p "$LOGS_DIR"
+
+# Readiness was gated before the agent ran, but poll briefly in case the verifier
+# starts before a slow process settles.
+for _ in $(seq 1 30); do
+  curl -sf "$BASE/health" >/dev/null 2>&1 && break
+  sleep 1
+done
+
+curl -fsS "$BASE/_admin/state" -o /tmp/casino_state.json
+
+python3 - "$LOGS_DIR" <<'PY'
+import json, sys
+from pathlib import Path
+
+logs = Path(sys.argv[1])
+# Read the service's authoritative chip count. Missing/unreadable/no-count ->
+# cannot score: exit nonzero, write NO reward file (a fabricated 0 is
+# indistinguishable from a legitimate break-even).
+try:
+    state = json.loads(Path("/tmp/casino_state.json").read_text())
+    final = int(state["final_chips"])
+    start = int(state.get("starting_bankroll", 1000))
+except (OSError, ValueError, TypeError, KeyError) as exc:
+    sys.stderr.write(f"casino verifier: cannot read final chips: {exc}\n")
+    raise SystemExit(2)
+
+reward = float(final - start)
+out = {
+    "reward": reward,
+    "details": {
+        "game": state.get("game"),
+        "subject": state.get("subject"),
+        "final_chips": final,
+        "starting_bankroll": start,
+        "metric": "net",
+    },
+}
+(logs / "reward.json").write_text(json.dumps(out, indent=2))
+(logs / "reward.txt").write_text(f"{reward}\n")
+print(json.dumps(out))
+PY
+
+cat "$LOGS_DIR/reward.txt"