diff --git a/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py b/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py index 0e7cc015a..262f8631f 100755 --- a/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py +++ b/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py @@ -24,7 +24,6 @@ from pathlib import Path from typing import Any - TOKEN_KEYS = { "input_tokens", "output_tokens", @@ -109,9 +108,12 @@ def has_token_usage(value: Any) -> bool: for obj in iter_dicts(value): for key in TOKEN_KEYS: token_value = obj.get(key) - if isinstance(token_value, (int, float)) and not isinstance(token_value, bool): - if token_value > 0: - return True + if ( + isinstance(token_value, (int, float)) + and not isinstance(token_value, bool) + and token_value > 0 + ): + return True return False @@ -215,7 +217,9 @@ def validate_acp(rows: list[dict[str, Any]], path: Path) -> list[str]: return issues -def validate_llm(rows: list[dict[str, Any]], path: Path) -> tuple[list[str], dict[str, Any]]: +def validate_llm( + rows: list[dict[str, Any]], path: Path +) -> tuple[list[str], dict[str, Any]]: issues: list[str] = [] request_count = 0 response_count = 0 @@ -270,7 +274,9 @@ def load_run_config(root: Path) -> dict[str, Any] | None: return None -def validate_rollout(root: Path, *, allow_oracle_without_llm: bool = False) -> dict[str, Any]: +def validate_rollout( + root: Path, *, allow_oracle_without_llm: bool = False +) -> dict[str, Any]: result_path = root / "result.json" result, result_error = read_json(result_path) issues: list[str] = [] @@ -319,7 +325,12 @@ def validate_rollout(root: Path, *, allow_oracle_without_llm: bool = False) -> d error_text = " ".join( str(result.get(key) or "") - for key in ("error", "verifier_error", "error_category", "verifier_error_category") + for key in ( + "error", + "verifier_error", + "error_category", + "verifier_error_category", + ) ).lower() if any(marker in error_text for marker in INFRA_ERROR_MARKERS): issues.append("result carries infra/provider error markers") diff --git a/benchmarks/casinobench/Dockerfile b/benchmarks/casinobench/Dockerfile new file mode 100644 index 000000000..cb0e0f01f --- /dev/null +++ b/benchmarks/casinobench/Dockerfile @@ -0,0 +1,7 @@ +# Run image for the casinobench benchmark = the prebuilt base (agent backends + +# casino CLI + the in-sandbox casino-service). The manifest's `image` key +# (ghcr.io/yiminnn/casinobench-base:2.0.6) selects it as a prebuilt, so this Dockerfile is not +# rebuilt at run time — it exists so the sandbox's environment_dir validates and +# so `docker build -t ghcr.io/yiminnn/casinobench-base:2.0.6 benchmarks/casinobench` works too. +# Build the base first: see docker/casinobench-base.Dockerfile + README.md. +FROM ghcr.io/yiminnn/casinobench-base:2.0.6 diff --git a/benchmarks/casinobench/README.md b/benchmarks/casinobench/README.md new file mode 100644 index 000000000..0ee2ca638 --- /dev/null +++ b/benchmarks/casinobench/README.md @@ -0,0 +1,89 @@ +# CasinoBench — in-sandbox env-0 casino benchmark + +CasinoBench is a stateful, single-service BenchFlow benchmark. A casino +**mock-service** (FastAPI/uvicorn) runs *inside the rollout's own sandbox* — the +env-0 / ClawsBench pattern — and the agent plays through the `casino` seven-tool +CLI over `localhost:9001`. Each game is one task; the game is selected per task +by the `CASINOBENCH_GAME` env var. The service is the chip authority, so the +verifier just reads the live final standing and scores net chips. There is **no +host-subprocess wiring** — the service is declared in the manifest and started +by BenchFlow's Environment plane in-sandbox. + +``` +benchmarks/casinobench/ +├── environment.toml # the in-sandbox manifest (the whole framework seam) +├── docker/casinobench-base.Dockerfile # casinobench-base: working service + ACP backends + casino CLI +├── verifier/test.sh # shared scorer: curl /_admin/state -> net chips -> reward +└── tasks/ + └── blackjack/ + ├── task.md # CASINOBENCH_GAME=six-deck-blackjack-s17, manifest -> ../../environment.toml + └── tests/test.sh # symlink -> ../../../verifier/test.sh +``` + +## The base image + +`casino-service` (the env-0 mock-service) and the ACP seats live in **one** +image so competing agents can share a sandbox. The image is built in two steps: + +```bash +CB=~/casinobench # your casinobench engine checkout (proprietary, separate repo) + +# 1. assemble the agent-seat build context (gitignored — see agent_env/.gitignore) +cp src/benchflow/agents/deepagents_acp_shim.py examples/casino/agent_env/deepagents-acp-shim +cp -r "$CB/packages/environments/casino" examples/casino/agent_env/casino-pkg +cp -r "$CB" examples/casino/agent_env/casinobench-engine + +# 2. the seat image (ACP backends + casino CLI; service still a --no-deps stub) +docker build -t casino-agent-seat:latest examples/casino/agent_env \ + -f examples/casino/agent_env/Dockerfile + +# 3. casinobench-base: extends the seat image with the engine + service extra so +# `casino-service` actually runs. This is the manifest's run `image`. +docker build -t env0acrdd8632.azurecr.io/casinobench-base:2.0.1 examples/casino/agent_env \ + -f benchmarks/casinobench/docker/casinobench-base.Dockerfile +``` + +Verify the service is real (not the broken `--no-deps` stub): + +```bash +docker run --rm env0acrdd8632.azurecr.io/casinobench-base:2.0.1 casino-service --help # exits 0 +``` + +## Run it + +```bash +bench eval run \ + --tasks-dir benchmarks/casinobench/tasks/blackjack \ + --environment-manifest benchmarks/casinobench/environment.toml \ + --agents roster.yaml +``` + +`roster.yaml` lists the seats (each an ACP backend baked into the base image): + +```yaml +agents: + - { name: claude, agent: claude-agent-acp, model: claude-haiku-4-5 } + - { name: codex, agent: codex-acp, model: gpt-5.5 } +``` + +Single-agent runs work too — swap `--agents roster.yaml` for `--agent +claude-agent-acp --model claude-haiku-4-5`. + +## How a trial flows + +1. The Environment plane reads `environment.toml`, runs `env0acrdd8632.azurecr.io/casinobench-base:2.0.1`, + forwards `CASINOBENCH_GAME` / `CASINOBENCH_HANDS` / `BENCHFLOW_SEED`, starts + `casino-service` on `:9001`, and health-gates it on `/health`. +2. The agent plays via the `casino` CLI (`lobby` → `join` → `observe`/`act` …). + The CLI is a thin HTTP client of `$CASINO_URL` (default `http://localhost:9001`). +3. `tests/test.sh` (the shared `verifier/test.sh`) curls `/_admin/state` for the + final chips and writes the net-chips reward to `/logs/verifier/reward.txt` + (and `reward.json`). A failed read aborts with no reward file so a verifier + error is recorded rather than a fabricated `0`. + +## Adding another game + +Copy `tasks/blackjack/` to `tasks//`, set `CASINOBENCH_GAME` to a +registered game id (`casino lobby` lists them — e.g. `european-roulette`, +`punto-banco-baccarat`, `jacks-or-better-video-poker`, `infinite-deck-blackjack`), +and keep the `tests/test.sh` symlink to the shared verifier. diff --git a/benchmarks/casinobench/docker/casinobench-base.Dockerfile b/benchmarks/casinobench/docker/casinobench-base.Dockerfile new file mode 100644 index 000000000..b46c3dacf --- /dev/null +++ b/benchmarks/casinobench/docker/casinobench-base.Dockerfile @@ -0,0 +1,48 @@ +# casinobench-base — the in-sandbox CasinoBench image. +# +# Extends the agent-seat image (examples/casino/agent_env, tagged +# `casino-agent-seat:latest`) so ONE shared sandbox carries both halves: +# - the ACP agent backends (codex-acp / claude-agent-acp / deepagents) + the +# `casino` seven-tool CLI — inherited from casino-agent-seat, untouched; +# - a WORKING `casino-service` (the env-0 HTTP mock-service on :9001). +# +# Why this image exists: casino-agent-seat installs env_0_casino with +# `pip install --no-deps`, so `casino-service` is a broken stub — it crashes on +# from env_0_casino.app import create_app (fastapi + the casinobench engine +# were never installed). Here we bake the casinobench engine + its `service` +# extra (fastapi / uvicorn / click / httpx) so `casino-service --help` exits 0 +# and the service actually serves. env_0_casino itself is already installed in +# the base, so its console scripts (`casino`, `casino-service`) just light up. +# +# Build context = examples/casino/agent_env (same as casino-agent-seat). Assemble +# it first (see benchmarks/casinobench/README.md): the deepagents shim, the +# env_0_casino package as `casino-pkg/`, AND the casinobench engine checkout as +# `casinobench-engine/`. Then: +# docker build -t casino-agent-seat:latest examples/casino/agent_env \ +# -f examples/casino/agent_env/Dockerfile +# docker build -t casinobench-base:latest examples/casino/agent_env \ +# -f benchmarks/casinobench/docker/casinobench-base.Dockerfile +FROM casino-agent-seat:latest + +# The casinobench engine (pure-Python, deterministic kernel — zero runtime deps) +# plus its `service` extra. `[service]` == fastapi + uvicorn[standard] + click + +# httpx, exactly the deps env_0_casino's server.py imports. The engine is +# proprietary (github.com/benchflow-ai/casinobench), so it is vendored into the +# build context rather than pulled from a public index. +COPY casinobench-engine /opt/casinobench-engine +RUN pip install --no-cache-dir "/opt/casinobench-engine[service]" && \ + python -c "import fastapi, uvicorn, casinobench.catalog" && \ + casino-service --help >/dev/null && \ + casino --help >/dev/null && \ + echo "casino-service + casino cli ok" + +# Reinstall env_0_casino from the (patched) build-context copy so the `casino` CLI +# carries the cwd-based seat fallback: in the shared-sandbox floor each seat runs +# in /work/, so cwd IS the seat id even when the agent runtime doesn't +# propagate CASINOBENCH_SEAT_ID to its casino subprocess. +COPY casino-pkg /opt/casino-pkg-patched +RUN pip install --no-cache-dir --no-deps --force-reinstall /opt/casino-pkg-patched && \ + python -c "import inspect, env_0_casino.cli as c; assert 'seat identity' in inspect.getsource(c._seat), 'cwd-seat patch missing'" && \ + echo "casino cli cwd-seat fallback ok" + +WORKDIR /app diff --git a/benchmarks/casinobench/environment.toml b/benchmarks/casinobench/environment.toml new file mode 100644 index 000000000..3c35b986e --- /dev/null +++ b/benchmarks/casinobench/environment.toml @@ -0,0 +1,46 @@ +# CasinoBench — Environment-plane manifest (in-sandbox, ClawsBench-style). +# +# The casino mock-service runs INSIDE the rollout's own sandbox (env-0 style), +# NOT as a host subprocess. BenchFlow's ManifestEnvironment starts it before the +# agent, health-gates it on /health, and the agent reaches it on localhost:9001 +# via the `casino` CLI. The ACP backends are baked into the same base image so +# competing seats can run in the same shared sandbox. Run a task with: +# bench eval run --tasks-dir benchmarks/casinobench/tasks/blackjack \ +# --environment-manifest benchmarks/casinobench/environment.toml \ +# --agents roster.yaml + +# One ready-to-run image serves every game (selected by CASINOBENCH_GAME), so +# this is `image` (the run target), not `base_image` (which only names what +# per-task images are built FROM and is NOT runnable on its own — +# resolve_manifest_image returns it as None). +[environment] +name = "casinobench" +# Published, versioned base image. The casinobench publish-base-image CI builds it +# on each release tag and pushes to BOTH the env0 ACR (this tag — the registry +# Daytona is authed to pull) and ghcr (for org-authed consumers). --sandbox daytona +# pulls this ACR tag via Image.base(); --sandbox docker uses the same tag built +# locally (docker build -t …), so there's no drift between "built +# locally" and "what Daytona pulls". +image = "ghcr.io/yiminnn/casinobench-base:2.0.6" +owns_lifecycle = false # the framework starts the service below +isolation = "per_task" + +[environment.task_selection] +mechanism = "env_var" +key = "CASINOBENCH_GAME" + +[environment.readiness] +timeout_sec = 60 + +# Forward the per-task game/seed into the service process. +# BENCHFLOW_SEED is deliberately NOT forwarded: a seat with the world seed can +# compute the deterministic roulette spin from (seed, table_id, hand) and bet the +# winning number. The multiplayer World now draws a secret random seed of its own. +[environment.forward_env] +keys = ["CASINOBENCH_GAME", "CASINOBENCH_HANDS"] + +[[environment.services]] +name = "casino" +command = "casino-service" # uvicorn on $CASINO_PORT (9001) +port = 9001 +health_path = "/health" diff --git a/benchmarks/casinobench/tasks/blackjack/environment/Dockerfile b/benchmarks/casinobench/tasks/blackjack/environment/Dockerfile new file mode 100644 index 000000000..601739bb5 --- /dev/null +++ b/benchmarks/casinobench/tasks/blackjack/environment/Dockerfile @@ -0,0 +1,7 @@ +# Task-package contract requires environment/Dockerfile. CasinoBench is +# manifest-backed: when run with --environment-manifest, the manifest's run +# `image` (ghcr.io/yiminnn/casinobench-base:2.0.6) is the sandbox target and THIS file is +# bypassed. It is a trivial passthrough so the per-game task carries no per-task +# build (the game is chosen at runtime by CASINOBENCH_GAME, not baked here), and +# so the non-manifest path still gets the working casino-service image. +FROM ghcr.io/yiminnn/casinobench-base:2.0.6 diff --git a/benchmarks/casinobench/tasks/blackjack/task.md b/benchmarks/casinobench/tasks/blackjack/task.md new file mode 100644 index 000000000..239085f5b --- /dev/null +++ b/benchmarks/casinobench/tasks/blackjack/task.md @@ -0,0 +1,53 @@ +--- +schema_version: '1.0' +metadata: + author_name: benchflow + category: games + tags: + - casino + - casinobench + - house + - blackjack +# The in-sandbox env-0 service is provisioned by the Environment plane from the +# manifest below; the game is selected by CASINOBENCH_GAME (env_var task_selection). +benchflow: + environment: + manifest: ../../environment.toml +agent: + timeout_sec: 1800.0 +verifier: + type: test-script + timeout_sec: 120.0 + service: main + # reward = net chips (final - starting): did-nothing -> 0, loss < 0, profit > 0. + reward_range: [-1000.0, 5000.0] +environment: + network_mode: public + cpus: 1 + memory_mb: 2048 + env: + CASINOBENCH_GAME: 'six-deck-blackjack-s17' + CASINOBENCH_HANDS: '10' +oracle: + env: {} +--- + +## prompt + +You have a seat at the CasinoBench casino. Play **blackjack** (the +`six-deck-blackjack-s17` table: six decks, dealer stands on all 17, double-down +allowed) and finish with as many chips as you can. + +The casino runs as a mock service inside this sandbox; reach it with the +`casino` CLI (the seven tools): + + casino lobby # open games, tables, your bankroll + casino rules six-deck-blackjack-s17 # the rules + casino join six-deck-blackjack-s17 # take your seat + # then repeat until observe reports "done": true + casino observe # request_id + your view + legal actions + casino act '' + casino cashier # your bankroll and realized PnL + +Choose actions only from the `legal_actions` returned by `casino observe`. Your +score is your net chips at the end, so play to win and manage risk. diff --git a/benchmarks/casinobench/tasks/blackjack/tests/test.sh b/benchmarks/casinobench/tasks/blackjack/tests/test.sh new file mode 120000 index 000000000..f58fd431a --- /dev/null +++ b/benchmarks/casinobench/tasks/blackjack/tests/test.sh @@ -0,0 +1 @@ +../../../verifier/test.sh \ No newline at end of file diff --git a/benchmarks/casinobench/verifier/test.sh b/benchmarks/casinobench/verifier/test.sh new file mode 100755 index 000000000..79302a7c4 --- /dev/null +++ b/benchmarks/casinobench/verifier/test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# CasinoBench verifier (env-0 style). The casino mock-service runs INSIDE this +# same sandbox on localhost:9001 (started by the BenchFlow Environment plane from +# benchmarks/casinobench/environment.toml). The service is the chip authority — +# the agent plays only through the `casino` CLI and cannot fake the count — so we +# read the live standing (final chips) from /_admin/state and score it. No replay. +# +# Reward = NET chips (final - starting): did-nothing -> 0, a loss is negative, a +# profit positive. A failed READ must NOT score a fabricated 0 (indistinguishable +# from a real break-even): abort nonzero with no reward file so BenchFlow records +# a verifier error instead. +# +# This file is the single shared verifier for every casinobench game task; each +# task points its `tests/test.sh` here (symlink). It writes the bare scalar to +# /logs/verifier/reward.txt and the structured map to reward.json. +set -euo pipefail +BASE="${CASINO_URL:-http://localhost:9001}" +LOGS_DIR="${LOGS_DIR:-/logs/verifier}" +mkdir -p "$LOGS_DIR" + +# Readiness was gated before the agent ran, but poll briefly in case the verifier +# starts before a slow process settles. +for _ in $(seq 1 30); do + curl -sf "$BASE/health" >/dev/null 2>&1 && break + sleep 1 +done + +curl -fsS "$BASE/_admin/state" -o /tmp/casino_state.json + +python3 - "$LOGS_DIR" <<'PY' +import json, sys +from pathlib import Path + +logs = Path(sys.argv[1]) +# Read the service's authoritative chip count. Missing/unreadable/no-count -> +# cannot score: exit nonzero, write NO reward file (a fabricated 0 is +# indistinguishable from a legitimate break-even). +try: + state = json.loads(Path("/tmp/casino_state.json").read_text()) + final = int(state["final_chips"]) + start = int(state.get("starting_bankroll", 1000)) +except (OSError, ValueError, TypeError, KeyError) as exc: + sys.stderr.write(f"casino verifier: cannot read final chips: {exc}\n") + raise SystemExit(2) + +reward = float(final - start) +out = { + "reward": reward, + "details": { + "game": state.get("game"), + "subject": state.get("subject"), + "final_chips": final, + "starting_bankroll": start, + "metric": "net", + }, +} +(logs / "reward.json").write_text(json.dumps(out, indent=2)) +(logs / "reward.txt").write_text(f"{reward}\n") +print(json.dumps(out)) +PY + +cat "$LOGS_DIR/reward.txt" diff --git a/benchmarks/medical-assistant/README.md b/benchmarks/medical-assistant/README.md new file mode 100644 index 000000000..cf61fb9a2 --- /dev/null +++ b/benchmarks/medical-assistant/README.md @@ -0,0 +1,62 @@ +# Multi-Agent Medical Assistant — BenchFlow Adapter + +BenchFlow adapter that **hosts the [Multi-Agent-Medical-Assistant](https://github.com/souvikmajumder26/Multi-Agent-Medical-Assistant)** +agent pattern as a first-class BenchFlow benchmark: it runs in a Docker sandbox +with its LLM routed through BenchFlow's LiteLLM proxy, so per-rollout usage/cost +and the raw-LLM trajectory are captured, and the agent never sees the raw +provider key. + +## What is hosted + +The upstream is a LangGraph **supervisor → specialists** agent. This adapter +reproduces its control flow as a registered BenchFlow agent (`medical-assistant`, +an ACP shim at `src/benchflow/agents/medical_acp_shim.py`): + +``` +supervisor (router) + ├── retrieve_kb (RAG-style knowledge-base specialist) + └── web_search (fallback specialist) + → answer (emits a CONFIDENCE; low confidence → confidence-gated + handoff back to web_search) + → guardrail (output safety check) +``` + +Each graph node is streamed back as its own ACP `tool_call` step, so the +multi-agent structure (which specialist ran, in what order) is visible in the +captured trajectory. + +### Scope vs. the full upstream app + +The upstream app's heavy stack — **Azure OpenAI embeddings, Qdrant RAG, torch CV +imaging weights, Tavily web search** — is **out of scope** on the deepseek-only +proxy path. The RAG corpus is replaced by a small in-process knowledge base so +the full *router → specialists → confidence handoff → guardrail* control flow runs +end-to-end. The CV/imaging specialists and live web search are not included. See +`benchmark.yaml` (`hosting.not_included`). + +## Tasks & verification + +Three clinical drug-safety questions (`tasks/`), each verified **deterministically**: +the agent writes its answer to `/app/answer.md`, and `tests/test.sh` scores it +against the task's `ground_truth.json` keyword groups (OR within a group, AND +across groups). Reward = matched groups / total groups → `/logs/verifier/reward.txt`. + +| Task | Question | +|------|----------| +| `metformin-side-effects` | Main side effects of metformin | +| `aspirin-secondary-prevention` | When low-dose aspirin is used, and its main risk | +| `ibuprofen-renal-caution` | Cautions for ibuprofen use | + +## Run + +```bash +set -a; . ~/sb-run.env; set +a # provides DEEPSEEK_API_KEY for the proxy +python benchmarks/medical-assistant/run_medical_assistant.py + +# or via the CLI: +bench eval run --config benchmarks/medical-assistant/medical-assistant-deepseek.yaml +``` + +Every rollout runs in its own Docker sandbox (`environment: docker`). The agent is +installed into the sandbox (uv venv + `langgraph` + `langchain-openai`) and its +`deepseek-v4-pro` calls go through the LiteLLM proxy. diff --git a/benchmarks/medical-assistant/benchmark.yaml b/benchmarks/medical-assistant/benchmark.yaml new file mode 100644 index 000000000..140738479 --- /dev/null +++ b/benchmarks/medical-assistant/benchmark.yaml @@ -0,0 +1,57 @@ +# benchmark.yaml — standard benchmark descriptor for BenchFlow. +# +# Every benchmark in benchmarks// ships this file. It declares what the +# benchmark is, where it comes from, how tasks are verified, and how it runs. + +name: medical-assistant +description: "Multi-Agent Medical Assistant — a LangGraph supervisor→specialists agent (router → KB/web specialists → confidence-gated handoff → output guardrail) hosted by BenchFlow and answering clinical drug-safety questions" +url: https://github.com/souvikmajumder26/Multi-Agent-Medical-Assistant +repo: https://github.com/souvikmajumder26/Multi-Agent-Medical-Assistant +author: "souvikmajumder26 (pattern); BenchFlow (adapter)" +converted_by: BenchFlow + +# ── Tasks ──────────────────────────────────────────────────────────── +tasks: + count: 3 + categories: + - medical-qa + tags: [medical-assistant, langgraph, multi-agent, qa, drug-safety] + splits: + main: 3 + +# ── What is hosted ─────────────────────────────────────────────────── +# The agent is the Multi-Agent-Medical-Assistant PATTERN: a real langgraph +# StateGraph with a supervisor/router, a KB (RAG-style) specialist, a web-search +# specialist, a confidence-gated handoff, and an output guardrail. It runs as a +# BenchFlow ACP-shim agent (agent: medical-assistant) whose LLM is deepseek-v4-pro +# routed through the LiteLLM proxy. +hosting: + agent: medical-assistant # registered in src/benchflow/agents/registry.py + shim: src/benchflow/agents/medical_acp_shim.py + graph_nodes: [supervisor, retrieve_kb, web_search, answer, guardrail] + # The upstream's heavy stack (Azure embeddings, Qdrant, torch CV weights, + # Tavily) is OUT OF SCOPE on the deepseek-only proxy path — the RAG corpus is + # replaced by a small in-process knowledge base so the full router→specialists→ + # handoff→guardrail control flow runs end-to-end. The CV/imaging specialists and + # live web search are not included. + faithful_to_upstream: control-flow # router + specialists + confidence handoff + guardrail + not_included: [azure-embeddings, qdrant-rag, cv-imaging-weights, tavily-web-search] + +# ── Verification ───────────────────────────────────────────────────── +verification: + method: deterministic_script + reward: proportional # matched_keyword_groups / total_groups + aggregation: per-task + detail: > + Each task ships ground_truth.json with keyword groups (OR within a group, + AND across groups). The agent writes its answer to /app/answer.md; tests/test.sh + scores it and writes reward to /logs/verifier/reward.txt. No LLM judge. + anti_cheat: false + +# ── Environment ────────────────────────────────────────────────────── +environment: + base_image: "python:3.12-slim" + platform: linux/amd64 + allow_internet: true # agent install (PyPI/uv) + proxy reachability + cpus: 1 + memory_mb: 2048 diff --git a/benchmarks/medical-assistant/medical-assistant-deepseek.yaml b/benchmarks/medical-assistant/medical-assistant-deepseek.yaml new file mode 100644 index 000000000..41119915b --- /dev/null +++ b/benchmarks/medical-assistant/medical-assistant-deepseek.yaml @@ -0,0 +1,12 @@ +# Job config — run the medical-assistant benchmark with deepseek-v4-pro, +# every node's LLM routed through BenchFlow's LiteLLM proxy, in a Docker sandbox. +# +# bench eval run --config benchmarks/medical-assistant/medical-assistant-deepseek.yaml +# or: +# python benchmarks/medical-assistant/run_medical_assistant.py + +tasks_dir: benchmarks/medical-assistant/tasks +agent: medical-assistant +model: deepseek/deepseek-v4-pro +environment: docker # each rollout runs in its own Docker sandbox +concurrency: 2 diff --git a/benchmarks/medical-assistant/run_medical_assistant.py b/benchmarks/medical-assistant/run_medical_assistant.py new file mode 100644 index 000000000..0552c569e --- /dev/null +++ b/benchmarks/medical-assistant/run_medical_assistant.py @@ -0,0 +1,55 @@ +"""Run the medical-assistant benchmark via BenchFlow. + + set -a; . ~/sb-run.env; set +a # provides DEEPSEEK_API_KEY for the proxy + python benchmarks/medical-assistant/run_medical_assistant.py + + # or, equivalently, via the CLI: + bench eval run --config benchmarks/medical-assistant/medical-assistant-deepseek.yaml + +Every rollout runs in its own Docker sandbox (environment: docker). The +medical-assistant agent (a LangGraph supervisor->specialists graph) is installed +into the sandbox and its deepseek-v4-pro calls are routed through BenchFlow's +LiteLLM proxy, so usage/cost + the raw-LLM trajectory are captured per rollout. +""" + +import argparse +import asyncio +import logging +import sys +from pathlib import Path + +_SCRIPT_DIR = Path(__file__).resolve().parent +_REPO_ROOT = _SCRIPT_DIR.parents[1] +_SRC_ROOT = _REPO_ROOT / "src" +if str(_SRC_ROOT) not in sys.path: + sys.path.insert(0, str(_SRC_ROOT)) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run the medical-assistant benchmark.") + parser.add_argument( + "config", + nargs="?", + default=str(_SCRIPT_DIR / "medical-assistant-deepseek.yaml"), + help="BenchFlow evaluation YAML config.", + ) + return parser.parse_args() + + +async def main() -> None: + from benchflow.evaluation import Evaluation + + args = _parse_args() + job = Evaluation.from_yaml(args.config) + job._tasks_dir = _SCRIPT_DIR / "tasks" # absolute path → cwd-independent + # Dedicated jobs_dir so we never resume into another agent's results. + job._jobs_dir = _REPO_ROOT / "out" / "medical-bench" / "jobs" + job._job_name = "medical-assistant-deepseek" + result = await job.run() + print(f"\nScore: {result.passed}/{result.total} ({result.score:.1%})") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/environment/Dockerfile b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/environment/Dockerfile new file mode 100644 index 000000000..98916b10f --- /dev/null +++ b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/environment/Dockerfile @@ -0,0 +1,12 @@ +# medical-assistant Q&A task — minimal sandbox. +# +# The medical-assistant agent (a LangGraph supervisor -> specialists graph) is +# installed at runtime by BenchFlow's registry install_cmd (uv venv + langgraph + +# langchain-openai), so this image only needs python3 (for the verifier), an +# `agent` user (the default sandbox_user), and writable /app + /logs dirs. The +# agent writes its answer to /app/answer.md, which the verifier scores. +FROM python:3.12-slim +RUN useradd -m -u 1000 agent || true +RUN mkdir -p /app /logs/verifier /logs/agent /logs/artifacts \ + && chown -R agent:agent /app /logs +WORKDIR /app diff --git a/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/instruction.md b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/instruction.md new file mode 100644 index 000000000..584c58499 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/instruction.md @@ -0,0 +1,3 @@ +You are a clinical decision-support assistant. Answer the following question accurately and concisely for a clinician, noting the key risks. + +Question: When is low-dose aspirin used, and what is its main risk? diff --git a/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/task.toml b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/task.toml new file mode 100644 index 000000000..9c9006d68 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "benchflow" +difficulty = "easy" +category = "medical-qa" +tags = ["medical-assistant", "langgraph", "multi-agent", "qa"] + +[agent] +timeout_sec = 600 + +[verifier] +timeout_sec = 120 + +[environment] +cpus = 1 +memory_mb = 2048 +allow_internet = true diff --git a/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/tests/ground_truth.json b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/tests/ground_truth.json new file mode 100644 index 000000000..4435344e3 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/tests/ground_truth.json @@ -0,0 +1,15 @@ +{ + "question": "When is low-dose aspirin used, and what is its main risk?", + "keyword_groups": [ + [ + "cardiovascular", + "secondary prevention", + "antiplatelet", + "heart", + "stroke" + ], + [ + "bleed" + ] + ] +} diff --git a/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/tests/test.sh b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/tests/test.sh new file mode 100755 index 000000000..24a577b56 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/aspirin-secondary-prevention/tests/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Deterministic verifier: score /app/answer.md against this task's keyword groups +# (OR within a group, AND across groups). reward = matched_groups / total_groups. +set -uo pipefail +LOGS_DIR="${LOGS_DIR:-/logs/verifier}" +mkdir -p "$LOGS_DIR" +ANSWER="${ANSWER_FILE:-/app/answer.md}" +GT="$(dirname "$0")/ground_truth.json" +python3 - "$ANSWER" "$GT" "$LOGS_DIR/reward.txt" <<'PYEOF' +import json, os, sys +answer_path, gt_path, reward_path = sys.argv[1], sys.argv[2], sys.argv[3] +text = "" +if os.path.exists(answer_path): + text = open(answer_path, encoding="utf-8", errors="ignore").read().lower() +groups = json.load(open(gt_path))["keyword_groups"] +hit = sum(1 for grp in groups if any(kw.lower() in text for kw in grp)) +reward = hit / len(groups) if groups else 0.0 +open(reward_path, "w").write(f"{reward:.4f}\n") +print(f"matched {hit}/{len(groups)} keyword groups -> reward {reward:.4f}") +if not text: + print("WARN: /app/answer.md missing or empty") +PYEOF +cat "$LOGS_DIR/reward.txt" diff --git a/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/environment/Dockerfile b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/environment/Dockerfile new file mode 100644 index 000000000..98916b10f --- /dev/null +++ b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/environment/Dockerfile @@ -0,0 +1,12 @@ +# medical-assistant Q&A task — minimal sandbox. +# +# The medical-assistant agent (a LangGraph supervisor -> specialists graph) is +# installed at runtime by BenchFlow's registry install_cmd (uv venv + langgraph + +# langchain-openai), so this image only needs python3 (for the verifier), an +# `agent` user (the default sandbox_user), and writable /app + /logs dirs. The +# agent writes its answer to /app/answer.md, which the verifier scores. +FROM python:3.12-slim +RUN useradd -m -u 1000 agent || true +RUN mkdir -p /app /logs/verifier /logs/agent /logs/artifacts \ + && chown -R agent:agent /app /logs +WORKDIR /app diff --git a/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/instruction.md b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/instruction.md new file mode 100644 index 000000000..92d8f3ebb --- /dev/null +++ b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/instruction.md @@ -0,0 +1,3 @@ +You are a clinical decision-support assistant. Answer the following question accurately and concisely for a clinician, noting the key risks. + +Question: What cautions apply to ibuprofen use? diff --git a/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/task.toml b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/task.toml new file mode 100644 index 000000000..9c9006d68 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "benchflow" +difficulty = "easy" +category = "medical-qa" +tags = ["medical-assistant", "langgraph", "multi-agent", "qa"] + +[agent] +timeout_sec = 600 + +[verifier] +timeout_sec = 120 + +[environment] +cpus = 1 +memory_mb = 2048 +allow_internet = true diff --git a/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/tests/ground_truth.json b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/tests/ground_truth.json new file mode 100644 index 000000000..eb8b1f802 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/tests/ground_truth.json @@ -0,0 +1,7 @@ +{ + "question": "What cautions apply to ibuprofen use?", + "keyword_groups": [ + ["renal", "kidney", "nephro"], + ["anticoagulant", "bleed"] + ] +} diff --git a/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/tests/test.sh b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/tests/test.sh new file mode 100755 index 000000000..24a577b56 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/ibuprofen-renal-caution/tests/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Deterministic verifier: score /app/answer.md against this task's keyword groups +# (OR within a group, AND across groups). reward = matched_groups / total_groups. +set -uo pipefail +LOGS_DIR="${LOGS_DIR:-/logs/verifier}" +mkdir -p "$LOGS_DIR" +ANSWER="${ANSWER_FILE:-/app/answer.md}" +GT="$(dirname "$0")/ground_truth.json" +python3 - "$ANSWER" "$GT" "$LOGS_DIR/reward.txt" <<'PYEOF' +import json, os, sys +answer_path, gt_path, reward_path = sys.argv[1], sys.argv[2], sys.argv[3] +text = "" +if os.path.exists(answer_path): + text = open(answer_path, encoding="utf-8", errors="ignore").read().lower() +groups = json.load(open(gt_path))["keyword_groups"] +hit = sum(1 for grp in groups if any(kw.lower() in text for kw in grp)) +reward = hit / len(groups) if groups else 0.0 +open(reward_path, "w").write(f"{reward:.4f}\n") +print(f"matched {hit}/{len(groups)} keyword groups -> reward {reward:.4f}") +if not text: + print("WARN: /app/answer.md missing or empty") +PYEOF +cat "$LOGS_DIR/reward.txt" diff --git a/benchmarks/medical-assistant/tasks/metformin-side-effects/environment/Dockerfile b/benchmarks/medical-assistant/tasks/metformin-side-effects/environment/Dockerfile new file mode 100644 index 000000000..98916b10f --- /dev/null +++ b/benchmarks/medical-assistant/tasks/metformin-side-effects/environment/Dockerfile @@ -0,0 +1,12 @@ +# medical-assistant Q&A task — minimal sandbox. +# +# The medical-assistant agent (a LangGraph supervisor -> specialists graph) is +# installed at runtime by BenchFlow's registry install_cmd (uv venv + langgraph + +# langchain-openai), so this image only needs python3 (for the verifier), an +# `agent` user (the default sandbox_user), and writable /app + /logs dirs. The +# agent writes its answer to /app/answer.md, which the verifier scores. +FROM python:3.12-slim +RUN useradd -m -u 1000 agent || true +RUN mkdir -p /app /logs/verifier /logs/agent /logs/artifacts \ + && chown -R agent:agent /app /logs +WORKDIR /app diff --git a/benchmarks/medical-assistant/tasks/metformin-side-effects/instruction.md b/benchmarks/medical-assistant/tasks/metformin-side-effects/instruction.md new file mode 100644 index 000000000..e5a8ec4f6 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/metformin-side-effects/instruction.md @@ -0,0 +1,3 @@ +You are a clinical decision-support assistant. Answer the following question accurately and concisely for a clinician, noting the key risks. + +Question: What are the main side effects of metformin? diff --git a/benchmarks/medical-assistant/tasks/metformin-side-effects/task.toml b/benchmarks/medical-assistant/tasks/metformin-side-effects/task.toml new file mode 100644 index 000000000..9c9006d68 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/metformin-side-effects/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "benchflow" +difficulty = "easy" +category = "medical-qa" +tags = ["medical-assistant", "langgraph", "multi-agent", "qa"] + +[agent] +timeout_sec = 600 + +[verifier] +timeout_sec = 120 + +[environment] +cpus = 1 +memory_mb = 2048 +allow_internet = true diff --git a/benchmarks/medical-assistant/tasks/metformin-side-effects/tests/ground_truth.json b/benchmarks/medical-assistant/tasks/metformin-side-effects/tests/ground_truth.json new file mode 100644 index 000000000..dcf0ca341 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/metformin-side-effects/tests/ground_truth.json @@ -0,0 +1,16 @@ +{ + "question": "What are the main side effects of metformin?", + "keyword_groups": [ + [ + "gastrointestinal", + "gi upset", + "gi ", + "nausea", + "diarrhea", + "diarrhoea" + ], + [ + "lactic acidosis" + ] + ] +} diff --git a/benchmarks/medical-assistant/tasks/metformin-side-effects/tests/test.sh b/benchmarks/medical-assistant/tasks/metformin-side-effects/tests/test.sh new file mode 100755 index 000000000..24a577b56 --- /dev/null +++ b/benchmarks/medical-assistant/tasks/metformin-side-effects/tests/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Deterministic verifier: score /app/answer.md against this task's keyword groups +# (OR within a group, AND across groups). reward = matched_groups / total_groups. +set -uo pipefail +LOGS_DIR="${LOGS_DIR:-/logs/verifier}" +mkdir -p "$LOGS_DIR" +ANSWER="${ANSWER_FILE:-/app/answer.md}" +GT="$(dirname "$0")/ground_truth.json" +python3 - "$ANSWER" "$GT" "$LOGS_DIR/reward.txt" <<'PYEOF' +import json, os, sys +answer_path, gt_path, reward_path = sys.argv[1], sys.argv[2], sys.argv[3] +text = "" +if os.path.exists(answer_path): + text = open(answer_path, encoding="utf-8", errors="ignore").read().lower() +groups = json.load(open(gt_path))["keyword_groups"] +hit = sum(1 for grp in groups if any(kw.lower() in text for kw in grp)) +reward = hit / len(groups) if groups else 0.0 +open(reward_path, "w").write(f"{reward:.4f}\n") +print(f"matched {hit}/{len(groups)} keyword groups -> reward {reward:.4f}") +if not text: + print("WARN: /app/answer.md missing or empty") +PYEOF +cat "$LOGS_DIR/reward.txt" diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a112b52ec..97ea85ea9 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -224,6 +224,15 @@ bench eval run -d skillsbench@1.1 --agent gemini --model gemini-3.1-flash-lite-p | `--source-env-sampling-arg` | — | Verifiers sampling argument as `KEY=VALUE`; repeatable (for example `reasoning_effort=minimal`) | | `--agent` | `claude-agent-acp` | Agent name | | `--model` | Agent default | Model ID | +| `--agents` | — | Roster file for a concurrent multi-agent floor; mutually exclusive with `--agent` | +| `--drive` | `auto-loop` | Multi-agent floor drive: `auto-loop` or `service-rounds` | +| `--deadline` | `1200` | Multi-agent floor soft deadline in seconds; `0` disables it up to the 24h safety cap | +| `--game` | — | Multi-agent floor task-selection value; defaults from `--tasks-dir` when present | +| `--url-env` | — | Multi-agent floor env var that receives the in-sandbox service URL | +| `--seat-env` | — | Multi-agent floor env var that receives each seat id | +| `--standings-path` | — | Multi-agent floor service path for final `{seat: score}` standings | +| `--events-path` | — | Multi-agent floor service path for event log snapshot output | +| `--service-env` | — | Multi-agent floor service environment variable as `KEY=VALUE`; repeatable | | `--reasoning-effort` | — | Agent reasoning/thinking effort when the agent exposes one (e.g. `max`) | | `--sandbox` | `docker` | Sandbox: docker, daytona, or modal | | `--usage-tracking` | `auto` | Token usage telemetry policy: `auto`, `required`, or `off` | diff --git a/docs/reference/multi-agent-trajectory.md b/docs/reference/multi-agent-trajectory.md new file mode 100644 index 000000000..b69d9cf60 --- /dev/null +++ b/docs/reference/multi-agent-trajectory.md @@ -0,0 +1,268 @@ +# Multi-agent trajectory tracking through the LiteLLM proxy + +Status: **design / target** (runtime-deferred, see `task-standard.md` G7). This +document records the industry comparison that motivates the design and the +contract BenchFlow should adopt so that a multi-agent workflow hosted through the +provider proxy produces **one structured trace tree** — agent identity *and* +agent-to-agent relationships — instead of one undifferentiated +`llm_trajectory.jsonl`. + +## Problem + +BenchFlow routes an agent's raw LLM calls through a loopback LiteLLM proxy. The +callback (`src/benchflow/providers/litellm_logging.py`) records `model + messages` +per call and keeps only `model_group` from request `metadata` +(`litellm_logging.py:130,150`) — every other tag is dropped. So when a +multi-agent workflow (a LangGraph supervisor→specialists graph, an Omnigent +session with sub-agents, concurrent arena seats) shares one proxy, the result is a +**flat, undifferentiated** log: you can see *that* N calls happened, not *which +agent* made each, nor how the agents relate. + +The earlier medical-bench workaround — **one proxy per agent → one file per agent** +(`out/medical-hosted//trajectory/llm_trajectory.jsonl`) — preserves +identity *by filename* but **destroys the relationship structure** (which agent +spawned/handed-off-to which) that every observability tool treats as first-class. +It also doesn't scale (one subprocess proxy per agent) and can't represent a +dynamic spawn tree. It was a demonstration, not the design. + +## What the industry does (surveyed, verified) + +A deep multi-source survey (OpenTelemetry GenAI semconv, LangSmith/LangGraph, +Langfuse, OpenLLMetry/Traceloop, Langtrace, and framework SDKs) returns one +**unanimous** structural answer: + +| System | Data model | Per-agent identity carrier | Relationship mechanism | Session/seat grouping | Capture | +|---|---|---|---|---|---| +| **OTel GenAI semconv** | one trace = tree of **typed** spans | `gen_ai.agent.id` / `gen_ai.agent.name` + span name `invoke_agent {name}` + `gen_ai.operation.name` | parent/child span **nesting** (`execute_tool` nests under `invoke_agent`; agents nest under `invoke_workflow`) | `gen_ai.conversation.id` (Conditionally Required; **never synthesize** a fallback) | in-process instrumentation; cross-process via `traceparent` | +| **LangSmith / LangGraph** | run tree | `run_type` + `name` (no `run_type=agent`; an agent is a `chain` run) | `parent_run_id` + `dotted_order` + `child_run_ids` | `thread_id`; `trace_id` = root run id | LangChain callbacks (`run_id`/`parent_run_id`) | +| **Langfuse** | trace + nested observations | typed observations (generation/span/event) + `name` | `parent_observation_id` | `session_id` | SDK / decorators / OTel | +| **OpenLLMetry / Traceloop** | OTel spans | `traceloop.span.kind ∈ {workflow,task,agent,tool}` + `entity.name` + `workflow.name` | OTel context nesting via `run_id`/`parent_run_id` | — | SDK monkey-patch + LangChain `BaseCallbackHandler`; injects `extra_headers` to **propagate** context | +| **Langtrace** | OTel spans ("adhere to OTEL") | OTel `gen_ai.*` attributes | OTel parent nesting | — | OTel instrumentation | +| **OpenAI Agents SDK** | trace + spans | agent span; **handoffs** are edges | root span via `Runner` | `group_id` | SDK tracing | +| **LiteLLM proxy** (BenchFlow today) | flat callback log | `metadata` body field + headers — but callback keeps only `model_group` | none natively (must add a parent pointer in metadata) | `metadata` | proxy callback (`StandardLoggingPayload`) | + +Three load-bearing facts, each confirmed 3-0 across independent verifiers: + +1. **A multi-agent run is ONE trace = a tree of typed spans joined by explicit + parent pointers — never a flat event log.** Universal across OTel, LangSmith, + Langfuse, Langtrace, OpenLLMetry. +2. **Per-agent identity is carried *on each call* (span attributes + span name), + so a single shared stream is differentiated per-call, not per-stream.** Agent / + tool / LLM-call / orchestration are first-class, **distinct span types** + (`gen_ai.operation.name`: `chat`, `invoke_agent`, `execute_tool`, + `invoke_workflow`, …; LangSmith `run_type`: chain/llm/tool/retriever/…). +3. **Relationships are captured at call time via parent/child nesting, not inferred + post-hoc** — `execute_tool` under `invoke_agent`; LangChain `run_id` → + `parent_run_id`. Session/seat grouping uses a single real conversation id; the + OTel spec **forbids synthetic fallbacks** (no UUID / trace-id / content hash). + +**Critical caveat for BenchFlow:** none of these tools solve the +"shared collector loses the tag" problem *with an HTTP proxy*. They instrument +**in-process** (framework callbacks, SDK monkey-patching) where the active +agent/parent context is already known, and propagate it across process boundaries +via OTel context (`traceparent`). BenchFlow's proxy sits *outside* the agent +process, so the agent context must be **explicitly attached to each request** — the +proxy cannot recover it otherwise. (Also: OTel GenAI agent conventions are +*experimental* / SHOULD-level; only `gen_ai.operation.name` and +`gen_ai.provider.name` are strictly Required. `gen_ai.agent.*` officially live on +agent-lifecycle spans, not every raw chat span — attaching them per raw call is a +deliberate, reasonable BenchFlow extension.) + +## The design: one pooled proxy + per-call metadata → one trace tree + +Adopt the dominant industry shape, adapted for an out-of-process proxy: + +**1. Each LLM request carries agent context in `metadata`** (the LiteLLM request +body field, which the proxy forwards to the logging callback — see *Verification* +below). A minimal, OTel-aligned schema: + +```jsonc +"metadata": { + "bf.agent_id": "answer", // stable id of the calling agent/node (~ gen_ai.agent.id) + "bf.agent_name": "answer", // human label (~ gen_ai.agent.name) + "bf.span_kind": "chat", // chat | invoke_agent | execute_tool | invoke_workflow (~ gen_ai.operation.name) + "bf.parent_agent_id": "supervisor", // parent pointer → reconstructs the tree (~ parent_run_id) + "bf.session_id": "medical-run-1", // real conversation/seat id, NEVER synthesized (~ gen_ai.conversation.id) + "bf.run_id": "answer#2" // this call's id, so children can point at it +} +``` + +**2. The callback records a span row** instead of today's bare model+messages line: +`litellm_logging.py:_base_record` stops discarding `metadata` and persists +`bf.agent_id` / `bf.agent_name` / `bf.span_kind` / `bf.parent_agent_id` / +`bf.session_id` / `bf.run_id` alongside the existing `model_group`. Every proxied +call becomes one typed span with a parent pointer. + +**3. The trajectory becomes a tree.** `trajectory_from_litellm_callback_log` +reconstructs parent/child structure from `bf.parent_agent_id` / `bf.run_id` +exactly as LangSmith does from `parent_run_id` / `dotted_order`. One +`llm_trajectory.jsonl` then holds the whole multi-agent run, splittable per agent +**and** navigable as a tree — no per-agent files, no lost relationships. + +Why this over "one proxy per agent": separate files preserve identity by filename +only and throw away the parent/child + handoff structure that is the *point* of a +multi-agent trace. The pooled-proxy + per-call-tag model is what every surveyed +tool does. + +### The uniform adapter + +BenchFlow cannot adopt a single framework as "the" multi-agent host (Omnigent, +the closest candidate, explicitly does **not** host LangGraph/CrewAI/AutoGen — see +below). The uniform layer is instead a **thin BenchFlow-side contract**: each +framework's native per-agent + parent context is mapped onto the one `metadata` +schema by a small per-framework shim, before the call leaves the agent process. + +| Framework | Native per-agent + parent context the shim maps from | +|---|---| +| LangChain / LangGraph | `BaseCallbackHandler` `run_id` / `parent_run_id` / node name → `bf.*` (already exposed) | +| OpenAI Agents SDK | `Runner` / root span + agent name + handoff edges | +| Omnigent | its internal conversation tree (`parent_conversation_id`/`root_conversation_id`/`agent_id`) → `bf.*` | +| custom (e.g. our medical slice) | the node passes its own name as `bf.agent_id` when it builds the `ChatOpenAI` call | +| AutoGen / CrewAI / Swarm | **under-evidenced** — per-agent + handoff exposure to an interceptor not yet confirmed; needs a per-framework spike before claiming support | + +The shim's only job is the mapping. Identity and relationships already exist +inside every framework; BenchFlow just needs them attached to the request. + +### Unified `bf.*` vocabulary (consolidated with the adapter proposal) + +The adapter proposal (PR #847) independently specified a richer per-call +attribution set. To avoid forking two schemas, that vocabulary is folded into the +one `bf.*` namespace. The callback captures **any** `bf.*` key generically (it +strips the `bf.` prefix from every metadata key), so the extended dimensions flow +through with **no code change** (verified in `tests/test_litellm_logging.py`). + +| `bf.*` key | status | meaning | OTel / #847 analogue | +|---|---|---|---| +| `agent_id` / `agent_name` | implemented | the calling agent/node | `gen_ai.agent.id` / `.name` | +| `parent_agent_id` | implemented | parent pointer (tree edge) | `parent_run_id` / `framework_parent_id` | +| `run_id` | implemented | this call's id | `llm.call_id` | +| `session_id` | implemented | conversation/seat/rollout id (never synthesized) | `gen_ai.conversation.id` / `rollout_id` | +| `span_kind` | implemented | `chat`/`invoke_agent`/`execute_tool`/`invoke_workflow` — the `relation` hook | `gen_ai.operation.name` | +| `role` | extended (#847) | declared role (planner/implementer/reviewer) | `role` | +| `scene` / `turn_index` | extended (#847) | scene + turn within a scene | `scene` / `turn` | +| `team_id` | extended (#847) | team / sub-graph grouping | `team_id` | +| `framework` / `framework_node_id` | extended (#847) | framework name + native node id | `framework` / `framework_node_id` | +| `trace_id` | extended (#847) | cross-process / cross-framework trace correlation | `trace_id` | + +### Uniform adapter protocol (from #847) + +A hosted framework is wrapped by a thin adapter — a *normalizer*, not a +reimplementation — with four operations: + +- `detect(task_dir, spec)` — can this adapter host the workflow? +- `prepare(ctx)` — install deps, write the LiteLLM env, compile the launch command. +- `run(ctx)` — run the external workflow **inside the BenchFlow sandbox**. +- `collect(ctx)` — gather native logs, normalize to BenchFlow events + graph edges, + return an `AdapterTraceBundle` (framework-native raw logs under + `trajectory/raw//` + coverage diagnostics: attribution quality, missing + metadata, unsupported relations, redaction state). + +Declared under a `benchflow.multi_agent` block (target / `benchflow:`-namespaced): +`capture_raw_llm: required` **fails closed** when zero LLM calls are captured; +`relationship_graph: required` persists `trajectory/agent_graph.json` (agent / team / +sub-graph nodes + edges carrying a `relation` ∈ {delegates, supervises, reviews, +handoff, parallel_child, fan_in, …}). When a framework **cannot** inject per-call +metadata, fall back to **one LiteLLM virtual key / route alias per role** as the +attribution channel. Report multi-agent lift only against a **matched single-agent +baseline** (same task set, tools, answer contract, usage + logging). + +The implemented `build_agent_tree()` is the minimal in-memory realization of this +graph (one parent pointer); `agent_graph.json` is its richer persisted form, with +`bf.span_kind` as the existing per-edge `relation` hook. + +## Does Omnigent satisfy this? (assessed separately, verified) + +Short answer: **not as wired today.** Omnigent the *framework* is a strong +multi-agent foundation, but the integration does not carry that across the +BenchFlow boundary. Per-requirement: + +- **Multi-agent support** — framework yes (recursive `AgentSpec.sub_agents`, + `sys_session_send` spawns children, parent-linked conversation tree); the BF + adapter drives a **single one-shot harness** (`omnigent run -p`) and only `pi` + is wired end-to-end. *Partial.* +- **Per-agent attribution through the proxy** — **fails.** Omnigent's HTTP adapter + sends only `{model, messages, tools?, stream?, **extra}` with `Content-Type` + + `Authorization` headers — **no agent id on the wire** — and BenchFlow's callback + would drop it anyway. `agent_id` lives only in Omnigent's DB. +- **Relationships** — Omnigent models a real parent-linked tree + shared OTel trace + internally (`db_models.py`), but it is **not populated by the one-shot CLI path** + (each `omnigent run -p` is a fresh, unlinked conversation, no `traceparent`) and + is never exported to BenchFlow. *Partial / not wired.* +- **Proxy routing of ALL traffic** — only `pi` (OpenAI-wire) is proven proxied; + native CLI sub-harnesses (Claude Code/Codex) use a separate + `HARNESS_*_GATEWAY_BASE_URL` / `claude_gateway_shim` channel the adapter never + sets, and sub-agent spawns pass no gateway override → **bypass risk**. +- **Uniform host for many frameworks** — hosts coding-agent CLIs + `claude_sdk` + + `agents_sdk`, but LangGraph/CrewAI/AutoGen/LangChain/DeepAgents are explicitly + "Not natively supported." *Not a universal host.* + +Two paths to make Omnigent conformant (complementary): **(A)** inject `agent_id` +into proxy metadata + have the callback record it (the contract above) — best for +per-call cost/attribution; **(B)** run Omnigent in server/session mode and export +its native conversation tree, joined to proxy usage by `agent_id` — best for the +relationship graph. `agent_id` from (A) is the clean join key for (B). + +## Implementation phases + +- **P0 ✅ DONE — callback records attribution.** `litellm_logging.py:_base_record` + now extracts every `bf.*` key from request `metadata` into `request.body['bf']` + (prefix stripped) instead of keeping only `model_group`. The trajectory importer + carries it through unchanged. Unit-tested in `tests/test_litellm_logging.py`. +- **P1 ✅ DONE — medical bench as proof, one proxy.** `examples/medical`'s `_llm` + tags each node's call with `bf.*` via `extra_body` (the verified client + mechanism); `trajectories/build_agent_tree()` reconstructs the unmixed agent + tree. Verified end-to-end on **three backends**, all producing the identical + tree `supervisor(1) → answer(2) → guardrail(1)`, `unmixed_ok: true`: + `run_agent_tree.py` (local, docker-proxy) and `run_in_sandbox.py` (agent running + **inside** a docker container and a remote daytona sandbox, through the proxy). + Unit-tested in `tests/trajectories/test_agent_tree.py`. +- **P2 — tree-shaped canonical trajectory.** Add a multi-agent trajectory kind that + reconstructs + emits the tree (parent pointers / dotted-order); fix + `n_tool_calls` to count real tool spans. +- **P3 — OTLP export option.** Optionally emit OTLP spans so existing backends + (Langfuse, Arize Phoenix/OpenInference, OpenLLMetry collectors) can render the + tree, instead of (or alongside) the bespoke JSONL. +- **Omnigent track.** Wire `HARNESS_*_GATEWAY_*` at the proxy to close the bypass; + drive server/session mode; export the conversation tree. +- **Concurrent-seats track ✅ DONE — native multi-agent floor.** `bench eval run + --agents agents.yaml` (`src/benchflow/arena/`) runs N agents on ONE shared task + + service CONCURRENTLY in ONE shared sandbox, each in `/work/`, each with its + own ACP trajectory and — for proxy seats — a separate raw `llm_trajectory.jsonl` + from that seat's own proxy (`session_id=floor-`). This answers the "concurrent + seats" open question below: **distinct per-seat files**, separated at the proxy by a + per-seat `bf.session_id`, rather than sibling sub-trees under one trace (which remains + the model for a single agent's supervisor→specialist calls). Agents resolve from all + three benchflow-ai/agents paths (raw ACP / ai-sdk / omnigent) or a BYOA manifest, and + carry a per-agent instruction file (`CLAUDE.md`/`GEMINI.md`/`AGENTS.md`). See + `examples/arena/README.md`. Unit-tested in `tests/test_{agents_manifest,agent_driver, + agent_instructions,concurrent_floor,arena_cli}.py`. + +## Verification (the research's #1 open question) + +The whole design rests on: *does a request-body `metadata` field survive to the +LiteLLM proxy logging callback?* This was the survey's top open question. It is +**confirmed empirically** against BenchFlow's own loopback proxy: a single chat +completion sent with `metadata: {bf.agent_id, bf.agent_name, bf.span_kind, +bf.parent_agent_id, bf.session_id, bf.run_id}` had **all six fields arrive intact** +at the callback (verdict PASS). The callback already *reads* +`litellm_params.metadata` (`litellm_logging.py:130`) — LiteLLM merges the request +body's `metadata` into it — so the only missing piece is *recording* the agent +fields instead of keeping just `model_group` (P0). + +## Open questions + +- AutoGen / CrewAI / OpenAI Swarm: how each exposes per-agent identity + explicit + handoff **edges** to an interceptor (needs a per-framework spike). +- Bespoke JSONL vs native OTLP export (P3) — what is lost/gained by staying custom. +- Concurrent seats vs supervisor→specialist: sibling sub-trees under one trace, OTel + span links, or distinct `bf.session_id` per seat? + +## Sources + +OpenTelemetry GenAI spans + agent spans +(`opentelemetry.io/docs/specs/semconv/gen-ai/`), LangSmith run-data-format +(`docs.langchain.com/langsmith/run-data-format`), Langfuse data model +(`langfuse.com/docs/observability/data-model`), OpenLLMetry/Traceloop semantic +conventions (`traceloop.com/docs/openllmetry`), Langtrace +(`github.com/Scale3-Labs/langtrace`). 27 sources fetched; 22 claims confirmed, 3 +refuted, across 110 research agents. diff --git a/docs/task-standard.md b/docs/task-standard.md index 152a98b47..0a05204c9 100644 --- a/docs/task-standard.md +++ b/docs/task-standard.md @@ -685,6 +685,43 @@ Richer team semantics such as role membership enforcement, summaries, handoff artifacts, parallel teams, branch routing, and full trajectory sharing are parsed as draft surface but must fail closed until a runtime owns them. +### Multi-agent trajectory tracking (target, G7) + +A multi-agent interaction (`multi-agent-sequential`, `arena-concurrent`, or a +hosted multi-agent framework) must produce **one trace tree**, not one +undifferentiated `llm_trajectory.jsonl`. Every surveyed observability standard +(OpenTelemetry GenAI, LangSmith, Langfuse, OpenLLMetry) models a run as a tree of +**typed spans joined by parent pointers**, with per-agent identity carried *on each +call* — never as a flat event log, and never as one-file-per-agent (which keeps +identity by filename but discards the relationships). See +[`reference/multi-agent-trajectory.md`](reference/multi-agent-trajectory.md). + +Because BenchFlow's provider proxy sits **outside** the agent process, agent +context cannot be recovered post-hoc: each proxied raw-LLM call MUST carry it in +the request `metadata` (verified to survive intact to the LiteLLM callback). The +contract: + +| `metadata` field | meaning | OTel analogue | +|---|---|---| +| `bf.agent_id` / `bf.agent_name` | the calling agent/node | `gen_ai.agent.id` / `gen_ai.agent.name` | +| `bf.span_kind` | `chat` \| `invoke_agent` \| `execute_tool` \| `invoke_workflow` | `gen_ai.operation.name` | +| `bf.parent_agent_id` / `bf.run_id` | parent pointer + this call's id (reconstruct the tree) | `parent_run_id` / `dotted_order` | +| `bf.session_id` | real conversation/seat id — **never synthesized** | `gen_ai.conversation.id` | + +The LiteLLM callback records these as a span row; the trajectory importer rebuilds +the tree from `bf.parent_agent_id` / `bf.run_id`. The callback captures **any** +`bf.*` key generically, so richer dimensions (`bf.role`, `bf.scene`, +`bf.turn_index`, `bf.team_id`, `bf.framework`, `bf.framework_node_id`, +`bf.trace_id`) need no further code. A **uniform adapter** (`detect` / `prepare` / +`run` / `collect` under a `benchflow.multi_agent` block, with `capture_raw_llm: +required` failing closed and `agent_graph.json` for relations) maps each framework's +native per-agent + parent context (LangGraph `run_id`/`parent_run_id`; OpenAI Agents +SDK root span; Omnigent's conversation tree) onto this one schema — BenchFlow does +not adopt any single framework as *the* host. Runtime is deferred (G7); the contract +is declared so multi-agent tasks parse without loss. See +[`reference/multi-agent-trajectory.md`](reference/multi-agent-trajectory.md) for the +full vocabulary + adapter protocol. + Simulated users and nudges should be explicit about runtime type: ```yaml @@ -847,6 +884,7 @@ Current implementation status: | `reward.json` precedence | yes | partial | prefer JSON when present and reject both-present mismatches | | metrics aggregate policy | yes | partial | `mean`, `weighted_mean`, and `weighted_sum`; richer engines remain target work | | `arena-concurrent` interaction (G4) | no | no | add interaction-mode schema now; A2A bridge for the agent-under-test + concurrently-running assessor at M2 | +| multi-agent trajectory tracking (G7) | no | no | per-call `bf.*` agent-attribution metadata on proxied raw-LLM calls (verified to reach the callback) → one structured trace tree; callback records agent/parent/conversation, importer rebuilds the tree; uniform per-framework metadata shim; M1/M2 | | hybrid reward envelope (G1) | partial | no | declared cross-surface product/sum of factors; M1 | | GAIN aggregation (G2) | no | no | dynamic live baseline + ceiling; M1 | | leaderboard-submission (G5) | partial | no | hosted / hidden external scorer with durable result record; M1 | diff --git a/examples/arena/README.md b/examples/arena/README.md new file mode 100644 index 000000000..22db794e7 --- /dev/null +++ b/examples/arena/README.md @@ -0,0 +1,79 @@ +# Native concurrent multi-agent floor (`bench eval run --agents`) + +Run **N agents on ONE shared task + its ONE service, concurrently, in ONE shared +sandbox** — each agent in its own `/work/` folder, each with its own ACP +trajectory and (proxy seats only) a separate raw `llm_trajectory.jsonl`. + +```bash +set -a; . ~/sb-run.env; set +a # API keys for proxy seats +uv run bench eval run \ + --agents examples/casino/agents.yaml \ + --environment-manifest benchmarks/casinobench/environment.toml \ + --sandbox docker --drive auto-loop \ + --url-env CASINO_URL --seat-env CASINOBENCH_SEAT_ID \ + --standings-path /_admin/standings --events-path /_admin/events \ + --service-env CASINO_MULTIPLAYER=1 \ + --jobs-dir out/native-floor/casino +``` + +## `agents.yaml` + +The roster is only the A/M axis. Each seat names a **prebuilt** benchflow agent +(`agent:`) **or** a **BYOA** agent manifest (`manifest:`) — exactly one — plus its +model and an optional per-agent instruction file. `count` fans a seat out into +`name-0..name-(n-1)`. Task, service, sandbox, output, drive, and prompt are normal +`bench eval run` flags. + +```yaml +agents: + - { agent: codex-acp, model: gpt-5.5, count: 2, instructions: prompts/aggressive.md } + - { agent: claude-agent-acp, model: claude-sonnet-4-6, count: 2, instructions: prompts/cautious.md } + - { agent: deepagents, model: deepseek/deepseek-v4-pro, instructions: prompts/aggressive.md } # proxy -> raw+acp + - { name: mine, manifest: agents/my.toml, model: gpt-5.5 } # BYOA (ACP manifest contract) +``` + +## What you get + +``` +out/native-floor/casino/ +├── roster.json # seat → agent / model / protocol / byoa +├── floor.json # per-seat status + (opt) standings + reward vector +└── /trajectory/ + ├── acp_trajectory.jsonl # every seat — the agent's tool calls + thinking + └── llm_trajectory.jsonl # PROXY seats only — the raw LiteLLM exchanges +``` + +### Instruction files (per agent family) + +The runner writes each seat's `instructions:` body into `/work//` +**before** launch — `CLAUDE.md` for claude-agent-acp, `GEMINI.md` for gemini, +`AGENTS.md` for everything else (`AgentConfig.instruction_filename`). + +### Raw-LLM coverage is partial by design + +Subscription seats (codex / claude **oauth**) call their provider directly, so +they produce an **ACP trajectory only** (`raw=false` in `floor.json`). Only +**proxy-routed** seats (an API key fronted by that seat's own LiteLLM proxy — +e.g. `deepagents` on deepseek) also get a separate raw `llm_trajectory.jsonl`. + +### Drive modes + +- `auto-loop` (default, verified) — one prompt; the agent runs its own + observe→act loop via the in-sandbox CLI. Multi-round happens inside the prompt. +- `service-rounds` (structural) — **the mock service drives the rounds**: the + runner polls the shared service per seat and re-prompts (nudges) the seat only + on `YOUR_TURN`, until `DONE`/deadline (re-entrant per round). The service + controls pacing; the agent acts once per nudge through its own tools. + +## Agent paths + +All three benchflow-ai/agents families collapse to one `AgentConfig`, so the +runner never branches on "which path": + +- **raw ACP** + **ai-sdk** → `protocol=acp` → `connect_acp` (the verified path). +- **omnigent** → `protocol=session-factory` → `Agent.connect`/`Session.prompt` + (structural; no session-factory agent is registered in this repo yet). + +**BYOA** = a `manifest.toml` following the data-only agent contract (ACP-only, +strict / no unknown fields); it is schema-validated then `register_agent`-ed, +indistinguishable downstream from a prebuilt. diff --git a/examples/arena/duel_deepseek.py b/examples/arena/duel_deepseek.py new file mode 100644 index 000000000..6cabac660 --- /dev/null +++ b/examples/arena/duel_deepseek.py @@ -0,0 +1,161 @@ +"""Real run of the arena scaffold with two deepseek-v4 seats. + +Two independent agents play one round of rock-paper-scissors **concurrently** +against a single shared, turn-gated environment — driven entirely by +``benchflow.arena.run_arena``. The environment here is an in-process object that +implements the same ``SeatClient`` turn-poll contract a networked co-tenant +service would (so the demo needs no FastAPI/sandbox); each seat's brain is a real +``deepseek-v4-pro`` call. + + set -a; . ./sb-run.env; set +a # DEEPSEEK_API_KEY (and optional _BASE_URL) + uv run python examples/arena/duel_deepseek.py + +This is the inter-agent (concurrent) axis: N agents, one shared world, per-seat +reward — none of which the framework hosts natively today. The scaffold is +opt-in and touches no existing scored path. +""" + +from __future__ import annotations + +import asyncio +import os +import random + +import httpx + +from benchflow.arena import Observation, SharedEnvReward, run_arena + +RPS_BEATS = {"rock": "scissors", "scissors": "paper", "paper": "rock"} + + +class DuelFloor: + """In-process shared env implementing the SeatClient contract: a turn-gated, + 2-seat rock-paper-scissors round. Seats lazy-join on first ``observe``; + throws stay private until both have acted, then it settles zero-sum.""" + + def __init__(self, stake: int = 100, start: int = 1000) -> None: + self.stake, self.start = stake, start + self.bankroll: dict[str, int] = {} + self.seated: list[str] = [] + self.throws: dict[str, str] = {} + self.pending: str | None = None + self.cur_rid: str | None = None + self.turn = 0 + self.formed = self.done = False + self.lock = asyncio.Lock() + + def _open(self, seat: str) -> None: + self.pending, self.turn = seat, self.turn + 1 + self.cur_rid = f"{seat}#{self.turn}" + + async def observe(self, seat_id: str) -> dict: + async with self.lock: + self.bankroll.setdefault(seat_id, self.start) + if not self.formed and seat_id not in self.seated: + self.seated.append(seat_id) + if not self.formed and len(self.seated) >= 2: + self.formed = True + self._open(self.seated[0]) + if self.done: + return {"status": "done", "bankroll": self.bankroll[seat_id]} + if not self.formed: + return {"status": "waiting"} + if self.pending != seat_id: + return {"status": "not_your_turn", "current_actor": self.pending} + return { + "status": "your_turn", "request_id": self.cur_rid, + "observation": { + "public": {"pot": self.stake, "game": "rock-paper-scissors"}, + "private": {}, + }, + "legal_actions": [ + {"verb": "throw", "args": {"hand": h}} + for h in ("rock", "paper", "scissors") + ], + } + + async def act(self, seat_id: str, request_id: str, action: dict) -> dict: + async with self.lock: + if not self.formed or self.pending != seat_id: + return {"ok": False, "status": "not_your_turn"} + if request_id != self.cur_rid: + return {"ok": False, "status": "stale_request_id"} + self.throws[seat_id] = str(action.get("args", {}).get("hand", "rock")) + idx = self.seated.index(seat_id) + if idx + 1 < len(self.seated): + self._open(self.seated[idx + 1]) + else: + self._settle() + return {"ok": True, "status": "applied"} + + def _settle(self) -> None: + a, b = self.seated + ta, tb = self.throws[a], self.throws[b] + if ta != tb: + win, lose = (a, b) if RPS_BEATS.get(ta) == tb else (b, a) + self.bankroll[win] += self.stake + self.bankroll[lose] -= self.stake + self.pending = self.cur_rid = None + self.done = True + + def standings(self) -> dict[str, int]: + return dict(self.bankroll) + + +class DeepSeekPolicy: + """A seat brain backed by a real deepseek-v4-pro call.""" + + def __init__(self, seat: str, http: httpx.AsyncClient, model: str = "deepseek-v4-pro") -> None: + self.seat, self.http, self.model = seat, http, model + self.base = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/") + self.key = os.environ["DEEPSEEK_API_KEY"] + self.choice: str | None = None + + async def act(self, obs: Observation) -> dict: + hands = [a["args"]["hand"] for a in obs.legal_actions] + prompt = ( + f"You are {self.seat} in ONE round of rock-paper-scissors for a pot of " + f"{obs.public.get('pot')} chips. Choose exactly one of: {', '.join(hands)}. " + "Reply with ONLY that single word." + ) + try: + r = await self.http.post( + f"{self.base}/chat/completions", + headers={"Authorization": f"Bearer {self.key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 256, "temperature": 0.8, + }, + timeout=90.0, + ) + text = (r.json()["choices"][0]["message"]["content"] or "").lower() + hand = next((h for h in hands if h in text), random.choice(hands)) + except Exception as exc: # a flaky call falls back to a random legal throw + print(f" [{self.seat}] deepseek call failed ({exc!r}); random fallback") + hand = random.choice(hands) + self.choice = hand + return {"verb": "throw", "args": {"hand": hand}} + + +async def _main() -> None: + if not os.environ.get("DEEPSEEK_API_KEY"): + raise SystemExit("DEEPSEEK_API_KEY required (source your env file first)") + floor = DuelFloor() + seats = ["seat-0", "seat-1"] + async with httpx.AsyncClient() as http: + policies = {s: DeepSeekPolicy(s, http) for s in seats} + print("running arena: 2 deepseek-v4-pro seats, one shared RPS table…", flush=True) + res = await run_arena( + seats, floor, lambda s: policies[s], deadline_s=120.0, poll_s=0.05, + ) + st = floor.standings() + print("\npicks :", {s: p.choice for s, p in policies.items()}) + print("standings :", st) + print("reward (pvp):", SharedEnvReward().score(st)) + print("seat status :", {s: r["status"] for s, r in res.items()}) + print("conserved :", sum(st.values()), "(== 2000)") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/examples/arena/floor_deepseek.py b/examples/arena/floor_deepseek.py new file mode 100644 index 000000000..8ee50359f --- /dev/null +++ b/examples/arena/floor_deepseek.py @@ -0,0 +1,157 @@ +"""Real arena run: N deepseek-v4 seats, routed through BenchFlow's provider proxy, +with a per-seat trajectory written for each. + +Each seat's raw LLM call goes through ``BENCHFLOW_PROVIDER_BASE_URL`` / +``BENCHFLOW_PROVIDER_API_KEY`` / ``BENCHFLOW_PROVIDER_MODEL`` (the proxy the SDK +injects in a real eval — there it also writes ``llm_trajectory.jsonl`` and +attributes usage per seat via the ``x-bf-seat`` tag). The bench's own per-seat +decision trajectory is written to ``out/arena-floor/.trajectory.jsonl``. + + # point the provider vars at the proxy (or, standalone, at the model API): + export BENCHFLOW_PROVIDER_BASE_URL=https://api.deepseek.com + export BENCHFLOW_PROVIDER_API_KEY=$DEEPSEEK_API_KEY + export BENCHFLOW_PROVIDER_MODEL=deepseek-v4-pro + uv run python examples/arena/floor_deepseek.py 3 +""" + +from __future__ import annotations + +import asyncio +import json +import re +import sys +from pathlib import Path + +import httpx + +from benchflow.arena import ( + ProxyChatPolicy, + SeatTrajectory, + SharedEnvReward, + provider_config, + run_arena, +) + + +class HighCardFloor: + """N-seat turn-gated env (SeatClient): each seat antes ``stake`` and picks + 0-9; the highest pick wins the pot (ties split). Seats lazy-join on observe.""" + + def __init__(self, n_seats: int, stake: int = 50, start: int = 1000) -> None: + self.n, self.stake, self.start = n_seats, stake, start + self.bankroll: dict[str, int] = {} + self.seated: list[str] = [] + self.picks: dict[str, int] = {} + self.pending: str | None = None + self.cur_rid: str | None = None + self.turn = 0 + self.formed = self.done = False + self.lock = asyncio.Lock() + + def _open(self, seat: str) -> None: + self.pending, self.turn = seat, self.turn + 1 + self.cur_rid = f"{seat}#{self.turn}" + + async def observe(self, seat_id: str) -> dict: + async with self.lock: + self.bankroll.setdefault(seat_id, self.start) + if not self.formed and seat_id not in self.seated: + self.seated.append(seat_id) + if not self.formed and len(self.seated) >= self.n: + self.formed = True + self._open(self.seated[0]) + if self.done: + return {"status": "done", "bankroll": self.bankroll[seat_id]} + if not self.formed: + return {"status": "waiting"} + if self.pending != seat_id: + return {"status": "not_your_turn", "current_actor": self.pending} + return { + "status": "your_turn", "request_id": self.cur_rid, + "observation": {"public": {"pot": self.stake * self.n}, "private": {}}, + "legal_actions": [{"verb": "pick", "args": {"n": k}} for k in range(10)], + } + + async def act(self, seat_id: str, request_id: str, action: dict) -> dict: + async with self.lock: + if not self.formed or self.pending != seat_id: + return {"ok": False, "status": "not_your_turn"} + if request_id != self.cur_rid: + return {"ok": False, "status": "stale_request_id"} + self.picks[seat_id] = int(action.get("args", {}).get("n", 0)) + idx = self.seated.index(seat_id) + if idx + 1 < len(self.seated): + self._open(self.seated[idx + 1]) + else: + self._settle() + return {"ok": True, "status": "applied"} + + def _settle(self) -> None: + hi = max(self.picks.values()) + winners = [s for s, v in self.picks.items() if v == hi] + share = (self.stake * self.n) // len(winners) + for s in self.seated: + self.bankroll[s] -= self.stake + for w in winners: + self.bankroll[w] += share + self.pending = self.cur_rid = None + self.done = True + + def standings(self) -> dict[str, int]: + return dict(self.bankroll) + + +def render_for(seat: str): + def render(obs) -> str: + ns = [a["args"]["n"] for a in obs.legal_actions] + return ( + f"You are {seat} in a one-round high-card game for a pot of " + f"{obs.public.get('pot')} chips. Pick ONE number from {ns[0]}..{ns[-1]}; " + "the single highest pick wins the whole pot (ties split it). " + "Reply with ONLY your number." + ) + return render + + +def pick_number(text: str, legal: list[dict]) -> dict: + m = re.search(r"\d", text or "") + n = int(m.group()) if m else None + for a in legal: + if a["args"]["n"] == n: + return a + import random + return random.choice(legal) + + +async def _main() -> None: + n = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + base, key, model = provider_config() + if not key: + raise SystemExit("no provider key (BENCHFLOW_PROVIDER_API_KEY / DEEPSEEK_API_KEY)") + run_dir = Path("out/arena-floor") + tr = SeatTrajectory(run_dir) + floor = HighCardFloor(n) + seats = [f"seat-{i}" for i in range(n)] + print(f"arena: {n} seats · provider {base} · model {model}", flush=True) + async with httpx.AsyncClient() as http: + policies = { + s: ProxyChatPolicy(s, http, render=render_for(s), pick=pick_number, + temperature=0.9, recorder=tr) + for s in seats + } + res = await run_arena(seats, floor, lambda s: policies[s], + deadline_s=120.0, poll_s=0.05) + st = floor.standings() + print("picks :", floor.picks) + print("standings :", st) + print("reward (pvp):", SharedEnvReward().score(st)) + print("seat status :", {s: r["status"] for s, r in res.items()}) + print("conserved :", sum(st.values()), f"(== {n * 1000})") + print(f"trajectories: {run_dir}/.trajectory.jsonl") + for s in seats: + rec = json.loads(tr.path(s).read_text().strip().splitlines()[-1]) + print(f" {s}: pick={rec['action']['args']['n']} llm.usage={rec['llm']['usage']}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/examples/arena/run_per_seat_proxy.py b/examples/arena/run_per_seat_proxy.py new file mode 100644 index 000000000..c479e7530 --- /dev/null +++ b/examples/arena/run_per_seat_proxy.py @@ -0,0 +1,97 @@ +"""Separate trajectories per agent — ONE LiteLLM proxy PER seat. + +The shared-proxy run writes a single `llm_trajectory.jsonl` with every seat's +calls mixed in (and the callback records only model+messages, so a per-call agent +tag does NOT survive). To track each concurrent agent separately, give each its +OWN `ensure_litellm_runtime` → its own callback log → its own +`/trajectory/llm_trajectory.jsonl` + its own usage/cost. This is exactly how +a BenchFlow multi-role rollout isolates roles. + + set -a; . ./sb-run.env; set +a + uv run python examples/arena/run_per_seat_proxy.py 3 +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from pathlib import Path + +import httpx + +from benchflow.arena import ProxyChatPolicy, SeatTrajectory, run_arena +from benchflow.providers import ( + ensure_litellm_runtime, + extract_usage, + stop_provider_runtime, +) + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from floor_deepseek import HighCardFloor, pick_number, render_for + + +async def _main() -> None: + if not os.environ.get("DEEPSEEK_API_KEY"): + raise SystemExit("DEEPSEEK_API_KEY required") + n = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + seats = [f"seat-{i}" for i in range(n)] + run_dir = Path("out/arena-per-seat") + + # 1) ONE proxy per seat — separate callback log → separate trajectory + usage + runtimes: dict = {} + envs: dict = {} + print(f"starting {n} per-seat LiteLLM proxies…", flush=True) + for s in seats: + env, rt = await ensure_litellm_runtime( + agent="deepagents", + agent_env={"DEEPSEEK_API_KEY": os.environ["DEEPSEEK_API_KEY"]}, + model="deepseek/deepseek-v4-pro", runtime=None, environment="local", + session_id=f"arena-{s}", + ) + runtimes[s], envs[s] = rt, env + print(f" {s}: {env['BENCHFLOW_PROVIDER_BASE_URL']}", flush=True) + + floor = HighCardFloor(n) + tr = SeatTrajectory(run_dir) + usages: dict = {} + try: + async with httpx.AsyncClient() as http: + policies = { + s: ProxyChatPolicy( + s, http, render=render_for(s), pick=pick_number, + base=envs[s]["BENCHFLOW_PROVIDER_BASE_URL"], # this seat's proxy + api_key=envs[s]["BENCHFLOW_PROVIDER_API_KEY"], + model=envs[s]["BENCHFLOW_PROVIDER_MODEL"], + temperature=0.9, max_tokens=256, recorder=tr, + ) + for s in seats + } + res = await run_arena(seats, floor, lambda s: policies[s], + deadline_s=180.0, poll_s=0.05) + await asyncio.sleep(1.5) # let each proxy's async callback flush + finally: + for s in seats: + await stop_provider_runtime(runtimes[s]) # stop FIRST → parses callback log + for s in seats: # then read each proxy's usage + trajectory (populated on stop) + usages[s] = extract_usage(runtimes[s]) + traj = getattr(getattr(runtimes[s], "server", None), "trajectory", None) + if traj is not None and traj.exchanges: + d = run_dir / s / "trajectory" + d.mkdir(parents=True, exist_ok=True) + (d / "llm_trajectory.jsonl").write_text(traj.to_jsonl(redact_keys=True)) + + st = floor.standings() + print("\nstandings :", st, "· conserved", sum(st.values()), + "· status", {s: r["status"] for s, r in res.items()}) + print("=== SEPARATE per-agent trajectories + usage ===") + for s in seats: + p = run_dir / s / "trajectory" / "llm_trajectory.jsonl" + ex = len(p.read_text().splitlines()) if p.exists() else 0 + u = usages.get(s, {}) + print(f" {s}: {p} ({ex} exchange) " + f"tokens={u.get('total_tokens')} cost=${u.get('cost_usd')}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/examples/arena/run_through_proxy.py b/examples/arena/run_through_proxy.py new file mode 100644 index 000000000..c5706a14b --- /dev/null +++ b/examples/arena/run_through_proxy.py @@ -0,0 +1,100 @@ +"""Real run THROUGH the BenchFlow LiteLLM proxy. + +Three deepseek-v4 seats play one shared high-card round concurrently via +``run_arena``; each seat's raw LLM call is routed through a loopback LiteLLM proxy +started by ``ensure_litellm_runtime`` — so per-agent **usage/cost** and the proxy's +**llm trajectory** are captured by BenchFlow, and the seats never see the raw +provider key (the proxy isolation invariant). A per-seat *decision* trajectory is +also written to ``out/arena-floor-proxy/.trajectory.jsonl``. + + set -a; . ./sb-run.env; set +a # DEEPSEEK_API_KEY (the real upstream key) + uv run python examples/arena/run_through_proxy.py +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from pathlib import Path + +import httpx + +from benchflow.arena import ProxyChatPolicy, SeatTrajectory, SharedEnvReward, run_arena +from benchflow.providers import ( + ensure_litellm_runtime, + extract_usage, + stop_provider_runtime, +) + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from floor_deepseek import HighCardFloor, pick_number, render_for + + +async def _main() -> None: + if not os.environ.get("DEEPSEEK_API_KEY"): + raise SystemExit("DEEPSEEK_API_KEY required (the real upstream key)") + n = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + seats = [f"seat-{i}" for i in range(n)] + + print("starting BenchFlow LiteLLM proxy (environment=local)…", flush=True) + agent_env, runtime = await ensure_litellm_runtime( + agent="deepagents", + agent_env={"DEEPSEEK_API_KEY": os.environ["DEEPSEEK_API_KEY"]}, + model="deepseek/deepseek-v4-pro", # provider-prefixed → routes to deepseek + runtime=None, + environment="local", + session_id="arena-floor", + ) + os.environ.update(agent_env) # each seat now reads BENCHFLOW_PROVIDER_* → the proxy + print(" proxy base :", agent_env.get("BENCHFLOW_PROVIDER_BASE_URL")) + print(" model alias:", agent_env.get("BENCHFLOW_PROVIDER_MODEL")) + print(" raw key hidden from seats:", + "DEEPSEEK_API_KEY" not in agent_env, flush=True) + + run_dir = Path("out/arena-floor-proxy") + tr = SeatTrajectory(run_dir) + floor = HighCardFloor(n) + res: dict = {} + try: + async with httpx.AsyncClient() as http: + policies = { + s: ProxyChatPolicy(s, http, render=render_for(s), pick=pick_number, + temperature=0.9, max_tokens=2048, recorder=tr) + for s in seats + } + res = await run_arena(seats, floor, lambda s: policies[s], + deadline_s=180.0, poll_s=0.05) + finally: + await asyncio.sleep(1.5) # let the proxy's async callback flush the last call + await stop_provider_runtime(runtime) # parses the proxy callback log + usage = extract_usage(runtime) # aggregate tokens/cost after stop + + # persist the proxy's raw-LLM trajectory in BenchFlow's canonical format + # (mirrors rollout._write_llm_trajectory). + proxy_traj = getattr(getattr(runtime, "server", None), "trajectory", None) + if proxy_traj is not None and proxy_traj.exchanges: + traj_dir = run_dir / "trajectory" + traj_dir.mkdir(parents=True, exist_ok=True) + (traj_dir / "llm_trajectory.jsonl").write_text(proxy_traj.to_jsonl(redact_keys=True)) + + st = floor.standings() + print("\npicks :", floor.picks) + print("standings :", st) + print("reward (pvp):", SharedEnvReward().score(st)) + print("seat status :", {s: r["status"] for s, r in res.items()}) + print("conserved :", sum(st.values()), f"(== {n * 1000})") + print("proxy usage :", json.dumps(usage)) # tokens + cost from the proxy callback log + if proxy_traj is not None and proxy_traj.exchanges: + print(f"llm_trajectory: {run_dir}/trajectory/llm_trajectory.jsonl " + f"({len(proxy_traj.exchanges)} raw exchanges, canonical format)") + print(f"decision traj : {run_dir}/.trajectory.jsonl") + for s in seats: + line = tr.path(s).read_text().strip().splitlines()[-1] + rec = json.loads(line) + print(f" {s}: pick={rec['action']['args']['n']} llm.usage={rec['llm']['usage']}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/examples/casino/PROMPT.md b/examples/casino/PROMPT.md new file mode 100644 index 000000000..41346915d --- /dev/null +++ b/examples/casino/PROMPT.md @@ -0,0 +1,21 @@ +Play the casino games and win as many chips as you can, using the `casino` +command (your seat is already configured): + + casino lobby — open games, your bankroll, queue state + casino rules — a game's rules + casino join — take a seat (or queue) at a game + casino observe [--wait N] — {request_id, observation, legal_actions, events} + casino act '' — play ONE of the legal actions + casino cashier — your bankroll + casino leave — leave your table or queue + +House etiquette (enforced by the casino): +- To wait for your turn or for opponents, use `casino observe --wait 30` — + it blocks until something happens. Never busy-loop plain observe. +- If a game queue hasn't matched after a couple of minutes, `casino leave` + and pick another game (the casino will also time you out of stale queues). +- If you sit silent on your turn too long the casino plays a default action + for you; repeated silence sits you out. You may stop playing at any time — + say so and stop. + +Play through the `casino` CLI. Begin with `casino lobby`. diff --git a/examples/casino/README.md b/examples/casino/README.md new file mode 100644 index 000000000..131f693f9 --- /dev/null +++ b/examples/casino/README.md @@ -0,0 +1,45 @@ +# Real multi-agent casino floor (arena-concurrent, on BenchFlow) + +N **real autonomous ACP agents** play ONE shared [casinobench](https://github.com/benchflow-ai/casinobench) +World concurrently — the live realization of the deferred `arena-concurrent` mode. +Each seat is a benchflow ACP agent in its OWN sandbox, all competing on one +leaderboard; each agent's raw + ACP trajectory is captured per seat. + +- `run_floor.py` — starts casinobench's shared World on the host, then runs a + roster of seats concurrently (`asyncio.gather`), each a `connect_acp` agent in a + `DockerSandbox` reaching the World over the docker bridge. Subscription agents + (codex / claude-code) get their auth uploaded per seat and produce + `acp_trajectory.jsonl`; deepseek/proxy seats also get a per-seat raw + `llm_trajectory.jsonl`. +- `town_snapshot.py` — serves a live Stanford-Town-style floor viewer: + casinobench's `render_html` canvas board (agents walking to game stations) in + live mode, with a click-to-open per-agent **run dossier** injected. Polls the + World, falls back to the persisted run when it ends, and feeds same-origin JSON + so a Cloudflare tunnel can publish it. +- `agent_env/` — the seat image (`casino-agent-seat`): Node + `codex-acp` + + `claude-agent-acp` (via benchflow's install commands) + the `casino` seven-tool + CLI. The deepagents shim and the casino CLI package are **assembled** into the + build context (gitignored — see `agent_env/.gitignore`). + +## Setup +This example depends on a local casinobench checkout (a separate repo). Assemble +the seat-image build context, then build it: + +```bash +CB=~/casinobench # your casinobench checkout +cp src/benchflow/agents/deepagents_acp_shim.py examples/casino/agent_env/deepagents-acp-shim +cp -r "$CB/packages/environments/casino" examples/casino/agent_env/casino-pkg +docker build -t casino-agent-seat:latest examples/casino/agent_env + +set -a; . ~/sb-run.env; set +a # DEEPSEEK_API_KEY (proxy seats) +# codex/claude seats use the host's ~/.codex/auth.json + ~/.claude/.credentials.json subscriptions +uv run python examples/casino/run_floor.py --world-port 9100 +# in another shell, publish the live viewer: +cd "$CB" && uv run python /examples/casino/town_snapshot.py \ + http://127.0.0.1:9100 /out/casino-floor/all-games ./serve & +cloudflared tunnel --url http://localhost:8899 # serving ./serve +``` + +The roster (agents × models) and the seat prompt are at the top of `run_floor.py`. +Only the models a subscription actually exposes work (e.g. codex→`gpt-5.5`, +claude→`claude-sonnet-4-6`/`claude-haiku-4-5`); others are rejected by the plan. diff --git a/examples/casino/agent_env/.gitignore b/examples/casino/agent_env/.gitignore new file mode 100644 index 000000000..ff978ff54 --- /dev/null +++ b/examples/casino/agent_env/.gitignore @@ -0,0 +1,7 @@ +# Assembled into the build context at setup time, not checked in (see ../README.md): +# deepagents-acp-shim — copied from src/benchflow/agents/deepagents_acp_shim.py +# casino-pkg/ — copied from /packages/environments/casino +# casinobench-engine/ — copied from (proprietary engine; base image) +deepagents-acp-shim +casino-pkg/ +casinobench-engine/ diff --git a/examples/casino/agent_env/Dockerfile b/examples/casino/agent_env/Dockerfile new file mode 100644 index 000000000..784a9a205 --- /dev/null +++ b/examples/casino/agent_env/Dockerfile @@ -0,0 +1,50 @@ +# Agent-seat image for the multi-agent casino floor. +# +# Bakes in everything ONE deepagents ACP seat needs so `connect_acp` can run it +# with no per-seat install: the deepagents venv + ACP shim (the autonomous agent), +# and the `casino` seven-tool CLI (the play surface — the agent shells out to +# `casino observe` / `casino act` against $CASINO_URL). The graph/world itself is +# the shared casinobench World service, reached over HTTP. +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates git && rm -rf /var/lib/apt/lists/* + +# uv (pinned interpreter for deepagents, exactly as benchflow's install_cmd does) +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.local/bin:${PATH}" + +# deepagents harness venv (the autonomous ReAct agent) + the OpenAI-compatible +# chat model dep, verified to import. +RUN uv venv --python 3.12 /opt/benchflow/deepagents-venv && \ + uv pip install -q --python /opt/benchflow/deepagents-venv/bin/python \ + deepagents langchain-openai && \ + /opt/benchflow/deepagents-venv/bin/python -c "import deepagents, langchain_openai; print('deepagents ok')" + +# the ACP shim benchflow launches on stdio +COPY deepagents-acp-shim /opt/benchflow/bin/deepagents-acp-shim +RUN chmod a+rx /opt/benchflow/bin/deepagents-acp-shim && \ + chmod -R a+rX /opt/benchflow/deepagents-venv + +# the casino seven-tool CLI on PATH (the agent's play surface). The CLI only +# makes HTTP calls to $CASINO_URL (click + httpx) — the casinobench engine is a +# server-side dep of the package, not the CLI, so install --no-deps + the two +# runtime imports the CLI actually uses. +COPY casino-pkg /opt/casino-pkg +RUN pip install --no-cache-dir "click>=8.0" "httpx>=0.27.0" && \ + pip install --no-cache-dir --no-deps /opt/casino-pkg && \ + casino --help >/dev/null && echo "casino cli ok" + +# the node ACP agents (codex-acp + claude-agent-acp) baked in via benchflow's own +# install commands — Node 22.20.0 + the npm packages + wrappers at +# /opt/benchflow/bin/{codex-acp,claude-agent-acp}. Subscription auth is uploaded +# per-seat at run time (not baked). connect_acp's force_build=False then needs no +# per-seat install. +RUN apt-get update && apt-get install -y --no-install-recommends tar xz-utils && \ + rm -rf /var/lib/apt/lists/* +COPY install-node-agents.sh /opt/install-node-agents.sh +RUN sh /opt/install-node-agents.sh && \ + test -x /opt/benchflow/bin/codex-acp && test -x /opt/benchflow/bin/claude-agent-acp && \ + echo "node acp agents ok" + +WORKDIR /app diff --git a/examples/casino/agent_env/install-node-agents.sh b/examples/casino/agent_env/install-node-agents.sh new file mode 100644 index 000000000..25b69eeb0 --- /dev/null +++ b/examples/casino/agent_env/install-node-agents.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e +export PATH="/opt/benchflow/node/bin:/opt/benchflow/bin:$PATH" +echo "=== installing codex-acp ===" +export DEBIAN_FRONTEND=noninteractive; BF_NODE_DIR=/opt/benchflow/node; BF_NODE_VERSION=22.20.0; if [ ! -x "$BF_NODE_DIR/bin/node" ]; then if ! command -v curl >/dev/null 2>&1 || ! command -v tar >/dev/null 2>&1 || ! command -v xz >/dev/null 2>&1; then if command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq curl ca-certificates tar xz-utils; elif command -v dnf >/dev/null 2>&1; then dnf -y install curl ca-certificates tar xz; elif command -v apk >/dev/null 2>&1; then apk add --no-cache curl ca-certificates tar xz; else echo 'BenchFlow JS agent bootstrap requires curl, tar, and xz' >&2; exit 127; fi; fi; arch="$(uname -m)"; case "$arch" in x86_64|amd64) node_arch=x64 ;; aarch64|arm64) node_arch=arm64 ;; *) echo "Unsupported architecture for Node.js: $arch" >&2; exit 1 ;; esac; tmp="$(mktemp -d)"; mkdir -p /opt/benchflow; curl -fsSLo "$tmp/node.tar.xz" "https://nodejs.org/dist/v${BF_NODE_VERSION}/node-v${BF_NODE_VERSION}-linux-${node_arch}.tar.xz"; rm -rf "$BF_NODE_DIR"; mkdir -p "$BF_NODE_DIR"; tar -xJf "$tmp/node.tar.xz" -C "$BF_NODE_DIR" --strip-components=1 --no-same-owner; rm -rf "$tmp"; fi; export PATH="/opt/benchflow/node/bin:$PATH"; "$BF_NODE_DIR/bin/node" --version; "$BF_NODE_DIR/bin/npm" --version && mkdir -p /opt/benchflow/js-agents /opt/benchflow/bin && export PATH="/opt/benchflow/bin:/opt/benchflow/js-agents/bin:/opt/benchflow/node/bin:$PATH" && ( /opt/benchflow/node/bin/npm install -g --prefix /opt/benchflow/js-agents @agentclientprotocol/codex-acp@0.0.45 ) && printf '%s\n' '#!/bin/sh' 'exec /opt/benchflow/node/bin/node /opt/benchflow/js-agents/bin/codex-acp "$@"' > /opt/benchflow/bin/codex-acp && chmod +x /opt/benchflow/bin/codex-acp && chmod -R a+rX /opt/benchflow && [ -x /opt/benchflow/js-agents/bin/codex-acp ] && [ -x /opt/benchflow/bin/codex-acp ] +echo "=== installing claude-agent-acp ===" +export DEBIAN_FRONTEND=noninteractive; BF_NODE_DIR=/opt/benchflow/node; BF_NODE_VERSION=22.20.0; if [ ! -x "$BF_NODE_DIR/bin/node" ]; then if ! command -v curl >/dev/null 2>&1 || ! command -v tar >/dev/null 2>&1 || ! command -v xz >/dev/null 2>&1; then if command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq curl ca-certificates tar xz-utils; elif command -v dnf >/dev/null 2>&1; then dnf -y install curl ca-certificates tar xz; elif command -v apk >/dev/null 2>&1; then apk add --no-cache curl ca-certificates tar xz; else echo 'BenchFlow JS agent bootstrap requires curl, tar, and xz' >&2; exit 127; fi; fi; arch="$(uname -m)"; case "$arch" in x86_64|amd64) node_arch=x64 ;; aarch64|arm64) node_arch=arm64 ;; *) echo "Unsupported architecture for Node.js: $arch" >&2; exit 1 ;; esac; tmp="$(mktemp -d)"; mkdir -p /opt/benchflow; curl -fsSLo "$tmp/node.tar.xz" "https://nodejs.org/dist/v${BF_NODE_VERSION}/node-v${BF_NODE_VERSION}-linux-${node_arch}.tar.xz"; rm -rf "$BF_NODE_DIR"; mkdir -p "$BF_NODE_DIR"; tar -xJf "$tmp/node.tar.xz" -C "$BF_NODE_DIR" --strip-components=1 --no-same-owner; rm -rf "$tmp"; fi; export PATH="/opt/benchflow/node/bin:$PATH"; "$BF_NODE_DIR/bin/node" --version; "$BF_NODE_DIR/bin/npm" --version && mkdir -p /opt/benchflow/js-agents /opt/benchflow/bin && export PATH="/opt/benchflow/bin:/opt/benchflow/js-agents/bin:/opt/benchflow/node/bin:$PATH" && ( /opt/benchflow/node/bin/npm install -g --prefix /opt/benchflow/js-agents @agentclientprotocol/claude-agent-acp@0.40.0 ) && printf '%s\n' '#!/bin/sh' 'exec /opt/benchflow/node/bin/node /opt/benchflow/js-agents/bin/claude-agent-acp "$@"' > /opt/benchflow/bin/claude-agent-acp && chmod +x /opt/benchflow/bin/claude-agent-acp && chmod -R a+rX /opt/benchflow && [ -x /opt/benchflow/js-agents/bin/claude-agent-acp ] && [ -x /opt/benchflow/bin/claude-agent-acp ] diff --git a/examples/casino/agents.yaml b/examples/casino/agents.yaml new file mode 100644 index 000000000..823232bc1 --- /dev/null +++ b/examples/casino/agents.yaml @@ -0,0 +1,22 @@ +# Native concurrent multi-agent casino roster for `bench eval run --agents`. +# +# set -a; . ~/sb-run.env; set +a # DEEPSEEK_API_KEY for the proxy seat +# bench eval run \ +# --agents examples/casino/agents.yaml \ +# --environment-manifest benchmarks/casinobench/environment.toml \ +# --sandbox docker --drive auto-loop \ +# --url-env CASINO_URL --seat-env CASINOBENCH_SEAT_ID \ +# --standings-path /_admin/standings --events-path /_admin/events \ +# --service-env CASINO_MULTIPLAYER=1 \ +# --jobs-dir out/native-floor/casino +# +# This file is intentionally agents-only. Task/service/sandbox/out/drive/prompt +# are standard `bench eval run` flags, not roster fields. + +agents: + # No `name:` means the seat/player id defaults to -. + # Subscription seats (codex/claude oauth) produce ACP trajectories only. + - { agent: codex-acp, model: gpt-5.5, count: 2, instructions: prompts/aggressive.md } + - { agent: claude-agent-acp, model: claude-sonnet-4-6, count: 2, instructions: prompts/cautious.md } + # Proxy seats route through a per-seat LiteLLM proxy and produce raw + ACP trajectories. + - { agent: deepagents, model: deepseek/deepseek-v4-pro, instructions: prompts/aggressive.md } diff --git a/examples/casino/build_town.py b/examples/casino/build_town.py new file mode 100644 index 000000000..e28e46607 --- /dev/null +++ b/examples/casino/build_town.py @@ -0,0 +1,49 @@ +"""Build the Stanford-Town-style casino viewer for a concurrent floor run. + +Unlike casinobench's build.py (single --trajectory), this merges EVERY seat's +acp_trajectory.jsonl into one seq-keyed thinking map, so the town floor shows all +agents moving between tables WITH each one's reasoning overlaid on its actions. + + uv run python examples/casino/build_town.py +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from casinobench.catalog import default_registry +from casinobench.event_log import EventLog +from casinobench.thinking import thinking_for_run +from casinobench.viewer_data import to_viewer_data +from casinobench.viewer_html import render_html + + +def main() -> int: + run_dir, out = Path(sys.argv[1]), Path(sys.argv[2]) + events = list(EventLog.from_jsonl((run_dir / "events.jsonl").read_text()).events) + + fj = json.loads((run_dir / "floor.json").read_text()) + standings = fj.get("standings", {}) + players = sorted(standings) or sorted({e.actor for e in events if e.actor}) + starting = int(fj.get("starting_bankroll", 1000)) + game_config = fj.get("game_config") if isinstance(fj.get("game_config"), dict) else {"stake": 50} + + merged: dict[int, str] = {} + for seat in players: + tp = run_dir / seat / "trajectory" / "acp_trajectory.jsonl" + if tp.exists(): + merged.update(thinking_for_run(events, tp, subject=seat)) + + run = to_viewer_data(events, players, starting, default_registry(), + game_config=game_config, thinking=merged) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(render_html(run)) + print(f"wrote {out}: {len(run['events'])} events, {len(run['games'])} games, " + f"{len(players)} players, thinking={len(merged)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/casino/live_viewer.py b/examples/casino/live_viewer.py new file mode 100644 index 000000000..2264afcd0 --- /dev/null +++ b/examples/casino/live_viewer.py @@ -0,0 +1,80 @@ +"""Live Casino Floor viewer: poll a running World + rebuild the browser HTML. + +Polls the shared World's /_admin endpoints every few seconds, writes a run dir, +rebuilds casinobench's `casino-town.html` from it (with a meta-refresh so the +browser auto-reloads), into the directory served by a Cloudflare tunnel. + + uv run python examples/casino/live_viewer.py +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import time +from pathlib import Path + +import httpx + +CASINOBENCH = "/home/liu.10379/casinobench" + + +def main() -> None: + world = sys.argv[1].rstrip("/") + serve_dir = Path(sys.argv[2]) + serve_dir.mkdir(parents=True, exist_ok=True) + run = serve_dir / "_run" + run.mkdir(exist_ok=True) + out = serve_dir / "index.html" + + while True: + try: + ev = httpx.get(f"{world}/_admin/events", timeout=8).json().get("jsonl", "") + state = httpx.get(f"{world}/_admin/state", timeout=8).json() + standings = httpx.get(f"{world}/_admin/standings", timeout=8).json() + (run / "events.jsonl").write_text(ev) + (run / "standings.json").write_text(json.dumps(standings)) + (run / "run.json").write_text( + json.dumps( + { + "final_bankrolls": standings, + "game_config": state.get("game_config") or {"stake": 50}, + "players": sorted(standings.keys()), + "starting_bankroll": int(state.get("starting_bankroll", 1000)), + "subject": state.get("subject", "agent"), + } + ) + ) + r = subprocess.run( + [ + "uv", + "run", + "python", + "viewer/build.py", + "--from", + str(run), + "--out", + str(out), + ], + cwd=CASINOBENCH, + capture_output=True, + text=True, + timeout=60, + ) + if out.exists(): + html = out.read_text() + if 'http-equiv="refresh"' not in html: + html = html.replace( + "", '', 1 + ) + out.write_text(html) + else: + print("build:", (r.stderr or r.stdout or "")[-200:], flush=True) + except Exception as exc: + print("live_viewer:", type(exc).__name__, str(exc)[:120], flush=True) + time.sleep(6) + + +if __name__ == "__main__": + main() diff --git a/examples/casino/prompts/aggressive.md b/examples/casino/prompts/aggressive.md new file mode 100644 index 000000000..85e0ee9b2 --- /dev/null +++ b/examples/casino/prompts/aggressive.md @@ -0,0 +1,5 @@ +# Table style: aggressive + +You play to maximize chips. Prefer high-variance lines: bet up when the odds +favor you, pressure marginal spots, and keep moving between games to find soft +tables. Don't sit out — act every turn it's yours. diff --git a/examples/casino/prompts/cautious.md b/examples/casino/prompts/cautious.md new file mode 100644 index 000000000..6d0d180ab --- /dev/null +++ b/examples/casino/prompts/cautious.md @@ -0,0 +1,5 @@ +# Table style: cautious + +You protect your bankroll. Take +EV spots only, fold marginal hands, and size +bets to survive variance. Walk away from a game when the edge is gone. Slow and +steady compounds — but still act every turn it's yours. diff --git a/examples/casino/roster-10.yaml b/examples/casino/roster-10.yaml new file mode 100644 index 000000000..ee7cd93a1 --- /dev/null +++ b/examples/casino/roster-10.yaml @@ -0,0 +1,8 @@ +# 10-seat heterogeneous floor: 5 runtimes, 3 providers, both auth modes. +agents: + - { agent: deepagents, model: deepseek/deepseek-v4-flash, count: 3, instructions: prompts/aggressive.md } + - { agent: pi-acp, model: deepseek/deepseek-v4-pro, count: 2, instructions: prompts/aggressive.md } + - { agent: opencode, model: deepseek/deepseek-v4-pro, count: 2, instructions: prompts/aggressive.md } + - { agent: openhands, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } + - { agent: codex-acp, model: gpt-5.5, count: 1, instructions: prompts/aggressive.md } + - { agent: claude-agent-acp, model: claude-opus-4-8, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-5.yaml b/examples/casino/roster-5.yaml new file mode 100644 index 000000000..de611a12a --- /dev/null +++ b/examples/casino/roster-5.yaml @@ -0,0 +1,3 @@ +# 5-agent floor for daytona↔docker parity: deepagents (proxy) on deepseek-v4-flash +agents: + - { agent: deepagents, model: deepseek/deepseek-v4-flash, count: 5, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-mixed5.yaml b/examples/casino/roster-mixed5.yaml new file mode 100644 index 000000000..eddcf02e3 --- /dev/null +++ b/examples/casino/roster-mixed5.yaml @@ -0,0 +1,7 @@ +# heterogeneous 5-agent floor: 3 proxy seats (deepseek-v4-pro) + 2 native-auth seats +agents: + - { agent: pi-acp, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } + - { agent: openhands, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } + - { agent: opencode, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } + - { agent: codex-acp, model: gpt-5.5, count: 1, instructions: prompts/aggressive.md } + - { agent: claude-agent-acp, model: claude-opus-4-8, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-oh-dsmoke.yaml b/examples/casino/roster-oh-dsmoke.yaml new file mode 100644 index 000000000..579538a93 --- /dev/null +++ b/examples/casino/roster-oh-dsmoke.yaml @@ -0,0 +1,2 @@ +agents: + - { agent: openhands, model: deepseek/deepseek-v4-flash, count: 3, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-oh-smoke.yaml b/examples/casino/roster-oh-smoke.yaml new file mode 100644 index 000000000..76861e4ad --- /dev/null +++ b/examples/casino/roster-oh-smoke.yaml @@ -0,0 +1,2 @@ +agents: + - { agent: openhands, model: deepseek/deepseek-v4-flash, count: 2, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-openhands.yaml b/examples/casino/roster-openhands.yaml new file mode 100644 index 000000000..99c2c7a4a --- /dev/null +++ b/examples/casino/roster-openhands.yaml @@ -0,0 +1,3 @@ +# 10 openhands seats on deepseek-v4-flash (API-key/proxy seats → no subscription limit) +agents: + - { agent: openhands, model: deepseek/deepseek-v4-flash, count: 10, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-smoke-cc.yaml b/examples/casino/roster-smoke-cc.yaml new file mode 100644 index 000000000..0c21ef90c --- /dev/null +++ b/examples/casino/roster-smoke-cc.yaml @@ -0,0 +1,3 @@ +# 1-seat smoke: claude-agent-acp + claude-fable-5 +agents: + - { agent: claude-agent-acp, model: claude-opus-4-8, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-smoke-cx.yaml b/examples/casino/roster-smoke-cx.yaml new file mode 100644 index 000000000..c5838cd6f --- /dev/null +++ b/examples/casino/roster-smoke-cx.yaml @@ -0,0 +1,3 @@ +# 1-seat smoke: codex-acp + gpt-5.5 +agents: + - { agent: codex-acp, model: gpt-5.5, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-smoke-oc.yaml b/examples/casino/roster-smoke-oc.yaml new file mode 100644 index 000000000..beb6d63f0 --- /dev/null +++ b/examples/casino/roster-smoke-oc.yaml @@ -0,0 +1,3 @@ +# 1-seat smoke: opencode + deepseek/deepseek-v4-pro +agents: + - { agent: opencode, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-smoke-oh.yaml b/examples/casino/roster-smoke-oh.yaml new file mode 100644 index 000000000..362262005 --- /dev/null +++ b/examples/casino/roster-smoke-oh.yaml @@ -0,0 +1,3 @@ +# 1-seat smoke: openhands + deepseek/deepseek-v4-pro +agents: + - { agent: openhands, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster-smoke-pi.yaml b/examples/casino/roster-smoke-pi.yaml new file mode 100644 index 000000000..7fac4c427 --- /dev/null +++ b/examples/casino/roster-smoke-pi.yaml @@ -0,0 +1,3 @@ +# 1-seat smoke: pi-acp + deepseek/deepseek-v4-pro +agents: + - { agent: pi-acp, model: deepseek/deepseek-v4-pro, count: 1, instructions: prompts/aggressive.md } diff --git a/examples/casino/roster.yaml b/examples/casino/roster.yaml new file mode 100644 index 000000000..160a8a399 --- /dev/null +++ b/examples/casino/roster.yaml @@ -0,0 +1,29 @@ +# Pure roster — the file form of repeated --agent/--model (the A/M axis only). +# Task / service / sandbox / out / drive come from the standard `bench eval run` +# flags, NOT this file. `instructions:` paths are relative to this file. +# +# set -a; . ~/sb-run.env; set +a +# bench eval run \ +# --agents examples/casino/roster.yaml \ +# --environment-manifest benchmarks/casinobench/environment.toml \ +# --tasks-dir benchmarks/casinobench/tasks/blackjack \ +# --sandbox docker --drive auto-loop \ +# --url-env CASINO_URL --seat-env CASINOBENCH_SEAT_ID \ +# --standings-path /_admin/standings --events-path /_admin/events \ +# --service-env CASINO_MULTIPLAYER=1 \ +# --jobs-dir out/floor/blackjack +# +# The floor flags (--url-env/--seat-env/--standings-path/--events-path/--service-env) +# are GENERAL: --service-env KEY=VALUE (repeatable) passes benchmark-specific env to +# the in-sandbox service — casino uses CASINO_MULTIPLAYER=1; another env-0 benchmark +# passes its own. No casino literal lives in benchflow. + +agents: + # No `name:` → the seat/player id is - (e.g. codex-acp-gpt-5.5), + # so the floor + viewer show which agent + model each player is. + # subscription seats (oauth) → ACP trajectory only + - { agent: codex-acp, model: gpt-5.5, count: 2, instructions: prompts/aggressive.md } + - { agent: claude-agent-acp, model: claude-sonnet-4-6, count: 2, instructions: prompts/cautious.md } + # proxy seat (API key → per-seat LiteLLM proxy) → raw + ACP trajectory. + # deepagents is in the seat image; openhands would need adding to install-node-agents.sh. + - { agent: deepagents, model: deepseek/deepseek-v4-pro, instructions: prompts/aggressive.md } diff --git a/examples/casino/run_floor.py b/examples/casino/run_floor.py new file mode 100644 index 000000000..942ecb816 --- /dev/null +++ b/examples/casino/run_floor.py @@ -0,0 +1,295 @@ +"""Real multi-agent casino floor on BenchFlow — heterogeneous, all games. + +N autonomous ACP agents play ONE shared casinobench World concurrently, each in +its OWN DockerSandbox (the casino-agent-seat image: node ACP agents + the `casino` +seven-tool CLI). The default roster is 4 subscription seats: + - 2x codex-acp on gpt-5.5 (ChatGPT subscription) + - 2x claude-agent-acp on sonnet-4-6 (Claude subscription) + +Each subscription agent calls its provider directly (oauth) → it produces an +`acp_trajectory.jsonl` (its `casino` tool-calls/thinking); there is NO raw +`llm_trajectory` for subscription seats (that needs an API key fronted by the +proxy — only proxy-routed deepseek seats get one). The World runs on the host; +agents reach it over the docker bridge gateway (the same path the proxy uses). + + set -a; . ~/sb-run.env; set +a + uv run python examples/casino/run_floor.py +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import os +import socket +import subprocess +from pathlib import Path + +import httpx + +from benchflow.acp.runtime import AgentPromptTimeoutError, connect_acp, execute_prompts +from benchflow.agents.credentials import upload_subscription_auth +from benchflow.agents.registry import AGENTS +from benchflow.providers import ( + ensure_litellm_runtime, + extract_usage, + stop_provider_runtime, +) +from benchflow.providers.litellm_runtime import _docker_host_address +from benchflow.sandbox.docker import DockerSandbox +from benchflow.task.config import SandboxConfig +from benchflow.trajectories._capture import TrajectoryWriter, make_trajectory_sink + +HERE = Path(__file__).resolve().parent +AGENT_ENV_DIR = HERE / "agent_env" +CASINOBENCH = Path(os.environ.get("CASINOBENCH_DIR", Path.home() / "casinobench")) +BRIDGE = _docker_host_address() # the host gateway agent containers can reach + +# roster: (agent, model, label, count) → seats