From 7deb4b2cfae1508d908cd9c7609dcde28e34b8d1 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Thu, 2 Jul 2026 21:53:03 -0700
Subject: [PATCH 01/11] fix(adapters): resolve Toolathlon MCP tokens at spawn +
 exit-code verifier scoring

Follow-up to #885. Makes the materialized Toolathlon tasks actually runnable.

1. Token spawn-time resolution. The ${token.X}->${TOOLATHLON_X} env rewrite
   could never resolve per-task or runtime-computed tokens (Drive folder ids
   written by preprocess, groundtruth allowlists, per-student Canvas tokens):
   MCP env/args pass through verbatim and args cannot be resolved from env at
   all. New _toolathlon_container.py ships into each container: `write-config`
   bakes the global token_key_session.py from injected TOOLATHLON_* secrets;
   `launch` wraps every MCP server and resolves ${token.X}/${config}/paths
   across argv AND env at spawn, mirroring upstream utils/mcp/tool_servers.py
   (global merged with per-task override). Dep-free via an addict shim; absolute
   interpreter + PATH fallback so FastMCP can spawn it.

2. Pre-create declared server dirs. arxiv_local (--storage-path) and the emails
   server create working dirs lazily; an agent that reaches the goal another way
   leaves them absent and evaluators that os.listdir them crash. The launcher
   mkdir -p's an explicit allow-list of directory-valued flags.

3. Exit-code verifier scoring. Upstream evaluators score by exit code and
   several signal task-failure by raising (e.g. raise ValueError("Some tests
   FAILED")) rather than returning False. Treat any non-zero evaluator exit as
   reward 0 (healthy fail); only escalate to a verifier error on signatures that
   cannot be an agent failure (ModuleNotFoundError/ImportError/PermissionError).

4. Tolerate task_config entries naming local harness tools (e.g. web_search)
   with no MCP yaml instead of failing the whole adaptation.

Validated on Daytona+OpenHands+Azure gpt-5.5: 37/58 headless tasks healthy on
first pass; find-alita-paper and google-cloud/maps/github/hf probes all reward
1.0. 9 adapter tests, ruff + ty clean.
---
 src/benchflow/adapters/_toolathlon.py         | 212 ++++++++++++----
 .../adapters/_toolathlon_container.py         | 227 ++++++++++++++++++
 tests/test_source_adapters.py                 | 213 +++++++++++++++-
 3 files changed, 597 insertions(+), 55 deletions(-)
 create mode 100644 src/benchflow/adapters/_toolathlon_container.py

diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py
index 1fd809b2..0e892651 100644
--- a/src/benchflow/adapters/_toolathlon.py
+++ b/src/benchflow/adapters/_toolathlon.py
@@ -3,19 +3,22 @@
 from __future__ import annotations
 
 import json
+import logging
 import re
 import shlex
 import shutil
 from dataclasses import dataclass
+from importlib import resources
 from pathlib import Path
 from typing import Any
 
 import tomli_w
 import yaml
 
+logger = logging.getLogger(__name__)
+
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
-_TOKEN_PLACEHOLDER_RE = re.compile(r"\$\{token\.([A-Za-z0-9_]+)\}")
 _TOOLATHLON_UVX_PACKAGE_PINS = {
     "office-word-mcp-server": "office-word-mcp-server==1.1.11",
 }
@@ -27,7 +30,6 @@ class _ToolathlonCredentialSpec:
     env: str
     b64_env: str
     trigger_servers: tuple[str, ...] = ()
-    token_defaults: tuple[tuple[str, str], ...] = ()
     copy_paths: tuple[str, ...] = ()
 
 
@@ -37,28 +39,12 @@ class _ToolathlonCredentialSpec:
         env="TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON",
         b64_env="TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON_B64",
         trigger_servers=("google-cloud",),
-        token_defaults=(
-            (
-                "gcp_service_account_path",
-                "/workspace/configs/gcp-service_account.keys.json",
-            ),
-        ),
     ),
     _ToolathlonCredentialSpec(
         path="configs/google_credentials.json",
         env="TOOLATHLON_GOOGLE_CREDENTIALS_JSON",
         b64_env="TOOLATHLON_GOOGLE_CREDENTIALS_JSON_B64",
         trigger_servers=("google_calendar", "google_sheet"),
-        token_defaults=(
-            (
-                "google_oauth2_credentials_path",
-                "/workspace/configs/google_credentials.json",
-            ),
-            (
-                "google_oauth2_token_path",
-                "/workspace/agent_workspace/.toolathlon/google_credentials.json",
-            ),
-        ),
         copy_paths=(
             "agent_workspace/.toolathlon/google_credentials.json",
             "agent_workspace/.toolathlon/calendar_credentials.json",
@@ -77,11 +63,59 @@ class _ToolathlonCredentialSpec:
 _TOOLATHLON_CREDENTIAL_REF_RE = re.compile(
     "|".join(re.escape(spec.path) for spec in _TOOLATHLON_CREDENTIAL_SPECS)
 )
-_TOOLATHLON_TOKEN_DEFAULTS = {
-    token: value
-    for spec in _TOOLATHLON_CREDENTIAL_SPECS
-    for token, value in spec.token_defaults
-}
+
+# Secret ``token.X`` values baked into the container-side global
+# ``token_key_session.py`` (see ``_toolathlon_container._GLOBAL_TOKENS``),
+# resolved host-side from BenchFlow's environment/.env at setup time.
+_TOOLATHLON_TOKEN_SECRET_ENVS = (
+    "TOOLATHLON_GCP_PROJECT_ID",
+    "TOOLATHLON_MAPS_API_KEY",
+    "TOOLATHLON_SERPER_API_KEY",
+    "TOOLATHLON_GITHUB_TOKEN",
+    "TOOLATHLON_HF_TOKEN",
+    "TOOLATHLON_WANDB_API_KEY",
+    "TOOLATHLON_NOTION_KEY",
+    "TOOLATHLON_NOTION_KEY_EVAL",
+    "TOOLATHLON_GOOGLE_CLIENT_ID",
+    "TOOLATHLON_GOOGLE_CLIENT_SECRET",
+    "TOOLATHLON_GOOGLE_REFRESH_TOKEN",
+    "TOOLATHLON_SNOWFLAKE_ACCOUNT",
+    "TOOLATHLON_SNOWFLAKE_USER",
+    "TOOLATHLON_SNOWFLAKE_ROLE",
+    "TOOLATHLON_SNOWFLAKE_WAREHOUSE",
+    "TOOLATHLON_SNOWFLAKE_DATABASE",
+    "TOOLATHLON_SNOWFLAKE_SCHEMA",
+)
+
+_TOOLATHLON_CONTAINER_MODULE_PATH = "/workspace/.toolathlon/toolathlon_container.py"
+
+# MCP server flags whose value is a working DIRECTORY the server (and the task's
+# evaluator) expect to exist. Upstream servers create these lazily, so an agent
+# that never exercises the server leaves the directory absent and evaluators
+# that ``os.listdir`` it crash. The launcher ``mkdir -p``s these. Explicit
+# allow-list, not a heuristic: every entry takes a directory (never a file).
+_TOOLATHLON_MCP_DIR_ARG_FLAGS = frozenset(
+    {
+        "--storage-path",
+        "--attachment_download_path",
+        "--attachment_upload_path",
+        "--email_export_path",
+    }
+)
+
+
+def _toolathlon_container_python(variant: str) -> str:
+    # Absolute so FastMCP can spawn the launcher even without PATH in the MCP
+    # server subprocess environment.
+    return "/opt/venv/bin/python3" if variant == "gym" else "/usr/bin/python3"
+
+
+def _toolathlon_container_module_source() -> str:
+    return (
+        resources.files("benchflow.adapters")
+        .joinpath("_toolathlon_container.py")
+        .read_text()
+    )
 
 
 def _safe_name(value: str) -> str:
@@ -131,10 +165,24 @@ def materialize_toolathlon(ctx: Any, output_dir: Path, *, variant: str) -> None:
         credential_refs = _toolathlon_credential_refs_for_task(
             upstream_task, task_config, variant=variant
         )
-        mcp_servers = [
-            _toolathlon_mcp_server(ctx, server_name, variant=variant)
-            for server_name in task_config.get("needed_mcp_servers", [])
-        ]
+        mcp_servers = []
+        for server_name in task_config.get("needed_mcp_servers", []):
+            try:
+                mcp_servers.append(
+                    _toolathlon_mcp_server(
+                        ctx, server_name, variant=variant, task_name=task_name
+                    )
+                )
+            except FileNotFoundError:
+                # ``task_config`` may name a local harness tool (e.g.
+                # ``web_search``) rather than an MCP server — upstream resolves
+                # those from its agent-tool registry, so no configs/mcp_servers
+                # yaml exists. Drop it instead of failing the adaptation.
+                logger.warning(
+                    "Toolathlon task %s: no MCP config for %r; dropping server",
+                    task_name,
+                    server_name,
+                )
 
         _write_text(task_dir / "instruction.md", _toolathlon_instruction(upstream_task))
         _write_toml(
@@ -218,6 +266,14 @@ def _toolathlon_task_toml(
 ) -> dict[str, Any]:
     benchmark_name = "toolathlon-gym" if variant == "gym" else "toolathlon"
     setup_commands: list[dict[str, Any]] = []
+    setup_commands.append(
+        {
+            "command": _toolathlon_token_setup_command(variant),
+            "cwd": "/workspace",
+            "timeout_sec": 60.0,
+            "env": _toolathlon_token_setup_env(),
+        }
+    )
     credential_command = _toolathlon_credential_setup_command(credential_refs)
     if credential_command:
         setup_commands.append(
@@ -370,6 +426,50 @@ def _toolathlon_credential_setup_command(credential_refs: set[str]) -> str | Non
     )
 
 
+def _toolathlon_token_setup_env() -> dict[str, str]:
+    """Env for the token-setup command: the container helper (base64) plus the
+    secret token values, resolved host-side from BenchFlow's environment/.env."""
+    import base64
+
+    module_b64 = base64.b64encode(
+        _toolathlon_container_module_source().encode()
+    ).decode()
+    env: dict[str, str] = {"TOOLATHLON_CONTAINER_MODULE_B64": module_b64}
+    for name in _TOOLATHLON_TOKEN_SECRET_ENVS:
+        env[name] = f"${{{name}:-}}"
+    return env
+
+
+def _toolathlon_token_setup_command(variant: str) -> str:
+    """Stage the container helper and generate the global token_key_session.py.
+
+    The helper is decoded from ``TOOLATHLON_CONTAINER_MODULE_B64`` and its
+    ``write-config`` mode bakes the injected secret tokens into
+    ``/workspace/configs/token_key_session.py`` (gitignored upstream, so absent
+    from the image). MCP servers later resolve ``${token.X}`` against it via the
+    same helper's ``launch`` mode.
+    """
+    python_cmd = _toolathlon_container_python(variant)
+    module_path = shlex.quote(_TOOLATHLON_CONTAINER_MODULE_PATH)
+    return "\n".join(
+        [
+            "set -e",
+            "mkdir -p /workspace/.toolathlon",
+            f"{python_cmd} - <<'PY'",
+            "import base64, os, pathlib",
+            f"target = pathlib.Path({_TOOLATHLON_CONTAINER_MODULE_PATH!r})",
+            "target.write_bytes(",
+            "    base64.b64decode(os.environ['TOOLATHLON_CONTAINER_MODULE_B64'])",
+            ")",
+            "target.chmod(0o755)",
+            "PY",
+            "chmod -R a+rX /workspace/.toolathlon",
+            f"{python_cmd} {module_path} write-config",
+            "chmod 0644 /workspace/configs/token_key_session.py",
+        ]
+    )
+
+
 def _toolathlon_setup_command(*, task_name: str, variant: str) -> str:
     python_cmd = (
         "/opt/venv/bin/python3" if variant == "gym" else "/usr/local/bin/uv run python"
@@ -455,8 +555,16 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
             'if [ "$status" -eq 0 ]; then',
             "  printf '{\"reward\": 1.0}\\n' > /logs/verifier/reward.json",
             "  printf '1.0\\n' > /logs/verifier/reward.txt",
-            "elif grep -q 'Traceback (most recent call last):' \"$EVAL_LOG\"; then",
-            '  echo "BenchFlow Toolathlon verifier setup error: evaluator crashed" >&2',
+            # Upstream evaluators score by exit code: a non-zero exit is a task
+            # FAIL (reward 0), and several signal failure by *raising* (e.g.
+            # ``raise ValueError('Some tests FAILED')``) rather than returning
+            # False. So a traceback alone is not an infra error. Only escalate on
+            # signatures that can't be an agent failure — a broken evaluator
+            # ENVIRONMENT (missing module/dependency, permission denied) — which
+            # a rerun cannot fix and which should surface, not silently score 0.
+            "elif grep -qE "
+            "'ModuleNotFoundError|ImportError|PermissionError: ' \"$EVAL_LOG\"; then",
+            '  echo "BenchFlow Toolathlon verifier setup error: evaluator environment failure" >&2',
             '  exit "$status"',
             "else",
             "  printf '{\"reward\": 0.0}\\n' > /logs/verifier/reward.json",
@@ -477,11 +585,15 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
 
 
 def _toolathlon_mcp_server(
-    ctx: Any, server_name: str, *, variant: str
+    ctx: Any, server_name: str, *, variant: str, task_name: str
 ) -> dict[str, Any]:
     config_path = _toolathlon_mcp_config_path(ctx, server_name)
     data = yaml.safe_load(config_path.read_text())
     params = data.get("params") or {}
+    # Static path placeholders (``${agent_workspace}`` …) resolve to fixed
+    # in-container paths now; ``${token.X}`` are left intact and resolved at
+    # spawn time by the container launcher, the only layer that sees per-task
+    # and runtime-computed token values.
     command = _replace_toolathlon_placeholders(
         str(params.get("command", "")), variant=variant
     )
@@ -502,21 +614,34 @@ def _toolathlon_mcp_server(
         )
     cwd = params.get("cwd")
     cwd = _replace_toolathlon_placeholders(str(cwd), variant=variant) if cwd else None
-    if variant == "official" and command in {"uv", "uvx"}:
+    if command in {"uv", "uvx"}:
         command = f"/usr/local/bin/{command}"
         if args:
             args = [_TOOLATHLON_UVX_PACKAGE_PINS.get(arg, arg) for arg in args]
+    # Wrap the real server in the container launcher so ``${token.X}`` in argv
+    # and env resolve at spawn time. ``TOOLATHLON_TASK_DIR`` tells the launcher
+    # which task's token_key_session.py overrides the global one.
+    env = dict(env)
+    env["TOOLATHLON_TASK_DIR"] = f"/workspace/tasks/finalpool/{task_name}"
+    ensure_dirs = [
+        args[i + 1]
+        for i, arg in enumerate(args[:-1])
+        if arg in _TOOLATHLON_MCP_DIR_ARG_FLAGS
+    ]
+    if ensure_dirs:
+        # ``:``-joined for the Linux container launcher (not os.pathsep, which
+        # would follow the host running the adapter).
+        env["TOOLATHLON_ENSURE_DIRS"] = ":".join(ensure_dirs)
     payload: dict[str, Any] = {
         "name": str(data.get("name") or server_name),
         "transport": "stdio",
-        "command": command,
-        "args": args,
+        "command": _toolathlon_container_python(variant),
+        "args": [_TOOLATHLON_CONTAINER_MODULE_PATH, "launch", command, *args],
+        "env": env,
         "exclude_tags": [_NOOP_EXCLUDE_TAG],
     }
     if cwd:
         payload["cwd"] = cwd
-    if env:
-        payload["env"] = env
     return payload
 
 
@@ -577,6 +702,13 @@ def _toolathlon_mcp_config_path(ctx: Any, server_name: str) -> Path:
 
 
 def _replace_toolathlon_placeholders(value: str, *, variant: str) -> str:
+    """Resolve static path placeholders to fixed in-container paths.
+
+    ``${token.X}`` placeholders are intentionally left untouched: they are
+    resolved at MCP spawn time by the container launcher (see
+    ``_toolathlon_container.py``), the only layer that can see per-task and
+    runtime-computed token values.
+    """
     local_servers = (
         "/opt/local_servers" if variant == "gym" else "/workspace/local_servers"
     )
@@ -588,15 +720,7 @@ def _replace_toolathlon_placeholders(value: str, *, variant: str) -> str:
     }
     for old, new in replacements.items():
         value = value.replace(old, new)
-
-    def token_replacement(match: re.Match[str]) -> str:
-        token_name = match.group(1)
-        if variant == "official" and token_name in _TOOLATHLON_TOKEN_DEFAULTS:
-            return _TOOLATHLON_TOKEN_DEFAULTS[token_name]
-        name = token_name.upper()
-        return f"${{TOOLATHLON_{name}}}"
-
-    return _TOKEN_PLACEHOLDER_RE.sub(token_replacement, value)
+    return value
 
 
 def _toolathlon_dockerfile(ctx: Any, *, variant: str) -> str:
diff --git a/src/benchflow/adapters/_toolathlon_container.py b/src/benchflow/adapters/_toolathlon_container.py
new file mode 100644
index 00000000..ac9b2077
--- /dev/null
+++ b/src/benchflow/adapters/_toolathlon_container.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Toolathlon container-side helper — runs INSIDE the task sandbox, never
+imported by host BenchFlow.
+
+The BenchFlow Toolathlon adapter ships this file into each task container and
+invokes it in two roles:
+
+``write-config``
+    Generate ``/workspace/configs/token_key_session.py`` (the global
+    ``all_token_key_session`` dict) from ``TOOLATHLON_*`` environment variables
+    injected at setup time. Upstream gitignores this file; without it the MCP
+    servers have no credentials.
+
+``launch <command> [args...]``
+    Wrap an MCP server. Resolve ``${token.X}`` / ``${config.X}`` / static path
+    placeholders across the server's argv **and** environment, then ``exec`` it.
+    Resolution mirrors upstream ``utils/mcp/tool_servers.py`` — the global
+    ``all_token_key_session`` merged with, and overridden by, the task's own
+    ``token_key_session.py``. This is the only layer that sees per-task and
+    runtime-computed tokens (Drive folder ids written by preprocess,
+    groundtruth-derived allowlists, per-student Canvas tokens, …), so it is
+    where they must be resolved — the host adapter cannot know them.
+
+Kept dependency-free (plain ``python3``, with an ``addict`` shim) so it needs
+neither the upstream venv nor a ``uv`` round-trip on every MCP spawn.
+"""
+
+from __future__ import annotations
+
+import importlib.machinery
+import importlib.util
+import os
+import re
+import sys
+import types
+from pathlib import Path
+
+_PLACEHOLDER = re.compile(r"\$\{([^}]+)\}")
+
+
+def _workspace() -> Path:
+    return Path(os.environ.get("TOOLATHLON_WORKSPACE", "/workspace"))
+
+
+class _AttrDict(dict):
+    """dict with attribute access, enough to stand in for ``addict.Dict`` when
+    executing upstream token files without the real dependency."""
+
+    def __getattr__(self, name: str):  # pragma: no cover - trivial
+        return self.get(name)
+
+    def __setattr__(self, name: str, value: object) -> None:  # pragma: no cover
+        self[name] = value
+
+
+def _ensure_addict_importable() -> None:
+    """Upstream token files do ``from addict import Dict``. Fall back to a
+    plain-dict shim when the real package is absent so this helper stays
+    dependency-free."""
+    if "addict" in sys.modules:
+        return
+    try:
+        import addict  # noqa: F401
+    except Exception:
+        shim = types.ModuleType("addict")
+        shim.__spec__ = importlib.machinery.ModuleSpec("addict", loader=None)
+        shim.__dict__["Dict"] = _AttrDict
+        sys.modules["addict"] = shim
+
+
+def _load_all_token_key_session(path: Path) -> dict:
+    if not path.is_file():
+        return {}
+    _ensure_addict_importable()
+    spec = importlib.util.spec_from_file_location(f"_tks_{abs(hash(path))}", path)
+    if spec is None or spec.loader is None:
+        return {}
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return dict(getattr(module, "all_token_key_session", {}) or {})
+
+
+# --------------------------------------------------------------------------- #
+# launch: resolve placeholders across an MCP server's argv + env, then exec.
+# --------------------------------------------------------------------------- #
+def _template_vars(task_dir: Path) -> dict[str, str]:
+    workspace = _workspace()
+    variables: dict[str, str] = {
+        "agent_workspace": str(workspace / "agent_workspace"),
+        "local_servers_paths": str(workspace / "local_servers"),
+        "local_binary_paths": str(workspace / "local_binary"),
+        "task_dir": str(workspace / "tasks" / "finalpool"),
+    }
+    tokens = _load_all_token_key_session(workspace / "configs" / "token_key_session.py")
+    tokens.update(_load_all_token_key_session(task_dir / "token_key_session.py"))
+    for key, value in tokens.items():
+        if isinstance(value, (str, int, float, bool)):
+            variables[f"token.{key}"] = str(value)
+    return variables
+
+
+def _resolve(value: str, variables: dict[str, str]) -> str:
+    return _PLACEHOLDER.sub(
+        lambda m: variables.get(m.group(1), m.group(0)),
+        value,
+    )
+
+
+def _launch(argv: list[str]) -> int:
+    if not argv:
+        sys.stderr.write("toolathlon launch: no command given\n")
+        return 2
+    task_dir = Path(os.environ.get("TOOLATHLON_TASK_DIR", str(_workspace())))
+    variables = _template_vars(task_dir)
+    resolved_argv = [_resolve(arg, variables) for arg in argv]
+    resolved_env = dict(os.environ)
+    for key, value in list(resolved_env.items()):
+        if "${" in value:
+            resolved_env[key] = _resolve(value, variables)
+    # Create directories the server declares (e.g. arxiv --storage-path, emails
+    # attachment dirs) so servers that create them lazily — and evaluators that
+    # list them — do not fail when the agent never exercises that path.
+    for directory in resolved_env.get("TOOLATHLON_ENSURE_DIRS", "").split(":"):
+        if directory:
+            Path(directory).mkdir(parents=True, exist_ok=True)
+    # FastMCP may spawn this launcher without inheriting PATH; guarantee a sane
+    # default so the wrapped server (uvx / npx / node) is still resolvable.
+    if not resolved_env.get("PATH"):
+        resolved_env["PATH"] = "/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin"
+    os.execvpe(resolved_argv[0], resolved_argv, resolved_env)
+    return 0  # unreachable when exec succeeds
+
+
+# --------------------------------------------------------------------------- #
+# write-config: materialize the global token_key_session.py from env.
+# --------------------------------------------------------------------------- #
+# (token key, env var, default) — secrets come from env; non-secret defaults
+# reproduce configs/token_key_session_example.py. Per-task values (allowlists,
+# folder ids, canvas/woo/notion scopes) stay at their "null"/placeholder default
+# here and are overridden by each task's own token_key_session.py at launch.
+_GLOBAL_TOKENS: tuple[tuple[str, str | None, object], ...] = (
+    ("timezone", None, "Asia/Hong_Kong"),
+    ("serper_api_key", "TOOLATHLON_SERPER_API_KEY", "XX"),
+    ("google_cloud_console_api_key", "TOOLATHLON_MAPS_API_KEY", "XX"),
+    ("gcp_project_id", "TOOLATHLON_GCP_PROJECT_ID", "XX"),
+    (
+        "gcp_service_account_path",
+        None,
+        "/workspace/configs/gcp-service_account.keys.json",
+    ),
+    ("google_client_id", "TOOLATHLON_GOOGLE_CLIENT_ID", ""),
+    ("google_client_secret", "TOOLATHLON_GOOGLE_CLIENT_SECRET", ""),
+    ("google_refresh_token", "TOOLATHLON_GOOGLE_REFRESH_TOKEN", ""),
+    ("google_sheets_folder_id", None, "null"),
+    (
+        "google_oauth2_credentials_path",
+        None,
+        "/workspace/configs/google_credentials.json",
+    ),
+    (
+        "google_oauth2_token_path",
+        None,
+        "/workspace/agent_workspace/.toolathlon/google_credentials.json",
+    ),
+    ("google_cloud_allowed_buckets", None, "null"),
+    ("google_cloud_allowed_bigquery_datasets", None, "null"),
+    ("google_cloud_allowed_log_buckets", None, "null"),
+    ("google_cloud_allowed_instances", None, "null"),
+    ("github_token", "TOOLATHLON_GITHUB_TOKEN", "XX"),
+    ("github_allowed_repos", None, "null"),
+    ("github_read_only", None, "1"),
+    ("huggingface_token", "TOOLATHLON_HF_TOKEN", "XX"),
+    ("wandb_api_key", "TOOLATHLON_WANDB_API_KEY", "XX"),
+    ("notion_integration_key", "TOOLATHLON_NOTION_KEY", "XX"),
+    ("notion_integration_key_eval", "TOOLATHLON_NOTION_KEY_EVAL", "XX"),
+    ("snowflake_account", "TOOLATHLON_SNOWFLAKE_ACCOUNT", "XX"),
+    ("snowflake_warehouse", "TOOLATHLON_SNOWFLAKE_WAREHOUSE", "COMPUTE_WH"),
+    ("snowflake_role", "TOOLATHLON_SNOWFLAKE_ROLE", "ACCOUNTADMIN"),
+    ("snowflake_user", "TOOLATHLON_SNOWFLAKE_USER", "XX"),
+    (
+        "snowflake_private_key_path",
+        None,
+        "/workspace/configs/snowflake_rsa_key.p8",
+    ),
+    ("snowflake_database", "TOOLATHLON_SNOWFLAKE_DATABASE", "SNOWFLAKE"),
+    ("snowflake_schema", "TOOLATHLON_SNOWFLAKE_SCHEMA", "PUBLIC"),
+    ("snowflake_op_allowed_databases", None, "null"),
+    ("canvas_api_token", None, "canvas_token_victoria_14z"),
+    ("canvas_domain", None, "localhost:20001"),
+    ("woocommerce_api_key", None, "ck_woocommerce_token_PE0613bf053"),
+    ("woocommerce_api_secret", None, "cs_woocommerce_token_PE0613bf053"),
+    ("woocommerce_site_url", None, "http://localhost:10003/store100"),
+    ("kubeconfig_path", None, "deployment/k8s/configs/cluster1-config.yaml"),
+    ("emails_config_file", None, "configs/example_email_config.json"),
+)
+
+
+def _write_config() -> int:
+    tokens: dict[str, object] = {}
+    for key, env_var, default in _GLOBAL_TOKENS:
+        value = os.environ.get(env_var) if env_var else None
+        tokens[key] = value if value not in (None, "") else default
+    lines = ["all_token_key_session = {"]
+    lines += [f"    {key!r}: {value!r}," for key, value in tokens.items()]
+    lines.append("}")
+    target = _workspace() / "configs" / "token_key_session.py"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text("\n".join(lines) + "\n")
+    target.chmod(0o644)
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    if not argv:
+        sys.stderr.write("toolathlon_container: expected 'write-config' or 'launch'\n")
+        return 2
+    mode, rest = argv[0], argv[1:]
+    if mode == "write-config":
+        return _write_config()
+    if mode == "launch":
+        return _launch(rest)
+    sys.stderr.write(f"toolathlon_container: unknown mode {mode!r}\n")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index 3a0b340b..e14bf346 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -143,15 +143,29 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup(
     assert task.config.environment.mcp_servers[0].exclude_tags == [
         "__benchflow_exclude_no_tools__"
     ]
+    # The real server is wrapped in the container launcher so ${token.X} in
+    # argv/env resolves at spawn time against the per-task token file.
     word = task.config.environment.mcp_servers[1]
-    assert word.command == "/usr/local/bin/uvx"
+    assert word.command == "/usr/bin/python3"
     assert word.args == [
+        "/workspace/.toolathlon/toolathlon_container.py",
+        "launch",
+        "/usr/local/bin/uvx",
         "--from",
         "office-word-mcp-server==1.1.11",
         "word_mcp_server",
     ]
     assert word.cwd == "/workspace/agent_workspace"
-    setup_command = task.config.environment.setup_commands[0].command
+    assert (
+        word.env["TOOLATHLON_TASK_DIR"]
+        == "/workspace/tasks/finalpool/arrange-workspace"
+    )
+    # First setup command stages the container helper and writes the global
+    # token_key_session.py; the preprocess command runs last.
+    token_setup = task.config.environment.setup_commands[0].command
+    assert "toolathlon_container.py write-config" in token_setup
+    assert "TOOLATHLON_CONTAINER_MODULE_B64" in token_setup
+    setup_command = task.config.environment.setup_commands[-1].command
     dockerfile = (generated / "environment" / "Dockerfile").read_text()
     task_toml = (generated / "task.toml").read_text()
     test_sh = (generated / "tests" / "test.sh").read_text()
@@ -166,8 +180,11 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup(
     assert 'chmod -R go-rwx "$private"' in setup_command
     assert "BenchFlow Toolathlon verifier setup error" in test_sh
     assert "toolathlon_evaluator.log" in test_sh
-    assert "evaluator crashed" in test_sh
-    assert "Traceback (most recent call last):" in test_sh
+    # A non-zero evaluator exit is a task fail (reward 0), matching upstream's
+    # exit-code scoring; only a broken evaluator ENVIRONMENT escalates.
+    assert "evaluator environment failure" in test_sh
+    assert "ModuleNotFoundError|ImportError|PermissionError" in test_sh
+    assert "Traceback (most recent call last):" not in test_sh
     assert "/usr/local/bin/uv run python" in test_sh
 
 
@@ -232,7 +249,7 @@ def test_toolathlon_gym_adapter_normalizes_postgres_env(
         "chmod -R a+rwX /workspace/agent_workspace"
         in (generated / "task.toml").read_text()
     )
-    setup_command = task.config.environment.setup_commands[0].command
+    setup_command = task.config.environment.setup_commands[-1].command
     assert 'chmod -R go-rwx "$private"' in setup_command
     assert (
         "postgres:" in (generated / "environment" / "docker-compose.yaml").read_text()
@@ -360,8 +377,13 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs(
         }
     ]
     assert not (adapted.path / ".benchflow-source-adapter-skipped.json").exists()
-    assert len(task.config.environment.setup_commands) == 2
-    credential_setup = task.config.environment.setup_commands[0]
+    # setup_commands: [token-setup, credential-inject, preprocess].
+    assert len(task.config.environment.setup_commands) == 3
+    assert (
+        "toolathlon_container.py write-config"
+        in task.config.environment.setup_commands[0].command
+    )
+    credential_setup = task.config.environment.setup_commands[1]
     assert credential_setup.env == {
         "TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON": "${TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON:-}",
         "TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON_B64": "${TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON_B64:-}",
@@ -371,16 +393,26 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs(
     assert "target.chmod(0o644)" in credential_setup.command
     assert "/home/agent" not in credential_setup.command
     assert ".gmail-mcp" not in credential_setup.command
+    # ${token.X} env stays literal at materialization; the launcher resolves it
+    # at spawn time against the container's token_key_session files.
     sheet = Task(adapted.path / "sheet-only")
     assert sheet.config.metadata["required_credential_files"] == [
         "configs/google_credentials.json"
     ]
     sheet_server = sheet.config.environment.mcp_servers[0]
+    assert sheet_server.command == "/usr/bin/python3"
+    assert sheet_server.args[:3] == [
+        "/workspace/.toolathlon/toolathlon_container.py",
+        "launch",
+        "/usr/local/bin/uvx",
+    ]
     assert sheet_server.env["CREDENTIALS_PATH"] == (
-        "/workspace/configs/google_credentials.json"
+        "${token.google_oauth2_credentials_path}"
     )
-    assert sheet_server.env["TOKEN_PATH"] == (
-        "/workspace/agent_workspace/.toolathlon/google_credentials.json"
+    assert sheet_server.env["TOKEN_PATH"] == "${token.google_oauth2_token_path}"
+    assert (
+        sheet_server.env["TOOLATHLON_TASK_DIR"]
+        == "/workspace/tasks/finalpool/sheet-only"
     )
     calendar = Task(adapted.path / "calendar-only")
     assert calendar.config.metadata["required_credential_files"] == [
@@ -410,7 +442,7 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs(
     (workspace / "configs" / "gcp-oauth.keys.json").write_text(
         json.dumps(oauth_payload)
     )
-    calendar_command = calendar.config.environment.setup_commands[0].command.replace(
+    calendar_command = calendar.config.environment.setup_commands[1].command.replace(
         "/usr/local/bin/uv run python", sys.executable
     )
     result = subprocess.run(
@@ -448,3 +480,162 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs(
     assert not (workspace / ".mcp-home").exists()
     forms = Task(adapted.path / "forms-only")
     assert "required_credential_files" not in forms.config.metadata
+
+def _run_container_helper(
+    workspace: Path, argv: list[str], env: dict[str, str] | None = None
+) -> subprocess.CompletedProcess:
+    module = (
+        Path(__file__).resolve().parents[1]
+        / "src"
+        / "benchflow"
+        / "adapters"
+        / "_toolathlon_container.py"
+    )
+    full_env = {"PATH": os.environ["PATH"], "TOOLATHLON_WORKSPACE": str(workspace)}
+    full_env.update(env or {})
+    return subprocess.run(
+        [sys.executable, str(module), *argv],
+        env=full_env,
+        text=True,
+        capture_output=True,
+        timeout=30,
+    )
+
+def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None:
+    (tmp_path / "configs").mkdir()
+    result = _run_container_helper(
+        tmp_path,
+        ["write-config"],
+        {
+            "TOOLATHLON_GCP_PROJECT_ID": "proj-123",
+            "TOOLATHLON_MAPS_API_KEY": "maps-abc",
+            "TOOLATHLON_GITHUB_TOKEN": "gho_xyz",
+        },
+    )
+    assert result.returncode == 0, result.stderr
+    generated = (tmp_path / "configs" / "token_key_session.py").read_text()
+    ns: dict[str, object] = {}
+    exec(generated, ns)
+    tokens = ns["all_token_key_session"]
+    assert tokens["gcp_project_id"] == "proj-123"
+    assert tokens["google_cloud_console_api_key"] == "maps-abc"
+    assert tokens["github_token"] == "gho_xyz"
+    # Unset secrets fall back to their example defaults, not the literal env ref.
+    assert tokens["huggingface_token"] == "XX"
+    assert tokens["github_read_only"] == "1"
+
+def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
+    (tmp_path / "configs").mkdir()
+    (tmp_path / "configs" / "token_key_session.py").write_text(
+        "all_token_key_session = {"
+        "'gcp_project_id': 'global-proj',"
+        "'google_cloud_allowed_bigquery_datasets': 'null'}\n"
+    )
+    task_dir = tmp_path / "tasks" / "finalpool" / "demo"
+    task_dir.mkdir(parents=True)
+    # Per-task file overrides the global dataset and derives a value from a file
+    # written by "preprocess" — exactly the runtime-token case.
+    (task_dir / "files").mkdir()
+    (task_dir / "files" / "folder_id.txt").write_text("FID-999")
+    (task_dir / "token_key_session.py").write_text(
+        "import os\n"
+        "from addict import Dict\n"
+        "_here = os.path.dirname(__file__)\n"
+        "with open(os.path.join(_here, 'files', 'folder_id.txt')) as f:\n"
+        "    _fid = f.read().strip()\n"
+        "all_token_key_session = Dict("
+        "google_cloud_allowed_bigquery_datasets='demo_ds',"
+        "google_sheets_folder_id=_fid)\n"
+    )
+    result = _run_container_helper(
+        tmp_path,
+        [
+            "launch",
+            "/bin/echo",
+            "--project=${token.gcp_project_id}",
+            "--dataset=${token.google_cloud_allowed_bigquery_datasets}",
+            "--folder=${token.google_sheets_folder_id}",
+        ],
+        {"TOOLATHLON_TASK_DIR": str(task_dir)},
+    )
+    assert result.returncode == 0, result.stderr
+    out = result.stdout.strip()
+    assert "--project=global-proj" in out  # from global
+    assert "--dataset=demo_ds" in out  # per-task override wins
+    assert "--folder=FID-999" in out  # runtime file value
+
+def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None:
+    (tmp_path / "configs").mkdir()
+    (tmp_path / "configs" / "token_key_session.py").write_text(
+        "all_token_key_session = {'github_token': 'gho_secret'}\n"
+    )
+    result = _run_container_helper(
+        tmp_path,
+        ["launch", "/usr/bin/env"],
+        {
+            "TOOLATHLON_TASK_DIR": str(tmp_path),
+            "GITHUB_TOKEN": "${token.github_token}",
+        },
+    )
+    assert result.returncode == 0, result.stderr
+    assert "GITHUB_TOKEN=gho_secret" in result.stdout
+
+def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None:
+    (tmp_path / "configs").mkdir()
+    (tmp_path / "configs" / "token_key_session.py").write_text(
+        "all_token_key_session = {}\n"
+    )
+    storage = tmp_path / "agent_workspace" / "arxiv_local_storage"
+    assert not storage.exists()
+    result = _run_container_helper(
+        tmp_path,
+        ["launch", "/bin/echo", "ok"],
+        {
+            "TOOLATHLON_TASK_DIR": str(tmp_path),
+            "TOOLATHLON_ENSURE_DIRS": str(storage),
+        },
+    )
+    assert result.returncode == 0, result.stderr
+    assert storage.is_dir()
+
+def test_toolathlon_arxiv_server_declares_ensure_dirs(tmp_path: Path, monkeypatch) -> None:
+    """The arxiv_local server's --storage-path is passed to the launcher as a
+    directory to pre-create, so the evaluator's listdir cannot crash."""
+    monkeypatch.chdir(tmp_path)
+    repo = tmp_path / "Toolathlon"
+    (repo / ".git").mkdir(parents=True)
+    (repo / "global_preparation").mkdir()
+    (repo / "configs" / "mcp_servers").mkdir(parents=True)
+    (repo / "configs" / "users_data.json").write_text("{}")
+    (repo / "configs" / "mcp_servers" / "arxiv_local.yaml").write_text(
+        """
+type: stdio
+name: arxiv_local
+params:
+  command: uv
+  args:
+    - "run"
+    - "python"
+    - "-m"
+    - "utils.local_servers.arxiv_local_wrapper"
+    - "--storage-path"
+    - "${agent_workspace}/arxiv_local_storage"
+  cwd: "${agent_workspace}"
+"""
+    )
+    task_dir = repo / "tasks" / "finalpool" / "find-alita-paper"
+    (task_dir / "docs").mkdir(parents=True)
+    (task_dir / "docs" / "agent_system_prompt.md").write_text("W: !!<<<<||||workspace_dir||||>>>>!!\n")
+    (task_dir / "docs" / "task.md").write_text("Find the paper.\n")
+    (task_dir / "task_config.json").write_text(
+        json.dumps({"needed_mcp_servers": ["arxiv_local"], "needed_local_tools": []})
+    )
+    adapted = adapt_resolved_source_if_needed(
+        _resolved(repo, repo="hkust-nlp/Toolathlon", path="tasks/finalpool")
+    )
+    task = Task(adapted.path / "find-alita-paper")
+    arxiv = task.config.environment.mcp_servers[0]
+    assert (
+        arxiv.env["TOOLATHLON_ENSURE_DIRS"]
+        == "/workspace/agent_workspace/arxiv_local_storage"
+    )

From 8a7e19bf3227e2d52043649765e74a6771d1a33e Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Thu, 2 Jul 2026 21:53:46 -0700
Subject: [PATCH 02/11] chore(adapters): bump Toolathlon adapter version to
 invalidate materialization cache

---
 src/benchflow/adapters/source.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py
index 1d5f167e..758b919d 100644
--- a/src/benchflow/adapters/source.py
+++ b/src/benchflow/adapters/source.py
@@ -25,7 +25,7 @@
 from benchflow._utils.benchmark_repos import ResolvedSource
 from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root
 
-_ADAPTER_VERSION = "2026-07-02.8"
+_ADAPTER_VERSION = "2026-07-03.1"
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 logger = logging.getLogger(__name__)

From d34f38e6876157bd5cd3d9134d19ed5a48d510e4 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Thu, 2 Jul 2026 21:56:58 -0700
Subject: [PATCH 03/11] style(adapters): ruff format Toolathlon adapter + tests

---
 src/benchflow/adapters/_toolathlon_container.py |  4 +++-
 tests/test_source_adapters.py                   | 14 ++++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/benchflow/adapters/_toolathlon_container.py b/src/benchflow/adapters/_toolathlon_container.py
index ac9b2077..a2420516 100644
--- a/src/benchflow/adapters/_toolathlon_container.py
+++ b/src/benchflow/adapters/_toolathlon_container.py
@@ -126,7 +126,9 @@ def _launch(argv: list[str]) -> int:
     # FastMCP may spawn this launcher without inheriting PATH; guarantee a sane
     # default so the wrapped server (uvx / npx / node) is still resolvable.
     if not resolved_env.get("PATH"):
-        resolved_env["PATH"] = "/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin"
+        resolved_env["PATH"] = (
+            "/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin"
+        )
     os.execvpe(resolved_argv[0], resolved_argv, resolved_env)
     return 0  # unreachable when exec succeeds
 
diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index e14bf346..dca58192 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -481,6 +481,7 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs(
     forms = Task(adapted.path / "forms-only")
     assert "required_credential_files" not in forms.config.metadata
 
+
 def _run_container_helper(
     workspace: Path, argv: list[str], env: dict[str, str] | None = None
 ) -> subprocess.CompletedProcess:
@@ -501,6 +502,7 @@ def _run_container_helper(
         timeout=30,
     )
 
+
 def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None:
     (tmp_path / "configs").mkdir()
     result = _run_container_helper(
@@ -524,6 +526,7 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
     assert tokens["huggingface_token"] == "XX"
     assert tokens["github_read_only"] == "1"
 
+
 def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
@@ -564,6 +567,7 @@ def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
     assert "--dataset=demo_ds" in out  # per-task override wins
     assert "--folder=FID-999" in out  # runtime file value
 
+
 def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None:
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
@@ -580,6 +584,7 @@ def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None:
     assert result.returncode == 0, result.stderr
     assert "GITHUB_TOKEN=gho_secret" in result.stdout
 
+
 def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None:
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
@@ -598,7 +603,10 @@ def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None:
     assert result.returncode == 0, result.stderr
     assert storage.is_dir()
 
-def test_toolathlon_arxiv_server_declares_ensure_dirs(tmp_path: Path, monkeypatch) -> None:
+
+def test_toolathlon_arxiv_server_declares_ensure_dirs(
+    tmp_path: Path, monkeypatch
+) -> None:
     """The arxiv_local server's --storage-path is passed to the launcher as a
     directory to pre-create, so the evaluator's listdir cannot crash."""
     monkeypatch.chdir(tmp_path)
@@ -625,7 +633,9 @@ def test_toolathlon_arxiv_server_declares_ensure_dirs(tmp_path: Path, monkeypatc
     )
     task_dir = repo / "tasks" / "finalpool" / "find-alita-paper"
     (task_dir / "docs").mkdir(parents=True)
-    (task_dir / "docs" / "agent_system_prompt.md").write_text("W: !!<<<<||||workspace_dir||||>>>>!!\n")
+    (task_dir / "docs" / "agent_system_prompt.md").write_text(
+        "W: !!<<<<||||workspace_dir||||>>>>!!\n"
+    )
     (task_dir / "docs" / "task.md").write_text("Find the paper.\n")
     (task_dir / "task_config.json").write_text(
         json.dumps({"needed_mcp_servers": ["arxiv_local"], "needed_local_tools": []})

From 729529f2cafc3fd7c39200f2dad2827e6f5ea26a Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Thu, 2 Jul 2026 22:53:04 -0700
Subject: [PATCH 04/11] fix(adapters): generated Toolathlon token dict must
 support attribute access

Upstream preprocess/eval read all_token_key_session via attributes
(all_token_key_session.github_token) because upstream builds it with
addict.Dict. The generated global token_key_session.py used a plain dict, so
every github/HF/woocommerce/notion/snowflake task whose preprocess reads a
token that way crashed with AttributeError. Emit a self-contained
attribute-accessible dict subclass.
---
 .../adapters/_toolathlon_container.py         | 20 +++++++++++++++++--
 tests/test_source_adapters.py                 |  4 ++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/benchflow/adapters/_toolathlon_container.py b/src/benchflow/adapters/_toolathlon_container.py
index a2420516..e4899fb0 100644
--- a/src/benchflow/adapters/_toolathlon_container.py
+++ b/src/benchflow/adapters/_toolathlon_container.py
@@ -202,9 +202,25 @@ def _write_config() -> int:
     for key, env_var, default in _GLOBAL_TOKENS:
         value = os.environ.get(env_var) if env_var else None
         tokens[key] = value if value not in (None, "") else default
-    lines = ["all_token_key_session = {"]
+    # Upstream code reads this via ATTRIBUTE access
+    # (``all_token_key_session.github_token``) because upstream builds it with
+    # ``addict.Dict``. Emit a self-contained attribute-accessible dict so the
+    # generated file needs no third-party import in either the preprocess env or
+    # the launcher; missing keys return None (addict returns an empty Dict, but
+    # nothing here relies on that).
+    lines = [
+        "class _TokenDict(dict):",
+        "    def __getattr__(self, name):",
+        "        try:",
+        "            return self[name]",
+        "        except KeyError:",
+        "            return None",
+        "",
+        "",
+        "all_token_key_session = _TokenDict({",
+    ]
     lines += [f"    {key!r}: {value!r}," for key, value in tokens.items()]
-    lines.append("}")
+    lines.append("})")
     target = _workspace() / "configs" / "token_key_session.py"
     target.parent.mkdir(parents=True, exist_ok=True)
     target.write_text("\n".join(lines) + "\n")
diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index dca58192..b0f7a809 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -525,6 +525,10 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
     # Unset secrets fall back to their example defaults, not the literal env ref.
     assert tokens["huggingface_token"] == "XX"
     assert tokens["github_read_only"] == "1"
+    # Upstream preprocess/eval read tokens via ATTRIBUTE access (addict.Dict);
+    # the generated dict must support it (a plain dict would AttributeError).
+    assert tokens.github_token == "gho_xyz"
+    assert tokens.gcp_project_id == "proj-123"
 
 
 def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:

From 941b24dbfe05412b42417f0849f0e8b9fbc57788 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Thu, 2 Jul 2026 22:53:33 -0700
Subject: [PATCH 05/11] chore(adapters): bump version for token-dict
 attribute-access fix

---
 src/benchflow/adapters/source.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py
index 758b919d..534629d6 100644
--- a/src/benchflow/adapters/source.py
+++ b/src/benchflow/adapters/source.py
@@ -25,7 +25,7 @@
 from benchflow._utils.benchmark_repos import ResolvedSource
 from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root
 
-_ADAPTER_VERSION = "2026-07-03.1"
+_ADAPTER_VERSION = "2026-07-03.2"
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 logger = logging.getLogger(__name__)

From f4ee38c6d7df5ecbee6f5c3197e5ceb74886d441 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Thu, 2 Jul 2026 23:42:14 -0700
Subject: [PATCH 06/11] fix(adapters): run Toolathlon preprocess/eval as full
 module path + pass launch_time

Two upstream-parity fixes to the preprocess/verifier invocation:
- Run as tasks.finalpool.<task>.preprocess.main (and .evaluation.main) from
  /workspace, exactly as upstream (uv run -m <dotted path>), so 'from ..utils'
  relative imports resolve. Running it as a bare 'preprocess.main' broke them.
- Pass --launch_time (a single value shared by preprocess and the verifier via
  /workspace/.toolathlon/launch_time.txt), matching upstream, but only to
  scripts that declare it (argparse errors on unknown args otherwise).
---
 src/benchflow/adapters/_toolathlon.py | 42 ++++++++++++++++++++-------
 src/benchflow/adapters/source.py      |  2 +-
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py
index 0e892651..42803807 100644
--- a/src/benchflow/adapters/_toolathlon.py
+++ b/src/benchflow/adapters/_toolathlon.py
@@ -475,24 +475,38 @@ def _toolathlon_setup_command(*, task_name: str, variant: str) -> str:
         "/opt/venv/bin/python3" if variant == "gym" else "/usr/local/bin/uv run python"
     )
     task_path = f"/workspace/tasks/finalpool/{task_name}"
+    # Run preprocess as the full ``tasks.finalpool.<task>.preprocess.main`` module
+    # from /workspace — exactly as upstream does — so ``from ..utils`` relative
+    # imports resolve (running it as a bare ``preprocess.main`` breaks them).
+    preprocess_module = f"tasks.finalpool.{task_name}.preprocess.main"
     return "\n".join(
         [
             "set -e",
-            "mkdir -p /workspace/agent_workspace",
+            "mkdir -p /workspace/agent_workspace /workspace/.toolathlon",
             f"TASK_DIR={shlex.quote(task_path)}",
+            # A single launch_time shared by preprocess and the verifier, matching
+            # upstream's ``datetime.now().strftime('%Y-%m-%d %H:%M:%S %A')``.
+            "if [ ! -f /workspace/.toolathlon/launch_time.txt ]; then",
+            "  date -u '+%Y-%m-%d %H:%M:%S %A' > /workspace/.toolathlon/launch_time.txt",
+            "fi",
             'if [ -d "$TASK_DIR/initial_workspace" ]; then',
             '  cp -a "$TASK_DIR/initial_workspace/." /workspace/agent_workspace/',
             "fi",
             'if [ -f "$TASK_DIR/preprocess/main.py" ]; then',
             '  TASK_DIR="$TASK_DIR" AGENT_WORKSPACE=/workspace/agent_workspace '
-            'PYTHONPATH="$TASK_DIR:/workspace:${PYTHONPATH:-}" '
+            'PYTHONPATH="/workspace:${PYTHONPATH:-}" '
             f"{python_cmd} - <<'PY'",
             "import os, runpy, sys",
-            "task_dir = os.environ['TASK_DIR']",
-            "sys.path.insert(0, task_dir)",
             "sys.path.insert(0, '/workspace')",
-            "sys.argv = ['preprocess.main', '--agent_workspace', os.environ['AGENT_WORKSPACE']]",
-            "runpy.run_module('preprocess.main', run_name='__main__')",
+            "argv = ['preprocess.main', '--agent_workspace', os.environ['AGENT_WORKSPACE']]",
+            # Only pass --launch_time to scripts that declare it; argparse errors
+            # on unknown args for the ones that don't.
+            "src = open(os.environ['TASK_DIR'] + '/preprocess/main.py').read()",
+            "if 'launch_time' in src:",
+            "    lt = open('/workspace/.toolathlon/launch_time.txt').read().strip()",
+            "    argv += ['--launch_time', lt]",
+            "sys.argv = argv",
+            f"runpy.run_module({preprocess_module!r}, run_name='__main__')",
             "PY",
             "fi",
             'for private in "$TASK_DIR/groundtruth_workspace" "$TASK_DIR/evaluation"; do',
@@ -510,6 +524,9 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
     python_cmd = (
         "/opt/venv/bin/python3" if variant == "gym" else "/usr/local/bin/uv run python"
     )
+    # Full module path from /workspace (as upstream) so ``from ..`` relative
+    # imports in evaluators resolve.
+    eval_module = f"tasks.finalpool.{task_name}.evaluation.main"
     task_path = f"/workspace/tasks/finalpool/{task_name}"
     interpreter_check = []
     if variant == "official":
@@ -535,19 +552,22 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
             "set +e",
             'TASK_DIR="$TASK_DIR" AGENT_WORKSPACE="$AGENT_WORKSPACE" '
             'GROUNDTRUTH="$GROUNDTRUTH" RES_LOG="$RES_LOG" '
-            'PYTHONPATH="$TASK_DIR:/workspace:${PYTHONPATH:-}" '
+            'PYTHONPATH="/workspace:${PYTHONPATH:-}" '
             f"{python_cmd} - <<'PY' > \"$EVAL_LOG\" 2>&1",
             "import os, runpy, sys",
-            "task_dir = os.environ['TASK_DIR']",
-            "sys.path.insert(0, task_dir)",
             "sys.path.insert(0, '/workspace')",
-            "sys.argv = [",
+            "argv = [",
             "    'evaluation.main',",
             "    '--agent_workspace', os.environ['AGENT_WORKSPACE'],",
             "    '--groundtruth_workspace', os.environ['GROUNDTRUTH'],",
             "    '--res_log_file', os.environ['RES_LOG'],",
             "]",
-            "runpy.run_module('evaluation.main', run_name='__main__')",
+            "src = open(os.environ['TASK_DIR'] + '/evaluation/main.py').read()",
+            "lt_path = '/workspace/.toolathlon/launch_time.txt'",
+            "if 'launch_time' in src and os.path.exists(lt_path):",
+            "    argv += ['--launch_time', open(lt_path).read().strip()]",
+            "sys.argv = argv",
+            f"runpy.run_module({eval_module!r}, run_name='__main__')",
             "PY",
             "status=$?",
             "set -u",
diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py
index 534629d6..84b9e869 100644
--- a/src/benchflow/adapters/source.py
+++ b/src/benchflow/adapters/source.py
@@ -25,7 +25,7 @@
 from benchflow._utils.benchmark_repos import ResolvedSource
 from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root
 
-_ADAPTER_VERSION = "2026-07-03.2"
+_ADAPTER_VERSION = "2026-07-03.3"
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 logger = logging.getLogger(__name__)

From 1023e3cd4ecb5d2924baec34dd16440154fdf544 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Fri, 3 Jul 2026 00:35:48 -0700
Subject: [PATCH 07/11] fix(adapters): bake Notion source/eval page URLs into
 Toolathlon global config
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The notion preprocess (utils.app_specific.notion.notion_remove_and_duplicate)
reads source_notion_page_url / eval_notion_page_url off the global
all_token_key_session, but the container helper only baked the two integration
keys — the URLs resolved to None and every notion task's setup crashed before
duplicating the target subpage. Forward TOOLATHLON_NOTION_SOURCE_PAGE_URL and
TOOLATHLON_NOTION_EVAL_PAGE_URL through the token env allowlist and add both to
_GLOBAL_TOKENS. Bumps adapter version to invalidate the materialization cache.
---
 src/benchflow/adapters/_toolathlon.py           | 8 ++++++--
 src/benchflow/adapters/_toolathlon_container.py | 2 ++
 src/benchflow/adapters/source.py                | 2 +-
 tests/test_source_adapters.py                   | 9 +++++++++
 4 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py
index 42803807..c3cfe3df 100644
--- a/src/benchflow/adapters/_toolathlon.py
+++ b/src/benchflow/adapters/_toolathlon.py
@@ -64,9 +64,11 @@ class _ToolathlonCredentialSpec:
     "|".join(re.escape(spec.path) for spec in _TOOLATHLON_CREDENTIAL_SPECS)
 )
 
-# Secret ``token.X`` values baked into the container-side global
+# ``token.X`` values baked into the container-side global
 # ``token_key_session.py`` (see ``_toolathlon_container._GLOBAL_TOKENS``),
-# resolved host-side from BenchFlow's environment/.env at setup time.
+# resolved host-side from BenchFlow's environment/.env at setup time. Most are
+# secrets; a few (Snowflake warehouse/schema, the Notion source/eval page URLs)
+# are non-secret per-deployment config carried through the same channel.
 _TOOLATHLON_TOKEN_SECRET_ENVS = (
     "TOOLATHLON_GCP_PROJECT_ID",
     "TOOLATHLON_MAPS_API_KEY",
@@ -76,6 +78,8 @@ class _ToolathlonCredentialSpec:
     "TOOLATHLON_WANDB_API_KEY",
     "TOOLATHLON_NOTION_KEY",
     "TOOLATHLON_NOTION_KEY_EVAL",
+    "TOOLATHLON_NOTION_SOURCE_PAGE_URL",
+    "TOOLATHLON_NOTION_EVAL_PAGE_URL",
     "TOOLATHLON_GOOGLE_CLIENT_ID",
     "TOOLATHLON_GOOGLE_CLIENT_SECRET",
     "TOOLATHLON_GOOGLE_REFRESH_TOKEN",
diff --git a/src/benchflow/adapters/_toolathlon_container.py b/src/benchflow/adapters/_toolathlon_container.py
index e4899fb0..47e3621d 100644
--- a/src/benchflow/adapters/_toolathlon_container.py
+++ b/src/benchflow/adapters/_toolathlon_container.py
@@ -175,6 +175,8 @@ def _launch(argv: list[str]) -> int:
     ("wandb_api_key", "TOOLATHLON_WANDB_API_KEY", "XX"),
     ("notion_integration_key", "TOOLATHLON_NOTION_KEY", "XX"),
     ("notion_integration_key_eval", "TOOLATHLON_NOTION_KEY_EVAL", "XX"),
+    ("source_notion_page_url", "TOOLATHLON_NOTION_SOURCE_PAGE_URL", "XX"),
+    ("eval_notion_page_url", "TOOLATHLON_NOTION_EVAL_PAGE_URL", "XX"),
     ("snowflake_account", "TOOLATHLON_SNOWFLAKE_ACCOUNT", "XX"),
     ("snowflake_warehouse", "TOOLATHLON_SNOWFLAKE_WAREHOUSE", "COMPUTE_WH"),
     ("snowflake_role", "TOOLATHLON_SNOWFLAKE_ROLE", "ACCOUNTADMIN"),
diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py
index 84b9e869..8aaec69c 100644
--- a/src/benchflow/adapters/source.py
+++ b/src/benchflow/adapters/source.py
@@ -25,7 +25,7 @@
 from benchflow._utils.benchmark_repos import ResolvedSource
 from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root
 
-_ADAPTER_VERSION = "2026-07-03.3"
+_ADAPTER_VERSION = "2026-07-03.4"
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 logger = logging.getLogger(__name__)
diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index b0f7a809..e53403f9 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -512,6 +512,9 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
             "TOOLATHLON_GCP_PROJECT_ID": "proj-123",
             "TOOLATHLON_MAPS_API_KEY": "maps-abc",
             "TOOLATHLON_GITHUB_TOKEN": "gho_xyz",
+            "TOOLATHLON_NOTION_KEY": "ntn_main",
+            "TOOLATHLON_NOTION_SOURCE_PAGE_URL": "https://www.notion.so/src111",
+            "TOOLATHLON_NOTION_EVAL_PAGE_URL": "https://www.notion.so/eval222",
         },
     )
     assert result.returncode == 0, result.stderr
@@ -522,6 +525,12 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
     assert tokens["gcp_project_id"] == "proj-123"
     assert tokens["google_cloud_console_api_key"] == "maps-abc"
     assert tokens["github_token"] == "gho_xyz"
+    # Notion preprocess reads the page URLs off the global config; without them
+    # baked in, ``notion_remove_and_duplicate`` gets ``None`` and every notion
+    # task's setup crashes.
+    assert tokens["notion_integration_key"] == "ntn_main"
+    assert tokens["source_notion_page_url"] == "https://www.notion.so/src111"
+    assert tokens["eval_notion_page_url"] == "https://www.notion.so/eval222"
     # Unset secrets fall back to their example defaults, not the literal env ref.
     assert tokens["huggingface_token"] == "XX"
     assert tokens["github_read_only"] == "1"

From efd94ff0b30a2e1877bb1a64038ce7af66cfa920 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Fri, 3 Jul 2026 00:40:41 -0700
Subject: [PATCH 08/11] fix(adapters): inject Toolathlon Snowflake private key
 (.p8) as a PEM credential
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Snowflake tasks need configs/snowflake_rsa_key.p8 for keypair auth, but the
credential-injection path validated every payload as JSON and chmod'd 0644 —
wrong for a PEM private key. Add content_format ('json'|'pem') and file_mode to
the credential spec: PEM payloads skip JSON validation and land at 0600. Register
the .p8 spec (b64_env TOOLATHLON_SNOWFLAKE_RSA_KEY_B64, trigger server snowflake).
Removes the last adapter blocker for the 4 Snowflake tasks (account still needs
a human trial signup).
---
 src/benchflow/adapters/_toolathlon.py | 41 +++++++++++++++++++--------
 tests/test_source_adapters.py         | 17 ++++++++++-
 2 files changed, 45 insertions(+), 13 deletions(-)

diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py
index c3cfe3df..7dc0d615 100644
--- a/src/benchflow/adapters/_toolathlon.py
+++ b/src/benchflow/adapters/_toolathlon.py
@@ -31,6 +31,11 @@ class _ToolathlonCredentialSpec:
     b64_env: str
     trigger_servers: tuple[str, ...] = ()
     copy_paths: tuple[str, ...] = ()
+    # "json" payloads are validated as JSON before writing; "pem" ones (e.g. a
+    # Snowflake private key) are opaque bytes written verbatim.
+    content_format: str = "json"
+    # Private keys must not be world-readable; JSON creds keep 0644.
+    file_mode: int = 0o644
 
 
 _TOOLATHLON_CREDENTIAL_SPECS = (
@@ -56,6 +61,14 @@ class _ToolathlonCredentialSpec:
         b64_env="TOOLATHLON_GCP_OAUTH_KEYS_JSON_B64",
         trigger_servers=("google_calendar",),
     ),
+    _ToolathlonCredentialSpec(
+        path="configs/snowflake_rsa_key.p8",
+        env="TOOLATHLON_SNOWFLAKE_RSA_KEY",
+        b64_env="TOOLATHLON_SNOWFLAKE_RSA_KEY_B64",
+        trigger_servers=("snowflake",),
+        content_format="pem",
+        file_mode=0o600,
+    ),
 )
 _TOOLATHLON_CREDENTIALS_BY_PATH = {
     spec.path: spec for spec in _TOOLATHLON_CREDENTIAL_SPECS
@@ -361,9 +374,11 @@ def _toolathlon_credential_setup_command(credential_refs: set[str]) -> str | Non
     specs = [
         {
             "path": spec.path,
-            "json_env": spec.env,
+            "raw_env": spec.env,
             "b64_env": spec.b64_env,
             "copy_paths": list(spec.copy_paths),
+            "content_format": spec.content_format,
+            "file_mode": spec.file_mode,
         }
         for ref in sorted(credential_refs)
         for spec in (_TOOLATHLON_CREDENTIALS_BY_PATH[ref],)
@@ -385,7 +400,7 @@ def _toolathlon_credential_setup_command(credential_refs: set[str]) -> str | Non
             "    if target.exists():",
             "        payload = target.read_bytes()",
             "    else:",
-            "        raw = os.environ.get(spec['json_env']) or ''",
+            "        raw = os.environ.get(spec['raw_env']) or ''",
             "        b64 = os.environ.get(spec['b64_env']) or ''",
             "        if raw:",
             "            payload = raw.encode()",
@@ -399,25 +414,27 @@ def _toolathlon_credential_setup_command(credential_refs: set[str]) -> str | Non
             "                sys.exit(66)",
             "        else:",
             "            missing.append(",
-            "                f\"{spec['path']} ({spec['json_env']} or {spec['b64_env']})\"",
+            "                f\"{spec['path']} ({spec['raw_env']} or {spec['b64_env']})\"",
             "            )",
             "            continue",
-            "    try:",
-            "        json.loads(payload.decode())",
-            "    except Exception as exc:",
-            "        sys.stderr.write(",
-            "            f\"BenchFlow Toolathlon credential setup error: invalid JSON for {spec['path']}: {exc}\\n\"",
-            "        )",
-            "        sys.exit(66)",
+            "    if spec['content_format'] == 'json':",
+            "        try:",
+            "            json.loads(payload.decode())",
+            "        except Exception as exc:",
+            "            sys.stderr.write(",
+            "                f\"BenchFlow Toolathlon credential setup error: invalid JSON for {spec['path']}: {exc}\\n\"",
+            "            )",
+            "            sys.exit(66)",
+            "    mode = spec['file_mode']",
             "    if not target.exists():",
             "        target.parent.mkdir(parents=True, exist_ok=True)",
             "        target.write_bytes(payload)",
-            "        target.chmod(0o644)",
+            "        target.chmod(mode)",
             "    for rel in spec.get('copy_paths', []):",
             "        copy = workspace / rel",
             "        copy.parent.mkdir(parents=True, exist_ok=True)",
             "        copy.write_bytes(payload)",
-            "        copy.chmod(0o644)",
+            "        copy.chmod(mode)",
             "if missing:",
             "    sys.stderr.write(",
             "        'BenchFlow Toolathlon credential setup error: missing '",
diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index e53403f9..5347fcd5 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -390,7 +390,9 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs(
     }
     assert "BenchFlow Toolathlon credential setup error" in credential_setup.command
     assert "configs/gcp-service_account.keys.json" in credential_setup.command
-    assert "target.chmod(0o644)" in credential_setup.command
+    # JSON creds keep 0644; the mode is data-driven off the spec now.
+    assert "target.chmod(mode)" in credential_setup.command
+    assert '"file_mode": 420' in credential_setup.command  # 0o644
     assert "/home/agent" not in credential_setup.command
     assert ".gmail-mcp" not in credential_setup.command
     # ${token.X} env stays literal at materialization; the launcher resolves it
@@ -540,6 +542,19 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
     assert tokens.gcp_project_id == "proj-123"
 
 
+def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None:
+    from benchflow.adapters._toolathlon import _toolathlon_credential_setup_command
+
+    cmd = _toolathlon_credential_setup_command({"configs/snowflake_rsa_key.p8"})
+    assert cmd is not None
+    # The Snowflake private key is PEM, not JSON: written verbatim (no json.loads
+    # gate, which would reject it) and locked to 0600 (== 384) as a private key.
+    assert '"content_format": "pem"' in cmd
+    assert '"file_mode": 384' in cmd
+    assert "TOOLATHLON_SNOWFLAKE_RSA_KEY_B64" in cmd
+    assert "if spec['content_format'] == 'json':" in cmd
+
+
 def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(

From 8671284cef022cc018fd1f20f7eccc066c14a0a6 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Fri, 3 Jul 2026 01:44:55 -0700
Subject: [PATCH 09/11] test: cover Toolathlon verifier scoring

---
 tests/test_source_adapters.py | 72 +++++++++++++++++++++++++++++++++--
 1 file changed, 69 insertions(+), 3 deletions(-)

diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index 5347fcd5..9ec851b4 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -81,7 +81,7 @@ def test_mcp_atlas_source_adapter_materializes_native_tasks(
 def test_toolathlon_source_adapter_materializes_mcp_and_setup(
     tmp_path: Path, monkeypatch
 ) -> None:
-    """Guards cd8e250b Toolathlon adapter work against raw finalpool task dirs."""
+    """Guards PR #887 and cd8e250b against raw finalpool task dirs."""
     monkeypatch.chdir(tmp_path)
     repo = tmp_path / "Toolathlon"
     (repo / ".git").mkdir(parents=True)
@@ -188,6 +188,68 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup(
     assert "/usr/local/bin/uv run python" in test_sh
 
 
+def _run_toolathlon_verifier_script(
+    tmp_path: Path, evaluator_source: str
+) -> subprocess.CompletedProcess:
+    from benchflow.adapters._toolathlon import _toolathlon_test_sh
+
+    workspace = tmp_path / "workspace"
+    logs = tmp_path / "logs" / "verifier"
+    eval_dir = workspace / "tasks" / "finalpool" / "demo" / "evaluation"
+    eval_dir.mkdir(parents=True)
+    (workspace / "agent_workspace").mkdir(parents=True)
+    (workspace / "tasks" / "finalpool" / "demo" / "groundtruth_workspace").mkdir()
+    for package in (
+        workspace / "tasks",
+        workspace / "tasks" / "finalpool",
+        workspace / "tasks" / "finalpool" / "demo",
+        eval_dir,
+    ):
+        (package / "__init__.py").write_text("")
+    (eval_dir / "main.py").write_text(evaluator_source)
+
+    script = _toolathlon_test_sh(task_name="demo", variant="gym")
+    script = script.replace("/workspace", str(workspace))
+    script = script.replace("/logs/verifier", str(logs))
+    script = script.replace("/opt/venv/bin/python3", sys.executable)
+    script_path = tmp_path / "test.sh"
+    script_path.write_text(script)
+
+    return subprocess.run(
+        ["bash", str(script_path)],
+        text=True,
+        capture_output=True,
+        timeout=30,
+    )
+
+
+def test_toolathlon_verifier_scores_task_failure_exit_zero(tmp_path: Path) -> None:
+    """Guards PR #887: evaluator task failures score 0 without infra failure."""
+    result = _run_toolathlon_verifier_script(
+        tmp_path,
+        "raise ValueError('Some tests FAILED')\n",
+    )
+
+    assert result.returncode == 0, result.stderr
+    reward_dir = tmp_path / "logs" / "verifier"
+    assert (reward_dir / "reward.txt").read_text() == "0.0\n"
+    assert json.loads((reward_dir / "reward.json").read_text()) == {"reward": 0.0}
+    assert "ValueError: Some tests FAILED" in result.stdout
+
+
+def test_toolathlon_verifier_escalates_import_failure(tmp_path: Path) -> None:
+    """Guards PR #887: broken evaluator environments fail the verifier."""
+    result = _run_toolathlon_verifier_script(
+        tmp_path,
+        "import definitely_missing_toolathlon_dependency\n",
+    )
+
+    assert result.returncode != 0
+    assert "ModuleNotFoundError" in result.stdout
+    assert "evaluator environment failure" in result.stderr
+    assert not (tmp_path / "logs" / "verifier" / "reward.txt").exists()
+
+
 def test_toolathlon_gym_adapter_normalizes_postgres_env(
     tmp_path: Path, monkeypatch
 ) -> None:
@@ -506,6 +568,7 @@ def _run_container_helper(
 
 
 def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None:
+    """Guards PR #887: container config bakes runtime secrets and attributes."""
     (tmp_path / "configs").mkdir()
     result = _run_container_helper(
         tmp_path,
@@ -543,6 +606,7 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
 
 
 def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None:
+    """Guards PR #887: PEM credentials bypass JSON parsing and lock mode."""
     from benchflow.adapters._toolathlon import _toolathlon_credential_setup_command
 
     cmd = _toolathlon_credential_setup_command({"configs/snowflake_rsa_key.p8"})
@@ -556,6 +620,7 @@ def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None
 
 
 def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
+    """Guards PR #887: launcher resolves global and per-task token refs."""
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
         "all_token_key_session = {"
@@ -597,6 +662,7 @@ def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
 
 
 def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None:
+    """Guards PR #887: launcher resolves token refs inside child env."""
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
         "all_token_key_session = {'github_token': 'gho_secret'}\n"
@@ -614,6 +680,7 @@ def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None:
 
 
 def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None:
+    """Guards PR #887: launcher creates server storage dirs before spawn."""
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
         "all_token_key_session = {}\n"
@@ -635,8 +702,7 @@ def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None:
 def test_toolathlon_arxiv_server_declares_ensure_dirs(
     tmp_path: Path, monkeypatch
 ) -> None:
-    """The arxiv_local server's --storage-path is passed to the launcher as a
-    directory to pre-create, so the evaluator's listdir cannot crash."""
+    """Guards PR #887: arxiv_local storage is pre-created for evaluators."""
     monkeypatch.chdir(tmp_path)
     repo = tmp_path / "Toolathlon"
     (repo / ".git").mkdir(parents=True)

From 2fe7eba3073703f0d8afb419fe7052d5ab688d25 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Fri, 3 Jul 2026 06:09:26 -0700
Subject: [PATCH 10/11] fix(adapters): harden Toolathlon launch-time detection

---
 src/benchflow/adapters/_toolathlon.py         | 32 ++++++--
 .../adapters/_toolathlon_container.py         |  4 +-
 src/benchflow/adapters/source.py              |  2 +-
 tests/test_source_adapters.py                 | 81 +++++++++++++++++++
 4 files changed, 108 insertions(+), 11 deletions(-)

diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py
index 7dc0d615..158e4b60 100644
--- a/src/benchflow/adapters/_toolathlon.py
+++ b/src/benchflow/adapters/_toolathlon.py
@@ -22,6 +22,21 @@
 _TOOLATHLON_UVX_PACKAGE_PINS = {
     "office-word-mcp-server": "office-word-mcp-server==1.1.11",
 }
+_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES = (
+    "def _declares_launch_time(path):",
+    "    tree = ast.parse(open(path, encoding='utf-8').read(), filename=path)",
+    "    for node in ast.walk(tree):",
+    "        if not isinstance(node, ast.Call):",
+    "            continue",
+    "        if not isinstance(node.func, ast.Attribute):",
+    "            continue",
+    "        if node.func.attr != 'add_argument':",
+    "            continue",
+    "        for arg in node.args:",
+    "            if isinstance(arg, ast.Constant) and arg.value == '--launch_time':",
+    "                return True",
+    "    return False",
+)
 
 
 @dataclass(frozen=True)
@@ -517,13 +532,14 @@ def _toolathlon_setup_command(*, task_name: str, variant: str) -> str:
             '  TASK_DIR="$TASK_DIR" AGENT_WORKSPACE=/workspace/agent_workspace '
             'PYTHONPATH="/workspace:${PYTHONPATH:-}" '
             f"{python_cmd} - <<'PY'",
-            "import os, runpy, sys",
+            "import ast, os, runpy, sys",
             "sys.path.insert(0, '/workspace')",
             "argv = ['preprocess.main', '--agent_workspace', os.environ['AGENT_WORKSPACE']]",
-            # Only pass --launch_time to scripts that declare it; argparse errors
-            # on unknown args for the ones that don't.
-            "src = open(os.environ['TASK_DIR'] + '/preprocess/main.py').read()",
-            "if 'launch_time' in src:",
+            *_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES,
+            # Only pass --launch_time to scripts that declare that exact CLI
+            # option; argparse errors on unknown args for scripts that merely
+            # mention launch_time in comments or unrelated code.
+            "if _declares_launch_time(os.environ['TASK_DIR'] + '/preprocess/main.py'):",
             "    lt = open('/workspace/.toolathlon/launch_time.txt').read().strip()",
             "    argv += ['--launch_time', lt]",
             "sys.argv = argv",
@@ -575,7 +591,7 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
             'GROUNDTRUTH="$GROUNDTRUTH" RES_LOG="$RES_LOG" '
             'PYTHONPATH="/workspace:${PYTHONPATH:-}" '
             f"{python_cmd} - <<'PY' > \"$EVAL_LOG\" 2>&1",
-            "import os, runpy, sys",
+            "import ast, os, runpy, sys",
             "sys.path.insert(0, '/workspace')",
             "argv = [",
             "    'evaluation.main',",
@@ -583,9 +599,9 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
             "    '--groundtruth_workspace', os.environ['GROUNDTRUTH'],",
             "    '--res_log_file', os.environ['RES_LOG'],",
             "]",
-            "src = open(os.environ['TASK_DIR'] + '/evaluation/main.py').read()",
+            *_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES,
             "lt_path = '/workspace/.toolathlon/launch_time.txt'",
-            "if 'launch_time' in src and os.path.exists(lt_path):",
+            "if _declares_launch_time(os.environ['TASK_DIR'] + '/evaluation/main.py') and os.path.exists(lt_path):",
             "    argv += ['--launch_time', open(lt_path).read().strip()]",
             "sys.argv = argv",
             f"runpy.run_module({eval_module!r}, run_name='__main__')",
diff --git a/src/benchflow/adapters/_toolathlon_container.py b/src/benchflow/adapters/_toolathlon_container.py
index 47e3621d..b17374e7 100644
--- a/src/benchflow/adapters/_toolathlon_container.py
+++ b/src/benchflow/adapters/_toolathlon_container.py
@@ -194,8 +194,8 @@ def _launch(argv: list[str]) -> int:
     ("woocommerce_api_key", None, "ck_woocommerce_token_PE0613bf053"),
     ("woocommerce_api_secret", None, "cs_woocommerce_token_PE0613bf053"),
     ("woocommerce_site_url", None, "http://localhost:10003/store100"),
-    ("kubeconfig_path", None, "deployment/k8s/configs/cluster1-config.yaml"),
-    ("emails_config_file", None, "configs/example_email_config.json"),
+    ("kubeconfig_path", None, "/workspace/deployment/k8s/configs/cluster1-config.yaml"),
+    ("emails_config_file", None, "/workspace/configs/example_email_config.json"),
 )
 
 
diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py
index 8aaec69c..1e1cabb1 100644
--- a/src/benchflow/adapters/source.py
+++ b/src/benchflow/adapters/source.py
@@ -25,7 +25,7 @@
 from benchflow._utils.benchmark_repos import ResolvedSource
 from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root
 
-_ADAPTER_VERSION = "2026-07-03.4"
+_ADAPTER_VERSION = "2026-07-03.5"
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 logger = logging.getLogger(__name__)
diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py
index 9ec851b4..8b1cd08b 100644
--- a/tests/test_source_adapters.py
+++ b/tests/test_source_adapters.py
@@ -185,9 +185,60 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup(
     assert "evaluator environment failure" in test_sh
     assert "ModuleNotFoundError|ImportError|PermissionError" in test_sh
     assert "Traceback (most recent call last):" not in test_sh
+    assert "ast.walk" in setup_command
+    assert "ast.walk" in test_sh
     assert "/usr/local/bin/uv run python" in test_sh
 
 
+def _run_toolathlon_setup_script(
+    tmp_path: Path, preprocess_source: str
+) -> subprocess.CompletedProcess:
+    from benchflow.adapters._toolathlon import _toolathlon_setup_command
+
+    workspace = tmp_path / "workspace"
+    task_dir = workspace / "tasks" / "finalpool" / "demo"
+    preprocess_dir = task_dir / "preprocess"
+    preprocess_dir.mkdir(parents=True)
+    (workspace / "agent_workspace").mkdir(parents=True)
+    for package in (
+        workspace / "tasks",
+        workspace / "tasks" / "finalpool",
+        task_dir,
+        preprocess_dir,
+    ):
+        (package / "__init__.py").write_text("")
+    (preprocess_dir / "main.py").write_text(preprocess_source)
+
+    script = _toolathlon_setup_command(task_name="demo", variant="gym")
+    script = script.replace("/workspace", str(workspace))
+    script = script.replace("/opt/venv/bin/python3", sys.executable)
+    script_path = tmp_path / "setup.sh"
+    script_path.write_text(script)
+
+    return subprocess.run(
+        ["bash", str(script_path)],
+        text=True,
+        capture_output=True,
+        timeout=30,
+    )
+
+
+def test_toolathlon_preprocess_ignores_launch_time_in_comments(
+    tmp_path: Path,
+) -> None:
+    """Guards PR #887: launch_time comments do not receive the CLI flag."""
+    result = _run_toolathlon_setup_script(
+        tmp_path,
+        "import argparse\n"
+        "# launch_time is documented here but not accepted by this script.\n"
+        "parser = argparse.ArgumentParser()\n"
+        "parser.add_argument('--agent_workspace', required=True)\n"
+        "parser.parse_args()\n",
+    )
+    assert result.returncode == 0, result.stderr
+    assert "unrecognized arguments: --launch_time" not in result.stderr
+
+
 def _run_toolathlon_verifier_script(
     tmp_path: Path, evaluator_source: str
 ) -> subprocess.CompletedProcess:
@@ -250,6 +301,29 @@ def test_toolathlon_verifier_escalates_import_failure(tmp_path: Path) -> None:
     assert not (tmp_path / "logs" / "verifier" / "reward.txt").exists()
 
 
+def test_toolathlon_verifier_ignores_launch_time_in_comments(
+    tmp_path: Path,
+) -> None:
+    """Guards PR #887: launch_time comments do not downgrade verifier reward."""
+    result = _run_toolathlon_verifier_script(
+        tmp_path,
+        "import argparse\n"
+        "# launch_time is documented here but not accepted by this evaluator.\n"
+        "parser = argparse.ArgumentParser()\n"
+        "parser.add_argument('--agent_workspace', required=True)\n"
+        "parser.add_argument('--groundtruth_workspace', required=True)\n"
+        "parser.add_argument('--res_log_file', required=True)\n"
+        "parser.parse_args()\n",
+    )
+    assert result.returncode == 0, result.stderr
+    logs = tmp_path / "logs" / "verifier"
+    assert json.loads((logs / "reward.json").read_text()) == {"reward": 1.0}
+    assert (
+        "unrecognized arguments: --launch_time"
+        not in (logs / "toolathlon_evaluator.log").read_text()
+    )
+
+
 def test_toolathlon_gym_adapter_normalizes_postgres_env(
     tmp_path: Path, monkeypatch
 ) -> None:
@@ -599,6 +673,13 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
     # Unset secrets fall back to their example defaults, not the literal env ref.
     assert tokens["huggingface_token"] == "XX"
     assert tokens["github_read_only"] == "1"
+    assert (
+        tokens["kubeconfig_path"]
+        == "/workspace/deployment/k8s/configs/cluster1-config.yaml"
+    )
+    assert (
+        tokens["emails_config_file"] == "/workspace/configs/example_email_config.json"
+    )
     # Upstream preprocess/eval read tokens via ATTRIBUTE access (addict.Dict);
     # the generated dict must support it (a plain dict would AttributeError).
     assert tokens.github_token == "gho_xyz"

From ff08340cffa62c48088abbf1c7cd1ffd590b5afa Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Sun, 5 Jul 2026 01:43:26 -0700
Subject: [PATCH 11/11] Fix integration light provider fallbacks

---
 .github/workflows/integration-light.yml      | 10 +++++++-
 tests/test_integration_provider_selection.py | 24 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-light.yml b/.github/workflows/integration-light.yml
index 2abd67fe..97b0bb7b 100644
--- a/.github/workflows/integration-light.yml
+++ b/.github/workflows/integration-light.yml
@@ -169,10 +169,18 @@ jobs:
     permissions:
       contents: read
     env:
-      # Low-value provider candidates only. The selector probes these in order.
+      # Provider candidates. The selector probes these in documented order and
+      # stops at the first usable key.
       DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
       DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}
+      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}
+      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}
+      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}
+      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}
       GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}
+      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}
       # The L1 smoke task. workflow_dispatch can override it.
       SMOKE_TASK: ${{ github.event.inputs.task || 'jax-computing-basics' }}
diff --git a/tests/test_integration_provider_selection.py b/tests/test_integration_provider_selection.py
index 4c6e7f65..afcf8670 100644
--- a/tests/test_integration_provider_selection.py
+++ b/tests/test_integration_provider_selection.py
@@ -3,10 +3,15 @@
 import sys
 from pathlib import Path
 
+import yaml
+
 SCRIPT = (
     Path(__file__).resolve().parents[1]
     / ".github/scripts/select_integration_provider.py"
 )
+INTEGRATION_LIGHT_WORKFLOW = (
+    Path(__file__).resolve().parents[1] / ".github/workflows/integration-light.yml"
+)
 spec = importlib.util.spec_from_file_location("select_integration_provider", SCRIPT)
 assert spec is not None
 selector = importlib.util.module_from_spec(spec)
@@ -55,3 +60,22 @@ def test_github_env_writer_uses_multiline_format(tmp_path, monkeypatch):
     assert path.read_text() == (
         "OPENAI_API_KEY<<__BENCHFLOW_ENV__\nsecret\nvalue\n__BENCHFLOW_ENV__\n"
     )
+
+
+def test_integration_light_exposes_provider_fallback_envs():
+    """Guards the CI unblock from PR #887 so light smoke can use fallbacks."""
+    workflow = yaml.safe_load(INTEGRATION_LIGHT_WORKFLOW.read_text())
+    env = workflow["jobs"]["rollout-smoke"]["env"]
+
+    assert {
+        "DEEPSEEK_API_KEY",
+        "DEEPSEEK_BASE_URL",
+        "GLM_API_KEY",
+        "GLM_BASE_URL",
+        "QWEN_API_KEY",
+        "QWEN_BASE_URL",
+        "LITELLM_API_KEY",
+        "LITELLM_BASE_URL",
+        "OPENAI_API_KEY",
+        "GITHUB_MODELS_TOKEN",
+    } <= set(env)