fix(adapters): Toolathlon spawn-time token resolution + exit-code verifier scoring

.github/workflows/integration-light.yml

@@ -169,13 +169,19 @@ jobs:
     permissions:
       contents: read
     env:
-      # Low-value provider candidates only. The selector probes these in order.
+      # Provider candidates. The selector probes these in documented order and
+      # stops at the first usable key.
       DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
       DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}
       GLM_API_KEY: ${{ secrets.GLM_API_KEY }}
       GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}
       GLM_MODEL: ${{ secrets.GLM_MODEL }}
+      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}
+      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}
       GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}
+      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}
       # The L1 smoke task. workflow_dispatch can override it.
       SMOKE_TASK: ${{ github.event.inputs.task || 'jax-computing-basics' }}

src/benchflow/adapters/_toolathlon.py

@@ -29,6 +29,21 @@
     # OAuth dir injected by the notion mcp-auth setup command.
     "mcp-remote": "mcp-remote@0.1.37",
 }
+_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES = (
+    "def _declares_launch_time(path):",
+    "    tree = ast.parse(open(path, encoding='utf-8').read(), filename=path)",
+    "    for node in ast.walk(tree):",
+    "        if not isinstance(node, ast.Call):",
+    "            continue",
+    "        if not isinstance(node.func, ast.Attribute):",
+    "            continue",
+    "        if node.func.attr != 'add_argument':",
+    "            continue",
+    "        for arg in node.args:",
+    "            if isinstance(arg, ast.Constant) and arg.value == '--launch_time':",
+    "                return True",
+    "    return False",
+)
 # Absolute location the notion_official server reads its OAuth token from (the
 # upstream config uses a cwd-relative ./configs/.mcp-auth).
 _TOOLATHLON_NOTION_MCP_AUTH_DIR = "/workspace/configs/.mcp-auth"
@@ -678,13 +693,14 @@ def _toolathlon_setup_command(*, task_name: str, variant: str) -> str:
             '  TASK_DIR="$TASK_DIR" AGENT_WORKSPACE=/workspace/agent_workspace '
             'PYTHONPATH="/workspace:${PYTHONPATH:-}" '
             f"{python_cmd} - <<'PY'",
-            "import os, runpy, sys",
+            "import ast, os, runpy, sys",
             "sys.path.insert(0, '/workspace')",
             "argv = ['preprocess.main', '--agent_workspace', os.environ['AGENT_WORKSPACE']]",
-            # Only pass --launch_time to scripts that declare it; argparse errors
-            # on unknown args for the ones that don't.
-            "src = open(os.environ['TASK_DIR'] + '/preprocess/main.py').read()",
-            "if 'launch_time' in src:",
+            *_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES,
+            # Only pass --launch_time to scripts that declare that exact CLI
+            # option; argparse errors on unknown args for scripts that merely
+            # mention launch_time in comments or unrelated code.
+            "if _declares_launch_time(os.environ['TASK_DIR'] + '/preprocess/main.py'):",
             "    lt = open('/workspace/.toolathlon/launch_time.txt').read().strip()",
             "    argv += ['--launch_time', lt]",
             "sys.argv = argv",
@@ -738,17 +754,17 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str:
             'GROUNDTRUTH="$GROUNDTRUTH" RES_LOG="$RES_LOG" '
             'PYTHONPATH="/workspace:${PYTHONPATH:-}" '
             f"{python_cmd} - <<'PY' > \"$EVAL_LOG\" 2>&1",
-            "import os, runpy, sys",
+            "import ast, os, runpy, sys",
             "sys.path.insert(0, '/workspace')",
             "argv = [",
             "    'evaluation.main',",
             "    '--agent_workspace', os.environ['AGENT_WORKSPACE'],",
             "    '--groundtruth_workspace', os.environ['GROUNDTRUTH'],",
             "    '--res_log_file', os.environ['RES_LOG'],",
             "]",
-            "src = open(os.environ['TASK_DIR'] + '/evaluation/main.py').read()",
+            *_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES,
             "lt_path = '/workspace/.toolathlon/launch_time.txt'",
-            "if 'launch_time' in src and os.path.exists(lt_path):",
+            "if _declares_launch_time(os.environ['TASK_DIR'] + '/evaluation/main.py') and os.path.exists(lt_path):",
             "    argv += ['--launch_time', open(lt_path).read().strip()]",
             "sys.argv = argv",
             f"runpy.run_module({eval_module!r}, run_name='__main__')",

src/benchflow/adapters/_toolathlon_container.py

@@ -336,8 +336,8 @@ def _launch(argv: list[str]) -> int:
     ("woocommerce_api_key", None, "ck_woocommerce_token_PE0613bf053"),
     ("woocommerce_api_secret", None, "cs_woocommerce_token_PE0613bf053"),
     ("woocommerce_site_url", None, "http://localhost:10003/store100"),
-    ("kubeconfig_path", None, "deployment/k8s/configs/cluster1-config.yaml"),
-    ("emails_config_file", None, "configs/example_email_config.json"),
+    ("kubeconfig_path", None, "/workspace/deployment/k8s/configs/cluster1-config.yaml"),
+    ("emails_config_file", None, "/workspace/configs/example_email_config.json"),
 )
 
 

src/benchflow/adapters/source.py

@@ -25,7 +25,7 @@
 from benchflow._utils.benchmark_repos import ResolvedSource
 from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root
 
-_ADAPTER_VERSION = "2026-07-05.5"
+_ADAPTER_VERSION = "2026-07-05.6"
 _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__"
 _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 logger = logging.getLogger(__name__)

tests/test_integration_provider_selection.py

@@ -3,10 +3,15 @@
 import sys
 from pathlib import Path
 
+import yaml
+
 SCRIPT = (
     Path(__file__).resolve().parents[1]
     / ".github/scripts/select_integration_provider.py"
 )
+INTEGRATION_LIGHT_WORKFLOW = (
+    Path(__file__).resolve().parents[1] / ".github/workflows/integration-light.yml"
+)
 spec = importlib.util.spec_from_file_location("select_integration_provider", SCRIPT)
 assert spec is not None
 selector = importlib.util.module_from_spec(spec)
@@ -55,3 +60,22 @@ def test_github_env_writer_uses_multiline_format(tmp_path, monkeypatch):
     assert path.read_text() == (
         "OPENAI_API_KEY<<__BENCHFLOW_ENV__\nsecret\nvalue\n__BENCHFLOW_ENV__\n"
     )
+
+
+def test_integration_light_exposes_provider_fallback_envs():
+    """Guards the CI unblock from PR #887 so light smoke can use fallbacks."""
+    workflow = yaml.safe_load(INTEGRATION_LIGHT_WORKFLOW.read_text())
+    env = workflow["jobs"]["rollout-smoke"]["env"]
+
+    assert {
+        "DEEPSEEK_API_KEY",
+        "DEEPSEEK_BASE_URL",
+        "GLM_API_KEY",
+        "GLM_BASE_URL",
+        "QWEN_API_KEY",
+        "QWEN_BASE_URL",
+        "LITELLM_API_KEY",
+        "LITELLM_BASE_URL",
+        "OPENAI_API_KEY",
+        "GITHUB_MODELS_TOKEN",
+    } <= set(env)

tests/test_source_adapters.py

@@ -196,13 +196,160 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup(
     assert "evaluator environment failure" in test_sh
     assert "ModuleNotFoundError|ImportError|PermissionError" in test_sh
     assert "Traceback (most recent call last):" not in test_sh
+    assert "ast.walk" in setup_command
+    assert "ast.walk" in test_sh
     assert "/usr/local/bin/uv run python" in test_sh
     assert "toolathlon_container.py write-task-tokens" in setup_command
     assert setup_command.index("write-task-tokens") < setup_command.index(
         'chmod -R go-rwx "$private"'
     )
 
 
+def _run_toolathlon_setup_script(
+    tmp_path: Path, preprocess_source: str
+) -> subprocess.CompletedProcess:
+    from benchflow.adapters._toolathlon import (
+        _toolathlon_container_module_source,
+        _toolathlon_setup_command,
+    )
+
+    workspace = tmp_path / "workspace"
+    task_dir = workspace / "tasks" / "finalpool" / "demo"
+    preprocess_dir = task_dir / "preprocess"
+    preprocess_dir.mkdir(parents=True)
+    (workspace / "agent_workspace").mkdir(parents=True)
+    for package in (
+        workspace / "tasks",
+        workspace / "tasks" / "finalpool",
+        task_dir,
+        preprocess_dir,
+    ):
+        (package / "__init__.py").write_text("")
+    (preprocess_dir / "main.py").write_text(preprocess_source)
+    helper = workspace / ".toolathlon" / "toolathlon_container.py"
+    helper.parent.mkdir()
+    helper.write_text(_toolathlon_container_module_source())
+    helper.chmod(0o755)
+
+    script = _toolathlon_setup_command(task_name="demo", variant="gym")
+    script = script.replace("/workspace", str(workspace))
+    script = script.replace("/opt/venv/bin/python3", sys.executable)
+    script_path = tmp_path / "setup.sh"
+    script_path.write_text(script)
+
+    env = os.environ.copy()
+    env["TOOLATHLON_WORKSPACE"] = str(workspace)
+
+    return subprocess.run(
+        ["bash", str(script_path)],
+        text=True,
+        capture_output=True,
+        env=env,
+        timeout=30,
+    )
+
+
+def test_toolathlon_preprocess_ignores_launch_time_in_comments(
+    tmp_path: Path,
+) -> None:
+    """Guards PR #887: launch_time comments do not receive the CLI flag."""
+    result = _run_toolathlon_setup_script(
+        tmp_path,
+        "import argparse\n"
+        "# launch_time is documented here but not accepted by this script.\n"
+        "parser = argparse.ArgumentParser()\n"
+        "parser.add_argument('--agent_workspace', required=True)\n"
+        "parser.parse_args()\n",
+    )
+    assert result.returncode == 0, result.stderr
+    assert "unrecognized arguments: --launch_time" not in result.stderr
+
+
+def _run_toolathlon_verifier_script(
+    tmp_path: Path, evaluator_source: str
+) -> subprocess.CompletedProcess:
+    from benchflow.adapters._toolathlon import _toolathlon_test_sh
+
+    workspace = tmp_path / "workspace"
+    logs = tmp_path / "logs" / "verifier"
+    eval_dir = workspace / "tasks" / "finalpool" / "demo" / "evaluation"
+    eval_dir.mkdir(parents=True)
+    (workspace / "agent_workspace").mkdir(parents=True)
+    (workspace / "tasks" / "finalpool" / "demo" / "groundtruth_workspace").mkdir()
+    for package in (
+        workspace / "tasks",
+        workspace / "tasks" / "finalpool",
+        workspace / "tasks" / "finalpool" / "demo",
+        eval_dir,
+    ):
+        (package / "__init__.py").write_text("")
+    (eval_dir / "main.py").write_text(evaluator_source)
+
+    script = _toolathlon_test_sh(task_name="demo", variant="gym")
+    script = script.replace("/workspace", str(workspace))
+    script = script.replace("/logs/verifier", str(logs))
+    script = script.replace("/opt/venv/bin/python3", sys.executable)
+    script_path = tmp_path / "test.sh"
+    script_path.write_text(script)
+
+    return subprocess.run(
+        ["bash", str(script_path)],
+        text=True,
+        capture_output=True,
+        timeout=30,
+    )
+
+
+def test_toolathlon_verifier_scores_task_failure_exit_zero(tmp_path: Path) -> None:
+    """Guards PR #887: evaluator task failures score 0 without infra failure."""
+    result = _run_toolathlon_verifier_script(
+        tmp_path,
+        "raise ValueError('Some tests FAILED')\n",
+    )
+
+    assert result.returncode == 0, result.stderr
+    reward_dir = tmp_path / "logs" / "verifier"
+    assert (reward_dir / "reward.txt").read_text() == "0.0\n"
+    assert json.loads((reward_dir / "reward.json").read_text()) == {"reward": 0.0}
+    assert "ValueError: Some tests FAILED" in result.stdout
+
+
+def test_toolathlon_verifier_escalates_import_failure(tmp_path: Path) -> None:
+    """Guards PR #887: broken evaluator environments fail the verifier."""
+    result = _run_toolathlon_verifier_script(
+        tmp_path,
+        "import definitely_missing_toolathlon_dependency\n",
+    )
+
+    assert result.returncode != 0
+    assert "ModuleNotFoundError" in result.stdout
+    assert "evaluator environment failure" in result.stderr
+    assert not (tmp_path / "logs" / "verifier" / "reward.txt").exists()
+
+
+def test_toolathlon_verifier_ignores_launch_time_in_comments(
+    tmp_path: Path,
+) -> None:
+    """Guards PR #887: launch_time comments do not downgrade verifier reward."""
+    result = _run_toolathlon_verifier_script(
+        tmp_path,
+        "import argparse\n"
+        "# launch_time is documented here but not accepted by this evaluator.\n"
+        "parser = argparse.ArgumentParser()\n"
+        "parser.add_argument('--agent_workspace', required=True)\n"
+        "parser.add_argument('--groundtruth_workspace', required=True)\n"
+        "parser.add_argument('--res_log_file', required=True)\n"
+        "parser.parse_args()\n",
+    )
+    assert result.returncode == 0, result.stderr
+    logs = tmp_path / "logs" / "verifier"
+    assert json.loads((logs / "reward.json").read_text()) == {"reward": 1.0}
+    assert (
+        "unrecognized arguments: --launch_time"
+        not in (logs / "toolathlon_evaluator.log").read_text()
+    )
+
+
 def _write_toolathlon_repo_skeleton(repo: Path) -> Path:
     """Minimal upstream Toolathlon layout; returns tasks/finalpool."""
     (repo / ".git").mkdir(parents=True)
@@ -852,6 +999,7 @@ def _run_container_helper(
 
 
 def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None:
+    """Guards PR #887: container config bakes runtime secrets and attributes."""
     (tmp_path / "configs").mkdir()
     result = _run_container_helper(
         tmp_path,
@@ -882,13 +1030,21 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None
     # Unset secrets fall back to their example defaults, not the literal env ref.
     assert tokens["huggingface_token"] == "XX"
     assert tokens["github_read_only"] == "1"
+    assert (
+        tokens["kubeconfig_path"]
+        == "/workspace/deployment/k8s/configs/cluster1-config.yaml"
+    )
+    assert (
+        tokens["emails_config_file"] == "/workspace/configs/example_email_config.json"
+    )
     # Upstream preprocess/eval read tokens via ATTRIBUTE access (addict.Dict);
     # the generated dict must support it (a plain dict would AttributeError).
     assert tokens.github_token == "gho_xyz"
     assert tokens.gcp_project_id == "proj-123"
 
 
 def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None:
+    """Guards PR #887: PEM credentials bypass JSON parsing and lock mode."""
     from benchflow.adapters._toolathlon import _toolathlon_credential_setup_command
 
     cmd = _toolathlon_credential_setup_command({"configs/snowflake_rsa_key.p8"})
@@ -902,6 +1058,7 @@ def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None
 
 
 def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None:
+    """Guards PR #887: launcher resolves global and per-task token refs."""
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
         "all_token_key_session = {"
@@ -1023,6 +1180,7 @@ def test_toolathlon_container_launch_wraps_google_cloud_mcp(tmp_path: Path) -> N
 
 
 def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None:
+    """Guards PR #887: launcher resolves token refs inside child env."""
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
         "all_token_key_session = {'github_token': 'gho_secret'}\n"
@@ -1065,6 +1223,7 @@ def test_toolathlon_container_launch_prefers_generated_kubeconfig(
 
 
 def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None:
+    """Guards PR #887: launcher creates server storage dirs before spawn."""
     (tmp_path / "configs").mkdir()
     (tmp_path / "configs" / "token_key_session.py").write_text(
         "all_token_key_session = {}\n"
@@ -1113,8 +1272,7 @@ def test_toolathlon_container_launch_filters_non_json_stdout(
 def test_toolathlon_arxiv_server_declares_ensure_dirs(
     tmp_path: Path, monkeypatch
 ) -> None:
-    """The arxiv_local server's --storage-path is passed to the launcher as a
-    directory to pre-create, so the evaluator's listdir cannot crash."""
+    """Guards PR #887: arxiv_local storage is pre-created for evaluators."""
     monkeypatch.chdir(tmp_path)
     repo = tmp_path / "Toolathlon"
     (repo / ".git").mkdir(parents=True)