diff --git a/.github/workflows/integration-light.yml b/.github/workflows/integration-light.yml index 39585b2e..6bf14b8c 100644 --- a/.github/workflows/integration-light.yml +++ b/.github/workflows/integration-light.yml @@ -169,13 +169,19 @@ jobs: permissions: contents: read env: - # Low-value provider candidates only. The selector probes these in order. + # Provider candidates. The selector probes these in documented order and + # stops at the first usable key. DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }} GLM_API_KEY: ${{ secrets.GLM_API_KEY }} GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }} GLM_MODEL: ${{ secrets.GLM_MODEL }} + QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }} + QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }} + LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }} # The L1 smoke task. workflow_dispatch can override it. SMOKE_TASK: ${{ github.event.inputs.task || 'jax-computing-basics' }} diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py index 4f3e2544..569cb2a6 100644 --- a/src/benchflow/adapters/_toolathlon.py +++ b/src/benchflow/adapters/_toolathlon.py @@ -29,6 +29,21 @@ # OAuth dir injected by the notion mcp-auth setup command. "mcp-remote": "mcp-remote@0.1.37", } +_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES = ( + "def _declares_launch_time(path):", + " tree = ast.parse(open(path, encoding='utf-8').read(), filename=path)", + " for node in ast.walk(tree):", + " if not isinstance(node, ast.Call):", + " continue", + " if not isinstance(node.func, ast.Attribute):", + " continue", + " if node.func.attr != 'add_argument':", + " continue", + " for arg in node.args:", + " if isinstance(arg, ast.Constant) and arg.value == '--launch_time':", + " return True", + " return False", +) # Absolute location the notion_official server reads its OAuth token from (the # upstream config uses a cwd-relative ./configs/.mcp-auth). _TOOLATHLON_NOTION_MCP_AUTH_DIR = "/workspace/configs/.mcp-auth" @@ -678,13 +693,14 @@ def _toolathlon_setup_command(*, task_name: str, variant: str) -> str: ' TASK_DIR="$TASK_DIR" AGENT_WORKSPACE=/workspace/agent_workspace ' 'PYTHONPATH="/workspace:${PYTHONPATH:-}" ' f"{python_cmd} - <<'PY'", - "import os, runpy, sys", + "import ast, os, runpy, sys", "sys.path.insert(0, '/workspace')", "argv = ['preprocess.main', '--agent_workspace', os.environ['AGENT_WORKSPACE']]", - # Only pass --launch_time to scripts that declare it; argparse errors - # on unknown args for the ones that don't. - "src = open(os.environ['TASK_DIR'] + '/preprocess/main.py').read()", - "if 'launch_time' in src:", + *_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES, + # Only pass --launch_time to scripts that declare that exact CLI + # option; argparse errors on unknown args for scripts that merely + # mention launch_time in comments or unrelated code. + "if _declares_launch_time(os.environ['TASK_DIR'] + '/preprocess/main.py'):", " lt = open('/workspace/.toolathlon/launch_time.txt').read().strip()", " argv += ['--launch_time', lt]", "sys.argv = argv", @@ -738,7 +754,7 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str: 'GROUNDTRUTH="$GROUNDTRUTH" RES_LOG="$RES_LOG" ' 'PYTHONPATH="/workspace:${PYTHONPATH:-}" ' f"{python_cmd} - <<'PY' > \"$EVAL_LOG\" 2>&1", - "import os, runpy, sys", + "import ast, os, runpy, sys", "sys.path.insert(0, '/workspace')", "argv = [", " 'evaluation.main',", @@ -746,9 +762,9 @@ def _toolathlon_test_sh(*, task_name: str, variant: str) -> str: " '--groundtruth_workspace', os.environ['GROUNDTRUTH'],", " '--res_log_file', os.environ['RES_LOG'],", "]", - "src = open(os.environ['TASK_DIR'] + '/evaluation/main.py').read()", + *_TOOLATHLON_LAUNCH_TIME_DETECTOR_LINES, "lt_path = '/workspace/.toolathlon/launch_time.txt'", - "if 'launch_time' in src and os.path.exists(lt_path):", + "if _declares_launch_time(os.environ['TASK_DIR'] + '/evaluation/main.py') and os.path.exists(lt_path):", " argv += ['--launch_time', open(lt_path).read().strip()]", "sys.argv = argv", f"runpy.run_module({eval_module!r}, run_name='__main__')", diff --git a/src/benchflow/adapters/_toolathlon_container.py b/src/benchflow/adapters/_toolathlon_container.py index 87f1e721..5d87bf08 100644 --- a/src/benchflow/adapters/_toolathlon_container.py +++ b/src/benchflow/adapters/_toolathlon_container.py @@ -336,8 +336,8 @@ def _launch(argv: list[str]) -> int: ("woocommerce_api_key", None, "ck_woocommerce_token_PE0613bf053"), ("woocommerce_api_secret", None, "cs_woocommerce_token_PE0613bf053"), ("woocommerce_site_url", None, "http://localhost:10003/store100"), - ("kubeconfig_path", None, "deployment/k8s/configs/cluster1-config.yaml"), - ("emails_config_file", None, "configs/example_email_config.json"), + ("kubeconfig_path", None, "/workspace/deployment/k8s/configs/cluster1-config.yaml"), + ("emails_config_file", None, "/workspace/configs/example_email_config.json"), ) diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py index 5ed29d31..d5727f43 100644 --- a/src/benchflow/adapters/source.py +++ b/src/benchflow/adapters/source.py @@ -25,7 +25,7 @@ from benchflow._utils.benchmark_repos import ResolvedSource from benchflow.adapters._toolathlon import materialize_toolathlon, toolathlon_tasks_root -_ADAPTER_VERSION = "2026-07-05.5" +_ADAPTER_VERSION = "2026-07-05.6" _NOOP_EXCLUDE_TAG = "__benchflow_exclude_no_tools__" _SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+") logger = logging.getLogger(__name__) diff --git a/tests/test_integration_provider_selection.py b/tests/test_integration_provider_selection.py index 4c6e7f65..afcf8670 100644 --- a/tests/test_integration_provider_selection.py +++ b/tests/test_integration_provider_selection.py @@ -3,10 +3,15 @@ import sys from pathlib import Path +import yaml + SCRIPT = ( Path(__file__).resolve().parents[1] / ".github/scripts/select_integration_provider.py" ) +INTEGRATION_LIGHT_WORKFLOW = ( + Path(__file__).resolve().parents[1] / ".github/workflows/integration-light.yml" +) spec = importlib.util.spec_from_file_location("select_integration_provider", SCRIPT) assert spec is not None selector = importlib.util.module_from_spec(spec) @@ -55,3 +60,22 @@ def test_github_env_writer_uses_multiline_format(tmp_path, monkeypatch): assert path.read_text() == ( "OPENAI_API_KEY<<__BENCHFLOW_ENV__\nsecret\nvalue\n__BENCHFLOW_ENV__\n" ) + + +def test_integration_light_exposes_provider_fallback_envs(): + """Guards the CI unblock from PR #887 so light smoke can use fallbacks.""" + workflow = yaml.safe_load(INTEGRATION_LIGHT_WORKFLOW.read_text()) + env = workflow["jobs"]["rollout-smoke"]["env"] + + assert { + "DEEPSEEK_API_KEY", + "DEEPSEEK_BASE_URL", + "GLM_API_KEY", + "GLM_BASE_URL", + "QWEN_API_KEY", + "QWEN_BASE_URL", + "LITELLM_API_KEY", + "LITELLM_BASE_URL", + "OPENAI_API_KEY", + "GITHUB_MODELS_TOKEN", + } <= set(env) diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py index ce7b519b..50fdea40 100644 --- a/tests/test_source_adapters.py +++ b/tests/test_source_adapters.py @@ -196,6 +196,8 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup( assert "evaluator environment failure" in test_sh assert "ModuleNotFoundError|ImportError|PermissionError" in test_sh assert "Traceback (most recent call last):" not in test_sh + assert "ast.walk" in setup_command + assert "ast.walk" in test_sh assert "/usr/local/bin/uv run python" in test_sh assert "toolathlon_container.py write-task-tokens" in setup_command assert setup_command.index("write-task-tokens") < setup_command.index( @@ -203,6 +205,151 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup( ) +def _run_toolathlon_setup_script( + tmp_path: Path, preprocess_source: str +) -> subprocess.CompletedProcess: + from benchflow.adapters._toolathlon import ( + _toolathlon_container_module_source, + _toolathlon_setup_command, + ) + + workspace = tmp_path / "workspace" + task_dir = workspace / "tasks" / "finalpool" / "demo" + preprocess_dir = task_dir / "preprocess" + preprocess_dir.mkdir(parents=True) + (workspace / "agent_workspace").mkdir(parents=True) + for package in ( + workspace / "tasks", + workspace / "tasks" / "finalpool", + task_dir, + preprocess_dir, + ): + (package / "__init__.py").write_text("") + (preprocess_dir / "main.py").write_text(preprocess_source) + helper = workspace / ".toolathlon" / "toolathlon_container.py" + helper.parent.mkdir() + helper.write_text(_toolathlon_container_module_source()) + helper.chmod(0o755) + + script = _toolathlon_setup_command(task_name="demo", variant="gym") + script = script.replace("/workspace", str(workspace)) + script = script.replace("/opt/venv/bin/python3", sys.executable) + script_path = tmp_path / "setup.sh" + script_path.write_text(script) + + env = os.environ.copy() + env["TOOLATHLON_WORKSPACE"] = str(workspace) + + return subprocess.run( + ["bash", str(script_path)], + text=True, + capture_output=True, + env=env, + timeout=30, + ) + + +def test_toolathlon_preprocess_ignores_launch_time_in_comments( + tmp_path: Path, +) -> None: + """Guards PR #887: launch_time comments do not receive the CLI flag.""" + result = _run_toolathlon_setup_script( + tmp_path, + "import argparse\n" + "# launch_time is documented here but not accepted by this script.\n" + "parser = argparse.ArgumentParser()\n" + "parser.add_argument('--agent_workspace', required=True)\n" + "parser.parse_args()\n", + ) + assert result.returncode == 0, result.stderr + assert "unrecognized arguments: --launch_time" not in result.stderr + + +def _run_toolathlon_verifier_script( + tmp_path: Path, evaluator_source: str +) -> subprocess.CompletedProcess: + from benchflow.adapters._toolathlon import _toolathlon_test_sh + + workspace = tmp_path / "workspace" + logs = tmp_path / "logs" / "verifier" + eval_dir = workspace / "tasks" / "finalpool" / "demo" / "evaluation" + eval_dir.mkdir(parents=True) + (workspace / "agent_workspace").mkdir(parents=True) + (workspace / "tasks" / "finalpool" / "demo" / "groundtruth_workspace").mkdir() + for package in ( + workspace / "tasks", + workspace / "tasks" / "finalpool", + workspace / "tasks" / "finalpool" / "demo", + eval_dir, + ): + (package / "__init__.py").write_text("") + (eval_dir / "main.py").write_text(evaluator_source) + + script = _toolathlon_test_sh(task_name="demo", variant="gym") + script = script.replace("/workspace", str(workspace)) + script = script.replace("/logs/verifier", str(logs)) + script = script.replace("/opt/venv/bin/python3", sys.executable) + script_path = tmp_path / "test.sh" + script_path.write_text(script) + + return subprocess.run( + ["bash", str(script_path)], + text=True, + capture_output=True, + timeout=30, + ) + + +def test_toolathlon_verifier_scores_task_failure_exit_zero(tmp_path: Path) -> None: + """Guards PR #887: evaluator task failures score 0 without infra failure.""" + result = _run_toolathlon_verifier_script( + tmp_path, + "raise ValueError('Some tests FAILED')\n", + ) + + assert result.returncode == 0, result.stderr + reward_dir = tmp_path / "logs" / "verifier" + assert (reward_dir / "reward.txt").read_text() == "0.0\n" + assert json.loads((reward_dir / "reward.json").read_text()) == {"reward": 0.0} + assert "ValueError: Some tests FAILED" in result.stdout + + +def test_toolathlon_verifier_escalates_import_failure(tmp_path: Path) -> None: + """Guards PR #887: broken evaluator environments fail the verifier.""" + result = _run_toolathlon_verifier_script( + tmp_path, + "import definitely_missing_toolathlon_dependency\n", + ) + + assert result.returncode != 0 + assert "ModuleNotFoundError" in result.stdout + assert "evaluator environment failure" in result.stderr + assert not (tmp_path / "logs" / "verifier" / "reward.txt").exists() + + +def test_toolathlon_verifier_ignores_launch_time_in_comments( + tmp_path: Path, +) -> None: + """Guards PR #887: launch_time comments do not downgrade verifier reward.""" + result = _run_toolathlon_verifier_script( + tmp_path, + "import argparse\n" + "# launch_time is documented here but not accepted by this evaluator.\n" + "parser = argparse.ArgumentParser()\n" + "parser.add_argument('--agent_workspace', required=True)\n" + "parser.add_argument('--groundtruth_workspace', required=True)\n" + "parser.add_argument('--res_log_file', required=True)\n" + "parser.parse_args()\n", + ) + assert result.returncode == 0, result.stderr + logs = tmp_path / "logs" / "verifier" + assert json.loads((logs / "reward.json").read_text()) == {"reward": 1.0} + assert ( + "unrecognized arguments: --launch_time" + not in (logs / "toolathlon_evaluator.log").read_text() + ) + + def _write_toolathlon_repo_skeleton(repo: Path) -> Path: """Minimal upstream Toolathlon layout; returns tasks/finalpool.""" (repo / ".git").mkdir(parents=True) @@ -852,6 +999,7 @@ def _run_container_helper( def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None: + """Guards PR #887: container config bakes runtime secrets and attributes.""" (tmp_path / "configs").mkdir() result = _run_container_helper( tmp_path, @@ -882,6 +1030,13 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None # Unset secrets fall back to their example defaults, not the literal env ref. assert tokens["huggingface_token"] == "XX" assert tokens["github_read_only"] == "1" + assert ( + tokens["kubeconfig_path"] + == "/workspace/deployment/k8s/configs/cluster1-config.yaml" + ) + assert ( + tokens["emails_config_file"] == "/workspace/configs/example_email_config.json" + ) # Upstream preprocess/eval read tokens via ATTRIBUTE access (addict.Dict); # the generated dict must support it (a plain dict would AttributeError). assert tokens.github_token == "gho_xyz" @@ -889,6 +1044,7 @@ def test_toolathlon_container_write_config_bakes_secrets(tmp_path: Path) -> None def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None: + """Guards PR #887: PEM credentials bypass JSON parsing and lock mode.""" from benchflow.adapters._toolathlon import _toolathlon_credential_setup_command cmd = _toolathlon_credential_setup_command({"configs/snowflake_rsa_key.p8"}) @@ -902,6 +1058,7 @@ def test_toolathlon_credential_setup_pem_key_skips_json_and_locks_mode() -> None def test_toolathlon_container_launch_resolves_tokens(tmp_path: Path) -> None: + """Guards PR #887: launcher resolves global and per-task token refs.""" (tmp_path / "configs").mkdir() (tmp_path / "configs" / "token_key_session.py").write_text( "all_token_key_session = {" @@ -1023,6 +1180,7 @@ def test_toolathlon_container_launch_wraps_google_cloud_mcp(tmp_path: Path) -> N def test_toolathlon_container_launch_resolves_env(tmp_path: Path) -> None: + """Guards PR #887: launcher resolves token refs inside child env.""" (tmp_path / "configs").mkdir() (tmp_path / "configs" / "token_key_session.py").write_text( "all_token_key_session = {'github_token': 'gho_secret'}\n" @@ -1065,6 +1223,7 @@ def test_toolathlon_container_launch_prefers_generated_kubeconfig( def test_toolathlon_container_launch_ensures_dirs(tmp_path: Path) -> None: + """Guards PR #887: launcher creates server storage dirs before spawn.""" (tmp_path / "configs").mkdir() (tmp_path / "configs" / "token_key_session.py").write_text( "all_token_key_session = {}\n" @@ -1113,8 +1272,7 @@ def test_toolathlon_container_launch_filters_non_json_stdout( def test_toolathlon_arxiv_server_declares_ensure_dirs( tmp_path: Path, monkeypatch ) -> None: - """The arxiv_local server's --storage-path is passed to the launcher as a - directory to pre-create, so the evaluator's listdir cannot crash.""" + """Guards PR #887: arxiv_local storage is pre-created for evaluators.""" monkeypatch.chdir(tmp_path) repo = tmp_path / "Toolathlon" (repo / ".git").mkdir(parents=True)