diff --git a/src/benchflow/agents/install.py b/src/benchflow/agents/install.py index a65d8d6d..dd934358 100644 --- a/src/benchflow/agents/install.py +++ b/src/benchflow/agents/install.py @@ -255,6 +255,28 @@ async def install_agent( diagnostics=diag.stdout or "", log_path=str(install_log), ) + if agent_cfg and agent_cfg.install_setup_cmd: + setup_cmd = agent_cfg.install_setup_cmd + setup_result = await env.exec(setup_cmd, timeout_sec=install_timeout) + setup_parts = [f"$ {setup_cmd}\n"] + if setup_result.stdout: + setup_parts.extend(["=== stdout ===\n", setup_result.stdout]) + if not setup_result.stdout.endswith("\n"): + setup_parts.append("\n") + if setup_result.stderr: + setup_parts.extend(["=== stderr ===\n", setup_result.stderr]) + if not setup_result.stderr.endswith("\n"): + setup_parts.append("\n") + with install_log.open("a") as handle: + handle.write("".join(setup_parts)) + if setup_result.return_code != 0: + raise AgentInstallError( + agent=agent_base, + return_code=setup_result.return_code, + stdout="".join(setup_parts), + diagnostics="post-install setup failed", + log_path=str(install_log), + ) return agent_cfg diff --git a/src/benchflow/agents/manifest.py b/src/benchflow/agents/manifest.py index daebc8cc..83fb495f 100644 --- a/src/benchflow/agents/manifest.py +++ b/src/benchflow/agents/manifest.py @@ -91,6 +91,8 @@ "disallow_web_tools_launch_suffix", "task_mcp_transport", "task_mcp_config_path", + "install_setup_cmd", + "launch_override_cmd", } ) diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 09c56fb0..fbd3db68 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -120,6 +120,85 @@ def _apt_install(*packages: str) -> str: _OPENHANDS_CLI_GIT_REV = "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271" _OPENHANDS_SDK_VERSION = "1.28.1" _OPENHANDS_TOOLS_VERSION = "1.28.1" +_OPENHANDS_SETTINGS_WRITER_PATH = "/opt/benchflow/bin/openhands-settings-writer" +_OPENHANDS_SETTINGS_WRITER = r"""import json +import os +import sys +from pathlib import Path + + +def optional_int(name): + value = os.environ.get(name, "").strip() + if not value: + return None + parsed = int(value) + if parsed <= 0: + raise ValueError(f"{name} must be positive") + return parsed + + +def optional_bool(name): + value = os.environ.get(name, "").strip().lower() + if not value: + return None + if value in {"1", "true", "yes"}: + return True + if value in {"0", "false", "no"}: + return False + raise ValueError(f"{name} must be a boolean") + + +llm = { + "model": os.environ["LLM_MODEL"], + "api_key": os.environ["LLM_API_KEY"], + "usage_id": "agent", +} +for env_name, field_name in ( + ("LLM_BASE_URL", "base_url"), + ("LLM_API_VERSION", "api_version"), +): + value = os.environ.get(env_name, "").strip() + if value: + llm[field_name] = value + +for env_name, field_name in ( + ("LLM_NATIVE_TOOL_CALLING", "native_tool_calling"), + ("LLM_CACHING_PROMPT", "caching_prompt"), + ("LLM_DROP_PARAMS", "drop_params"), + ("LLM_MODIFY_PARAMS", "modify_params"), +): + value = optional_bool(env_name) + if value is not None: + llm[field_name] = value + +context_limit = optional_int("BENCHFLOW_OPENHANDS_CONTEXT_LIMIT") +output_limit = optional_int("BENCHFLOW_OPENHANDS_OUTPUT_LIMIT") +if context_limit is not None: + llm["max_input_tokens"] = context_limit +if output_limit is not None: + llm["max_output_tokens"] = output_limit + +condenser = { + "llm": {**llm, "usage_id": "condenser"}, + "max_size": 80, + "keep_first": 4, + "kind": "LLMSummarizingCondenser", +} +if context_limit is not None and output_limit is not None: + reserve = optional_int("BENCHFLOW_OPENHANDS_CONTEXT_RESERVE") or 4096 + condenser_limit = context_limit - output_limit - reserve + if condenser_limit <= 0: + raise ValueError("OpenHands context budget leaves no room for input") + condenser["max_tokens"] = condenser_limit + +settings = { + "llm": llm, + "tools": [], + "condenser": condenser, + "kind": "Agent", +} +Path(sys.argv[1]).write_text(json.dumps(settings, separators=(",", ":"))) +""" _JS_AGENT_PATH = ( f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:" f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH" @@ -508,6 +587,11 @@ class AgentConfig: task_mcp_transport: str = "acp" # Native-config target path, relative to $HOME unless absolute. task_mcp_config_path: str = "" + # Host-owned install step for provider/harness compatibility shims that + # cannot be represented in the data-only agent manifest contract. + install_setup_cmd: str = "" + # Host-owned launch override paired with install_setup_cmd. + launch_override_cmd: str = "" # Agent registry — all supported agents @@ -966,6 +1050,16 @@ class AgentConfig: "fi && " "openhands acp --always-approve --override-with-envs" ), + install_setup_cmd=_install_python_script( + _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER + ), + launch_override_cmd=( + 'export PATH="$HOME/.local/bin:$PATH" && ' + "mkdir -p ~/.openhands && " + f"python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " + "~/.openhands/agent_settings.json && " + "openhands acp --always-approve --override-with-envs" + ), protocol="acp", requires_env=["LLM_API_KEY"], api_protocol="", diff --git a/src/benchflow/cli/agent.py b/src/benchflow/cli/agent.py index 3c8be945..36bef6c1 100644 --- a/src/benchflow/cli/agent.py +++ b/src/benchflow/cli/agent.py @@ -84,7 +84,8 @@ def agent_show( console.print(f" Aliases: {', '.join(aliases)}") console.print(f" Description: {cfg.description}") console.print(f" Protocol: {cfg.protocol}") - console.print(f" Launch: {cfg.launch_cmd}") + launch_cmd = cfg.launch_override_cmd or cfg.launch_cmd + console.print(f" Launch: {launch_cmd}") console.print(f" Requires: {_format_requires(cfg) or '(none)'}") console.print(f" Provider auth: {_PROVIDER_AUTH_MESSAGE}") if cfg.subscription_auth: diff --git a/src/benchflow/rollout_planes.py b/src/benchflow/rollout_planes.py index 6d2272d9..5aff1011 100644 --- a/src/benchflow/rollout_planes.py +++ b/src/benchflow/rollout_planes.py @@ -54,10 +54,14 @@ class DefaultRolloutPlanes: """Default bindings for the four concrete planes.""" def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: - launch = AGENT_LAUNCH.get(agent, agent) + agent_cfg = AGENTS.get(agent) + launch = ( + agent_cfg.launch_override_cmd + if agent_cfg and agent_cfg.launch_override_cmd + else AGENT_LAUNCH.get(agent, agent) + ) if not disallow_web_tools: return launch - agent_cfg = AGENTS.get(agent) if agent_cfg and agent_cfg.disallow_web_tools_launch_suffix: return launch + agent_cfg.disallow_web_tools_launch_suffix return launch diff --git a/src/benchflow/runtime.py b/src/benchflow/runtime.py index 1c24c832..40c111b5 100644 --- a/src/benchflow/runtime.py +++ b/src/benchflow/runtime.py @@ -195,7 +195,7 @@ def launch_cmd(self) -> str: config = self.config if config is None: return self.name - return config.launch_cmd + return config.launch_override_cmd or config.launch_cmd def __repr__(self) -> str: return f"Agent({self.name!r}, model={self.model!r})" diff --git a/tests/test_agent_cli.py b/tests/test_agent_cli.py index 7fe2a4a8..0575849f 100644 --- a/tests/test_agent_cli.py +++ b/tests/test_agent_cli.py @@ -27,3 +27,13 @@ def test_agent_show_mentions_provider_specific_azure_auth() -> None: assert "Provider auth:" in output assert "AZURE_API_KEY" in output assert "AZURE_API_ENDPOINT" in output + + +def test_agent_show_uses_launch_override_when_present() -> None: + """Guards PR #927: agent diagnostics show the effective runtime launch.""" + result = CliRunner().invoke(app, ["agent", "show", "openhands"]) + + assert result.exit_code == 0 + output = click.unstyle(result.output) + assert "Launch:" in output + assert "/opt/benchflow/bin/openhands-settings-writer" in output diff --git a/tests/test_openhands_context_budget.py b/tests/test_openhands_context_budget.py new file mode 100644 index 00000000..2b95b6a5 --- /dev/null +++ b/tests/test_openhands_context_budget.py @@ -0,0 +1,68 @@ +import json +import os +import subprocess +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from benchflow.agents.install import install_agent +from benchflow.agents.registry import _OPENHANDS_SETTINGS_WRITER, AGENTS +from benchflow.rollout_planes import DefaultRolloutPlanes + + +def test_openhands_settings_reserve_context_for_output(tmp_path): + target = tmp_path / "agent_settings.json" + env = { + **os.environ, + "LLM_MODEL": "openai/qwen35-9b-base", + "LLM_API_KEY": "placeholder", + "LLM_BASE_URL": "http://example.test/v1", + "LLM_NATIVE_TOOL_CALLING": "true", + "LLM_CACHING_PROMPT": "false", + "LLM_DROP_PARAMS": "true", + "LLM_MODIFY_PARAMS": "true", + "BENCHFLOW_OPENHANDS_CONTEXT_LIMIT": "262144", + "BENCHFLOW_OPENHANDS_OUTPUT_LIMIT": "32768", + "BENCHFLOW_OPENHANDS_CONTEXT_RESERVE": "4096", + } + + completed = subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + capture_output=True, + text=True, + check=True, + ) + settings = json.loads(target.read_text()) + + assert completed.stdout == "" + assert settings["kind"] == "Agent" + assert settings["llm"]["max_input_tokens"] == 262144 + assert settings["llm"]["max_output_tokens"] == 32768 + assert settings["llm"]["caching_prompt"] is False + assert settings["condenser"]["kind"] == "LLMSummarizingCondenser" + assert settings["condenser"]["max_tokens"] == 225280 + assert settings["condenser"]["llm"]["usage_id"] == "condenser" + + +def test_openhands_launch_installs_and_runs_settings_writer(): + path = "/opt/benchflow/bin/openhands-settings-writer" + launch = DefaultRolloutPlanes().agent_launch("openhands", disallow_web_tools=False) + assert path in AGENTS["openhands"].install_setup_cmd + assert path in AGENTS["openhands"].launch_override_cmd + assert "mkdir -p ~/.openhands" in AGENTS["openhands"].launch_override_cmd + assert path in launch + assert launch == AGENTS["openhands"].launch_override_cmd + + +@pytest.mark.asyncio +async def test_openhands_install_runs_root_owned_settings_setup(tmp_path): + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0, stdout="", stderr="")) + + await install_agent(env, "openhands", tmp_path) + + assert env.exec.await_count == 2 + setup_cmd = env.exec.await_args_list[1].args[0] + assert "/opt/benchflow/bin/openhands-settings-writer" in setup_cmd