From b4dcafe2b7aa1386b801e5b6f1e6d77353b224f7 Mon Sep 17 00:00:00 2001 From: Yimin Liu <47510889+Yiminnn@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:18:33 -0700 Subject: [PATCH 1/4] Preserve OpenHands context budget --- src/benchflow/agents/registry.py | 95 +++++++++++++++++++++++--- tests/test_openhands_context_budget.py | 48 +++++++++++++ 2 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 tests/test_openhands_context_budget.py diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 0f00b33a..6e8b7b5c 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -120,6 +120,85 @@ def _apt_install(*packages: str) -> str: _OPENHANDS_CLI_GIT_REV = "3ca17446c5d9c1e35e054803478a3501ec251ecf" _OPENHANDS_SDK_VERSION = "1.22.1" _OPENHANDS_TOOLS_VERSION = "1.22.1" +_OPENHANDS_SETTINGS_WRITER_PATH = "/opt/benchflow/bin/openhands-settings-writer" +_OPENHANDS_SETTINGS_WRITER = r'''import json +import os +import sys +from pathlib import Path + + +def optional_int(name): + value = os.environ.get(name, "").strip() + if not value: + return None + parsed = int(value) + if parsed <= 0: + raise ValueError(f"{name} must be positive") + return parsed + + +def optional_bool(name): + value = os.environ.get(name, "").strip().lower() + if not value: + return None + if value in {"1", "true", "yes"}: + return True + if value in {"0", "false", "no"}: + return False + raise ValueError(f"{name} must be a boolean") + + +llm = { + "model": os.environ["LLM_MODEL"], + "api_key": os.environ["LLM_API_KEY"], + "usage_id": "agent", +} +for env_name, field_name in ( + ("LLM_BASE_URL", "base_url"), + ("LLM_API_VERSION", "api_version"), +): + value = os.environ.get(env_name, "").strip() + if value: + llm[field_name] = value + +for env_name, field_name in ( + ("LLM_NATIVE_TOOL_CALLING", "native_tool_calling"), + ("LLM_CACHING_PROMPT", "caching_prompt"), + ("LLM_DROP_PARAMS", "drop_params"), + ("LLM_MODIFY_PARAMS", "modify_params"), +): + value = optional_bool(env_name) + if value is not None: + llm[field_name] = value + +context_limit = optional_int("BENCHFLOW_OPENHANDS_CONTEXT_LIMIT") +output_limit = optional_int("BENCHFLOW_OPENHANDS_OUTPUT_LIMIT") +if context_limit is not None: + llm["max_input_tokens"] = context_limit +if output_limit is not None: + llm["max_output_tokens"] = output_limit + +condenser = { + "llm": {**llm, "usage_id": "condenser"}, + "max_size": 80, + "keep_first": 4, + "kind": "LLMSummarizingCondenser", +} +if context_limit is not None and output_limit is not None: + reserve = optional_int("BENCHFLOW_OPENHANDS_CONTEXT_RESERVE") or 4096 + condenser_limit = context_limit - output_limit - reserve + if condenser_limit <= 0: + raise ValueError("OpenHands context budget leaves no room for input") + condenser["max_tokens"] = condenser_limit + +settings = { + "llm": llm, + "tools": [], + "condenser": condenser, + "kind": "Agent", +} +Path(sys.argv[1]).write_text(json.dumps(settings, separators=(",", ":"))) +''' _JS_AGENT_PATH = ( f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:" f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH" @@ -923,20 +1002,16 @@ class AgentConfig: 'echo \'{"llm":{"model":"placeholder","api_key":"placeholder"}}\' ' "> ~/.openhands/agent_settings.json && " "command -v openhands >/dev/null 2>&1" + + " && " + + _install_python_script( + _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER + ) ), launch_cmd=( 'export PATH="$HOME/.local/bin:$PATH" && ' "mkdir -p ~/.openhands && " - # Write llm settings including base_url so the BenchFlow LiteLLM - # gateway (LLM_BASE_URL) is honored. OpenHands' --override-with-envs - # does not reliably apply base_url; it is omitted when unset. - '{ printf \'{"llm":{"model":"%s","api_key":"%s"\' ' - '"$LLM_MODEL" "$LLM_API_KEY"; ' - 'if [ -n "$LLM_BASE_URL" ]; then ' - 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' - 'if [ -n "$LLM_API_VERSION" ]; then ' - 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' - "printf '}}'; } > ~/.openhands/agent_settings.json && " + f"python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " + "~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), protocol="acp", diff --git a/tests/test_openhands_context_budget.py b/tests/test_openhands_context_budget.py new file mode 100644 index 00000000..5f5ef29b --- /dev/null +++ b/tests/test_openhands_context_budget.py @@ -0,0 +1,48 @@ +import json +import os +import subprocess +import sys + +from benchflow.agents.registry import AGENTS, _OPENHANDS_SETTINGS_WRITER + + +def test_openhands_settings_reserve_context_for_output(tmp_path): + target = tmp_path / "agent_settings.json" + env = { + **os.environ, + "LLM_MODEL": "openai/qwen35-9b-base", + "LLM_API_KEY": "placeholder", + "LLM_BASE_URL": "http://example.test/v1", + "LLM_NATIVE_TOOL_CALLING": "true", + "LLM_CACHING_PROMPT": "false", + "LLM_DROP_PARAMS": "true", + "LLM_MODIFY_PARAMS": "true", + "BENCHFLOW_OPENHANDS_CONTEXT_LIMIT": "262144", + "BENCHFLOW_OPENHANDS_OUTPUT_LIMIT": "32768", + "BENCHFLOW_OPENHANDS_CONTEXT_RESERVE": "4096", + } + + completed = subprocess.run( + [sys.executable, "-c", _OPENHANDS_SETTINGS_WRITER, str(target)], + env=env, + capture_output=True, + text=True, + check=True, + ) + settings = json.loads(target.read_text()) + + assert completed.stdout == "" + assert settings["kind"] == "Agent" + assert settings["llm"]["max_input_tokens"] == 262144 + assert settings["llm"]["max_output_tokens"] == 32768 + assert settings["llm"]["caching_prompt"] is False + assert settings["condenser"]["kind"] == "LLMSummarizingCondenser" + assert settings["condenser"]["max_tokens"] == 225280 + assert settings["condenser"]["llm"]["usage_id"] == "condenser" + + +def test_openhands_launch_installs_and_runs_settings_writer(): + config = AGENTS["openhands"] + path = "/opt/benchflow/bin/openhands-settings-writer" + assert path in config.install_cmd + assert path in config.launch_cmd From 3260ce4cfa36403e879ddeee5442468e14d03a8d Mon Sep 17 00:00:00 2001 From: Yimin Liu <47510889+Yiminnn@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:21:19 -0700 Subject: [PATCH 2/4] Sort OpenHands context test imports --- tests/test_openhands_context_budget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_openhands_context_budget.py b/tests/test_openhands_context_budget.py index 5f5ef29b..70324db6 100644 --- a/tests/test_openhands_context_budget.py +++ b/tests/test_openhands_context_budget.py @@ -3,7 +3,7 @@ import subprocess import sys -from benchflow.agents.registry import AGENTS, _OPENHANDS_SETTINGS_WRITER +from benchflow.agents.registry import _OPENHANDS_SETTINGS_WRITER, AGENTS def test_openhands_settings_reserve_context_for_output(tmp_path): From 4ebb0e2c6986177348b2bc82b06dc14427a52614 Mon Sep 17 00:00:00 2001 From: Yimin Liu <47510889+Yiminnn@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:25:22 -0700 Subject: [PATCH 3/4] Keep OpenHands setup manifest-neutral --- src/benchflow/agents/manifest.py | 1 + src/benchflow/agents/registry.py | 31 ++++++++++++++++++++------ src/benchflow/rollout_planes.py | 8 +++++-- src/benchflow/runtime.py | 2 +- tests/test_openhands_context_budget.py | 9 +++++--- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/benchflow/agents/manifest.py b/src/benchflow/agents/manifest.py index daebc8cc..e3334982 100644 --- a/src/benchflow/agents/manifest.py +++ b/src/benchflow/agents/manifest.py @@ -91,6 +91,7 @@ "disallow_web_tools_launch_suffix", "task_mcp_transport", "task_mcp_config_path", + "launch_override_cmd", } ) diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 6e8b7b5c..7efc9be7 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -121,7 +121,7 @@ def _apt_install(*packages: str) -> str: _OPENHANDS_SDK_VERSION = "1.22.1" _OPENHANDS_TOOLS_VERSION = "1.22.1" _OPENHANDS_SETTINGS_WRITER_PATH = "/opt/benchflow/bin/openhands-settings-writer" -_OPENHANDS_SETTINGS_WRITER = r'''import json +_OPENHANDS_SETTINGS_WRITER = r"""import json import os import sys from pathlib import Path @@ -198,7 +198,7 @@ def optional_bool(name): "kind": "Agent", } Path(sys.argv[1]).write_text(json.dumps(settings, separators=(",", ":"))) -''' +""" _JS_AGENT_PATH = ( f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:" f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH" @@ -587,6 +587,9 @@ class AgentConfig: task_mcp_transport: str = "acp" # Native-config target path, relative to $HOME unless absolute. task_mcp_config_path: str = "" + # Host-owned launch override for provider/harness compatibility shims that + # cannot be represented in the data-only agent manifest contract. + launch_override_cmd: str = "" # Agent registry — all supported agents @@ -1002,15 +1005,29 @@ class AgentConfig: 'echo \'{"llm":{"model":"placeholder","api_key":"placeholder"}}\' ' "> ~/.openhands/agent_settings.json && " "command -v openhands >/dev/null 2>&1" - + " && " - + _install_python_script( - _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER - ) ), launch_cmd=( 'export PATH="$HOME/.local/bin:$PATH" && ' "mkdir -p ~/.openhands && " - f"python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " + # Write llm settings including base_url so the BenchFlow LiteLLM + # gateway (LLM_BASE_URL) is honored. OpenHands' --override-with-envs + # does not reliably apply base_url; it is omitted when unset. + '{ printf \'{"llm":{"model":"%s","api_key":"%s"\' ' + '"$LLM_MODEL" "$LLM_API_KEY"; ' + 'if [ -n "$LLM_BASE_URL" ]; then ' + 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' + 'if [ -n "$LLM_API_VERSION" ]; then ' + 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' + "printf '}}'; } > ~/.openhands/agent_settings.json && " + "openhands acp --always-approve --override-with-envs" + ), + launch_override_cmd=( + 'export PATH="$HOME/.local/bin:$PATH" && ' + "mkdir -p ~/.openhands && " + + _install_python_script( + _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER + ) + + f" && python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " "~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), diff --git a/src/benchflow/rollout_planes.py b/src/benchflow/rollout_planes.py index 6d2272d9..5aff1011 100644 --- a/src/benchflow/rollout_planes.py +++ b/src/benchflow/rollout_planes.py @@ -54,10 +54,14 @@ class DefaultRolloutPlanes: """Default bindings for the four concrete planes.""" def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: - launch = AGENT_LAUNCH.get(agent, agent) + agent_cfg = AGENTS.get(agent) + launch = ( + agent_cfg.launch_override_cmd + if agent_cfg and agent_cfg.launch_override_cmd + else AGENT_LAUNCH.get(agent, agent) + ) if not disallow_web_tools: return launch - agent_cfg = AGENTS.get(agent) if agent_cfg and agent_cfg.disallow_web_tools_launch_suffix: return launch + agent_cfg.disallow_web_tools_launch_suffix return launch diff --git a/src/benchflow/runtime.py b/src/benchflow/runtime.py index 1c24c832..40c111b5 100644 --- a/src/benchflow/runtime.py +++ b/src/benchflow/runtime.py @@ -195,7 +195,7 @@ def launch_cmd(self) -> str: config = self.config if config is None: return self.name - return config.launch_cmd + return config.launch_override_cmd or config.launch_cmd def __repr__(self) -> str: return f"Agent({self.name!r}, model={self.model!r})" diff --git a/tests/test_openhands_context_budget.py b/tests/test_openhands_context_budget.py index 70324db6..ff120757 100644 --- a/tests/test_openhands_context_budget.py +++ b/tests/test_openhands_context_budget.py @@ -4,6 +4,7 @@ import sys from benchflow.agents.registry import _OPENHANDS_SETTINGS_WRITER, AGENTS +from benchflow.rollout_planes import DefaultRolloutPlanes def test_openhands_settings_reserve_context_for_output(tmp_path): @@ -42,7 +43,9 @@ def test_openhands_settings_reserve_context_for_output(tmp_path): def test_openhands_launch_installs_and_runs_settings_writer(): - config = AGENTS["openhands"] path = "/opt/benchflow/bin/openhands-settings-writer" - assert path in config.install_cmd - assert path in config.launch_cmd + launch = DefaultRolloutPlanes().agent_launch("openhands", disallow_web_tools=False) + assert path in AGENTS["openhands"].launch_override_cmd + assert "mkdir -p ~/.openhands" in AGENTS["openhands"].launch_override_cmd + assert path in launch + assert launch == AGENTS["openhands"].launch_override_cmd From ff29b1637b97f30f025ad544df79e51d0aac3a01 Mon Sep 17 00:00:00 2001 From: Yimin Liu <47510889+Yiminnn@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:34:31 -0700 Subject: [PATCH 4/4] Install OpenHands context shim before launch --- src/benchflow/agents/install.py | 22 ++++++++++++++++++++++ src/benchflow/agents/manifest.py | 1 + src/benchflow/agents/registry.py | 12 +++++++----- tests/test_openhands_context_budget.py | 17 +++++++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/benchflow/agents/install.py b/src/benchflow/agents/install.py index a65d8d6d..dd934358 100644 --- a/src/benchflow/agents/install.py +++ b/src/benchflow/agents/install.py @@ -255,6 +255,28 @@ async def install_agent( diagnostics=diag.stdout or "", log_path=str(install_log), ) + if agent_cfg and agent_cfg.install_setup_cmd: + setup_cmd = agent_cfg.install_setup_cmd + setup_result = await env.exec(setup_cmd, timeout_sec=install_timeout) + setup_parts = [f"$ {setup_cmd}\n"] + if setup_result.stdout: + setup_parts.extend(["=== stdout ===\n", setup_result.stdout]) + if not setup_result.stdout.endswith("\n"): + setup_parts.append("\n") + if setup_result.stderr: + setup_parts.extend(["=== stderr ===\n", setup_result.stderr]) + if not setup_result.stderr.endswith("\n"): + setup_parts.append("\n") + with install_log.open("a") as handle: + handle.write("".join(setup_parts)) + if setup_result.return_code != 0: + raise AgentInstallError( + agent=agent_base, + return_code=setup_result.return_code, + stdout="".join(setup_parts), + diagnostics="post-install setup failed", + log_path=str(install_log), + ) return agent_cfg diff --git a/src/benchflow/agents/manifest.py b/src/benchflow/agents/manifest.py index e3334982..83fb495f 100644 --- a/src/benchflow/agents/manifest.py +++ b/src/benchflow/agents/manifest.py @@ -91,6 +91,7 @@ "disallow_web_tools_launch_suffix", "task_mcp_transport", "task_mcp_config_path", + "install_setup_cmd", "launch_override_cmd", } ) diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 7efc9be7..776616c5 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -587,8 +587,10 @@ class AgentConfig: task_mcp_transport: str = "acp" # Native-config target path, relative to $HOME unless absolute. task_mcp_config_path: str = "" - # Host-owned launch override for provider/harness compatibility shims that + # Host-owned install step for provider/harness compatibility shims that # cannot be represented in the data-only agent manifest contract. + install_setup_cmd: str = "" + # Host-owned launch override paired with install_setup_cmd. launch_override_cmd: str = "" @@ -1021,13 +1023,13 @@ class AgentConfig: "printf '}}'; } > ~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), + install_setup_cmd=_install_python_script( + _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER + ), launch_override_cmd=( 'export PATH="$HOME/.local/bin:$PATH" && ' "mkdir -p ~/.openhands && " - + _install_python_script( - _OPENHANDS_SETTINGS_WRITER_PATH, _OPENHANDS_SETTINGS_WRITER - ) - + f" && python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " + f"python3 {_OPENHANDS_SETTINGS_WRITER_PATH} " "~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), diff --git a/tests/test_openhands_context_budget.py b/tests/test_openhands_context_budget.py index ff120757..2b95b6a5 100644 --- a/tests/test_openhands_context_budget.py +++ b/tests/test_openhands_context_budget.py @@ -2,7 +2,11 @@ import os import subprocess import sys +from unittest.mock import AsyncMock, MagicMock +import pytest + +from benchflow.agents.install import install_agent from benchflow.agents.registry import _OPENHANDS_SETTINGS_WRITER, AGENTS from benchflow.rollout_planes import DefaultRolloutPlanes @@ -45,7 +49,20 @@ def test_openhands_settings_reserve_context_for_output(tmp_path): def test_openhands_launch_installs_and_runs_settings_writer(): path = "/opt/benchflow/bin/openhands-settings-writer" launch = DefaultRolloutPlanes().agent_launch("openhands", disallow_web_tools=False) + assert path in AGENTS["openhands"].install_setup_cmd assert path in AGENTS["openhands"].launch_override_cmd assert "mkdir -p ~/.openhands" in AGENTS["openhands"].launch_override_cmd assert path in launch assert launch == AGENTS["openhands"].launch_override_cmd + + +@pytest.mark.asyncio +async def test_openhands_install_runs_root_owned_settings_setup(tmp_path): + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0, stdout="", stderr="")) + + await install_agent(env, "openhands", tmp_path) + + assert env.exec.await_count == 2 + setup_cmd = env.exec.await_args_list[1].args[0] + assert "/opt/benchflow/bin/openhands-settings-writer" in setup_cmd