From adccf0dba9d72b627aea4474fd5b698581280256 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 10 Jul 2026 17:44:42 -0700 Subject: [PATCH 1/4] Fix OpenHands Azure reasoning effort propagation --- src/benchflow/agents/registry.py | 4 ++++ src/benchflow/providers/litellm_config.py | 25 +++++++++++++++++++++++ tests/test_agent_registry.py | 10 +++++++++ tests/test_litellm_config.py | 14 +++++++++++++ 4 files changed, 53 insertions(+) diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 8e2eedf2..97b4dfa5 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -932,6 +932,10 @@ class AgentConfig: 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' 'if [ -n "$LLM_API_VERSION" ]; then ' 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' + 'if [ -n "$LLM_REASONING_EFFORT" ]; then ' + 'printf \',"reasoning_effort":"%s",' + '"litellm_extra_body":{"reasoning_effort":"%s"}\' ' + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"; fi; ' "printf '}}'; } > ~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), diff --git a/src/benchflow/providers/litellm_config.py b/src/benchflow/providers/litellm_config.py index 47b5247d..7db2f837 100644 --- a/src/benchflow/providers/litellm_config.py +++ b/src/benchflow/providers/litellm_config.py @@ -18,9 +18,13 @@ AZURE_API_VERSION_ENV = "AZURE_API_VERSION" AZURE_DEFAULT_API_VERSION = "preview" BEDROCK_THINKING_EFFORT_ENV = "BENCHFLOW_BEDROCK_THINKING_EFFORT" +PROVIDER_REASONING_EFFORT_ENV = "BENCHFLOW_REASONING_EFFORT" LITELLM_MODEL_ALIAS_ENV = "BENCHFLOW_LITELLM_MODEL_ALIAS" LITELLM_MODEL_VIA_ENV = "BENCHFLOW_LITELLM_MODEL_VIA_ENV" LITELLM_MASTER_KEY_ENV = "BENCHFLOW_LITELLM_MASTER_KEY" +_PROVIDER_REASONING_EFFORTS = frozenset( + {"none", "minimal", "low", "medium", "high", "xhigh", "max"} +) # Per-token USD prices for models LiteLLM's built-in ``model_cost`` does not # already know (custom OpenAI-compatible endpoints such as private vLLM servers @@ -164,6 +168,24 @@ def _registered_api_key_ref(cfg: ProviderConfig) -> str | None: return None +def _provider_reasoning_effort(env: dict[str, str]) -> str | None: + """Return an explicitly requested gateway-side reasoning effort.""" + raw = ( + env.get(PROVIDER_REASONING_EFFORT_ENV) + or env.get("LLM_REASONING_EFFORT") + or "" + ) + effort = raw.strip().lower() + if not effort: + return None + if effort not in _PROVIDER_REASONING_EFFORTS: + allowed = ", ".join(sorted(_PROVIDER_REASONING_EFFORTS)) + raise ValueError( + f"{PROVIDER_REASONING_EFFORT_ENV} must be one of: {allowed}" + ) + return effort + + def _bedrock_thinking_effort(model: str, env: dict[str, str]) -> str | None: if not _BEDROCK_ADAPTIVE_THINKING_RE.search(model): return None @@ -213,6 +235,9 @@ def _route_registered_provider( "api_base": api_base, "api_version": env.get(AZURE_API_VERSION_ENV, AZURE_DEFAULT_API_VERSION), } + effort = _provider_reasoning_effort(env) + if effort: + params["reasoning_effort"] = effort return LiteLLMRoute( requested_model=model, model_alias=safe_model_alias(model), diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index a5e73346..5063039b 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -202,6 +202,16 @@ def test_openhands_launch_cmd_writes_optional_azure_api_version(self): assert ',"api_version":"%s"' in cfg.launch_cmd assert '"$LLM_API_VERSION"' in cfg.launch_cmd + def test_openhands_launch_cmd_writes_optional_reasoning_effort(self): + """Guards GPT-5.6-SOL xhigh runs against OpenHands' default high effort.""" + cfg = AGENTS["openhands"] + assert 'if [ -n "$LLM_REASONING_EFFORT" ]' in cfg.launch_cmd + assert ',"reasoning_effort":"%s",' in cfg.launch_cmd + assert '"litellm_extra_body":{"reasoning_effort":"%s"}' in cfg.launch_cmd + assert ( + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"' in cfg.launch_cmd + ) + def test_harvey_lab_installs_python_deps_in_venv(self): """Guards the v0.5 stress failure where pip hit PEP 668 in Ubuntu.""" cfg = AGENTS["harvey-lab-harness"] diff --git a/tests/test_litellm_config.py b/tests/test_litellm_config.py index 6679c097..7cb2c8dc 100644 --- a/tests/test_litellm_config.py +++ b/tests/test_litellm_config.py @@ -49,6 +49,20 @@ def test_azure_openai_route_uses_resource_and_preview_version(): assert route.litellm_params["api_version"] == "preview" +def test_azure_openai_route_honors_openhands_reasoning_effort_env(): + """Guards GPT-5.6-SOL xhigh runs against proxy-alias capability guessing.""" + route = resolve_litellm_route( + "azure-foundry-openai/gpt-5.6-sol", + { + "AZURE_API_KEY": "key", + "AZURE_RESOURCE": "benchflow", + "LLM_REASONING_EFFORT": "xhigh", + }, + ) + + assert route.litellm_params["reasoning_effort"] == "xhigh" + + def test_azure_anthropic_route_uses_azure_ai_anthropic_surface(): route = resolve_litellm_route( "azure-foundry-anthropic/claude-opus-4-5", From 401f41203139231113c5c2009871d4b3586db0da Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 10 Jul 2026 17:47:12 -0700 Subject: [PATCH 2/4] Reference PR in reasoning-effort regression tests --- tests/test_agent_registry.py | 2 +- tests/test_litellm_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index 5063039b..adddbe3c 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -203,7 +203,7 @@ def test_openhands_launch_cmd_writes_optional_azure_api_version(self): assert '"$LLM_API_VERSION"' in cfg.launch_cmd def test_openhands_launch_cmd_writes_optional_reasoning_effort(self): - """Guards GPT-5.6-SOL xhigh runs against OpenHands' default high effort.""" + """Guards PR #911 against OpenHands silently using default high effort.""" cfg = AGENTS["openhands"] assert 'if [ -n "$LLM_REASONING_EFFORT" ]' in cfg.launch_cmd assert ',"reasoning_effort":"%s",' in cfg.launch_cmd diff --git a/tests/test_litellm_config.py b/tests/test_litellm_config.py index 7cb2c8dc..cd8f52ec 100644 --- a/tests/test_litellm_config.py +++ b/tests/test_litellm_config.py @@ -50,7 +50,7 @@ def test_azure_openai_route_uses_resource_and_preview_version(): def test_azure_openai_route_honors_openhands_reasoning_effort_env(): - """Guards GPT-5.6-SOL xhigh runs against proxy-alias capability guessing.""" + """Guards PR #911 against proxy-alias capability guessing.""" route = resolve_litellm_route( "azure-foundry-openai/gpt-5.6-sol", { From 9c114d36418c44d541c085bb87d6cc948be259c3 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 10 Jul 2026 17:55:34 -0700 Subject: [PATCH 3/4] Keep xhigh enforcement in the provider route --- src/benchflow/agents/registry.py | 4 ---- src/benchflow/providers/litellm_config.py | 8 ++------ tests/test_agent_registry.py | 10 ---------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 97b4dfa5..8e2eedf2 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -932,10 +932,6 @@ class AgentConfig: 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' 'if [ -n "$LLM_API_VERSION" ]; then ' 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' - 'if [ -n "$LLM_REASONING_EFFORT" ]; then ' - 'printf \',"reasoning_effort":"%s",' - '"litellm_extra_body":{"reasoning_effort":"%s"}\' ' - '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"; fi; ' "printf '}}'; } > ~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), diff --git a/src/benchflow/providers/litellm_config.py b/src/benchflow/providers/litellm_config.py index 7db2f837..18c0bda6 100644 --- a/src/benchflow/providers/litellm_config.py +++ b/src/benchflow/providers/litellm_config.py @@ -171,18 +171,14 @@ def _registered_api_key_ref(cfg: ProviderConfig) -> str | None: def _provider_reasoning_effort(env: dict[str, str]) -> str | None: """Return an explicitly requested gateway-side reasoning effort.""" raw = ( - env.get(PROVIDER_REASONING_EFFORT_ENV) - or env.get("LLM_REASONING_EFFORT") - or "" + env.get(PROVIDER_REASONING_EFFORT_ENV) or env.get("LLM_REASONING_EFFORT") or "" ) effort = raw.strip().lower() if not effort: return None if effort not in _PROVIDER_REASONING_EFFORTS: allowed = ", ".join(sorted(_PROVIDER_REASONING_EFFORTS)) - raise ValueError( - f"{PROVIDER_REASONING_EFFORT_ENV} must be one of: {allowed}" - ) + raise ValueError(f"{PROVIDER_REASONING_EFFORT_ENV} must be one of: {allowed}") return effort diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index adddbe3c..a5e73346 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -202,16 +202,6 @@ def test_openhands_launch_cmd_writes_optional_azure_api_version(self): assert ',"api_version":"%s"' in cfg.launch_cmd assert '"$LLM_API_VERSION"' in cfg.launch_cmd - def test_openhands_launch_cmd_writes_optional_reasoning_effort(self): - """Guards PR #911 against OpenHands silently using default high effort.""" - cfg = AGENTS["openhands"] - assert 'if [ -n "$LLM_REASONING_EFFORT" ]' in cfg.launch_cmd - assert ',"reasoning_effort":"%s",' in cfg.launch_cmd - assert '"litellm_extra_body":{"reasoning_effort":"%s"}' in cfg.launch_cmd - assert ( - '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"' in cfg.launch_cmd - ) - def test_harvey_lab_installs_python_deps_in_venv(self): """Guards the v0.5 stress failure where pip hit PEP 668 in Ubuntu.""" cfg = AGENTS["harvey-lab-harness"] From cc784ef31baad99070dd6c829b8de937fd81ed0f Mon Sep 17 00:00:00 2001 From: Bingran You Date: Sat, 11 Jul 2026 13:31:16 -0700 Subject: [PATCH 4/4] Restore OpenHands xhigh request propagation --- src/benchflow/agents/registry.py | 4 ++++ tests/test_agent_registry.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 8e2eedf2..97b4dfa5 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -932,6 +932,10 @@ class AgentConfig: 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' 'if [ -n "$LLM_API_VERSION" ]; then ' 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' + 'if [ -n "$LLM_REASONING_EFFORT" ]; then ' + 'printf \',"reasoning_effort":"%s",' + '"litellm_extra_body":{"reasoning_effort":"%s"}\' ' + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"; fi; ' "printf '}}'; } > ~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index a5e73346..adddbe3c 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -202,6 +202,16 @@ def test_openhands_launch_cmd_writes_optional_azure_api_version(self): assert ',"api_version":"%s"' in cfg.launch_cmd assert '"$LLM_API_VERSION"' in cfg.launch_cmd + def test_openhands_launch_cmd_writes_optional_reasoning_effort(self): + """Guards PR #911 against OpenHands silently using default high effort.""" + cfg = AGENTS["openhands"] + assert 'if [ -n "$LLM_REASONING_EFFORT" ]' in cfg.launch_cmd + assert ',"reasoning_effort":"%s",' in cfg.launch_cmd + assert '"litellm_extra_body":{"reasoning_effort":"%s"}' in cfg.launch_cmd + assert ( + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"' in cfg.launch_cmd + ) + def test_harvey_lab_installs_python_deps_in_venv(self): """Guards the v0.5 stress failure where pip hit PEP 668 in Ubuntu.""" cfg = AGENTS["harvey-lab-harness"]