diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 8e2eedf2..97b4dfa5 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -932,6 +932,10 @@ class AgentConfig: 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' 'if [ -n "$LLM_API_VERSION" ]; then ' 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' + 'if [ -n "$LLM_REASONING_EFFORT" ]; then ' + 'printf \',"reasoning_effort":"%s",' + '"litellm_extra_body":{"reasoning_effort":"%s"}\' ' + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"; fi; ' "printf '}}'; } > ~/.openhands/agent_settings.json && " "openhands acp --always-approve --override-with-envs" ), diff --git a/src/benchflow/providers/litellm_config.py b/src/benchflow/providers/litellm_config.py index 47b5247d..18c0bda6 100644 --- a/src/benchflow/providers/litellm_config.py +++ b/src/benchflow/providers/litellm_config.py @@ -18,9 +18,13 @@ AZURE_API_VERSION_ENV = "AZURE_API_VERSION" AZURE_DEFAULT_API_VERSION = "preview" BEDROCK_THINKING_EFFORT_ENV = "BENCHFLOW_BEDROCK_THINKING_EFFORT" +PROVIDER_REASONING_EFFORT_ENV = "BENCHFLOW_REASONING_EFFORT" LITELLM_MODEL_ALIAS_ENV = "BENCHFLOW_LITELLM_MODEL_ALIAS" LITELLM_MODEL_VIA_ENV = "BENCHFLOW_LITELLM_MODEL_VIA_ENV" LITELLM_MASTER_KEY_ENV = "BENCHFLOW_LITELLM_MASTER_KEY" +_PROVIDER_REASONING_EFFORTS = frozenset( + {"none", "minimal", "low", "medium", "high", "xhigh", "max"} +) # Per-token USD prices for models LiteLLM's built-in ``model_cost`` does not # already know (custom OpenAI-compatible endpoints such as private vLLM servers @@ -164,6 +168,20 @@ def _registered_api_key_ref(cfg: ProviderConfig) -> str | None: return None +def _provider_reasoning_effort(env: dict[str, str]) -> str | None: + """Return an explicitly requested gateway-side reasoning effort.""" + raw = ( + env.get(PROVIDER_REASONING_EFFORT_ENV) or env.get("LLM_REASONING_EFFORT") or "" + ) + effort = raw.strip().lower() + if not effort: + return None + if effort not in _PROVIDER_REASONING_EFFORTS: + allowed = ", ".join(sorted(_PROVIDER_REASONING_EFFORTS)) + raise ValueError(f"{PROVIDER_REASONING_EFFORT_ENV} must be one of: {allowed}") + return effort + + def _bedrock_thinking_effort(model: str, env: dict[str, str]) -> str | None: if not _BEDROCK_ADAPTIVE_THINKING_RE.search(model): return None @@ -213,6 +231,9 @@ def _route_registered_provider( "api_base": api_base, "api_version": env.get(AZURE_API_VERSION_ENV, AZURE_DEFAULT_API_VERSION), } + effort = _provider_reasoning_effort(env) + if effort: + params["reasoning_effort"] = effort return LiteLLMRoute( requested_model=model, model_alias=safe_model_alias(model), diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index a5e73346..adddbe3c 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -202,6 +202,16 @@ def test_openhands_launch_cmd_writes_optional_azure_api_version(self): assert ',"api_version":"%s"' in cfg.launch_cmd assert '"$LLM_API_VERSION"' in cfg.launch_cmd + def test_openhands_launch_cmd_writes_optional_reasoning_effort(self): + """Guards PR #911 against OpenHands silently using default high effort.""" + cfg = AGENTS["openhands"] + assert 'if [ -n "$LLM_REASONING_EFFORT" ]' in cfg.launch_cmd + assert ',"reasoning_effort":"%s",' in cfg.launch_cmd + assert '"litellm_extra_body":{"reasoning_effort":"%s"}' in cfg.launch_cmd + assert ( + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"' in cfg.launch_cmd + ) + def test_harvey_lab_installs_python_deps_in_venv(self): """Guards the v0.5 stress failure where pip hit PEP 668 in Ubuntu.""" cfg = AGENTS["harvey-lab-harness"] diff --git a/tests/test_litellm_config.py b/tests/test_litellm_config.py index 6679c097..cd8f52ec 100644 --- a/tests/test_litellm_config.py +++ b/tests/test_litellm_config.py @@ -49,6 +49,20 @@ def test_azure_openai_route_uses_resource_and_preview_version(): assert route.litellm_params["api_version"] == "preview" +def test_azure_openai_route_honors_openhands_reasoning_effort_env(): + """Guards PR #911 against proxy-alias capability guessing.""" + route = resolve_litellm_route( + "azure-foundry-openai/gpt-5.6-sol", + { + "AZURE_API_KEY": "key", + "AZURE_RESOURCE": "benchflow", + "LLM_REASONING_EFFORT": "xhigh", + }, + ) + + assert route.litellm_params["reasoning_effort"] == "xhigh" + + def test_azure_anthropic_route_uses_azure_ai_anthropic_surface(): route = resolve_litellm_route( "azure-foundry-anthropic/claude-opus-4-5",