diff --git a/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py b/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py index 55ff8b9b..ce600bc1 100755 --- a/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py +++ b/.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py @@ -250,6 +250,7 @@ def validate_llm( issues: list[str] = [] request_count = 0 response_count = 0 + completed_candidates: list[tuple[int, str, bool]] = [] error_count = 0 usage_count = 0 for index, row in enumerate(rows, start=1): @@ -266,6 +267,27 @@ def validate_llm( body = response.get("body") if isinstance(body, dict): response_count += 1 + response_status = body.get("status") + completed = ( + response.get("status_code") == 200 + and response_status in {None, "completed"} + and not body.get("incomplete_details") + ) + if completed: + signature = json.dumps( + ( + request.get("body") + if isinstance(request, dict) + and isinstance(request.get("body"), dict) + else {} + ), + sort_keys=True, + separators=(",", ":"), + default=str, + ) + completed_candidates.append( + (index - 1, signature, has_token_usage(body)) + ) if has_token_usage(body): usage_count += 1 else: @@ -282,14 +304,153 @@ def validate_llm( ) if usage_count == 0: issues.append(f"{path}: no provider token usage in response bodies") + candidates_by_request: dict[str, list[tuple[int, bool]]] = {} + for index, signature, has_usage in completed_candidates: + candidates_by_request.setdefault(signature, []).append((index, has_usage)) + request_bodies = [ + json.dumps( + ( + request.get("body") + if isinstance(request := row.get("request"), dict) + and isinstance(request.get("body"), dict) + else {} + ), + sort_keys=True, + separators=(",", ":"), + default=str, + ) + for row in rows + ] + selected: list[tuple[int, bool]] = [] + for candidates in candidates_by_request.values(): + consumed = [ + candidate + for candidate in candidates + if response_consumed_by_later_request(rows, request_bodies, candidate[0]) + ] + if consumed: + selected.append(max(consumed)) + continue + safe_unconsumed = [ + candidate + for candidate in candidates + if response_safe_without_later_consumption(rows, candidate[0]) + ] + if safe_unconsumed: + selected.append(max(safe_unconsumed)) + selected.sort() + successful_exchange_indices = [index for index, _ in selected] + successful_response_count = len(selected) + successful_usage_count = sum(1 for _, has_usage in selected if has_usage) + if successful_response_count and successful_usage_count < successful_response_count: + issues.append( + f"{path}: completed responses with usage {successful_usage_count} < " + f"completed responses {successful_response_count}" + ) return issues, { "requests": request_count, "responses": response_count, + "successful_responses": successful_response_count, + "successful_exchange_indices": successful_exchange_indices, + "successful_responses_with_usage": successful_usage_count, + "deduplicated_completed_responses": ( + len(completed_candidates) - successful_response_count + ), "errors": error_count, "responses_with_usage": usage_count, } +def response_consumed_by_later_request( + rows: list[dict[str, Any]], + request_bodies: list[str], + exchange_idx: int, +) -> bool: + response = rows[exchange_idx].get("response") + body = response.get("body") if isinstance(response, dict) else {} + call_ids = response_call_ids(body if isinstance(body, dict) else {}) + return bool( + call_ids + and any( + any(call_id in request_body for call_id in call_ids) + for request_body in request_bodies[exchange_idx + 1 :] + ) + ) + + +def response_call_ids(body: dict[str, Any]) -> set[str]: + return {call_id for call_id, _ in response_tool_calls(body)} + + +def response_safe_without_later_consumption( + rows: list[dict[str, Any]], exchange_idx: int +) -> bool: + row = rows[exchange_idx] + response = row.get("response") + body = response.get("body") if isinstance(response, dict) else {} + calls = response_tool_calls(body if isinstance(body, dict) else {}) + if not calls or all(name == "finish" for _, name in calls): + return True + return not any( + isinstance(request := later.get("request"), dict) + and isinstance(request.get("body"), dict) + for later in rows[exchange_idx + 1 :] + ) + + +def response_tool_calls(body: dict[str, Any]) -> list[tuple[str, str | None]]: + calls = [ + ( + str(item.get("call_id") or item.get("id")), + str(item.get("name")) if item.get("name") else None, + ) + for item in body.get("output") or [] + if isinstance(item, dict) + and item.get("type") in {"function_call", "tool_call"} + and (item.get("call_id") or item.get("id")) + ] + choices = body.get("choices") + if isinstance(choices, list): + for choice in choices: + message = choice.get("message") if isinstance(choice, dict) else None + if not isinstance(message, dict): + continue + calls.extend( + ( + str(call.get("id") or call.get("tool_call_id")), + ( + str(function.get("name")) + if isinstance(function := call.get("function"), dict) + and function.get("name") + else str(call.get("name")) + if call.get("name") + else None + ), + ) + for call in message.get("tool_calls") or [] + if isinstance(call, dict) + and (call.get("id") or call.get("tool_call_id")) + ) + message = body.get("message") + if isinstance(message, dict): + calls.extend( + ( + str(call.get("id") or call.get("tool_call_id")), + ( + str(function.get("name")) + if isinstance(function := call.get("function"), dict) + and function.get("name") + else str(call.get("name")) + if call.get("name") + else None + ), + ) + for call in message.get("tool_calls") or [] + if isinstance(call, dict) and (call.get("id") or call.get("tool_call_id")) + ) + return calls + + def result_has_terminal_error(result: dict[str, Any]) -> bool: if result.get("error") or result.get("verifier_error"): return True @@ -446,10 +607,20 @@ def validate_results_row( ) trajectory = row.get("trajectory") if isinstance(trajectory, list): - if llm_summary and llm_summary.get("responses", 0) and len(trajectory) == 0: + successful_responses = ( + int(llm_summary.get("successful_responses", 0)) if llm_summary else 0 + ) + if successful_responses and len(trajectory) != successful_responses: issues.append( - f"{row_path}: no results trajectory steps from LLM exchanges" + f"{row_path}: results trajectory steps {len(trajectory)} != " + f"successful LLM responses {successful_responses}" ) + expected_indices = ( + set(llm_summary.get("successful_exchange_indices", [])) + if llm_summary + else set() + ) + observed_indices: set[int] = set() for idx, step in enumerate(trajectory): if not isinstance(step, dict): issues.append(f"{row_path}: trajectory[{idx}] must be an object") @@ -468,6 +639,34 @@ def validate_results_row( issues.append( f"{row_path}: trajectory[{idx}] source is not llm_trajectory" ) + if isinstance(extras, dict) and isinstance( + extras.get("exchange_index"), int + ): + observed_indices.add(extras["exchange_index"]) + if step.get("is_truncated") is True: + issues.append( + f"{row_path}: trajectory[{idx}] is incorrectly truncated" + ) + step_messages = [] + for field in ("prompt", "completion"): + value = step.get(field) + if isinstance(value, list): + step_messages.extend( + message for message in value if isinstance(message, dict) + ) + issues.extend( + validate_training_messages( + step_messages, + tools=tools, + row_path=f"{row_path}: trajectory[{idx}]", + ) + ) + if expected_indices and observed_indices != expected_indices: + issues.append( + f"{row_path}: results exchange indices " + f"{sorted(observed_indices)} != successful LLM exchange indices " + f"{sorted(expected_indices)}" + ) token_usage = row.get("token_usage") if not isinstance(token_usage, dict): issues.append(f"{row_path}: missing object token_usage") diff --git a/pyproject.toml b/pyproject.toml index 21bcc214..c6a6fb25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "rich>=13.0", # Dataset registry bench_version checks (PEP 440 specifier sets). "packaging>=24", - "litellm[proxy]==1.89.0", + "litellm[proxy]==1.91.0", "tomli-w>=1.0", "typer>=0.9", ] diff --git a/src/benchflow/acp/runtime.py b/src/benchflow/acp/runtime.py index 1d7e8e11..2555ea89 100644 --- a/src/benchflow/acp/runtime.py +++ b/src/benchflow/acp/runtime.py @@ -25,6 +25,7 @@ from benchflow.acp.client import ACPClient from benchflow.acp.container_transport import ContainerTransport +from benchflow.acp.selection import selected_acp_transport from benchflow.acp.types import McpServerSpec from benchflow.agents.protocol import ACPSessionAdapter from benchflow.agents.providers import ( @@ -38,8 +39,13 @@ AgentPromptTimeoutError, IdleTimeoutDiagnostic, IdleTimeoutError, + TransportClosedDiagnostic, + TransportClosedError, +) +from benchflow.sandbox.lockdown import ( + build_priv_drop_cmd, + enforce_agent_egress_firewall, ) -from benchflow.sandbox.lockdown import build_priv_drop_cmd from benchflow.sandbox.process import DaytonaProcess, DaytonaPtyProcess, DockerProcess from benchflow.trajectories._capture import _capture_session_trajectory @@ -51,6 +57,7 @@ "IdleTimeoutError", "connect_acp", "execute_prompts", + "selected_acp_transport", ] logger = logging.getLogger(__name__) @@ -59,6 +66,24 @@ _ACP_CONNECT_MAX_RETRIES = 3 _ACP_CONNECT_BASE_DELAY = 2.0 _PROMPT_CANCEL_DRAIN_TIMEOUT_SEC = 0.25 +_ACP_HANDSHAKE_TIMEOUT_SEC = 60 + + +async def _wait_for_acp_handshake(awaitable, *, phase: str): + try: + return await asyncio.wait_for(awaitable, timeout=_ACP_HANDSHAKE_TIMEOUT_SEC) + except TimeoutError as e: + msg = ( + f"ACP {phase} timed out after {_ACP_HANDSHAKE_TIMEOUT_SEC}s " + "before the first prompt" + ) + raise TransportClosedError( + msg, + TransportClosedDiagnostic( + raw_message=msg, + transport_diagnosis=f"acp_{phase}_timeout", + ), + ) from e # models.dev provider inference — used when acp_model_format="provider/model" @@ -484,9 +509,13 @@ async def connect_acp( if environment == "docker": live_proc = DockerProcess.from_sandbox_env(env) elif environment == "daytona": - if agent == "gemini": + transport_name = selected_acp_transport( + agent=agent, + environment=environment, + ) + if transport_name == "ssh": live_proc = await DaytonaProcess.from_sandbox_env(env) - logger.info("Using SSH transport for Gemini on Daytona") + logger.info("Using SSH transport for %s on Daytona", agent) else: live_proc = await DaytonaPtyProcess.from_sandbox_env(env) logger.info("Using PTY transport for Daytona sandbox") @@ -511,15 +540,18 @@ async def connect_acp( acp_client = ACPClient(transport) await acp_client.connect() - init_result = await asyncio.wait_for(acp_client.initialize(), timeout=60) + init_result = await _wait_for_acp_handshake( + acp_client.initialize(), + phase="initialize", + ) agent_name = ( init_result.agent_info.name if init_result.agent_info else agent ) logger.info(f"ACP agent: {agent_name}") - session = await asyncio.wait_for( + session = await _wait_for_acp_handshake( acp_client.session_new(cwd=agent_cwd, mcp_servers=mcp_servers), - timeout=60, + phase="session_new", ) logger.info(f"Session: {session.session_id}") break @@ -552,6 +584,7 @@ async def connect_acp( agent_env=agent_env, reasoning_effort=reasoning_effort, ) + await enforce_agent_egress_firewall(env, sandbox_user, agent_env) except Exception: with contextlib.suppress(Exception): await acp_client.close() diff --git a/src/benchflow/acp/selection.py b/src/benchflow/acp/selection.py new file mode 100644 index 00000000..804e295b --- /dev/null +++ b/src/benchflow/acp/selection.py @@ -0,0 +1,30 @@ +"""Pure ACP transport selection helpers shared by runtime and artifacts.""" + +import logging +import os + +logger = logging.getLogger(__name__) + +_DAYTONA_ACP_TRANSPORT_ENV = "BENCHFLOW_DAYTONA_ACP_TRANSPORT" +_DAYTONA_ACP_TRANSPORTS = {"pty", "ssh"} + + +def daytona_acp_transport() -> str: + value = os.environ.get(_DAYTONA_ACP_TRANSPORT_ENV, "pty").strip().lower() + if value in _DAYTONA_ACP_TRANSPORTS: + return value + logger.warning( + "Invalid %s=%r; using PTY transport", + _DAYTONA_ACP_TRANSPORT_ENV, + value, + ) + return "pty" + + +def selected_acp_transport(*, agent: str, environment: str) -> str: + """Return the concrete agent transport selected for artifact provenance.""" + if environment == "docker": + return "docker-stdio" + if environment == "daytona": + return "ssh" if agent == "gemini" else daytona_acp_transport() + return "provider-default" diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 0f00b33a..09c56fb0 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -117,9 +117,9 @@ def _apt_install(*packages: str) -> str: # OpenCode routes through the chat-completions path. Shared with # ``benchflow.acp.runtime._format_acp_model`` so set_model targets the same id. OPENCODE_PROXY_PROVIDER_ID = "benchflow" -_OPENHANDS_CLI_GIT_REV = "3ca17446c5d9c1e35e054803478a3501ec251ecf" -_OPENHANDS_SDK_VERSION = "1.22.1" -_OPENHANDS_TOOLS_VERSION = "1.22.1" +_OPENHANDS_CLI_GIT_REV = "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271" +_OPENHANDS_SDK_VERSION = "1.28.1" +_OPENHANDS_TOOLS_VERSION = "1.28.1" _JS_AGENT_PATH = ( f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:" f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH" @@ -700,7 +700,7 @@ class AgentConfig: skill_paths=["$HOME/.opencode/skills"], home_dirs=[".opencode"], install_cmd=( - _js_agent_install("opencode", "opencode-ai") + _js_agent_install("opencode", "opencode-ai@1.17.20") + " && " + _opencode_family_proxy_wrapper_install( "opencode", ".config/opencode/opencode.json" @@ -902,10 +902,11 @@ class AgentConfig: " curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null 2>&1 && " ' export PATH="$HOME/.local/bin:$PATH"; ' " fi && " - # Pin the OpenHands CLI source so the agent workflow cannot drift - # with GitHub main; only override the buggy sdk/tools 1.21.0 pins. - # SDK 1.22.x restores default-to-UNKNOWN for the synthetic - # `security_risk` tool field without the API drift seen in 1.26.x. + # Pin the OpenHands CLI source and its matching SDK/tools release + # together so the ACP runtime cannot drift or be forced onto an + # older, API-incompatible SDK. The 1.28.1 line retains the + # security_risk default fix and includes later long-run ACP, + # terminal, tool-executor, and event-log stability fixes. f"printf 'openhands-sdk=={_OPENHANDS_SDK_VERSION}\\n" f"openhands-tools=={_OPENHANDS_TOOLS_VERSION}\\n' " "> /tmp/oh-sdk-overrides.txt && " @@ -936,7 +937,33 @@ class AgentConfig: 'printf \',"base_url":"%s"\' "$LLM_BASE_URL"; fi; ' 'if [ -n "$LLM_API_VERSION" ]; then ' 'printf \',"api_version":"%s"\' "$LLM_API_VERSION"; fi; ' + 'if [ -n "$LLM_TIMEOUT" ]; then ' + 'case "$LLM_TIMEOUT" in *[!0-9]*) ' + 'echo "LLM_TIMEOUT must be a non-negative integer" >&2; exit 2;; esac; ' + 'printf \',"timeout":%s\' "$LLM_TIMEOUT"; fi; ' + 'case "$LLM_REASONING_EFFORT" in ' + 'max) printf \',"litellm_extra_body":{"reasoning":{"effort":"max"}}\' ;; ' + "none|low|medium|high|xhigh) " + 'printf \',"reasoning_effort":"%s",' + '"litellm_extra_body":{"reasoning_effort":"%s"}\' ' + '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT" ;; ' + '?*) printf \',"litellm_extra_body":{"reasoning_effort":"%s"}\' ' + '"$LLM_REASONING_EFFORT" ;; ' + "esac; " "printf '}}'; } > ~/.openhands/agent_settings.json && " + 'if [ "${BENCHFLOW_OPENHANDS_DISABLE_SUBAGENTS:-0}" = "1" ]; then ' + 'OH_BIN="$(command -v openhands)"; ' + 'OH_PY="$(dirname "$(readlink -f "$OH_BIN")")/python"; ' + '[ -x "$OH_PY" ] || { ' + 'echo "Cannot locate OpenHands tool interpreter" >&2; exit 127; }; ' + '"$OH_PY" -c \'from pathlib import Path; ' + "import openhands_cli.utils as u; " + "p=Path(u.__file__); s=p.read_text(); " + 'old=" Tool(name=task_tool_name),\\n"; ' + 'new=" # BenchFlow: delegation disabled for this run.\\n"; ' + "assert old in s or new in s; " + "p.write_text(s.replace(old,new,1))'; " + "fi && " "openhands acp --always-approve --override-with-envs" ), protocol="acp", diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index 159f3627..0d65fcb3 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -352,7 +352,13 @@ async def _read_callback_chunk(self, offset: int, limit: int) -> bytes: f"dd if={shlex.quote(self.log_path)} bs=1 skip={offset} count={limit} " "2>/dev/null | base64 -w 0" ) - result = await self.sandbox.exec(command, timeout_sec=10) + # Daytona otherwise retains one session wrapper shell for every + # one-second live-capture poll. Long MAX rollouts can accumulate + # thousands of orphaned shells and eventually wedge the agent's own + # terminal calls. Providers without a transient-exec path keep the + # existing behavior. + execute = getattr(self.sandbox, "exec_transient", self.sandbox.exec) + result = await execute(command, timeout_sec=10) encoded = (result.stdout or "").strip() if result.return_code != 0 or not encoded: return b"" diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index 3b5e6ba0..63909c2d 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -122,6 +122,7 @@ from benchflow.rollout._setup import ( _agent_process_kill_pattern as _agent_process_kill_pattern, ) +from benchflow.rollout._setup import _apply_prompt_prefix as _apply_prompt_prefix from benchflow.rollout._setup import _apply_web_policy as _apply_web_policy from benchflow.rollout._setup import ( _configured_task_workdir as _configured_task_workdir, @@ -870,7 +871,10 @@ async def setup(self) -> None: declared_sandbox_skills_dir=getattr(env_config, "skills_dir", None), ) self._task_skill_policy = task_skill_policy - self._resolved_prompts = _resolve_prompts(cfg.task_path, cfg.prompts) + self._resolved_prompts = _apply_prompt_prefix( + _resolve_prompts(cfg.task_path, cfg.prompts), + self._task.config.agent.prompt_prefix, + ) self._agent_launch = self._planes.agent_launch( cfg.primary_agent, disallow_web_tools=self._disallow_web_tools, diff --git a/src/benchflow/rollout/_results.py b/src/benchflow/rollout/_results.py index 024a07ad..588e5cbd 100644 --- a/src/benchflow/rollout/_results.py +++ b/src/benchflow/rollout/_results.py @@ -163,6 +163,7 @@ def _write_config( loop_strategy: LoopStrategySpec | None = None, ) -> None: """Write config.json to rollout_dir with secrets filtered out.""" + from benchflow.acp.selection import selected_acp_transport from benchflow.agents.install import effective_install_timeout artifact_source = artifact_source_provenance(source_provenance) @@ -180,6 +181,10 @@ def _write_config( "model": model, "reasoning_effort": reasoning_effort, "environment": environment, + "acp_transport": selected_acp_transport( + agent=agent, + environment=environment, + ), "environment_manifest": _environment_manifest_metadata(environment_manifest), **skill_policy.config_metadata(), "sandbox_user": sandbox_user, diff --git a/src/benchflow/rollout/_setup.py b/src/benchflow/rollout/_setup.py index cccdbce1..06a9a44f 100644 --- a/src/benchflow/rollout/_setup.py +++ b/src/benchflow/rollout/_setup.py @@ -287,6 +287,13 @@ def _resolve_prompts( return [p if p is not None else instruction for p in prompts] +def _apply_prompt_prefix(prompts: list[str], prompt_prefix: str | None) -> list[str]: + """Prepend a recorded, task-agnostic harness policy to resolved prompts.""" + if prompt_prefix is None: + return prompts + return [f"{prompt_prefix}\n\n{prompt}" for prompt in prompts] + + async def _start_env_and_upload( env: Any, task_path: Path, diff --git a/src/benchflow/sandbox/daytona.py b/src/benchflow/sandbox/daytona.py index 739e4888..da7cc4bb 100644 --- a/src/benchflow/sandbox/daytona.py +++ b/src/benchflow/sandbox/daytona.py @@ -9,6 +9,7 @@ import asyncio import atexit +import contextlib import importlib import logging import shlex @@ -605,6 +606,7 @@ async def _sandbox_exec( timeout_sec: int | None = None, shell: str = "bash -c", user: str | int | None = None, + cleanup_session: bool = False, ) -> ExecResult: """Run ``command`` as a Daytona session command and poll to completion. @@ -646,24 +648,32 @@ async def _sandbox_exec( user_arg = shlex.quote(user) command = f"su {user_arg} -s /bin/bash -c {shlex.quote(command)}" - response = await sandbox.process.execute_session_command( - session_id, - SessionExecuteRequest( - command=command, - run_async=True, - ), - timeout=timeout_sec, - ) + try: + response = await sandbox.process.execute_session_command( + session_id, + SessionExecuteRequest( + command=command, + run_async=True, + ), + timeout=timeout_sec, + ) - if response.cmd_id is None: - raise RuntimeError("Cannot find command ID.") + if response.cmd_id is None: + raise RuntimeError("Cannot find command ID.") - # Don't delete session; Daytona kills child processes - return await self._poll_response( - session_id, - response.cmd_id, - timeout_sec=timeout_sec, - ) + # Normal sandbox execs intentionally retain their Daytona session: + # deleting one can kill detached child processes that the caller + # started deliberately. Short-lived polling commands opt into + # cleanup_session=True and must not leave children behind. + return await self._poll_response( + session_id, + response.cmd_id, + timeout_sec=timeout_sec, + ) + finally: + if cleanup_session: + with contextlib.suppress(Exception): + await sandbox.process.delete_session(session_id) @_SDK_RETRY async def _sdk_upload_file(self, source_path: Path | str, target_path: str) -> None: @@ -830,6 +840,34 @@ async def exec( service=service, ) + async def exec_transient( + self, + command: str, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout_sec: int | None = None, + user: str | int | None = None, + service: str = "main", + ) -> ExecResult: + """Run a child-free command and delete its Daytona session afterward. + + Daytona session commands may leave their wrapper shell alive after the + command has completed. High-frequency polling must use this method so a + long rollout does not accumulate thousands of orphaned shells. Callers + must not use it for commands intended to leave background services + running, because deleting the session may terminate those children. + """ + user = self._resolve_user(user) + env = self._merge_env(env) + return await self._strategy.exec_transient( + command, + cwd=cwd, + env=env, + timeout_sec=timeout_sec, + user=user, + service=service, + ) + async def services(self) -> list[str]: return await self._strategy.services() diff --git a/src/benchflow/sandbox/daytona_dind.py b/src/benchflow/sandbox/daytona_dind.py index d8fd7edc..8a77b694 100644 --- a/src/benchflow/sandbox/daytona_dind.py +++ b/src/benchflow/sandbox/daytona_dind.py @@ -187,10 +187,16 @@ async def _vm_exec( cwd: str | None = None, env: dict[str, str] | None = None, timeout_sec: int | None = None, + cleanup_session: bool = False, ) -> ExecResult: """Run a command on the DinD sandbox VM using sh (Alpine-compatible).""" return await self._env._sandbox_exec( - command, cwd=cwd, env=env, timeout_sec=timeout_sec, shell="sh -c" + command, + cwd=cwd, + env=env, + timeout_sec=timeout_sec, + shell="sh -c", + cleanup_session=cleanup_session, ) def _compose_env_vars(self) -> dict[str, str]: @@ -254,11 +260,13 @@ async def _compose_exec( self, subcommand: list[str], timeout_sec: int | None = None, + cleanup_session: bool = False, ) -> ExecResult: return await self._vm_exec( self._compose_cmd(subcommand), env=self._compose_env_vars(), timeout_sec=timeout_sec, + cleanup_session=cleanup_session, ) async def _run_pre_compose_hook(self) -> None: @@ -475,6 +483,45 @@ async def exec( timeout_sec: int | None = None, user: str | int | None = None, service: str = "main", + ) -> ExecResult: + return await self._exec( + command, + cwd=cwd, + env=env, + timeout_sec=timeout_sec, + user=user, + service=service, + cleanup_session=False, + ) + + async def exec_transient( + self, + command: str, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout_sec: int | None = None, + user: str | int | None = None, + service: str = "main", + ) -> ExecResult: + return await self._exec( + command, + cwd=cwd, + env=env, + timeout_sec=timeout_sec, + user=user, + service=service, + cleanup_session=True, + ) + + async def _exec( + self, + command: str, + cwd: str | None, + env: dict[str, str] | None, + timeout_sec: int | None, + user: str | int | None, + service: str, + cleanup_session: bool, ) -> ExecResult: """Run a command in a compose service inside the DinD VM. @@ -504,6 +551,12 @@ async def exec( # ``set -a``/``. file``). parts.extend([service, "sh", "-c", command]) + if cleanup_session: + return await self._compose_exec( + parts, + timeout_sec=timeout_sec, + cleanup_session=True, + ) return await self._compose_exec(parts, timeout_sec=timeout_sec) async def upload_file(self, source_path: Path | str, target_path: str) -> None: diff --git a/src/benchflow/sandbox/daytona_reaper.py b/src/benchflow/sandbox/daytona_reaper.py index f356c0fe..2fe90ee7 100644 --- a/src/benchflow/sandbox/daytona_reaper.py +++ b/src/benchflow/sandbox/daytona_reaper.py @@ -10,6 +10,8 @@ from __future__ import annotations import logging +import os +import re from typing import Any logger = logging.getLogger("benchflow") @@ -33,6 +35,9 @@ # of age. The dotted key follows the Docker/Daytona label convention. _BENCHFLOW_MANAGED_LABEL = "benchflow.managed" _BENCHFLOW_MANAGED_VALUE = "1" +_BENCHFLOW_OWNER_LABEL = "benchflow.owner" +_BENCHFLOW_OWNER_ENV = "BENCHFLOW_DAYTONA_OWNER" +_BENCHFLOW_OWNER_MAX_LEN = 48 # The dotted prefix every benchflow-stamped label key shares. Used only by the # read-only orphan-leak guard below to recognize a sandbox that *looks* like # benchflow created it (carries the namespace) yet fails the strict ownership @@ -41,6 +46,19 @@ _BENCHFLOW_LABEL_NAMESPACE = "benchflow." +def _benchflow_owner_scope() -> str | None: + raw = os.environ.get(_BENCHFLOW_OWNER_ENV, "").strip() + if not raw: + return None + normalized = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip(".-") + return normalized[:_BENCHFLOW_OWNER_MAX_LEN] or None + + +def _benchflow_managed_value() -> str: + owner = _benchflow_owner_scope() + return _BENCHFLOW_MANAGED_VALUE if owner is None else f"1:{owner}" + + def _benchflow_owned_labels() -> dict[str, str]: """Return a fresh ownership-label dict for one sandbox-creation call. @@ -48,7 +66,11 @@ def _benchflow_owned_labels() -> dict[str, str]: in place (it injects the language label), so a shared dict would leak that mutation across creation sites. """ - return {_BENCHFLOW_MANAGED_LABEL: _BENCHFLOW_MANAGED_VALUE} + owner = _benchflow_owner_scope() + labels = {_BENCHFLOW_MANAGED_LABEL: _benchflow_managed_value()} + if owner is not None: + labels[_BENCHFLOW_OWNER_LABEL] = owner + return labels def _is_benchflow_owned(sb: Any) -> bool: @@ -63,15 +85,16 @@ def _is_benchflow_owned(sb: Any) -> bool: labels = getattr(sb, "labels", None) if not isinstance(labels, dict): return False - return labels.get(_BENCHFLOW_MANAGED_LABEL) == _BENCHFLOW_MANAGED_VALUE + return labels.get(_BENCHFLOW_MANAGED_LABEL) == _benchflow_managed_value() def _is_benchflow_label_orphan(sb: Any) -> bool: """Return whether *sb* looks benchflow-created but lacks the ownership label. True only when the sandbox carries at least one ``benchflow.``-namespaced - label key yet fails :func:`_is_benchflow_owned` (the exact - ``benchflow.managed=1`` pair is absent or has drifted to another value). + label key yet lacks any valid ownership marker. A sandbox with a scoped + ``benchflow.managed=1:`` value belongs to another BenchFlow operator, + so it is foreign rather than orphaned and must be silently skipped. Such a sandbox is almost certainly one benchflow created whose ownership label was lost — the age-based reaper's scope gate will now skip it forever, so it leaks. This is a *detection-only* predicate: the missing/altered label @@ -84,6 +107,9 @@ def _is_benchflow_label_orphan(sb: Any) -> bool: labels = getattr(sb, "labels", None) if not isinstance(labels, dict): return False + managed = labels.get(_BENCHFLOW_MANAGED_LABEL) + if isinstance(managed, str) and managed.startswith("1:"): + return False return any( isinstance(key, str) and key.startswith(_BENCHFLOW_LABEL_NAMESPACE) for key in labels @@ -171,7 +197,7 @@ def reap_stale_sandboxes( "verify and remove it manually if it is stale.", getattr(sb, "id", "?"), _BENCHFLOW_MANAGED_LABEL, - _BENCHFLOW_MANAGED_VALUE, + _benchflow_managed_value(), ) counts["skipped"] += 1 continue diff --git a/src/benchflow/sandbox/daytona_strategies.py b/src/benchflow/sandbox/daytona_strategies.py index 70790621..593d15aa 100644 --- a/src/benchflow/sandbox/daytona_strategies.py +++ b/src/benchflow/sandbox/daytona_strategies.py @@ -75,6 +75,25 @@ async def exec( service: str = "main", ) -> ExecResult: ... + async def exec_transient( + self, + command: str, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout_sec: int | None = None, + user: str | int | None = None, + service: str = "main", + ) -> ExecResult: + """Run a child-free command, cleaning its provider session if supported.""" + return await self.exec( + command, + cwd=cwd, + env=env, + timeout_sec=timeout_sec, + user=user, + service=service, + ) + @abstractmethod async def upload_file(self, source_path: Path | str, target_path: str) -> None: ... @@ -245,6 +264,25 @@ async def exec( command, cwd=cwd, env=env, timeout_sec=timeout_sec, user=user ) + async def exec_transient( + self, + command: str, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout_sec: int | None = None, + user: str | int | None = None, + service: str = "main", + ) -> ExecResult: + _reject_non_main_service(service) + return await self._env._sandbox_exec( + command, + cwd=cwd, + env=env, + timeout_sec=timeout_sec, + user=user, + cleanup_session=True, + ) + async def upload_file(self, source_path: Path | str, target_path: str) -> None: await self._env._sdk_upload_file(source_path, target_path) diff --git a/src/benchflow/sandbox/lockdown.py b/src/benchflow/sandbox/lockdown.py index 019d6bad..27363f49 100644 --- a/src/benchflow/sandbox/lockdown.py +++ b/src/benchflow/sandbox/lockdown.py @@ -18,6 +18,7 @@ import shlex from pathlib import Path from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit from benchflow.agents.registry import get_sandbox_home_dirs from benchflow.sandbox._cache_reclaim import build_reclaim_caches_cmd @@ -90,6 +91,41 @@ def _resolve_locked_paths( # Sandbox user + privilege drop +def _agent_egress_firewall_cmd(sandbox_user: str) -> str: + user = shlex.quote(sandbox_user) + return ( + "set -e; " + "if ! command -v iptables >/dev/null 2>&1; then " + "if command -v apt-get >/dev/null 2>&1; then " + "export DEBIAN_FRONTEND=noninteractive; " + "apt-get update -qq && apt-get install -y -qq iptables >/dev/null; " + "elif command -v dnf >/dev/null 2>&1; then " + "dnf -y install iptables >/dev/null; " + "elif command -v apk >/dev/null 2>&1; then " + "apk add --no-cache iptables >/dev/null; " + "else echo 'No supported iptables package manager' >&2; exit 86; fi; fi; " + f"agent_uid=$(id -u {user}) || exit 86; " + 'iptables -C OUTPUT -o lo -m owner --uid-owner "$agent_uid" ' + "-j ACCEPT 2>/dev/null || " + 'iptables -I OUTPUT 1 -o lo -m owner --uid-owner "$agent_uid" ' + "-j ACCEPT; " + 'iptables -C OUTPUT -m owner --uid-owner "$agent_uid" ' + "-j REJECT 2>/dev/null || " + 'iptables -A OUTPUT -m owner --uid-owner "$agent_uid" -j REJECT; ' + "if [ -s /proc/net/if_inet6 ]; then " + "command -v ip6tables >/dev/null 2>&1 || " + "{ echo 'IPv6 enabled but ip6tables unavailable' >&2; exit 86; }; " + 'ip6tables -C OUTPUT -o lo -m owner --uid-owner "$agent_uid" ' + "-j ACCEPT 2>/dev/null || " + 'ip6tables -I OUTPUT 1 -o lo -m owner --uid-owner "$agent_uid" ' + "-j ACCEPT; " + 'ip6tables -C OUTPUT -m owner --uid-owner "$agent_uid" ' + "-j REJECT 2>/dev/null || " + 'ip6tables -A OUTPUT -m owner --uid-owner "$agent_uid" -j REJECT; ' + "fi" + ) + + def build_priv_drop_cmd(agent_launch: str, sandbox_user: str) -> str: """Build a shell command that drops to sandbox_user via setpriv or su. @@ -107,6 +143,40 @@ def build_priv_drop_cmd(agent_launch: str, sandbox_user: str) -> str: ) +async def enforce_agent_egress_firewall( + env: Any, + sandbox_user: str | None, + agent_env: dict[str, str], +) -> None: + """Block sandbox-user external egress after ACP bootstrap, before prompting.""" + if not sandbox_user or agent_env.get("BENCHFLOW_DISALLOW_WEB_TOOLS") != "1": + return + + base_url = agent_env.get("LLM_BASE_URL", "") + parsed = urlsplit(base_url) + if ( + parsed.scheme != "http" + or parsed.hostname not in {"127.0.0.1", "localhost"} + or parsed.port is None + ): + raise RuntimeError( + "No-web agent requires an HTTP loopback LLM_BASE_URL with a port" + ) + + result = await env.exec( + _agent_egress_firewall_cmd(sandbox_user), + user="root", + timeout_sec=120, + ) + if _exec_return_code(result) != 0: + detail = _exec_failure_detail(result) + raise RuntimeError(f"Failed to enforce sandbox-user egress firewall.{detail}") + logger.info( + "Sandbox-user egress firewall active for %s (loopback allowed)", + sandbox_user, + ) + + def _legacy_root_tool_link_cmd(source: str, dest: str) -> str: """Link legacy root-only tool dirs into the sandbox home when needed.""" src = shlex.quote(source) @@ -142,7 +212,10 @@ async def setup_sandbox_user( f"if [ -d /root/$d ]; then " f"cp -a /root/$d/. {home}/$d/ 2>/dev/null || true; fi; done && " f"chown -R {sandbox_user}:{sandbox_user} {home} && " - f"chown -R {sandbox_user}:{sandbox_user} {shlex.quote(workspace)}", + f"chown -R {sandbox_user}:{sandbox_user} {shlex.quote(workspace)} && " + f"for d in /output /outputs; do " + f'if [ -d "$d" ] && [ ! -L "$d" ]; then ' + f'chown -R {sandbox_user}:{sandbox_user} "$d"; fi; done', timeout_sec=timeout_sec, ) logger.info(f"Sandbox user {sandbox_user} ready (workspace={workspace})") diff --git a/src/benchflow/sandbox/process.py b/src/benchflow/sandbox/process.py index b7bd6c88..6e7a48b5 100644 --- a/src/benchflow/sandbox/process.py +++ b/src/benchflow/sandbox/process.py @@ -26,6 +26,9 @@ _ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _DAYTONA_PTY_READLINE_TIMEOUT_ENV = "BENCHFLOW_DAYTONA_PTY_READLINE_TIMEOUT" _DAYTONA_PTY_READLINE_TIMEOUT_DEFAULT_SEC = 900.0 +_DAYTONA_SSH_ACCESS_TTL_MINUTES = 48 * 60 +_DAYTONA_SSH_SERVER_ALIVE_INTERVAL_SEC = 30 +_DAYTONA_SSH_SERVER_ALIVE_COUNT_MAX = 12 def _daytona_pty_readline_timeout_sec() -> float: @@ -418,6 +421,13 @@ def _write_ssh_config(ssh_user: str) -> str: f.write(f" User {ssh_user}\n") f.write(" StrictHostKeyChecking no\n") f.write(" UserKnownHostsFile /dev/null\n") + f.write( + f" ServerAliveInterval {_DAYTONA_SSH_SERVER_ALIVE_INTERVAL_SEC}\n" + ) + f.write( + f" ServerAliveCountMax {_DAYTONA_SSH_SERVER_ALIVE_COUNT_MAX}\n" + ) + f.write(" TCPKeepAlive yes\n") f.write(" LogLevel ERROR\n") os.chmod(path, 0o600) except Exception: @@ -616,7 +626,9 @@ async def start( ) try: - ssh_access = await self._sandbox.create_ssh_access() + ssh_access = await self._sandbox.create_ssh_access( + expires_in_minutes=_DAYTONA_SSH_ACCESS_TTL_MINUTES + ) ssh_config_path = self._write_ssh_config(ssh_access.token) self._ssh_config_path = ssh_config_path cmd = self._ssh_args(ssh_config_path, remote_cmd) diff --git a/src/benchflow/task/config.py b/src/benchflow/task/config.py index 7c1002cf..dfebbc44 100644 --- a/src/benchflow/task/config.py +++ b/src/benchflow/task/config.py @@ -374,6 +374,14 @@ def validate_verifier_environment(self) -> VerifierConfig: class AgentConfig(TaskConfigModel): """Agent harness ($H$) configuration.""" + prompt_prefix: str | None = Field( + default=None, + description=( + "Optional run-specific policy text prepended to each resolved task " + "prompt. Intended for generic harness constraints such as benchmark " + "integrity rules; must not contain task-specific solution content." + ), + ) timeout_sec: float | None = Field( default=None, gt=0, @@ -396,6 +404,16 @@ class AgentConfig(TaskConfigModel): description="Hostnames reachable when network_mode='allowlist'.", ) + @field_validator("prompt_prefix") + @classmethod + def validate_prompt_prefix(cls, value: str | None) -> str | None: + if value is None: + return None + value = value.strip() + if not value: + raise ValueError("prompt_prefix must contain non-whitespace text") + return value + @field_validator("allowed_hosts") @classmethod def validate_allowed_hosts(cls, hosts: list[str] | None) -> list[str] | None: diff --git a/src/benchflow/trajectories/export_prime_sft.py b/src/benchflow/trajectories/export_prime_sft.py index 94e222ba..4ea47a56 100644 --- a/src/benchflow/trajectories/export_prime_sft.py +++ b/src/benchflow/trajectories/export_prime_sft.py @@ -1122,6 +1122,10 @@ def normalize_prime_sft_exchange( ) if skip_reason: return None, skip_reason + repaired, _ = _align_legacy_tool_call_ids( + [("messages", message) for message in messages] + ) + messages = [message for _, message in repaired] if _has_tool_calls(messages) and not tool_defs: return None, "missing_tool_defs" try: diff --git a/src/benchflow/trajectories/results.py b/src/benchflow/trajectories/results.py index ffd12c6c..4f979a84 100644 --- a/src/benchflow/trajectories/results.py +++ b/src/benchflow/trajectories/results.py @@ -164,19 +164,41 @@ def _llm_steps_from_trajectory( exchanges = load_llm_trajectory_jsonl(path, strict=True) except PrimeSftTrajectoryJsonlError as exc: return [], [], f"Invalid LLM trajectory JSONL: {exc}" + training_success_indices = _training_success_exchange_indices(exchanges) + skipped_successful: list[str] = [] for exchange_idx, exchange in enumerate(exchanges): response = exchange.get("response") - if not isinstance(response, dict) or response.get("status_code") != 200: + if exchange_idx not in training_success_indices: continue - normalized, _ = normalize_prime_sft_exchange(exchange) + if not isinstance(response, dict): + skipped_successful.append( + f"exchange {exchange_idx}: selected response is not an object" + ) + continue + normalized, skip_reason = normalize_prime_sft_exchange(exchange) if normalized is None: + skipped_successful.append( + f"exchange {exchange_idx}: {skip_reason or 'normalization failed'}" + ) continue prompt = normalized.messages[:-1] completion = normalized.messages[-1:] if not completion: + skipped_successful.append(f"exchange {exchange_idx}: no completion") continue if normalized.tool_defs: - tool_defs = normalized.tool_defs + known_names = { + tool.get("function", {}).get("name") + for tool in tool_defs + if isinstance(tool, dict) and isinstance(tool.get("function"), dict) + } + tool_defs.extend( + tool + for tool in normalized.tool_defs + if isinstance(tool, dict) + and isinstance(tool.get("function"), dict) + and tool["function"].get("name") not in known_names + ) response_body = ( cast(dict[str, Any], response.get("body")) if isinstance(response.get("body"), dict) @@ -203,18 +225,176 @@ def _llm_steps_from_trajectory( "tokens": None, "reward": reward, "advantage": 0.0, - "is_truncated": bool( - response_body.get("incomplete_details") - or response_body.get("truncation") - or is_truncated - ), + "is_truncated": _response_is_truncated(response_body) or is_truncated, "trajectory_id": f"{trajectory_id_prefix}__llm_{exchange_idx}", "extras": extras, } steps.append(step) + if skipped_successful: + return ( + steps, + tool_defs, + "Successful LLM exchanges were omitted from results.jsonl: " + + "; ".join(skipped_successful), + ) return steps, tool_defs, None +def _response_is_truncated(response_body: dict[str, Any]) -> bool: + return bool( + response_body.get("incomplete_details") + or response_body.get("status") == "incomplete" + ) + + +def _response_is_training_success(response: Any) -> bool: + if not isinstance(response, dict) or response.get("status_code") != 200: + return False + body = response.get("body") + if not isinstance(body, dict): + return False + status = body.get("status") + if status is not None and status != "completed": + return False + return not bool(body.get("incomplete_details")) + + +def _training_success_exchange_indices( + exchanges: list[dict[str, Any]], +) -> set[int]: + request_bodies = [ + json.dumps( + ( + request.get("body") + if isinstance(request := exchange.get("request"), dict) + and isinstance(request.get("body"), dict) + else {} + ), + sort_keys=True, + separators=(",", ":"), + default=str, + ) + for exchange in exchanges + ] + candidates_by_request: dict[str, list[int]] = {} + for exchange_idx, exchange in enumerate(exchanges): + if not _response_is_training_success(exchange.get("response")): + continue + candidates_by_request.setdefault(request_bodies[exchange_idx], []).append( + exchange_idx + ) + selected: set[int] = set() + for candidates in candidates_by_request.values(): + consumed = [ + exchange_idx + for exchange_idx in candidates + if _response_consumed_by_later_request( + exchanges, request_bodies, exchange_idx + ) + ] + if consumed: + selected.add(max(consumed)) + continue + safe_unconsumed = [ + exchange_idx + for exchange_idx in candidates + if _response_safe_without_later_consumption(exchanges, exchange_idx) + ] + if safe_unconsumed: + selected.add(max(safe_unconsumed)) + return selected + + +def _response_consumed_by_later_request( + exchanges: list[dict[str, Any]], + request_bodies: list[str], + exchange_idx: int, +) -> bool: + response = exchanges[exchange_idx].get("response") + body = response.get("body") if isinstance(response, dict) else {} + call_ids = _response_call_ids(body if isinstance(body, dict) else {}) + return bool( + call_ids + and any( + any(call_id in request_body for call_id in call_ids) + for request_body in request_bodies[exchange_idx + 1 :] + ) + ) + + +def _response_call_ids(body: dict[str, Any]) -> set[str]: + return {call_id for call_id, _ in _response_tool_calls(body)} + + +def _response_safe_without_later_consumption( + exchanges: list[dict[str, Any]], exchange_idx: int +) -> bool: + exchange = exchanges[exchange_idx] + response = exchange.get("response") + body = response.get("body") if isinstance(response, dict) else {} + calls = _response_tool_calls(body if isinstance(body, dict) else {}) + if not calls or all(name == "finish" for _, name in calls): + return True + return not any( + isinstance(request := later.get("request"), dict) + and isinstance(request.get("body"), dict) + for later in exchanges[exchange_idx + 1 :] + ) + + +def _response_tool_calls(body: dict[str, Any]) -> list[tuple[str, str | None]]: + calls = [ + ( + str(item.get("call_id") or item.get("id")), + str(item.get("name")) if item.get("name") else None, + ) + for item in body.get("output") or [] + if isinstance(item, dict) + and item.get("type") in {"function_call", "tool_call"} + and (item.get("call_id") or item.get("id")) + ] + choices = body.get("choices") + if isinstance(choices, list): + for choice in choices: + message = choice.get("message") if isinstance(choice, dict) else None + if not isinstance(message, dict): + continue + calls.extend( + ( + str(call.get("id") or call.get("tool_call_id")), + ( + str(function.get("name")) + if isinstance(function := call.get("function"), dict) + and function.get("name") + else str(call.get("name")) + if call.get("name") + else None + ), + ) + for call in message.get("tool_calls") or [] + if isinstance(call, dict) + and (call.get("id") or call.get("tool_call_id")) + ) + message = body.get("message") + if isinstance(message, dict): + calls.extend( + ( + str(call.get("id") or call.get("tool_call_id")), + ( + str(function.get("name")) + if isinstance(function := call.get("function"), dict) + and function.get("name") + else str(call.get("name")) + if call.get("name") + else None + ), + ) + for call in message.get("tool_calls") or [] + if isinstance(call, dict) and (call.get("id") or call.get("tool_call_id")) + ) + return calls + + def _top_level_prompt_completion( steps: list[dict[str, Any]], prompts: list[str], @@ -294,7 +474,11 @@ def build_rollout_results_record( training_ready_reason = None if not training_ready: if llm_export_error: - training_ready_reason = "invalid_llm_trajectory_jsonl" + training_ready_reason = ( + "invalid_llm_trajectory_jsonl" + if llm_export_error.startswith("Invalid LLM trajectory JSONL:") + else "export_error" + ) elif validation_error: training_ready_reason = "invalid_prime_sft_row" elif effective_export_error: diff --git a/tests/test_acp.py b/tests/test_acp.py index 403246a9..175f9ba4 100644 --- a/tests/test_acp.py +++ b/tests/test_acp.py @@ -1252,6 +1252,88 @@ async def test_daytona_direct_uses_pty_transport(self, tmp_path): mock_pty.assert_awaited_once_with(mock_env) mock_ssh.assert_not_awaited() + @pytest.mark.asyncio + async def test_daytona_direct_can_opt_into_ssh_transport( + self, tmp_path, monkeypatch + ): + """Guards PR #921 fallback for PTY post-tool controller deadlocks.""" + from benchflow.acp.runtime import connect_acp + + monkeypatch.setenv("BENCHFLOW_DAYTONA_ACP_TRANSPORT", "ssh") + mock_acp = self._make_mocks() + mock_env = MagicMock() + mock_env.exec = AsyncMock(return_value=MagicMock(return_code=1, stdout="")) + + with ( + patch( + "benchflow.acp.runtime.DaytonaPtyProcess.from_sandbox_env", + new_callable=AsyncMock, + return_value=MagicMock(), + ) as mock_pty, + patch( + "benchflow.acp.runtime.DaytonaProcess.from_sandbox_env", + new_callable=AsyncMock, + return_value=MagicMock(), + ) as mock_ssh, + patch("benchflow.acp.runtime.ContainerTransport", return_value=MagicMock()), + patch("benchflow.acp.runtime.ACPClient", return_value=mock_acp), + ): + await connect_acp( + env=mock_env, + agent="openhands", + agent_launch="openhands acp", + agent_env={}, + sandbox_user=None, + model=None, + rollout_dir=tmp_path, + environment="daytona", + agent_cwd="/app", + ) + + mock_ssh.assert_awaited_once_with(mock_env) + mock_pty.assert_not_awaited() + + @pytest.mark.asyncio + async def test_invalid_daytona_transport_falls_back_to_pty( + self, tmp_path, monkeypatch + ): + """Guards PR #921 against invalid transport config disabling Daytona.""" + from benchflow.acp.runtime import connect_acp + + monkeypatch.setenv("BENCHFLOW_DAYTONA_ACP_TRANSPORT", "invalid") + mock_acp = self._make_mocks() + mock_env = MagicMock() + mock_env.exec = AsyncMock(return_value=MagicMock(return_code=1, stdout="")) + + with ( + patch( + "benchflow.acp.runtime.DaytonaPtyProcess.from_sandbox_env", + new_callable=AsyncMock, + return_value=MagicMock(), + ) as mock_pty, + patch( + "benchflow.acp.runtime.DaytonaProcess.from_sandbox_env", + new_callable=AsyncMock, + return_value=MagicMock(), + ) as mock_ssh, + patch("benchflow.acp.runtime.ContainerTransport", return_value=MagicMock()), + patch("benchflow.acp.runtime.ACPClient", return_value=mock_acp), + ): + await connect_acp( + env=mock_env, + agent="openhands", + agent_launch="openhands acp", + agent_env={}, + sandbox_user=None, + model=None, + rollout_dir=tmp_path, + environment="daytona", + agent_cwd="/app", + ) + + mock_pty.assert_awaited_once_with(mock_env) + mock_ssh.assert_not_awaited() + @pytest.mark.asyncio async def test_daytona_gemini_uses_ssh_transport(self, tmp_path): """Guards the Gemini regression introduced by PR #896's PTY migration.""" diff --git a/tests/test_acp_setup_failure_propagation.py b/tests/test_acp_setup_failure_propagation.py index b32c136e..6353cfb9 100644 --- a/tests/test_acp_setup_failure_propagation.py +++ b/tests/test_acp_setup_failure_propagation.py @@ -17,6 +17,7 @@ import pytest from benchflow.acp.client import ACPClient +from benchflow.diagnostics import TransportClosedError def _stock_acp_mock() -> AsyncMock: @@ -206,3 +207,96 @@ async def test_no_model_does_not_call_set_model(tmp_path) -> None: mock_acp.set_model.assert_not_awaited() mock_acp.close.assert_not_awaited() + + +async def test_initialize_timeout_is_transport_failure_not_agent_timeout( + tmp_path, +) -> None: + """Guards PR #921 against classifying ACP bootstrap stalls as task timeout.""" + from benchflow.acp.runtime import connect_acp + + mock_acp = _stock_acp_mock() + mock_acp.initialize = AsyncMock(side_effect=TimeoutError()) + mock_env = AsyncMock() + + with ( + patch( + "benchflow.acp.runtime.DockerProcess.from_sandbox_env", + return_value=MagicMock(), + ), + patch("benchflow.acp.runtime.ContainerTransport", return_value=MagicMock()), + patch("benchflow.acp.runtime.ACPClient", return_value=mock_acp), + patch("benchflow.acp.runtime.asyncio.sleep", new_callable=AsyncMock), + pytest.raises(TransportClosedError) as exc_info, + ): + await connect_acp( + env=mock_env, + agent="test-agent", + agent_launch="test-agent", + agent_env={}, + sandbox_user=None, + model=None, + rollout_dir=tmp_path, + environment="docker", + agent_cwd="/app", + ) + + assert exc_info.value.diagnostic.transport_diagnosis == "acp_initialize_timeout" + assert mock_acp.initialize.await_count == 4 + mock_acp.close.assert_awaited() + + +async def test_no_web_firewall_runs_after_session_new_before_return(tmp_path) -> None: + """Guards PR #921: bootstrap first, then fail-closed egress isolation.""" + from benchflow.acp.runtime import connect_acp + + events: list[str] = [] + mock_acp = _stock_acp_mock() + + async def session_new(*args, **kwargs): + events.append("session_new") + return MagicMock(session_id="s1") + + async def enforce(*args, **kwargs): + events.append("firewall") + + mock_acp.session_new = AsyncMock(side_effect=session_new) + mock_env = AsyncMock() + + with ( + patch( + "benchflow.acp.runtime.DockerProcess.from_sandbox_env", + return_value=MagicMock(), + ), + patch("benchflow.acp.runtime.ContainerTransport", return_value=MagicMock()), + patch("benchflow.acp.runtime.ACPClient", return_value=mock_acp), + patch( + "benchflow.acp.runtime.enforce_agent_egress_firewall", + new_callable=AsyncMock, + side_effect=enforce, + ) as mock_firewall, + ): + await connect_acp( + env=mock_env, + agent="openhands", + agent_launch="openhands acp", + agent_env={ + "BENCHFLOW_DISALLOW_WEB_TOOLS": "1", + "LLM_BASE_URL": "http://127.0.0.1:1234", + }, + sandbox_user="agent", + model=None, + rollout_dir=tmp_path, + environment="docker", + agent_cwd="/app", + ) + + assert events == ["session_new", "firewall"] + mock_firewall.assert_awaited_once_with( + mock_env, + "agent", + { + "BENCHFLOW_DISALLOW_WEB_TOOLS": "1", + "LLM_BASE_URL": "http://127.0.0.1:1234", + }, + ) diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index a5e73346..b26e4d31 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -5,6 +5,11 @@ test_registry_invariants.py — search there for the consolidated tripwire. """ +import json +import os +import subprocess +import sys + import pytest from benchflow.agents.env import resolve_provider_env @@ -144,7 +149,7 @@ def test_openhands_install_cmd_pins_cli_git_revision(self): "--overrides /tmp/oh-sdk-overrides.txt " "--from " "'git+https://github.com/OpenHands/OpenHands-CLI.git@" - "3ca17446c5d9c1e35e054803478a3501ec251ecf' " + "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271' " "openhands --python 3.12" in cfg.install_cmd ) assert "OpenHands/OpenHands-CLI.git@main" not in cfg.install_cmd @@ -152,12 +157,12 @@ def test_openhands_install_cmd_pins_cli_git_revision(self): assert "command -v git" in cfg.install_cmd assert "install.openhands.dev/install.sh" not in cfg.install_cmd - def test_openhands_install_cmd_overrides_buggy_sdk_pin(self): - """Guards PR #644 against Opus timeouts from OpenHands SDK 1.21.0.""" + def test_openhands_install_cmd_pins_matching_long_run_sdk(self): + """Guards PR #921 against restoring the unstable 1.22.1 ACP runtime.""" cfg = AGENTS["openhands"] - assert "openhands-sdk==1.22.1" in cfg.install_cmd - assert "openhands-tools==1.22.1" in cfg.install_cmd + assert "openhands-sdk==1.28.1" in cfg.install_cmd + assert "openhands-tools==1.28.1" in cfg.install_cmd assert "openhands-sdk>=1.22.0" not in cfg.install_cmd assert "--overrides /tmp/oh-sdk-overrides.txt" in cfg.install_cmd @@ -202,6 +207,139 @@ def test_openhands_launch_cmd_writes_optional_azure_api_version(self): assert ',"api_version":"%s"' in cfg.launch_cmd assert '"$LLM_API_VERSION"' in cfg.launch_cmd + def test_openhands_launch_cmd_writes_optional_reasoning_effort(self): + """Guards PR #911 against OpenHands silently using default high effort.""" + cfg = AGENTS["openhands"] + assert "none|low|medium|high|xhigh)" in cfg.launch_cmd + assert ',"reasoning_effort":"%s",' in cfg.launch_cmd + assert '"litellm_extra_body":{"reasoning_effort":"%s"}' in cfg.launch_cmd + assert '"$LLM_REASONING_EFFORT" "$LLM_REASONING_EFFORT"' in cfg.launch_cmd + + def test_openhands_launch_cmd_keeps_minimal_out_of_typed_effort(self, tmp_path): + """Guards PR #921: OpenHands' typed effort enum rejects minimal.""" + cfg = AGENTS["openhands"] + settings_cmd = cfg.launch_cmd.split(" && openhands acp", 1)[0] + env = { + **os.environ, + "HOME": str(tmp_path), + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_BASE_URL": "http://127.0.0.1:4000/v1", + "LLM_REASONING_EFFORT": "minimal", + } + subprocess.run(["bash", "-c", settings_cmd], env=env, check=True) + settings = json.loads( + (tmp_path / ".openhands" / "agent_settings.json").read_text() + ) + assert "reasoning_effort" not in settings["llm"] + assert settings["llm"]["litellm_extra_body"] == {"reasoning_effort": "minimal"} + + def test_openhands_launch_cmd_passes_max_via_untyped_responses_body(self, tmp_path): + """Guards PR #921: OpenHands' typed effort enum stops at xhigh.""" + cfg = AGENTS["openhands"] + assert 'case "$LLM_REASONING_EFFORT" in ' in cfg.launch_cmd + assert "max) printf" in cfg.launch_cmd + assert ',"litellm_extra_body":{"reasoning":{"effort":"max"}}' in cfg.launch_cmd + settings_cmd = cfg.launch_cmd.split(" && openhands acp", 1)[0] + env = { + **os.environ, + "HOME": str(tmp_path), + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_BASE_URL": "http://127.0.0.1:4000/v1", + "LLM_API_VERSION": "preview", + "LLM_REASONING_EFFORT": "max", + } + subprocess.run(["bash", "-c", settings_cmd], env=env, check=True) + settings = json.loads( + (tmp_path / ".openhands" / "agent_settings.json").read_text() + ) + assert "reasoning_effort" not in settings["llm"] + assert settings["llm"]["litellm_extra_body"] == {"reasoning": {"effort": "max"}} + + def test_openhands_launch_cmd_writes_optional_llm_timeout(self, tmp_path): + """Guards PR #921 against MAX responses exceeding OpenHands' 300s default.""" + cfg = AGENTS["openhands"] + assert 'if [ -n "$LLM_TIMEOUT" ]' in cfg.launch_cmd + assert ',"timeout":%s' in cfg.launch_cmd + settings_cmd = cfg.launch_cmd.split(" && openhands acp", 1)[0] + env = { + **os.environ, + "HOME": str(tmp_path), + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_TIMEOUT": "115200", + } + subprocess.run(["bash", "-c", settings_cmd], env=env, check=True) + settings = json.loads( + (tmp_path / ".openhands" / "agent_settings.json").read_text() + ) + assert settings["llm"]["timeout"] == 115200 + + def test_openhands_launch_cmd_can_disable_subagents(self, tmp_path): + """Guards PR #921 against the OpenHands post-tool delegation deadlock.""" + cfg = AGENTS["openhands"] + settings_cmd = cfg.launch_cmd.split(" && openhands acp", 1)[0] + tool_root = tmp_path / "tools" + package_root = tool_root / "openhands" / "site-packages" + package_dir = package_root / "openhands_cli" + package_dir.mkdir(parents=True) + (package_dir / "__init__.py").write_text("") + utils_path = package_dir / "utils.py" + utils_path.write_text( + "def get_default_cli_tools():\n" + " return [\n" + " Tool(name=task_tool_name),\n" + " ]\n" + ) + bin_dir = tool_root / "openhands" / "bin" + bin_dir.mkdir(parents=True) + python_wrapper = bin_dir / "python" + python_wrapper.write_text( + f'#!/bin/sh\nPYTHONPATH={package_root} exec {sys.executable} "$@"\n' + ) + python_wrapper.chmod(0o755) + openhands = bin_dir / "openhands" + openhands.write_text("#!/bin/sh\nexit 0\n") + openhands.chmod(0o755) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + (fake_bin / "openhands").symlink_to(openhands) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "BENCHFLOW_OPENHANDS_DISABLE_SUBAGENTS": "1", + } + + subprocess.run(["bash", "-c", settings_cmd], env=env, check=True) + + patched = utils_path.read_text() + assert "Tool(name=task_tool_name)" not in patched + assert "BenchFlow: delegation disabled for this run." in patched + + def test_openhands_launch_cmd_rejects_non_numeric_llm_timeout(self, tmp_path): + """Guards PR #921 against malformed timeout JSON in agent settings.""" + cfg = AGENTS["openhands"] + settings_cmd = cfg.launch_cmd.split(" && openhands acp", 1)[0] + env = { + **os.environ, + "HOME": str(tmp_path), + "LLM_MODEL": "openai/gpt-5.6-sol", + "LLM_API_KEY": "proxy-key", + "LLM_TIMEOUT": "none", + } + result = subprocess.run( + ["bash", "-c", settings_cmd], + env=env, + capture_output=True, + text=True, + ) + assert result.returncode == 2 + assert "LLM_TIMEOUT must be a non-negative integer" in result.stderr + def test_harvey_lab_installs_python_deps_in_venv(self): """Guards the v0.5 stress failure where pip hit PEP 668 in Ubuntu.""" cfg = AGENTS["harvey-lab-harness"] diff --git a/tests/test_agent_setup.py b/tests/test_agent_setup.py index f4d41c92..39359d16 100644 --- a/tests/test_agent_setup.py +++ b/tests/test_agent_setup.py @@ -734,7 +734,7 @@ async def test_install_agent_writes_command_stdout_and_stderr_on_failure( "uv tool install --force --refresh " "--overrides /tmp/oh-sdk-overrides.txt " "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@" - "3ca17446c5d9c1e35e054803478a3501ec251ecf' " + "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271' " "openhands --python 3.12" in log_text ) assert "=== stderr ===" in log_text diff --git a/tests/test_benchflow_experiment_review_validator.py b/tests/test_benchflow_experiment_review_validator.py index 3360afa6..4da4a1ce 100644 --- a/tests/test_benchflow_experiment_review_validator.py +++ b/tests/test_benchflow_experiment_review_validator.py @@ -238,3 +238,200 @@ def test_validator_rejects_unready_results_for_healthy_rollout(tmp_path: Path) - assert report["healthy"] is False assert any("training_ready=false" in issue for issue in report["issues"]) + + +def test_validator_rejects_results_with_dropped_successful_exchange( + tmp_path: Path, +) -> None: + """Guards PR #921 MAX review against silently omitted LLM exchanges.""" + validator = _load_validator() + rollout = _rollout(tmp_path) + llm_path = rollout / "trajectory" / "llm_trajectory.jsonl" + first = json.loads(llm_path.read_text()) + second = json.loads(llm_path.read_text()) + second["request"]["body"]["messages"][0]["content"] = "solve a different turn" + _write_jsonl(llm_path, [first, second]) + + report = validator.validate_rollout(rollout) + + assert report["healthy"] is False + assert any( + "results trajectory steps 1 != successful LLM responses 2" in issue + for issue in report["issues"] + ) + + +def test_validator_rejects_nested_truncated_training_step(tmp_path: Path) -> None: + """Guards PR #921 MAX review against `truncation=\"disabled\"` truthiness.""" + validator = _load_validator() + rollout = _rollout(tmp_path) + row = json.loads((rollout / "results.jsonl").read_text()) + row["trajectory"][0]["is_truncated"] = True + _write_jsonl(rollout / "results.jsonl", [row]) + + report = validator.validate_rollout(rollout) + + assert report["healthy"] is False + assert any( + "trajectory[0] is incorrectly truncated" in issue for issue in report["issues"] + ) + + +def test_validator_excludes_recovered_incomplete_response_from_expected_steps( + tmp_path: Path, +) -> None: + """Guards PR #921 MAX review content-filter recovery semantics.""" + validator = _load_validator() + rollout = _rollout(tmp_path) + llm_path = rollout / "trajectory" / "llm_trajectory.jsonl" + incomplete = json.loads(llm_path.read_text()) + incomplete["response"]["body"].update( + { + "status": "incomplete", + "incomplete_details": {"reason": "content_filter"}, + } + ) + completed = json.loads(llm_path.read_text()) + completed["response"]["body"].update( + {"status": "completed", "incomplete_details": None} + ) + _write_jsonl(llm_path, [incomplete, completed]) + row = json.loads((rollout / "results.jsonl").read_text()) + row["trajectory"][0]["extras"]["exchange_index"] = 1 + _write_jsonl(rollout / "results.jsonl", [row]) + + report = validator.validate_rollout(rollout) + + assert report["healthy"] is True + assert report["artifacts"]["llm"]["successful_responses"] == 1 + + +def test_validator_deduplicates_completed_retry_race(tmp_path: Path) -> None: + """Guards PR #921 MAX review against abandoned late responses.""" + validator = _load_validator() + rollout = _rollout(tmp_path) + llm_path = rollout / "trajectory" / "llm_trajectory.jsonl" + first = json.loads(llm_path.read_text()) + second = json.loads(llm_path.read_text()) + _write_jsonl(llm_path, [first, second]) + row = json.loads((rollout / "results.jsonl").read_text()) + row["trajectory"][0]["extras"]["exchange_index"] = 1 + _write_jsonl(rollout / "results.jsonl", [row]) + + report = validator.validate_rollout(rollout) + + assert report["healthy"] is True + assert report["artifacts"]["llm"]["successful_responses"] == 1 + assert report["artifacts"]["llm"]["deduplicated_completed_responses"] == 1 + + +def test_validator_prefers_duplicate_consumed_by_later_request( + tmp_path: Path, +) -> None: + """Guards PR #921 against keeping a late abandoned response.""" + validator = _load_validator() + rollout = _rollout(tmp_path) + llm_path = rollout / "trajectory" / "llm_trajectory.jsonl" + consumed = json.loads(llm_path.read_text()) + consumed["response"]["body"]["choices"][0]["message"] = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_consumed", + "type": "function", + "function": {"name": "finish", "arguments": "{}"}, + } + ], + } + abandoned = json.loads(json.dumps(consumed)) + abandoned["response"]["body"]["choices"][0]["message"]["tool_calls"][0]["id"] = ( + "call_abandoned" + ) + followup = json.loads(llm_path.read_text()) + followup["request"]["body"]["messages"].extend( + [ + consumed["response"]["body"]["choices"][0]["message"], + { + "role": "tool", + "tool_call_id": "call_consumed", + "content": "done", + }, + ] + ) + _write_jsonl(llm_path, [consumed, abandoned, followup]) + row = json.loads((rollout / "results.jsonl").read_text()) + row["trajectory"] = [ + { + **row["trajectory"][0], + "extras": {"source": "llm_trajectory", "exchange_index": 0}, + }, + { + **row["trajectory"][0], + "extras": {"source": "llm_trajectory", "exchange_index": 2}, + }, + ] + _write_jsonl(rollout / "results.jsonl", [row]) + + report = validator.validate_rollout(rollout) + + assert report["healthy"] is True + assert report["artifacts"]["llm"]["successful_exchange_indices"] == [0, 2] + + +def test_validator_excludes_unique_late_unconsumed_nonterminal_retry( + tmp_path: Path, +) -> None: + """Guards PR #921 against the adaptive-cruise exchange-59 regression.""" + validator = _load_validator() + rollout = _rollout(tmp_path) + llm_path = rollout / "trajectory" / "llm_trajectory.jsonl" + base = json.loads(llm_path.read_text()) + late = json.loads(json.dumps(base)) + late["request"]["body"]["messages"].append( + {"role": "user", "content": "Late retry request."} + ) + late["response"]["body"]["choices"][0]["message"] = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_late", + "type": "function", + "function": {"name": "terminal", "arguments": "{}"}, + } + ], + } + terminal = json.loads(json.dumps(base)) + terminal["request"]["body"]["messages"].append( + {"role": "user", "content": "Final response."} + ) + terminal["response"]["body"]["choices"][0]["message"] = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_finish", + "type": "function", + "function": {"name": "finish", "arguments": "{}"}, + } + ], + } + _write_jsonl(llm_path, [base, late, terminal]) + row = json.loads((rollout / "results.jsonl").read_text()) + row["trajectory"] = [ + { + **row["trajectory"][0], + "extras": {"source": "llm_trajectory", "exchange_index": 0}, + }, + { + **row["trajectory"][0], + "extras": {"source": "llm_trajectory", "exchange_index": 2}, + }, + ] + _write_jsonl(rollout / "results.jsonl", [row]) + + report = validator.validate_rollout(rollout) + + assert report["healthy"] is True + assert report["artifacts"]["llm"]["successful_exchange_indices"] == [0, 2] diff --git a/tests/test_config_override.py b/tests/test_config_override.py index 85c7068b..f5cc9a97 100644 --- a/tests/test_config_override.py +++ b/tests/test_config_override.py @@ -2,6 +2,9 @@ from __future__ import annotations +import json +from unittest.mock import MagicMock + import pytest from pydantic import ValidationError @@ -121,6 +124,19 @@ def test_apply_overrides_agent_and_preserves_siblings(): assert out.sandbox.cpus == 1 +def test_apply_overrides_agent_prompt_prefix_and_strips_whitespace(): + out = apply_config_override( + _cfg(), + {"agent": {"prompt_prefix": " Follow benchmark integrity rules. "}}, + ) + assert out.agent.prompt_prefix == "Follow benchmark integrity rules." + + +def test_apply_rejects_blank_agent_prompt_prefix(): + with pytest.raises(ValidationError, match="prompt_prefix"): + apply_config_override(_cfg(), {"agent": {"prompt_prefix": " "}}) + + def test_apply_overrides_sandbox_by_field_name(): # Regression: merging against by_alias=True made `sandbox` (alias # `environment`) un-overridable via its field name. Must work now. @@ -138,6 +154,47 @@ def test_apply_revalidates_and_rejects_bad_value(): apply_config_override(_cfg(), {"agent": {"timeout_sec": "nope"}}) +@pytest.mark.asyncio +async def test_rollout_applies_configured_prompt_prefix(tmp_path): + """Guards PR #921 against recording a prompt policy without sending it.""" + from benchflow.rollout import Rollout, RolloutConfig + + task = tmp_path / "task" + task.mkdir() + (task / "task.toml").write_text('version = "1.0"\n') + (task / "instruction.md").write_text("Solve the visible task.") + + planes = MagicMock() + planes.resolve_locked_paths.return_value = [] + planes.resolve_agent_env.return_value = {} + planes.agent_launch.return_value = "oracle" + planes.create_environment.return_value = MagicMock() + + rollout = Rollout( + RolloutConfig.from_legacy( + task_path=task, + agent="oracle", + jobs_dir=tmp_path / "jobs", + config_override={ + "agent": {"prompt_prefix": "Do not inspect hidden evaluators."} + }, + planes=planes, + ) + ) + await rollout.setup() + + assert rollout._resolved_prompts == [ + "Do not inspect hidden evaluators.\n\nSolve the visible task." + ] + assert ( + rollout._task.config.agent.prompt_prefix == "Do not inspect hidden evaluators." + ) + config = json.loads((rollout._rollout_dir / "config.json").read_text()) + assert config["config_override"]["patch"]["agent"]["prompt_prefix"] == ( + "Do not inspect hidden evaluators." + ) + + # ---- CLI threading: --config-override on the run-config-file path ---------- diff --git a/tests/test_daytona_command_polling.py b/tests/test_daytona_command_polling.py index 861c046f..b651618f 100644 --- a/tests/test_daytona_command_polling.py +++ b/tests/test_daytona_command_polling.py @@ -9,6 +9,51 @@ class TestDaytonaCommandPolling: + @pytest.mark.asyncio + async def test_transient_exec_deletes_completed_session(self) -> None: + """Guards the Daytona live-capture session fix in PR #921.""" + pytest.importorskip("daytona") + from benchflow.sandbox.daytona import ( + DaytonaSandbox, + _DaytonaDirect, + _load_daytona_sdk, + ) + + _load_daytona_sdk() + + class FakeProcess: + def __init__(self) -> None: + self.created: list[str] = [] + self.deleted: list[str] = [] + + async def create_session(self, session_id): + self.created.append(session_id) + + async def execute_session_command(self, session_id, request, timeout=None): + return SimpleNamespace(cmd_id="cmd-1") + + async def get_session_command(self, session_id, command_id): + return SimpleNamespace(id="cmd-1", exit_code=0) + + async def get_session_command_logs(self, session_id, command_id): + return SimpleNamespace(stdout="ok", stderr="") + + async def delete_session(self, session_id): + self.deleted.append(session_id) + + process = FakeProcess() + sandbox = DaytonaSandbox.__new__(DaytonaSandbox) + sandbox.default_user = None + sandbox._persistent_env = {} + sandbox._sandbox = SimpleNamespace(process=process) + sandbox._strategy = _DaytonaDirect(sandbox) + + result = await sandbox.exec_transient("echo ok", timeout_sec=5) + + assert result.return_code == 0 + assert result.stdout == "ok" + assert process.deleted == process.created + @pytest.mark.asyncio async def test_exec_times_out_when_daytona_command_never_exits(self) -> None: """Guards the v0.5 Daytona polling timeout regression.""" diff --git a/tests/test_daytona_reap.py b/tests/test_daytona_reap.py index f0566504..4242cc05 100644 --- a/tests/test_daytona_reap.py +++ b/tests/test_daytona_reap.py @@ -17,6 +17,7 @@ import pytest from benchflow.sandbox.daytona import ( + _benchflow_owned_labels, _is_benchflow_label_orphan, _is_benchflow_owned, reap_stale_sandboxes, @@ -303,6 +304,33 @@ def test_no_labels_attr_not_owned(self): def test_non_mapping_labels_not_owned(self): assert not _is_benchflow_owned(SimpleNamespace(labels=["benchflow.managed"])) + def test_scoped_owner_matches_only_current_operator(self, monkeypatch): + """Guards PR #921 against cross-operator Daytona cleanup.""" + monkeypatch.setenv("BENCHFLOW_DAYTONA_OWNER", "gpt56 max / openhands") + labels = _benchflow_owned_labels() + + assert labels == { + "benchflow.managed": "1:gpt56-max-openhands", + "benchflow.owner": "gpt56-max-openhands", + } + assert _is_benchflow_owned(SimpleNamespace(labels=labels)) + assert not _is_benchflow_owned( + SimpleNamespace( + labels={ + "benchflow.managed": "1:another-experiment", + "benchflow.owner": "another-experiment", + } + ) + ) + + def test_unscoped_legacy_owner_does_not_match_scoped_operator(self, monkeypatch): + """Guards PR #921 so old broad labels cannot enter a scoped reap.""" + monkeypatch.setenv("BENCHFLOW_DAYTONA_OWNER", "gpt56-sol-openhands-max") + + assert not _is_benchflow_owned( + SimpleNamespace(labels={"benchflow.managed": "1"}) + ) + class TestIsBenchflowLabelOrphan: """The read-only orphan-leak predicate (label-integrity, mutation surface). @@ -352,6 +380,51 @@ def test_no_labels_attr_not_orphan(self): def test_non_mapping_labels_not_orphan(self): assert not _is_benchflow_label_orphan(SimpleNamespace(labels=["benchflow.run"])) + def test_other_scoped_owner_is_foreign_not_orphan(self, monkeypatch): + """Guards PR #921 against warning or deleting another operator's run.""" + monkeypatch.setenv("BENCHFLOW_DAYTONA_OWNER", "our-experiment") + + assert not _is_benchflow_label_orphan( + SimpleNamespace( + labels={ + "benchflow.managed": "1:their-experiment", + "benchflow.owner": "their-experiment", + } + ) + ) + + +def test_reaper_skips_other_scoped_owner(monkeypatch): + """Guards PR #921 against cross-operator deletion on a shared API key.""" + monkeypatch.setenv("BENCHFLOW_DAYTONA_OWNER", "our-experiment") + client = FakeClient( + [ + _sb( + "ours", + "STARTED", + 9999, + labels={ + "benchflow.managed": "1:our-experiment", + "benchflow.owner": "our-experiment", + }, + ), + _sb( + "theirs", + "STARTED", + 9999, + labels={ + "benchflow.managed": "1:their-experiment", + "benchflow.owner": "their-experiment", + }, + ), + ] + ) + + counts = reap_stale_sandboxes(client) + + assert client.deleted == ["ours"] + assert counts == {"found": 2, "deleted": 1, "skipped": 1, "failed": 0} + class TestOrphanLeakGuard: """Reaper flags benchflow-namespaced-but-unlabeled sandboxes, never deletes. diff --git a/tests/test_live_llm_trajectory.py b/tests/test_live_llm_trajectory.py index 6b76ec31..b592f454 100644 --- a/tests/test_live_llm_trajectory.py +++ b/tests/test_live_llm_trajectory.py @@ -168,6 +168,19 @@ async def exec(self, command: str, timeout_sec: int): return SimpleNamespace(return_code=0, stdout=encoded) +class _TransientSandboxWithCallbackLog(_SandboxWithCallbackLog): + def __init__(self, data: bytes) -> None: + super().__init__(data) + self.transient_calls = 0 + + async def exec(self, command: str, timeout_sec: int): + raise AssertionError("Daytona callback polling must use transient exec") + + async def exec_transient(self, command: str, timeout_sec: int): + self.transient_calls += 1 + return await super().exec(command, timeout_sec) + + @pytest.mark.asyncio async def test_daytona_proxy_incrementally_mirrors_callback(tmp_path, monkeypatch): """Guards incremental Daytona callback mirroring from commit c86adfb.""" @@ -200,3 +213,26 @@ async def test_daytona_proxy_incrementally_mirrors_callback(tmp_path, monkeypatc await process._stop_live_capture() assert len(output_path.read_text().splitlines()) == 2 + + +@pytest.mark.asyncio +async def test_daytona_proxy_uses_transient_exec_for_callback_poll(tmp_path): + """Guards the Daytona live-capture session fix in PR #921.""" + sandbox = _TransientSandboxWithCallbackLog(_callback_line().encode()) + process = SandboxLiteLLMProcess( + sandbox=sandbox, + route=SimpleNamespace(), + runtime_dir="/tmp/runtime", + endpoint=LiteLLMEndpoint("http://agent", "http://local"), + log_path="/tmp/runtime/callback.jsonl", + pid_path="/tmp/runtime/pid", + stdout_path="/tmp/runtime/stdout", + stderr_path="/tmp/runtime/stderr", + session_id="run", + agent_name="openhands", + ) + + chunk = await process._read_callback_chunk(0, 24 * 1024) + + assert chunk + assert sandbox.transient_calls == 1 diff --git a/tests/test_process.py b/tests/test_process.py index 1d39e28a..8f26cbf4 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -3,6 +3,7 @@ import os import shlex import subprocess +from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -566,6 +567,35 @@ async def test_bootstrapped_env_file_is_removed_when_ssh_launch_fails(self): assert "secret" not in cleanup_cmd assert proc._ssh_config_path is None + def test_ssh_config_hardens_long_running_connections(self): + """Guards PR #921 against Daytona SSH idle-session transport loss.""" + path = DaytonaProcess._write_ssh_config("token-user") + try: + config = Path(path).read_text() + finally: + os.unlink(path) + + assert "ServerAliveInterval 30" in config + assert "ServerAliveCountMax 12" in config + assert "TCPKeepAlive yes" in config + + @pytest.mark.asyncio + async def test_ssh_access_outlives_max_agent_timeout(self): + """Guards PR #921 against Daytona's 60-minute SSH token default.""" + sandbox = _make_daytona_sandbox() + proc = DaytonaProcess(sandbox=sandbox, is_dind=False) + process = _FakeProcess(stdin=_FakeStdin()) + + with patch( + "asyncio.create_subprocess_exec", + new_callable=AsyncMock, + return_value=process, + ): + await proc.start(command="openhands acp") + + sandbox.create_ssh_access.assert_awaited_once_with(expires_in_minutes=2880) + await proc.close() + @pytest.mark.asyncio async def test_ssh_config_not_created_when_env_bootstrap_fails(self): """Guards temp SSH config cleanup when SDK bootstrap fails early.""" diff --git a/tests/test_sandbox_setup.py b/tests/test_sandbox_setup.py index f12e01f2..76b0f73d 100644 --- a/tests/test_sandbox_setup.py +++ b/tests/test_sandbox_setup.py @@ -89,6 +89,15 @@ async def test_setup_command_still_creates_user_prepares_home_and_chowns_workspa assert "chown -R agent:agent /home/agent" in cmd assert f"chown -R agent:agent {shlex.quote('/app')}" in cmd + @pytest.mark.asyncio + async def test_setup_command_grants_declared_top_level_output_roots(self): + """Guards PR #921 against root-owned /output breaking non-root agents.""" + cmd, _ = await _run_setup_sandbox_user() + + assert "for d in /output /outputs; do" in cmd + assert '[ -d "$d" ] && [ ! -L "$d" ]' in cmd + assert 'chown -R agent:agent "$d"' in cmd + @pytest.mark.asyncio async def test_setup_command_keeps_heavy_root_tool_dirs_on_shared_paths(self): """Legacy root-only tool dirs should use conditional symlinks, not duplication.""" diff --git a/tests/test_sdk_internals.py b/tests/test_sdk_internals.py index 7f076604..df571c90 100644 --- a/tests/test_sdk_internals.py +++ b/tests/test_sdk_internals.py @@ -308,6 +308,25 @@ def test_resolve_prompts_does_not_add_no_web_text(self, tmp_path): assert "Internet access is disabled" not in result[0] +class TestPromptPrefix: + def test_prefix_is_prepended_to_every_resolved_prompt(self): + from benchflow.rollout._setup import _apply_prompt_prefix + + assert _apply_prompt_prefix( + ["First task prompt", "Second task prompt"], + "Do not inspect hidden evaluator assets.", + ) == [ + "Do not inspect hidden evaluator assets.\n\nFirst task prompt", + "Do not inspect hidden evaluator assets.\n\nSecond task prompt", + ] + + def test_none_preserves_prompt_list_identity(self): + from benchflow.rollout._setup import _apply_prompt_prefix + + prompts = ["Task prompt"] + assert _apply_prompt_prefix(prompts, None) is prompts + + # _init_trial @@ -408,6 +427,7 @@ def test_config_json_written(self, tmp_path): "agent", "model", "environment", + "acp_transport", "skill_mode", "skill_source", "requested_skills_dir", @@ -430,6 +450,7 @@ def test_config_json_written(self, tmp_path): assert data["task_path"] == "foo" assert data["model"] == "claude-haiku-4-5-20251001" assert data["environment"] == "docker" + assert data["acp_transport"] == "docker-stdio" assert data["skill_mode"] == "no-skill" assert data["include_task_skills"] is False assert data["effective_skills_dir"] is None @@ -437,6 +458,26 @@ def test_config_json_written(self, tmp_path): assert data["timeout_sec"] == 300 assert data["scenes"] == [] + def test_config_json_records_daytona_ssh_transport(self, tmp_path, monkeypatch): + """Guards PR #921 so audits can prove the selected Daytona transport.""" + monkeypatch.setenv("BENCHFLOW_DAYTONA_ACP_TRANSPORT", "ssh") + + self._write( + tmp_path, + task_path=Path("/tasks/foo"), + agent="openhands", + model="azure-foundry-openai/gpt-5.6-sol", + environment="daytona", + sandbox_user="agent", + context_root=None, + timeout=115200, + started_at=datetime(2026, 7, 13, 17, 0), + agent_env={}, + ) + + data = json.loads((tmp_path / "config.json").read_text()) + assert data["acp_transport"] == "ssh" + def test_config_json_includes_scene_role_metadata(self, tmp_path): """Multi-role scene metadata is recorded without leaking env values.""" from benchflow import Role, Scene, Turn diff --git a/tests/test_sdk_lockdown.py b/tests/test_sdk_lockdown.py index aca4550d..1f4e2cfe 100644 --- a/tests/test_sdk_lockdown.py +++ b/tests/test_sdk_lockdown.py @@ -1,5 +1,6 @@ """Tests for path lockdown — _validate_locked_path, _resolve_locked_paths, lockdown_paths.""" +import subprocess from unittest.mock import AsyncMock, MagicMock import pytest @@ -384,11 +385,88 @@ def test_single_quotes_in_launch(self): """Single quotes in agent_launch don't break the command.""" cmd = build_priv_drop_cmd("agent --prompt 'hello world'", "agent") assert "hello world" in cmd - assert cmd.count("if ") == 1 - assert cmd.count(" fi") == 1 + subprocess.run(["bash", "-n", "-c", cmd], check=True) def test_custom_sandbox_user(self): cmd = build_priv_drop_cmd("my-agent", "bench-user") assert "--reuid=bench-user" in cmd assert "su -l bench-user" in cmd assert "/home/bench-user" in cmd + + def test_privilege_drop_does_not_install_firewall_during_acp_bootstrap(self): + """Guards PR #921 against blocking OpenHands before ACP initialize.""" + cmd = build_priv_drop_cmd("my-agent", "agent") + + assert "iptables" not in cmd + subprocess.run(["bash", "-n", "-c", cmd], check=True) + + +class TestAgentEgressFirewall: + """External egress is blocked after ACP bootstrap but before prompting.""" + + def test_firewall_allows_loopback_and_rejects_ipv4_and_ipv6_egress(self): + from benchflow.sandbox.lockdown import _agent_egress_firewall_cmd + + cmd = _agent_egress_firewall_cmd("agent") + + assert "apt-get install -y -qq iptables" in cmd + assert "dnf -y install iptables" in cmd + assert "apk add --no-cache iptables" in cmd + assert "agent_uid=$(id -u agent)" in cmd + assert '-o lo -m owner --uid-owner "$agent_uid" -j ACCEPT' in cmd + assert '--uid-owner "$agent_uid" -j REJECT' in cmd + assert "ip6tables -C OUTPUT -o lo" in cmd + assert "IPv6 enabled but ip6tables unavailable" in cmd + subprocess.run(["bash", "-n", "-c", cmd], check=True) + + async def test_policy_is_skipped_when_web_tools_are_allowed(self): + from benchflow.sandbox.lockdown import enforce_agent_egress_firewall + + env = MagicMock() + env.exec = AsyncMock() + + await enforce_agent_egress_firewall( + env, + "agent", + {"LLM_BASE_URL": "http://127.0.0.1:1234"}, + ) + + env.exec.assert_not_awaited() + + async def test_policy_requires_loopback_proxy_and_runs_as_root(self): + from benchflow.sandbox.lockdown import enforce_agent_egress_firewall + + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0)) + + await enforce_agent_egress_firewall( + env, + "agent", + { + "BENCHFLOW_DISALLOW_WEB_TOOLS": "1", + "LLM_BASE_URL": "http://127.0.0.1:1234", + }, + ) + + env.exec.assert_awaited_once() + (_cmd,) = env.exec.await_args.args + assert "iptables -C OUTPUT -o lo" in _cmd + assert env.exec.await_args.kwargs == {"user": "root", "timeout_sec": 120} + + async def test_policy_rejects_non_loopback_proxy(self): + from benchflow.sandbox.lockdown import enforce_agent_egress_firewall + + env = MagicMock() + env.exec = AsyncMock() + + with pytest.raises(RuntimeError, match="loopback LLM_BASE_URL"): + await enforce_agent_egress_firewall( + env, + "agent", + { + "BENCHFLOW_DISALLOW_WEB_TOOLS": "1", + "LLM_BASE_URL": "https://api.openai.com/v1", + }, + ) + + env.exec.assert_not_awaited() diff --git a/tests/test_trace_import_cli.py b/tests/test_trace_import_cli.py index b80b0a71..a4f8b748 100644 --- a/tests/test_trace_import_cli.py +++ b/tests/test_trace_import_cli.py @@ -52,7 +52,9 @@ def test_tasks_generate_rejects_removed_short_options( ) assert result.exit_code != 0 - assert f"No such option: {alias}" in click.unstyle(result.output) + output = click.unstyle(result.output) + assert "No such option" in output + assert alias in output def test_tasks_generate_dry_run_uses_generation_filters( diff --git a/tests/test_train_mode_artifact_emission.py b/tests/test_train_mode_artifact_emission.py index 22824d21..6aed4d37 100644 --- a/tests/test_train_mode_artifact_emission.py +++ b/tests/test_train_mode_artifact_emission.py @@ -28,6 +28,7 @@ from benchflow.trajectories.export_prime_sft import validate_prime_sft_jsonl from benchflow.trajectories.results import ( JOB_RESULTS_ERRORS_FILENAME, + _training_success_exchange_indices, write_job_results_jsonl, ) @@ -607,6 +608,337 @@ def test_results_jsonl_uses_canonical_prime_sft_normalization(tmp_path): assert validate_prime_sft_jsonl(artifact, expected_rows=1)["ok"] is True +def test_results_jsonl_keeps_all_repaired_exchanges_and_disabled_truncation( + tmp_path, +): + """Guards PR #921 MAX canary against silent exchange loss.""" + rollout_dir = tmp_path / "rollout-max-multi-exchange" + rollout_dir.mkdir() + traj_dir = rollout_dir / "trajectory" + traj_dir.mkdir() + first = _llm_exchange() + first["response"]["body"]["truncation"] = "disabled" + second = _llm_exchange( + messages=[ + {"role": "user", "content": "List files."}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "terminal", + "arguments": '{"command":"ls"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "README.md"}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "pyproject.toml", + }, + ], + assistant={"role": "assistant", "content": "Done."}, + ) + second["response"]["body"]["truncation"] = "disabled" + (traj_dir / "llm_trajectory.jsonl").write_text( + json.dumps(first) + "\n" + json.dumps(second) + "\n" + ) + + _build_rollout_result( + rollout_dir, + task_name="list-files", + rollout_name="r1", + agent="openhands", + agent_name="OpenHands", + model="openai-compatible-model", + n_tool_calls=1, + prompts=["List files."], + error=None, + verifier_error=None, + trajectory=_acp_trajectory(), + partial_trajectory=False, + trajectory_source="acp", + rewards={"reward": 1.0}, + started_at=datetime.now(), + timing={}, + total_tokens=26, + ) + + row = json.loads((rollout_dir / "results.jsonl").read_text()) + assert row["info"]["training_ready"] is True + assert len(row["trajectory"]) == 2 + assert all(step["is_truncated"] is False for step in row["trajectory"]) + assert {tool["function"]["name"] for tool in row["tool_defs"]} == {"terminal"} + assert row["completion"][0]["tool_calls"][0]["function"]["name"] == "terminal" + assert row["completion"][1] == { + "role": "tool", + "tool_call_id": "call_1", + "content": "README.md\npyproject.toml", + } + assert row["completion"][-1] == {"role": "assistant", "content": "Done."} + + +def test_results_jsonl_fails_closed_when_successful_exchange_is_omitted(tmp_path): + """Guards PR #921 MAX canary against green rows with dropped exchanges.""" + rollout_dir = tmp_path / "rollout-invalid-later-exchange" + rollout_dir.mkdir() + traj_dir = rollout_dir / "trajectory" + traj_dir.mkdir() + invalid = _llm_exchange( + messages=[ + {"role": "user", "content": "List files."}, + {"role": "tool", "tool_call_id": "orphan", "content": "README.md"}, + ], + assistant={"role": "assistant", "content": "Done."}, + ) + (traj_dir / "llm_trajectory.jsonl").write_text( + json.dumps(_llm_exchange()) + "\n" + json.dumps(invalid) + "\n" + ) + + _build_rollout_result( + rollout_dir, + task_name="list-files", + rollout_name="r1", + agent="openhands", + agent_name="OpenHands", + model="openai-compatible-model", + n_tool_calls=1, + prompts=["List files."], + error=None, + verifier_error=None, + trajectory=_acp_trajectory(), + partial_trajectory=False, + trajectory_source="acp", + rewards={"reward": 1.0}, + started_at=datetime.now(), + timing={}, + ) + + row = json.loads((rollout_dir / "results.jsonl").read_text()) + assert row["info"]["training_ready"] is False + assert row["info"]["training_ready_reason"] == "export_error" + assert "Successful LLM exchanges were omitted" in row["error"]["error_chain_str"] + + +def test_results_jsonl_excludes_recovered_incomplete_provider_exchange(tmp_path): + """Guards PR #921 MAX canary content-filter recovery semantics.""" + rollout_dir = tmp_path / "rollout-recovered-incomplete" + rollout_dir.mkdir() + traj_dir = rollout_dir / "trajectory" + traj_dir.mkdir() + incomplete = _llm_exchange() + incomplete["response"]["body"].update( + { + "status": "incomplete", + "incomplete_details": {"reason": "content_filter"}, + "truncation": "disabled", + } + ) + completed = _llm_exchange(assistant={"role": "assistant", "content": "Recovered."}) + completed["response"]["body"].update( + { + "status": "completed", + "incomplete_details": None, + "truncation": "disabled", + } + ) + (traj_dir / "llm_trajectory.jsonl").write_text( + json.dumps(incomplete) + "\n" + json.dumps(completed) + "\n" + ) + + _build_rollout_result( + rollout_dir, + task_name="list-files", + rollout_name="r1", + agent="openhands", + agent_name="OpenHands", + model="openai-compatible-model", + n_tool_calls=1, + prompts=["List files."], + error=None, + verifier_error=None, + trajectory=_acp_trajectory(), + partial_trajectory=False, + trajectory_source="acp", + rewards={"reward": 1.0}, + started_at=datetime.now(), + timing={}, + ) + + row = json.loads((rollout_dir / "results.jsonl").read_text()) + assert row["info"]["training_ready"] is True + assert len(row["trajectory"]) == 1 + assert row["trajectory"][0]["extras"]["exchange_index"] == 1 + assert row["trajectory"][0]["is_truncated"] is False + + +def test_results_jsonl_keeps_latest_completed_duplicate_request(tmp_path): + """Guards PR #921 MAX canary against late retry race responses.""" + rollout_dir = tmp_path / "rollout-duplicate-completed-request" + rollout_dir.mkdir() + traj_dir = rollout_dir / "trajectory" + traj_dir.mkdir() + abandoned = _llm_exchange( + assistant={"role": "assistant", "content": "Abandoned response."} + ) + consumed = _llm_exchange( + assistant={"role": "assistant", "content": "Consumed retry."} + ) + assert abandoned["request"]["body"] == consumed["request"]["body"] + (traj_dir / "llm_trajectory.jsonl").write_text( + json.dumps(abandoned) + "\n" + json.dumps(consumed) + "\n" + ) + + _build_rollout_result( + rollout_dir, + task_name="list-files", + rollout_name="r1", + agent="openhands", + agent_name="OpenHands", + model="openai-compatible-model", + n_tool_calls=1, + prompts=["List files."], + error=None, + verifier_error=None, + trajectory=_acp_trajectory(), + partial_trajectory=False, + trajectory_source="acp", + rewards={"reward": 1.0}, + started_at=datetime.now(), + timing={}, + ) + + row = json.loads((rollout_dir / "results.jsonl").read_text()) + assert row["info"]["training_ready"] is True + assert len(row["trajectory"]) == 1 + assert row["trajectory"][0]["extras"]["exchange_index"] == 1 + assert row["completion"] == [{"role": "assistant", "content": "Consumed retry."}] + + +def test_results_jsonl_prefers_duplicate_consumed_by_later_request(tmp_path): + """Guards PR #921 against keeping a late abandoned response.""" + rollout_dir = tmp_path / "rollout-consumed-duplicate-request" + rollout_dir.mkdir() + traj_dir = rollout_dir / "trajectory" + traj_dir.mkdir() + consumed = _llm_exchange() + consumed["response"]["body"]["choices"][0]["message"]["tool_calls"][0]["id"] = ( + "call_consumed" + ) + abandoned = _llm_exchange() + abandoned["response"]["body"]["choices"][0]["message"]["tool_calls"][0]["id"] = ( + "call_abandoned" + ) + followup = _llm_exchange( + messages=[ + {"role": "user", "content": "List files."}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_consumed", + "type": "function", + "function": { + "name": "terminal", + "arguments": '{"command":"ls"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_consumed", + "content": "README.md", + }, + ], + assistant={"role": "assistant", "content": "Done."}, + ) + assert consumed["request"]["body"] == abandoned["request"]["body"] + (traj_dir / "llm_trajectory.jsonl").write_text( + "\n".join(json.dumps(item) for item in (consumed, abandoned, followup)) + "\n" + ) + + _build_rollout_result( + rollout_dir, + task_name="list-files", + rollout_name="r1", + agent="openhands", + agent_name="OpenHands", + model="openai-compatible-model", + n_tool_calls=1, + prompts=["List files."], + error=None, + verifier_error=None, + trajectory=_acp_trajectory(), + partial_trajectory=False, + trajectory_source="acp", + rewards={"reward": 1.0}, + started_at=datetime.now(), + timing={}, + ) + + row = json.loads((rollout_dir / "results.jsonl").read_text()) + assert [step["extras"]["exchange_index"] for step in row["trajectory"]] == [ + 0, + 2, + ] + + +def test_results_drop_unique_late_unconsumed_nonterminal_retry(): + """Guards PR #921 against the adaptive-cruise exchange-59 regression.""" + consumed = _llm_exchange() + consumed["response"]["body"]["choices"][0]["message"]["tool_calls"][0]["id"] = ( + "call_consumed" + ) + followup = _llm_exchange( + messages=[ + {"role": "user", "content": "List files."}, + consumed["response"]["body"]["choices"][0]["message"], + { + "role": "tool", + "tool_call_id": "call_consumed", + "content": "README.md", + }, + ], + assistant={"role": "assistant", "content": "Continuing."}, + ) + late = _llm_exchange() + late["request"]["body"]["messages"].append( + {"role": "user", "content": "Late retry request."} + ) + late["response"]["body"]["choices"][0]["message"]["tool_calls"][0]["id"] = ( + "call_late" + ) + terminal = _llm_exchange( + assistant={ + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_finish", + "type": "function", + "function": {"name": "finish", "arguments": "{}"}, + } + ], + } + ) + terminal["request"]["body"]["messages"].append( + {"role": "user", "content": "Final response."} + ) + + assert _training_success_exchange_indices([consumed, followup, late, terminal]) == { + 0, + 1, + 3, + } + + def test_results_jsonl_preserves_llm_exchange_metadata(tmp_path): """Guards PR #925: trainer conversion retains LLM call purpose metadata.""" rollout_dir = tmp_path / "rollout-call-purpose" diff --git a/tests/trajectories/test_export_prime_sft.py b/tests/trajectories/test_export_prime_sft.py index 0aa407ba..3d8c1298 100644 --- a/tests/trajectories/test_export_prime_sft.py +++ b/tests/trajectories/test_export_prime_sft.py @@ -12,6 +12,7 @@ convert_benchflow_rollouts_to_prime_sft_rows, export_prime_sft_jsonl, load_llm_trajectory_jsonl, + normalize_prime_sft_exchange, validate_prime_sft_jsonl, ) @@ -1116,6 +1117,45 @@ def test_convert_openhands_responses_shape_preserves_tool_calls(tmp_path: Path) assert row["messages"][-1] == {"role": "assistant", "content": "Done."} +def test_normalize_exchange_merges_duplicate_tool_output_fragments() -> None: + """Guards PR #921 MAX canary exchange conversion against silent drops.""" + exchange = _exchange(final=False) + exchange["request"]["body"]["messages"] = [ + {"role": "user", "content": "List files."}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "bash", + "arguments": {"command": "ls"}, + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "README.md"}, + {"role": "tool", "tool_call_id": "call_1", "content": "pyproject.toml"}, + ] + + normalized, skip_reason = normalize_prime_sft_exchange(exchange) + + assert skip_reason is None + assert normalized is not None + tool_messages = [ + message for message in normalized.messages if message["role"] == "tool" + ] + assert tool_messages == [ + { + "role": "tool", + "tool_call_id": "call_1", + "content": "README.md\npyproject.toml", + } + ] + + def test_convert_succeeds_when_trajectory_written_via_to_jsonl_with_secrets( tmp_path: Path, ) -> None: diff --git a/uv.lock b/uv.lock index 4f6cfcfb..e5518010 100644 --- a/uv.lock +++ b/uv.lock @@ -373,7 +373,7 @@ requires-dist = [ { name = "google-genai", marker = "extra == 'judge'", specifier = ">=1.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "langchain-openai", marker = "extra == 'deepagents'", specifier = ">=0.2" }, - { name = "litellm", extras = ["proxy"], specifier = "==1.89.0" }, + { name = "litellm", extras = ["proxy"], specifier = "==1.91.0" }, { name = "modal", marker = "extra == 'sandbox-modal'", specifier = ">=0.73" }, { name = "openai", marker = "extra == 'judge'", specifier = ">=1.40" }, { name = "packaging", specifier = ">=24" }, @@ -621,14 +621,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -654,55 +654,55 @@ wheels = [ [[package]] name = "cryptography" -version = "46.0.7" +version = "48.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652, upload-time = "2026-04-08T01:57:54.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/5d/4a8f770695d73be252331e60e526291e3df0c9b27556a90a6b47bccca4c2/cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4", size = 7179869, upload-time = "2026-04-08T01:56:17.157Z" }, - { url = "https://files.pythonhosted.org/packages/5f/45/6d80dc379b0bbc1f9d1e429f42e4cb9e1d319c7a8201beffd967c516ea01/cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325", size = 4275492, upload-time = "2026-04-08T01:56:19.36Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9a/1765afe9f572e239c3469f2cb429f3ba7b31878c893b246b4b2994ffe2fe/cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308", size = 4426670, upload-time = "2026-04-08T01:56:21.415Z" }, - { url = "https://files.pythonhosted.org/packages/8f/3e/af9246aaf23cd4ee060699adab1e47ced3f5f7e7a8ffdd339f817b446462/cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77", size = 4280275, upload-time = "2026-04-08T01:56:23.539Z" }, - { url = "https://files.pythonhosted.org/packages/0f/54/6bbbfc5efe86f9d71041827b793c24811a017c6ac0fd12883e4caa86b8ed/cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1", size = 4928402, upload-time = "2026-04-08T01:56:25.624Z" }, - { url = "https://files.pythonhosted.org/packages/2d/cf/054b9d8220f81509939599c8bdbc0c408dbd2bdd41688616a20731371fe0/cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef", size = 4459985, upload-time = "2026-04-08T01:56:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/f9/46/4e4e9c6040fb01c7467d47217d2f882daddeb8828f7df800cb806d8a2288/cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de", size = 3990652, upload-time = "2026-04-08T01:56:29.095Z" }, - { url = "https://files.pythonhosted.org/packages/36/5f/313586c3be5a2fbe87e4c9a254207b860155a8e1f3cca99f9910008e7d08/cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83", size = 4279805, upload-time = "2026-04-08T01:56:30.928Z" }, - { url = "https://files.pythonhosted.org/packages/69/33/60dfc4595f334a2082749673386a4d05e4f0cf4df8248e63b2c3437585f2/cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb", size = 4892883, upload-time = "2026-04-08T01:56:32.614Z" }, - { url = "https://files.pythonhosted.org/packages/c7/0b/333ddab4270c4f5b972f980adef4faa66951a4aaf646ca067af597f15563/cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b", size = 4459756, upload-time = "2026-04-08T01:56:34.306Z" }, - { url = "https://files.pythonhosted.org/packages/d2/14/633913398b43b75f1234834170947957c6b623d1701ffc7a9600da907e89/cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85", size = 4410244, upload-time = "2026-04-08T01:56:35.977Z" }, - { url = "https://files.pythonhosted.org/packages/10/f2/19ceb3b3dc14009373432af0c13f46aa08e3ce334ec6eff13492e1812ccd/cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e", size = 4674868, upload-time = "2026-04-08T01:56:38.034Z" }, - { url = "https://files.pythonhosted.org/packages/1a/bb/a5c213c19ee94b15dfccc48f363738633a493812687f5567addbcbba9f6f/cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457", size = 3026504, upload-time = "2026-04-08T01:56:39.666Z" }, - { url = "https://files.pythonhosted.org/packages/2b/02/7788f9fefa1d060ca68717c3901ae7fffa21ee087a90b7f23c7a603c32ae/cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b", size = 3488363, upload-time = "2026-04-08T01:56:41.893Z" }, - { url = "https://files.pythonhosted.org/packages/7b/56/15619b210e689c5403bb0540e4cb7dbf11a6bf42e483b7644e471a2812b3/cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842", size = 7119671, upload-time = "2026-04-08T01:56:44Z" }, - { url = "https://files.pythonhosted.org/packages/74/66/e3ce040721b0b5599e175ba91ab08884c75928fbeb74597dd10ef13505d2/cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c", size = 4268551, upload-time = "2026-04-08T01:56:46.071Z" }, - { url = "https://files.pythonhosted.org/packages/03/11/5e395f961d6868269835dee1bafec6a1ac176505a167f68b7d8818431068/cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902", size = 4408887, upload-time = "2026-04-08T01:56:47.718Z" }, - { url = "https://files.pythonhosted.org/packages/40/53/8ed1cf4c3b9c8e611e7122fb56f1c32d09e1fff0f1d77e78d9ff7c82653e/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d", size = 4271354, upload-time = "2026-04-08T01:56:49.312Z" }, - { url = "https://files.pythonhosted.org/packages/50/46/cf71e26025c2e767c5609162c866a78e8a2915bbcfa408b7ca495c6140c4/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022", size = 4905845, upload-time = "2026-04-08T01:56:50.916Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ea/01276740375bac6249d0a971ebdf6b4dc9ead0ee0a34ef3b5a88c1a9b0d4/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce", size = 4444641, upload-time = "2026-04-08T01:56:52.882Z" }, - { url = "https://files.pythonhosted.org/packages/3d/4c/7d258f169ae71230f25d9f3d06caabcff8c3baf0978e2b7d65e0acac3827/cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f", size = 3967749, upload-time = "2026-04-08T01:56:54.597Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2a/2ea0767cad19e71b3530e4cad9605d0b5e338b6a1e72c37c9c1ceb86c333/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99", size = 4270942, upload-time = "2026-04-08T01:56:56.416Z" }, - { url = "https://files.pythonhosted.org/packages/41/3d/fe14df95a83319af25717677e956567a105bb6ab25641acaa093db79975d/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1", size = 4871079, upload-time = "2026-04-08T01:56:58.31Z" }, - { url = "https://files.pythonhosted.org/packages/9c/59/4a479e0f36f8f378d397f4eab4c850b4ffb79a2f0d58704b8fa0703ddc11/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2", size = 4443999, upload-time = "2026-04-08T01:57:00.508Z" }, - { url = "https://files.pythonhosted.org/packages/28/17/b59a741645822ec6d04732b43c5d35e4ef58be7bfa84a81e5ae6f05a1d33/cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e", size = 4399191, upload-time = "2026-04-08T01:57:02.654Z" }, - { url = "https://files.pythonhosted.org/packages/59/6a/bb2e166d6d0e0955f1e9ff70f10ec4b2824c9cfcdb4da772c7dd69cc7d80/cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee", size = 4655782, upload-time = "2026-04-08T01:57:04.592Z" }, - { url = "https://files.pythonhosted.org/packages/95/b6/3da51d48415bcb63b00dc17c2eff3a651b7c4fed484308d0f19b30e8cb2c/cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298", size = 3002227, upload-time = "2026-04-08T01:57:06.91Z" }, - { url = "https://files.pythonhosted.org/packages/32/a8/9f0e4ed57ec9cebe506e58db11ae472972ecb0c659e4d52bbaee80ca340a/cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb", size = 3475332, upload-time = "2026-04-08T01:57:08.807Z" }, - { url = "https://files.pythonhosted.org/packages/a7/7f/cd42fc3614386bc0c12f0cb3c4ae1fc2bbca5c9662dfed031514911d513d/cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4", size = 7165618, upload-time = "2026-04-08T01:57:10.645Z" }, - { url = "https://files.pythonhosted.org/packages/a5/d0/36a49f0262d2319139d2829f773f1b97ef8aef7f97e6e5bd21455e5a8fb5/cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7", size = 4270628, upload-time = "2026-04-08T01:57:12.885Z" }, - { url = "https://files.pythonhosted.org/packages/8a/6c/1a42450f464dda6ffbe578a911f773e54dd48c10f9895a23a7e88b3e7db5/cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832", size = 4415405, upload-time = "2026-04-08T01:57:14.923Z" }, - { url = "https://files.pythonhosted.org/packages/9a/92/4ed714dbe93a066dc1f4b4581a464d2d7dbec9046f7c8b7016f5286329e2/cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163", size = 4272715, upload-time = "2026-04-08T01:57:16.638Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e6/a26b84096eddd51494bba19111f8fffe976f6a09f132706f8f1bf03f51f7/cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2", size = 4918400, upload-time = "2026-04-08T01:57:19.021Z" }, - { url = "https://files.pythonhosted.org/packages/c7/08/ffd537b605568a148543ac3c2b239708ae0bd635064bab41359252ef88ed/cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067", size = 4450634, upload-time = "2026-04-08T01:57:21.185Z" }, - { url = "https://files.pythonhosted.org/packages/16/01/0cd51dd86ab5b9befe0d031e276510491976c3a80e9f6e31810cce46c4ad/cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0", size = 3985233, upload-time = "2026-04-08T01:57:22.862Z" }, - { url = "https://files.pythonhosted.org/packages/92/49/819d6ed3a7d9349c2939f81b500a738cb733ab62fbecdbc1e38e83d45e12/cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba", size = 4271955, upload-time = "2026-04-08T01:57:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/80/07/ad9b3c56ebb95ed2473d46df0847357e01583f4c52a85754d1a55e29e4d0/cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006", size = 4879888, upload-time = "2026-04-08T01:57:26.88Z" }, - { url = "https://files.pythonhosted.org/packages/b8/c7/201d3d58f30c4c2bdbe9b03844c291feb77c20511cc3586daf7edc12a47b/cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0", size = 4449961, upload-time = "2026-04-08T01:57:29.068Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ef/649750cbf96f3033c3c976e112265c33906f8e462291a33d77f90356548c/cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85", size = 4401696, upload-time = "2026-04-08T01:57:31.029Z" }, - { url = "https://files.pythonhosted.org/packages/41/52/a8908dcb1a389a459a29008c29966c1d552588d4ae6d43f3a1a4512e0ebe/cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e", size = 4664256, upload-time = "2026-04-08T01:57:33.144Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fa/f0ab06238e899cc3fb332623f337a7364f36f4bb3f2534c2bb95a35b132c/cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246", size = 3013001, upload-time = "2026-04-08T01:57:34.933Z" }, - { url = "https://files.pythonhosted.org/packages/d2/f1/00ce3bde3ca542d1acd8f8cfa38e446840945aa6363f9b74746394b14127/cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3", size = 3472985, upload-time = "2026-04-08T01:57:36.714Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, + { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, + { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, + { url = "https://files.pythonhosted.org/packages/89/c6/24266ac10c47f6cd2a865f4446062b466da1d1f10b27189eac00e61bf0c9/cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", size = 5300085, upload-time = "2026-06-09T22:31:58.703Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bb/cc4b78784f97efc8c5874c2a9743708d172be6663024b34a0467885ae0c8/cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", size = 4746137, upload-time = "2026-06-09T22:31:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/1f/52/0c44de3f5267f8fbe8e835138017522a333436166e406f0db9b9e6e3033f/cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", size = 4333867, upload-time = "2026-06-09T22:32:28.096Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/772d7adbfa931537bc401640b7cac9976bff689bda187833e5d63b428e49/cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", size = 4701805, upload-time = "2026-06-09T22:31:38.284Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a3/b06844f303873493c963caf581c04df31c7035e0c1b0f02c4814d319ec80/cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", size = 5258461, upload-time = "2026-06-09T22:31:04.187Z" }, + { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, + { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, + { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" }, + { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" }, + { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" }, + { url = "https://files.pythonhosted.org/packages/24/dc/d8ca05ffea724eec6d232ea6f18e74c269eb6bdfdcc9bfba689790d1325f/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:e361afba8918070d376df76f408a4f67fec0ee9cff81a99e48fe9a233ef59e17", size = 5290385, upload-time = "2026-06-09T22:31:15.212Z" }, + { url = "https://files.pythonhosted.org/packages/03/8c/3be6cb4da181f5bb6c19cf560c2359d60644a6b5fc5b57854e528f47b296/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d069066deead00ac7f090be101be875a06855908f7ec004c27b8fefb4acfb411", size = 4737082, upload-time = "2026-06-09T22:32:22.66Z" }, + { url = "https://files.pythonhosted.org/packages/aa/f6/d5f60a5a1434dbfd949e227fd0065d194c7e6b6ac526b17f5c06152b8231/cryptography-48.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:09f73a725d582cef64b91281a322cd798d14a33b2b6f2b7ad9531dc336d84c02", size = 4325328, upload-time = "2026-06-09T22:32:10.777Z" }, + { url = "https://files.pythonhosted.org/packages/17/b7/ba75dd947a14b6ad907b01ae8f6b5b348cdd1b48142f0063dee9e20c1d9d/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:15254441469dd6bf027039453288e2072124f8b6603563f5d759e1c9b69273fa", size = 4694530, upload-time = "2026-06-09T22:31:53.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/29/50d6b9e8aff12d8b67afaeb3569335e32dc83a5723e3bbded24fdac9f809/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:8ace4507d1e6533c125f4fac754f8bb8b6a74c08e92179dabd7e16571a3efbf3", size = 5245046, upload-time = "2026-06-09T22:31:25.774Z" }, + { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" }, + { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" }, + { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" }, + { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, + { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, + { url = "https://files.pythonhosted.org/packages/e5/0b/aa68b221dde92d09cb29a024ede17550ee21e77a404e59fc093c82bb51e1/cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", size = 5289970, upload-time = "2026-06-09T22:31:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/78/13/fba657f958d2af66ea959a4ba01212632089249d34af1ae48054136344d7/cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", size = 4731873, upload-time = "2026-06-09T22:31:22.253Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4c/9a964756d24a26b3e34dfcb16f961b89838786e6700b635b0d1e3adff4b6/cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", size = 4330804, upload-time = "2026-06-09T22:31:36.56Z" }, + { url = "https://files.pythonhosted.org/packages/4b/0f/a10f3a6eb12950a10e3a874070283aa2dd5875b2bfd15fad8a3e17b3f13e/cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46", size = 4696217, upload-time = "2026-06-09T22:31:13.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6f/5cd12f951165ea73ef85266775d97e4c763b2474ccfd816dd69d3a18d6f8/cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", size = 5245252, upload-time = "2026-06-09T22:32:02.193Z" }, + { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, + { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, + { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, ] [[package]] @@ -985,6 +985,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "expression" +version = "5.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/c7/bb061623b5815566bda69f5e9d156e38a97ebb383b8db3d2dedb26415466/expression-5.6.0.tar.gz", hash = "sha256:454f6fe138347194a43c7f878d958efe9b84b9cc770e462010c7a52e18058065", size = 59147, upload-time = "2025-02-19T09:37:37.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/a2/656b8bebe495117342a8676ccabf52b3885ce11a856c8dfe1fbbdc250d2d/expression-5.6.0-py3-none-any.whl", hash = "sha256:f5c62e38186c9287e088dee9cf3939b0bbde21cb4c59571872154a53d33dd7c0", size = 69673, upload-time = "2025-02-19T09:37:35.476Z" }, +] + [[package]] name = "fastapi" version = "0.136.3" @@ -1820,7 +1832,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.89.0" +version = "1.91.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1836,9 +1848,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/86/4b/15d4cb75f054933c1f19bcfd5683e139cdf792099b995ae55916b26094dc/litellm-1.89.0.tar.gz", hash = "sha256:eb1910a23497044b4375a0500c65f4c60d291a575d7b679c7566a5df9b9a5fcb", size = 14062606, upload-time = "2026-06-13T23:45:53.723Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/1e/90cfeada42170986a290feebaca0a90923b3c310cddeb0f8690abd239ad4/litellm-1.91.0.tar.gz", hash = "sha256:4fd469fe7356ba8fcc86f4efdf332e3426b760962ab12331fdaf1a01aeec065f", size = 14872290, upload-time = "2026-07-04T19:18:28.466Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/86/49cf94af8c51cacc15fd9bff1e6f9de1fb07ab10b8bb09961675ab389af4/litellm-1.89.0-py3-none-any.whl", hash = "sha256:63b33e2de386ab2a83fed7ed852c755e59d461a21b16c79fc17993f1b8c3d154", size = 15475805, upload-time = "2026-06-13T23:45:46.037Z" }, + { url = "https://files.pythonhosted.org/packages/a4/15/81fc2d162513803fe08b359c2e2a98f7a33195034fb94b005e263e272c6b/litellm-1.91.0-py3-none-any.whl", hash = "sha256:c3eb52dd2c6a5779e9efd67350f3640f9055d9c05d4b572fc1dd01a217e562ad", size = 16669331, upload-time = "2026-07-04T19:18:25.203Z" }, ] [package.optional-dependencies] @@ -1849,6 +1861,7 @@ proxy = [ { name = "backoff" }, { name = "boto3" }, { name = "cryptography" }, + { name = "expression" }, { name = "fastapi" }, { name = "fastapi-sso" }, { name = "granian" }, @@ -1876,11 +1889,11 @@ proxy = [ [[package]] name = "litellm-enterprise" -version = "0.1.42" +version = "0.1.44" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/93/fe/fb60dd7bbb88fa65818e70f5f21cdabfe1c5df2302d9dda9bc46ac4fe4d9/litellm_enterprise-0.1.42.tar.gz", hash = "sha256:ec2170d9627a715aa038537a07af8053e6826e115ff6162a41af25d2990f3b2e", size = 70623, upload-time = "2026-05-31T04:12:10.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/24/cff90fd913861cc6569cd6374c7a2da53de84ffb1ee421045ae1698e3467/litellm_enterprise-0.1.44.tar.gz", hash = "sha256:0712c743810192c3bb16f5163dd25a42eff73f99a85fb056f7756a48e73e7b8b", size = 70982, upload-time = "2026-06-26T17:01:24.162Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/45/a6a56c510779b633e18fc5403aa82388fd6f0833132725bcb756d68cb0f5/litellm_enterprise-0.1.42-py3-none-any.whl", hash = "sha256:1014e38445d7a79d3638061504def42640719903b89a493b68806ba6e2f75fe7", size = 137906, upload-time = "2026-05-31T04:12:09.714Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/a511aba9243879ccbf877381a68020c63bfc928e439788aec88dd3b7ee63/litellm_enterprise-0.1.44-py3-none-any.whl", hash = "sha256:5821cf9a313650edf1fb26a628d70e49b213507e556a95ee811112366edbbdbb", size = 138240, upload-time = "2026-06-26T17:01:23.198Z" }, ] [[package]]