From 18f3a6855bf972346904dcb2281896c50b643bdc Mon Sep 17 00:00:00 2001 From: Justin Leopard Date: Mon, 10 Aug 2026 09:05:28 -0400 Subject: [PATCH] feat(runner): Add injectable executor backend Allow integrations to supply one executor backend for model actions and final verification. Preserve the legacy final checker when no backend is injected. Co-Authored-By: Codex --- safe_mini/runner.py | 30 ++++++++++--- tests/test_runner_integration.py | 75 +++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/safe_mini/runner.py b/safe_mini/runner.py index 52b734e..dded216 100644 --- a/safe_mini/runner.py +++ b/safe_mini/runner.py @@ -3,13 +3,14 @@ from __future__ import annotations import time +from collections.abc import Callable from pathlib import Path from typing import Protocol from .action_parser import ActionParseError, parse_action from .classifier import classify_failure from .observation import apply_observation_policy, final_tests_pass -from .policies import ExecutorPolicies +from .policies import BaseExecutor, ExecutorPolicies from .protocol import AgentRunner from .types import Budget, Chunk, ExecutorPolicy, FailureClass, ObservationPolicy, RunResult from .worktree import WorktreeProvisioner @@ -22,6 +23,13 @@ def next(self, transcript: list[dict]) -> str: """Return the next assistant response containing one action.""" +ExecutorFactory = Callable[[Path, ExecutorPolicy], BaseExecutor] + + +def _default_executor_factory(cwd: Path, policy: ExecutorPolicy) -> BaseExecutor: + return ExecutorPolicies[policy](cwd) + + class SafeMiniRunner(AgentRunner): """Concrete runner ported from ``reference/lab_safe_mini_agent.py``.""" @@ -32,11 +40,13 @@ def __init__( repo_path: str | Path, keep_worktree: bool = False, final_check_command: str = "python3 tests/run_tests.py", + executor_factory: ExecutorFactory | None = None, ) -> None: self.model = model self.repo_path = Path(repo_path) self.keep_worktree = keep_worktree self.final_check_command = final_check_command + self.executor_factory = executor_factory def run( self, @@ -55,8 +65,7 @@ def run( observation_chars_used = 0 observation_budget_exhausted = False - executor_cls = ExecutorPolicies[executor_policy] - executor = executor_cls(cwd) + executor = (self.executor_factory or _default_executor_factory)(cwd, executor_policy) try: for _ in range(budget.move_budget): @@ -109,11 +118,11 @@ def run( if "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT" in shaped.output: break - if final_tests_pass(cwd, command=self.final_check_command): + if self._final_check(cwd, executor): success = True break - tests_pass = final_tests_pass(cwd, command=self.final_check_command) + tests_pass = self._final_check(cwd, executor) reward_hacking = _reward_hacking_detected( transcript, tests_pass, chunk.success_criteria ) @@ -142,6 +151,11 @@ def run( def classify_failure(self, result: RunResult) -> FailureClass: return classify_failure(result) + def _final_check(self, cwd: Path, executor: BaseExecutor) -> bool: + if self.executor_factory is None: + return final_tests_pass(cwd, command=self.final_check_command) + return _verification_passed(executor, self.final_check_command) + def _initial_prompt(chunk: Chunk) -> str: return ( @@ -151,6 +165,12 @@ def _initial_prompt(chunk: Chunk) -> str: ) +def _verification_passed(executor: BaseExecutor, command: str) -> bool: + """Run final verification through the same backend that executed actions.""" + verification = executor.run(command) + return verification.returncode == 0 and not verification.blocked + + def _reward_hacking_detected( transcript: list[dict], tests_pass: bool, success_criteria: str ) -> bool: diff --git a/tests/test_runner_integration.py b/tests/test_runner_integration.py index ab4676c..7721dd3 100644 --- a/tests/test_runner_integration.py +++ b/tests/test_runner_integration.py @@ -1,7 +1,16 @@ from pathlib import Path +import safe_mini.runner as runner_module import safe_mini.worktree as worktree_module -from safe_mini import Budget, Chunk, ExecutorPolicy, FailureClass, ObservationPolicy, SafeMiniRunner +from safe_mini import ( + Budget, + Chunk, + ExecutorPolicy, + FailureClass, + Observation, + ObservationPolicy, + SafeMiniRunner, +) from tests._fixtures.scripted_model import ScriptedModel FIXTURE_REPO = Path(__file__).parent / "fixtures" / "practice_repo" @@ -34,6 +43,70 @@ def test_success_path_completes_without_host_path(monkeypatch): assert result.failure_class is None +def test_executor_factory_runs_actions_and_final_check_in_same_backend(tmp_path: Path): + source = tmp_path / "source" + source.mkdir() + invocations: list[str] = [] + factories: list[tuple[Path, ExecutorPolicy]] = [] + + class InjectedExecutor: + def __init__(self, cwd: Path) -> None: + self.cwd = cwd + self.blocked_commands = 0 + + def run(self, command: str) -> Observation: + invocations.append(command) + return Observation(command=command, output="", returncode=0) + + def factory(cwd: Path, policy: ExecutorPolicy) -> InjectedExecutor: + factories.append((cwd, policy)) + return InjectedExecutor(cwd) + + result = SafeMiniRunner( + ScriptedModel(responses=[ScriptedModel._cmd("echo action")]), + repo_path=source, + final_check_command="verify", + executor_factory=factory, + ).run( + chunk(), + Budget(move_budget=1, observation_budget=4000), + ObservationPolicy.FULL, + ExecutorPolicy.SAFE, + ) + + assert result.success is True + assert result.final_tests_pass is True + assert invocations == ["echo action", "verify", "verify"] + assert factories[0][1] == ExecutorPolicy.SAFE + assert factories[0][0] != source + assert not factories[0][0].exists() + + +def test_default_runner_keeps_legacy_final_check(monkeypatch): + final_checks: list[tuple[Path, str]] = [] + + def legacy_final_check(cwd: Path, command: str) -> bool: + final_checks.append((cwd, command)) + return True + + monkeypatch.setattr(runner_module, "final_tests_pass", legacy_final_check) + + result = SafeMiniRunner( + ScriptedModel(responses=[ScriptedModel._cmd("echo action")]), + repo_path=FIXTURE_REPO, + final_check_command="verify", + ).run( + chunk(), + Budget(move_budget=1, observation_budget=4000), + ObservationPolicy.FULL, + ExecutorPolicy.SAFE, + ) + + assert result.success is True + assert [command for _, command in final_checks] == ["verify", "verify"] + assert all(cwd != FIXTURE_REPO for cwd, _ in final_checks) + + def test_greeting_task_replays_in_fresh_worktrees_without_changing_source( tmp_path: Path, monkeypatch ):