From 8bd7bb27d38dfa9b0f7bb96f5fef9cd87e9cd141 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 2 Jul 2026 02:16:23 +0200 Subject: [PATCH 1/4] fix(example): make candidate install opt-in + scrubbed; fix --draft-pr crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Safety fixes for the sdk-evolution example agent: - Candidate inspection pip-installed and imported the latest upstream SDK from PyPI on every default run (arbitrary code execution from freshly published code). It is now opt-in: --inspect-candidates defaults to False (and the RunOptions default too), and _collect_snapshots honors the flag instead of treating inspection as mandatory. When it does run, the venv/pip/import subprocesses get a scrubbed environment (throwaway HOME, PATH only) so a malicious candidate cannot read the caller's credentials/config. - The default --draft-pr flow raised RuntimeError finalizing the report because the default report dir is gitignored, so 'git add' of it returned rc=1. The finalize step now skips cleanly when the report path is gitignored (the report is already embedded in the PR body), and _create_autonomous_pr stages only uv.lock. Updates the example tests to the opt-in default and adds tests: default run does not install candidates; finalize skips a gitignored report dir. Refs REVIEW.md §3.1, §8. --- examples/sdk_evolution_agent/cli.py | 46 ++++++++--- examples/sdk_evolution_agent/models.py | 4 +- examples/sdk_evolution_agent/snapshots.py | 28 ++++++- tests/test_sdk_evolution_agent.py | 94 +++++++++++++++++++++-- 4 files changed, 152 insertions(+), 20 deletions(-) diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py index 93a25bf..26c2894 100644 --- a/examples/sdk_evolution_agent/cli.py +++ b/examples/sdk_evolution_agent/cli.py @@ -4,7 +4,6 @@ import argparse import re -from dataclasses import replace from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -98,10 +97,12 @@ def parse_args(argv: list[str] | None = None) -> RunOptions: parser.add_argument( "--inspect-candidates", action="store_true", - default=True, + default=False, help=( - "Inspect latest candidate SDK versions in temporary virtualenvs. " - "Always enabled for update candidates; accepted for compatibility." + "Opt in to inspecting candidate SDK versions by pip-installing and " + "importing them in a temporary, credential-scrubbed virtualenv. This " + "executes freshly downloaded upstream code, so it is OFF by default; " + "without it, snapshots use the already-installed versions only." ), ) parser.add_argument("--create-branch", action="store_true", help="Create a local branch first.") @@ -146,7 +147,6 @@ async def run_agent( ) -> Path: """Run the full local SDK evolution workflow.""" - options = replace(options, inspect_candidates=True) run_id = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") report_root = (options.workspace / options.report_dir / run_id).resolve() event_log_path = report_root / "events.jsonl" @@ -184,7 +184,7 @@ async def run_agent( command_runner=command_runner, ) update_versions = _refresh_update_versions(evidence) - snapshots = _collect_snapshots(evidence) + snapshots = _collect_snapshots(evidence, inspect_candidates=options.inspect_candidates) api_diffs = [to_jsonable(diff) for diff in diff_snapshot_groups(snapshots)] release_notes = [ to_jsonable(item) @@ -412,8 +412,10 @@ def _create_autonomous_pr( ) -> list[dict[str, Any]]: branch_name = options.branch_name or _current_branch(root, command_runner=command_runner) body = build_draft_pr_body(report_path.read_text(encoding="utf-8")) - relative_report = _relative_path(root, report_path.parent) - paths = ("uv.lock", relative_report) + # Only stage the lockfile. The report lives under the gitignored default + # report dir, so `git add`-ing it returned rc=1 and previously broke the + # --draft-pr flow; its content is already embedded in the PR body above. + paths = ("uv.lock",) results = [ to_jsonable(stage_paths(root, paths, command_runner=command_runner)), to_jsonable( @@ -452,6 +454,11 @@ def _commit_final_autonomous_pr_report( ) -> None: branch_name = options.branch_name or _current_branch(root, command_runner=command_runner) relative_report = _relative_path(root, report_path.parent) + if _path_is_git_ignored(root, relative_report, command_runner=command_runner): + # The default report dir is gitignored; its content is already embedded in + # the draft PR body, so there is nothing to commit. Previously this staged + # an ignored path (rc=1) and then raised. Skip cleanly instead. + return results = [ stage_paths(root, (relative_report,), command_runner=command_runner), commit_staged( @@ -468,6 +475,18 @@ def _commit_final_autonomous_pr_report( raise RuntimeError(f"failed to commit final autonomous PR report: {detail}") +def _path_is_git_ignored( + root: Path, path: str, *, command_runner: CommandRunner | None +) -> bool: + runner = command_runner + if runner is None: + from examples.sdk_evolution_agent.collectors import run_command + + runner = run_command + # `git check-ignore` exits 0 when the path is ignored, 1 when it is tracked. + return runner(("git", "check-ignore", path), cwd=root).returncode == 0 + + def _current_branch(root: Path, *, command_runner: CommandRunner | None) -> str: runner = command_runner or None if runner is None: @@ -485,8 +504,11 @@ def _relative_path(root: Path, path: Path) -> str: return str(path) -def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = True) -> list[Any]: - del inspect_candidates # Candidate inspection is mandatory for update candidates. +def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = False) -> list[Any]: + # Candidate inspection pip-installs and imports freshly downloaded upstream + # code, so it only runs when explicitly opted in (--inspect-candidates). When + # off, every snapshot uses the already-installed version via snapshot_current_api, + # which imports nothing new. snapshots = [] update_versions = _refresh_update_versions(evidence) refresh_preview_seen = evidence.get("refresh_preview") is not None @@ -497,10 +519,12 @@ def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = T locked = package.get("locked_version") installed = package.get("installed_version") baseline = locked or installed - if locked and installed and locked != installed: + if inspect_candidates and locked and installed and locked != installed: snapshots.append(snapshot_candidate_in_venv(name, str(locked))) else: snapshots.append(snapshot_current_api(name, version=baseline)) + if not inspect_candidates: + continue candidate = update_versions.get(name) if candidate is None and not refresh_preview_seen: latest = package.get("latest_version") diff --git a/examples/sdk_evolution_agent/models.py b/examples/sdk_evolution_agent/models.py index 8878709..9ce477e 100644 --- a/examples/sdk_evolution_agent/models.py +++ b/examples/sdk_evolution_agent/models.py @@ -148,7 +148,9 @@ class RunOptions: report_dir: Path = Path("reports/sdk-evolution") implementation_enabled: bool = False refresh_preview: bool = False - inspect_candidates: bool = True + # Off by default: candidate inspection pip-installs and imports freshly + # downloaded upstream code (see _collect_snapshots / --inspect-candidates). + inspect_candidates: bool = False create_branch: bool = False branch_name: str | None = None draft_pr: bool = False diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py index 548d8a3..3619d8d 100644 --- a/examples/sdk_evolution_agent/snapshots.py +++ b/examples/sdk_evolution_agent/snapshots.py @@ -5,6 +5,7 @@ import importlib import inspect import json +import os import subprocess import sys import tempfile @@ -96,6 +97,23 @@ def diff_snapshot_groups(snapshots: Sequence[ApiSnapshot]) -> tuple[ApiDiff, ... return tuple(diffs) +def _isolated_env(home: Path) -> dict[str, str]: + """A minimal environment for candidate subprocesses: PATH + a throwaway HOME. + + Deliberately omits the caller's credentials/config so freshly downloaded + upstream code executed during inspection cannot read them. + """ + + env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home)} + if sys.platform == "win32": + env["USERPROFILE"] = str(home) + for key in ("SYSTEMROOT", "SystemRoot", "COMSPEC", "PATHEXT", "TEMP", "TMP"): + value = os.environ.get(key) + if value: + env[key] = value + return env + + def snapshot_candidate_in_venv( package: str, version: str, @@ -108,7 +126,13 @@ def snapshot_candidate_in_venv( module_name = DEFAULT_MODULES.get(package, package.replace("-", "_")) with tempfile.TemporaryDirectory(prefix="ark-sdk-snapshot-") as directory: venv = Path(directory) / ".venv" - subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout) + # Scrub the environment for every subprocess that touches freshly downloaded + # upstream code: give it a throwaway HOME and only PATH, so a malicious or + # buggy candidate package cannot read the caller's credentials/config. + env = _isolated_env(Path(directory)) + subprocess.run( + (python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env + ) bin_dir = "Scripts" if sys.platform == "win32" else "bin" venv_python = venv / bin_dir / "python" subprocess.run( @@ -117,6 +141,7 @@ def snapshot_candidate_in_venv( text=True, capture_output=True, timeout=timeout, + env=env, ) completed = subprocess.run( ( @@ -131,6 +156,7 @@ def snapshot_candidate_in_venv( text=True, capture_output=True, timeout=timeout, + env=env, ) raw = json.loads(completed.stdout) return ApiSnapshot( diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 5d2ef33..6320a4b 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -497,10 +497,11 @@ def run_new(value: str, *, verbose: bool = False) -> str: assert diff.changed == ("run",) -def test_parse_args_inspects_candidates_by_default() -> None: - options = parse_args(["--runtime", "fake"]) - - assert options.inspect_candidates is True +def test_parse_args_candidate_inspection_is_opt_in() -> None: + # Candidate inspection pip-installs+imports upstream code, so it is off by + # default and only enabled with the explicit flag. + assert parse_args(["--runtime", "fake"]).inspect_candidates is False + assert parse_args(["--runtime", "fake", "--inspect-candidates"]).inspect_candidates is True def test_collect_snapshots_uses_lockfile_baseline_for_candidates( @@ -541,7 +542,7 @@ def candidate_snapshot(package: str, version: str) -> ApiSnapshot: } ] }, - inspect_candidates=False, + inspect_candidates=True, ) assert len(snapshots) == 2 @@ -596,6 +597,7 @@ def candidate_snapshot(package: str, version: str) -> ApiSnapshot: "stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n", }, }, + inspect_candidates=True, ) assert len(snapshots) == 3 @@ -643,6 +645,7 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot: "stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n", }, }, + inspect_candidates=True, ) assert calls == [ @@ -932,7 +935,7 @@ async def test_run_agent_report_only_generates_artifacts( packages=("claude-agent-sdk",), report_dir=Path("reports"), implementation_enabled=False, - inspect_candidates=False, + inspect_candidates=True, ), pypi_client=_fake_pypi, runtime=FixtureEvolutionRuntime(), @@ -956,6 +959,79 @@ async def test_run_agent_report_only_generates_artifacts( assert "Recursive self-adaptation impact" in report_path.read_text(encoding="utf-8") +@pytest.mark.asyncio +async def test_run_agent_does_not_install_candidates_by_default( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "pyproject.toml").write_text( + '[project.optional-dependencies]\nclaude = ["claude-agent-sdk>=0.2"]\n', + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("", encoding="utf-8") + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + lambda package, *, version=None: ApiSnapshot( + package=package, version=version, module=package.replace("-", "_") + ), + ) + + def forbidden_candidate(package: str, version: str) -> ApiSnapshot: + raise AssertionError("candidate install must not run without --inspect-candidates") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", forbidden_candidate + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.collect_release_notes", lambda packages, updates: [] + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.collect_behavior_evidence", + lambda packages, updates: {"results": [], "diffs": [], "summary": {"status": "pass"}}, + ) + + # Default run (no inspect_candidates) must never pip-install/import upstream code. + report_path = await run_agent( + RunOptions(workspace=tmp_path, runtime="fake", packages=("claude-agent-sdk",)), + pypi_client=_fake_pypi, + runtime=FixtureEvolutionRuntime(), + ) + + assert report_path.exists() + + +def test_finalize_report_skips_when_report_dir_is_gitignored(tmp_path: Path) -> None: + from examples.sdk_evolution_agent.cli import _commit_final_autonomous_pr_report + + commands: list[tuple[str, ...]] = [] + + def runner( + command: tuple[str, ...], + *, + cwd: Path | None = None, + env: dict[str, str] | None = None, + ) -> CommandResult: + del cwd, env + commands.append(command) + # check-ignore exits 0 => path IS ignored (the default report dir). + return CommandResult(command=command, returncode=0) + + report_path = tmp_path / "reports" / "sdk-evolution" / "run" / "report.md" + report_path.parent.mkdir(parents=True) + report_path.write_text("x", encoding="utf-8") + + # Must skip cleanly (no add/commit of a gitignored path) instead of raising. + _commit_final_autonomous_pr_report( + tmp_path, + report_path=report_path, + options=RunOptions(workspace=tmp_path, runtime="fake"), + command_runner=runner, + ) + + assert not any(command[:2] == ("git", "commit") for command in commands) + assert not any(command[:2] == ("git", "add") for command in commands) + + @pytest.mark.asyncio async def test_run_agent_autonomous_pr_path( tmp_path: Path, @@ -1019,6 +1095,9 @@ def runner( ) if command[:2] == ("uv", "lock"): return CommandResult(command=command, returncode=0, stdout="updated") + if command[:2] == ("git", "check-ignore"): + # Report dir is tracked in this scenario -> not ignored (exit 1). + return CommandResult(command=command, returncode=1, stdout="") return CommandResult(command=command, returncode=0, stdout="ok") report_path = await run_agent( @@ -1027,6 +1106,7 @@ def runner( runtime="fake", packages=("claude-agent-sdk",), report_dir=Path("reports"), + inspect_candidates=True, implementation_enabled=True, refresh_preview=True, create_branch=True, @@ -1130,7 +1210,7 @@ def test_parse_args_and_pr_body() -> None: assert options.runtime == "claude-agent-sdk" assert options.packages == ("claude-agent-sdk",) - assert options.inspect_candidates is True + assert options.inspect_candidates is False assert options.implementation_enabled is True assert options.draft_pr is True assert "No auto-merge" in body From 884ff4286f7a3d9f77d3cc766796b8872fb0ae4b Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 2 Jul 2026 11:35:41 +0200 Subject: [PATCH 2/4] fix(example): conform runtimes to the lifecycle protocol; handle string kinds FixtureEvolutionRuntime and the test RecordingRuntime now implement aclose/__aenter__/__aexit__, and example code labels runtimes via runtime_kind_value() instead of .kind.value, which raises AttributeError for namespaced third-party string kinds. --- examples/run_same_task.py | 10 ++++++---- examples/sdk_evolution_agent/stages.py | 18 +++++++++++++++--- tests/test_sdk_evolution_agent.py | 9 +++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/examples/run_same_task.py b/examples/run_same_task.py index c33968d..1b33963 100644 --- a/examples/run_same_task.py +++ b/examples/run_same_task.py @@ -4,7 +4,7 @@ import asyncio -from agent_runtime_kit import AgentTask, create_default_registry +from agent_runtime_kit import AgentTask, create_default_registry, runtime_kind_value from agent_runtime_kit.adapters import register_adapters @@ -14,15 +14,17 @@ async def main() -> None: task = AgentTask(goal="Summarize this repository in one paragraph.") for kind in registry.kinds(): - if kind.value == "fake": + # runtime_kind_value: kinds may be enum members or namespaced strings. + label = runtime_kind_value(kind) + if label == "fake": continue runtime = registry.resolve(kind) diagnostic = runtime.availability() if not diagnostic.available: - print(f"{kind.value}: unavailable - {diagnostic.message}") + print(f"{label}: unavailable - {diagnostic.message}") continue result = await runtime.run(task) - print(f"{kind.value}: {result.output}") + print(f"{label}: {result.output}") if __name__ == "__main__": diff --git a/examples/sdk_evolution_agent/stages.py b/examples/sdk_evolution_agent/stages.py index 843a468..15c9c53 100644 --- a/examples/sdk_evolution_agent/stages.py +++ b/examples/sdk_evolution_agent/stages.py @@ -19,6 +19,7 @@ PermissionProfile, RuntimeAvailability, RuntimeRegistry, + runtime_kind_value, ) from agent_runtime_kit.adapters import ( AntigravityAgentRuntime, @@ -93,6 +94,15 @@ async def cancel(self, task_id: str) -> None: del task_id + async def aclose(self) -> None: + """Fixture runtime holds no resources.""" + + async def __aenter__(self) -> FixtureEvolutionRuntime: + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object, /) -> None: + await self.aclose() + def build_registry() -> RuntimeRegistry: """Build the runtime registry used by the SDK evolution agent.""" @@ -159,10 +169,12 @@ async def run_stage( ) -> dict[str, Any]: """Run one AI stage through agent-runtime-kit and validate its output.""" + # runtime_kind_value: third-party runtimes may use plain string kinds. + kind_label = runtime_kind_value(runtime.kind) if not runtime.capabilities.structured_output: - raise StageExecutionError(f"{runtime.kind.value} cannot honor required output_schema") + raise StageExecutionError(f"{kind_label} cannot honor required output_schema") if not runtime.capabilities.working_directory: - raise StageExecutionError(f"{runtime.kind.value} cannot honor required working_directory") + raise StageExecutionError(f"{kind_label} cannot honor required working_directory") permissions = _stage_permissions(runtime, write_enabled=write_enabled) task = AgentTask( goal=json.dumps(payload, sort_keys=True, default=str), @@ -176,7 +188,7 @@ async def run_stage( try: result = await runtime.run(task) except Exception as exc: - raise StageExecutionError(f"{stage} failed through {runtime.kind.value}: {exc}") from exc + raise StageExecutionError(f"{stage} failed through {kind_label}: {exc}") from exc data = result.parsed_output if data is None: try: diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 6320a4b..290f0b2 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -1426,6 +1426,15 @@ async def run(self, task: AgentTask) -> AgentResult: async def cancel(self, task_id: str) -> None: del task_id + async def aclose(self) -> None: + return None + + async def __aenter__(self) -> RecordingRuntime: + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object, /) -> None: + await self.aclose() + class PermissiveRuntime(RecordingRuntime): async def run(self, task: AgentTask) -> AgentResult: From 910209790c00abce2f40a8974a6df824006ab8c1 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 2 Jul 2026 11:44:44 +0200 Subject: [PATCH 3/4] fix(example): --draft-pr skips report dirs git cannot commit, not just gitignored ones git check-ignore exits 128 for a path outside the repository; treating that as 'not ignored' staged a doomed out-of-repo path and raised. Only a definitively not-ignored path (exit 1) is now committed. Adds the rc=128 test, pins that a drifted lock without --inspect-candidates never installs candidates, and documents the proxy/CA trade-off of the scrubbed env. --- examples/sdk_evolution_agent/cli.py | 17 +++-- examples/sdk_evolution_agent/snapshots.py | 5 +- tests/test_sdk_evolution_agent.py | 81 +++++++++++++++++++++++ 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py index 26c2894..f32b3ce 100644 --- a/examples/sdk_evolution_agent/cli.py +++ b/examples/sdk_evolution_agent/cli.py @@ -454,10 +454,11 @@ def _commit_final_autonomous_pr_report( ) -> None: branch_name = options.branch_name or _current_branch(root, command_runner=command_runner) relative_report = _relative_path(root, report_path.parent) - if _path_is_git_ignored(root, relative_report, command_runner=command_runner): - # The default report dir is gitignored; its content is already embedded in - # the draft PR body, so there is nothing to commit. Previously this staged - # an ignored path (rc=1) and then raised. Skip cleanly instead. + if not _report_dir_committable(root, relative_report, command_runner=command_runner): + # Skipped when the report dir is gitignored (the default location) or + # outside the repository entirely: `git add` would fail on either, and + # the report content is already embedded in the draft PR body, so there + # is nothing that must be committed. return results = [ stage_paths(root, (relative_report,), command_runner=command_runner), @@ -475,7 +476,7 @@ def _commit_final_autonomous_pr_report( raise RuntimeError(f"failed to commit final autonomous PR report: {detail}") -def _path_is_git_ignored( +def _report_dir_committable( root: Path, path: str, *, command_runner: CommandRunner | None ) -> bool: runner = command_runner @@ -483,8 +484,10 @@ def _path_is_git_ignored( from examples.sdk_evolution_agent.collectors import run_command runner = run_command - # `git check-ignore` exits 0 when the path is ignored, 1 when it is tracked. - return runner(("git", "check-ignore", path), cwd=root).returncode == 0 + # `git check-ignore` exits 0 when the path is ignored, 1 when it is not + # ignored, and 128 when it cannot judge it (e.g. the path lies outside the + # repository). Only a definitively not-ignored path can be staged. + return runner(("git", "check-ignore", path), cwd=root).returncode == 1 def _current_branch(root: Path, *, command_runner: CommandRunner | None) -> str: diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py index 3619d8d..fef8acb 100644 --- a/examples/sdk_evolution_agent/snapshots.py +++ b/examples/sdk_evolution_agent/snapshots.py @@ -101,7 +101,10 @@ def _isolated_env(home: Path) -> dict[str, str]: """A minimal environment for candidate subprocesses: PATH + a throwaway HOME. Deliberately omits the caller's credentials/config so freshly downloaded - upstream code executed during inspection cannot read them. + upstream code executed during inspection cannot read them. That scrub also + drops proxy and CA overrides (HTTPS_PROXY, SSL_CERT_FILE, PIP_INDEX_URL...), + so behind a corporate TLS-intercepting proxy or private mirror the candidate + install may fail — an accepted trade-off of the isolation. """ env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home)} diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 290f0b2..addb4f5 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -654,6 +654,52 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot: ] +def test_collect_snapshots_without_opt_in_never_installs_even_when_lock_drifted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The security default: a drifted lock (locked != installed) plus pending + # updates must NOT trigger isolated-venv installs unless --inspect-candidates + # was passed; everything snapshots from the already-installed environment. + calls: list[tuple[str, str, str | None]] = [] + + def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot: + calls.append(("current", package, version)) + return ApiSnapshot(package=package, version=version, module=package.replace("-", "_")) + + def isolated_snapshot(package: str, version: str) -> ApiSnapshot: + calls.append(("isolated", package, version)) + return ApiSnapshot(package=package, version=version, module=package.replace("-", "_")) + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + current_snapshot, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", + isolated_snapshot, + ) + + _collect_snapshots( + { + "packages": [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.96", + "installed_version": "0.2.106", + "latest_version": "0.2.106", + }, + ], + "refresh_preview": { + "stdout": "", + "stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n", + }, + }, + inspect_candidates=False, + ) + + assert calls == [("current", "claude-agent-sdk", "0.2.96")] + + def test_candidate_api_diff_guard_blocks_missing_update_diff() -> None: guarded = with_candidate_api_diff_guard( { @@ -1032,6 +1078,41 @@ def runner( assert not any(command[:2] == ("git", "add") for command in commands) +def test_finalize_report_skips_when_report_dir_is_outside_the_repo(tmp_path: Path) -> None: + from examples.sdk_evolution_agent.cli import _commit_final_autonomous_pr_report + + commands: list[tuple[str, ...]] = [] + + def runner( + command: tuple[str, ...], + *, + cwd: Path | None = None, + env: dict[str, str] | None = None, + ) -> CommandResult: + del cwd, env + commands.append(command) + # check-ignore exits 128 when it cannot judge the path (outside the repo). + if command[:2] == ("git", "check-ignore"): + return CommandResult(command=command, returncode=128, stderr="fatal: outside repo") + return CommandResult(command=command, returncode=0) + + report_path = tmp_path / "elsewhere" / "run" / "report.md" + report_path.parent.mkdir(parents=True) + report_path.write_text("x", encoding="utf-8") + + # A --report-dir outside the workspace must skip cleanly, not stage a + # doomed out-of-repo path and raise. + _commit_final_autonomous_pr_report( + tmp_path / "repo", + report_path=report_path, + options=RunOptions(workspace=tmp_path / "repo", runtime="fake"), + command_runner=runner, + ) + + assert not any(command[:2] == ("git", "commit") for command in commands) + assert not any(command[:2] == ("git", "add") for command in commands) + + @pytest.mark.asyncio async def test_run_agent_autonomous_pr_path( tmp_path: Path, From 3a0e585b987e8f7ce28b5c83c2cfbf9ea5a4da44 Mon Sep 17 00:00:00 2001 From: Eloi Date: Thu, 2 Jul 2026 17:09:53 +0200 Subject: [PATCH 4/4] fix(example)!: gate+scrub behavior candidate probes; pin release-note links to github.com MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps in the example's handling of untrusted upstream content: - Behavior probes installed and imported candidate SDK versions unconditionally, with the caller's full environment — bypassing both the --inspect-candidates opt-in and the credential scrub the API snapshots already apply. The probes now honor the same gate, share snapshots.isolated_env (throwaway HOME, PATH only) for all three subprocesses, record skipped candidates explicitly, and a one-sided skip is excluded from diffs so the gate cannot read as a breaking change. Without the opt-in, a drifted lockfile baseline falls back to probing the installed environment. - Release-note links are extracted from fetched pages that can carry user-generated markup; a protocol-relative href (//evil.example/x/discussions/1) survived the regex and urljoined into an off-site GET. Followed links must now resolve to https://github.com. - A configured GitHub token whose GraphQL call fails (expired, insufficient scope) was silently swallowed into the unauthenticated HTML scrape. It now surfaces on the source record; tokenless runs still use the plain fetch as their normal mode. Docs updated: candidate inspection is documented as opt-in (the "always enabled" claim was stale) and the design doc records the inverted default. --- CHANGELOG.md | 12 +- docs/sdk-evolution-agent-design.md | 9 +- docs/sdk-evolution-agent.md | 15 +- examples/sdk_evolution_agent/behavior.py | 55 +++++- examples/sdk_evolution_agent/cli.py | 6 +- examples/sdk_evolution_agent/release_notes.py | 32 +++- examples/sdk_evolution_agent/snapshots.py | 7 +- tests/test_sdk_evolution_agent.py | 163 +++++++++++++++++- 8 files changed, 267 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 130f2e2..47077ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,8 +118,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 metadata cannot abort a run. - SDK evolution example: inspecting candidate SDK versions (which pip-installs and imports freshly downloaded upstream code) is now opt-in via - `--inspect-candidates` and runs in a credential-scrubbed environment, and - `--draft-pr` no longer fails when the report directory is gitignored. + `--inspect-candidates` and runs in a credential-scrubbed environment — and + the behavior probes now honor the same gate and scrub instead of installing + candidates unconditionally with the caller's full environment (skipped + candidates are recorded explicitly). `--draft-pr` no longer fails when the + report directory is gitignored. +- SDK evolution example: release-note links extracted from fetched pages are + only followed when they resolve to `https://github.com` (a protocol-relative + href in user-generated discussion markup could otherwise trigger an off-site + request), and a configured-but-failing GitHub GraphQL token now surfaces on + the source instead of silently downgrading to the unauthenticated scrape. ## 0.2.0 - 2026-06-23 diff --git a/docs/sdk-evolution-agent-design.md b/docs/sdk-evolution-agent-design.md index 986ab3c..6ad1166 100644 --- a/docs/sdk-evolution-agent-design.md +++ b/docs/sdk-evolution-agent-design.md @@ -197,9 +197,12 @@ python -m examples.sdk_evolution_agent \ --package google-antigravity ``` -`--inspect-candidates` should be effectively always on. The CLI can keep the -flag for compatibility, but update candidates without candidate API snapshots -are not actionable. +`--inspect-candidates` was originally envisioned as effectively always on, +since update candidates without candidate API snapshots are not actionable. +The shipped behavior deliberately inverts that default: candidate inspection +installs and imports freshly downloaded upstream code, so it is opt-in and +runs in a credential-scrubbed environment, with explicit `skip` records when +it is off. Implementation mode should remain explicitly gated: diff --git a/docs/sdk-evolution-agent.md b/docs/sdk-evolution-agent.md index b8b46f2..6d5f5aa 100644 --- a/docs/sdk-evolution-agent.md +++ b/docs/sdk-evolution-agent.md @@ -117,12 +117,15 @@ contains a different installed version, the agent inspects the locked baseline in a temporary isolated virtualenv instead of trusting the drifted environment. When a refresh preview is available, package update candidates come from the resolver's `uv lock --dry-run -P ...` output, not only from PyPI's `latest` -metadata. For each resolver update candidate, the agent installs the target -version in a temporary isolated virtualenv and writes an API snapshot plus -`api_diffs.json` entry. This avoids false downgrade diffs for packages whose -locked prerelease is newer than PyPI's stable latest field. Candidate inspection -is always enabled for update candidates; `--inspect-candidates` remains accepted -only for CLI compatibility. +metadata. With `--inspect-candidates`, the agent installs each resolver update +candidate in a temporary isolated virtualenv — with a credential-scrubbed +environment (throwaway `HOME`, `PATH` only) — and writes an API snapshot plus +`api_diffs.json` entry, and runs the behavior probes against the candidate the +same way. This avoids false downgrade diffs for packages whose locked +prerelease is newer than PyPI's stable latest field. Candidate inspection is +opt-in because it executes freshly downloaded upstream code; without the flag, +candidates are recorded as explicit `skip` entries rather than silently +missing evidence. If `uv lock --dry-run -P ...` reports an SDK update but the run cannot produce a candidate-version API diff for that package, implementation is blocked and the diff --git a/examples/sdk_evolution_agent/behavior.py b/examples/sdk_evolution_agent/behavior.py index 2847111..96317b4 100644 --- a/examples/sdk_evolution_agent/behavior.py +++ b/examples/sdk_evolution_agent/behavior.py @@ -15,14 +15,24 @@ from typing import Any from examples.sdk_evolution_agent.models import BehaviorDiff, BehaviorProbeResult -from examples.sdk_evolution_agent.snapshots import DEFAULT_MODULES +from examples.sdk_evolution_agent.snapshots import DEFAULT_MODULES, isolated_env def collect_behavior_evidence( packages: Sequence[Mapping[str, object]], update_versions: Mapping[str, str], + *, + inspect_candidates: bool = False, ) -> dict[str, Any]: - """Collect current/candidate behavior probes and compare them.""" + """Collect current/candidate behavior probes and compare them. + + Probing a version that is not installed means pip-installing and importing + freshly downloaded upstream code, so those venv probes run only when the + caller opted in via ``inspect_candidates`` (the ``--inspect-candidates`` CLI + flag) — the same gate the API snapshots use. Without the opt-in, candidates + get an explicit ``skip`` record instead of a silent evidence hole, and a + drifted lockfile baseline falls back to probing the installed environment. + """ results: list[BehaviorProbeResult] = [] for package in packages: @@ -32,13 +42,21 @@ def collect_behavior_evidence( locked_version = _string_or_none(package.get("locked_version")) installed_version = _string_or_none(package.get("installed_version")) current_version = locked_version or installed_version - if locked_version and installed_version and locked_version != installed_version: + if ( + inspect_candidates + and locked_version + and installed_version + and locked_version != installed_version + ): results.extend(probe_candidate_in_venv(name, locked_version, scope="current-baseline")) else: results.extend(probe_current_package(name, version=current_version)) candidate = update_versions.get(name) if candidate: - results.extend(probe_candidate_in_venv(name, candidate, scope="candidate")) + if inspect_candidates: + results.extend(probe_candidate_in_venv(name, candidate, scope="candidate")) + else: + results.append(_skipped_candidate_probe(name, candidate)) diffs = diff_behavior_results(results) return { "results": [result for result in results], @@ -69,7 +87,12 @@ def probe_candidate_in_venv( with tempfile.TemporaryDirectory(prefix="ark-sdk-behavior-") as directory: venv = Path(directory) / ".venv" - subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout) + # Scrub the environment for every subprocess that touches freshly + # downloaded upstream code (same scrub as the API snapshots): a + # throwaway HOME and only PATH, so a malicious or buggy candidate + # package cannot read the caller's credentials/config. + env = isolated_env(Path(directory)) + subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env) bin_dir = "Scripts" if sys.platform == "win32" else "bin" venv_python = venv / bin_dir / "python" subprocess.run( @@ -78,6 +101,7 @@ def probe_candidate_in_venv( text=True, capture_output=True, timeout=timeout, + env=env, ) completed = subprocess.run( (str(venv_python), "-c", _PROBE_SCRIPT, package, version, scope), @@ -85,6 +109,7 @@ def probe_candidate_in_venv( text=True, capture_output=True, timeout=timeout, + env=env, ) raw = json.loads(completed.stdout) return tuple(BehaviorProbeResult(**item) for item in raw) @@ -102,6 +127,10 @@ def diff_behavior_results(results: Sequence[BehaviorProbeResult]) -> tuple[Behav after = scopes.get("candidate") or scopes.get("isolated-venv") if before is None or after is None: continue + if (before.status == "skip") != (after.status == "skip"): + # One side was not probed (candidate installs are opt-in): absence + # of evidence is not a behavior change and must not read as one. + continue if before.status == after.status and _contract_details(before) == _contract_details(after): severity = "none" summary = "No behavior contract difference detected." @@ -347,6 +376,22 @@ def _failed( ) +def _skipped_candidate_probe(package: str, version: str) -> BehaviorProbeResult: + return BehaviorProbeResult( + package=package, + version=version, + scope="candidate", + probe="adapter-contract", + status="skip", + summary=( + "Candidate behavior probe skipped: probing a candidate installs and " + "imports freshly downloaded upstream code, which is opt-in. Rerun " + "with --inspect-candidates to collect this evidence." + ), + details={"reason": "candidate installs are opt-in (--inspect-candidates)"}, + ) + + def _string_or_none(value: object) -> str | None: if value is None: return None diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py index f32b3ce..b776339 100644 --- a/examples/sdk_evolution_agent/cli.py +++ b/examples/sdk_evolution_agent/cli.py @@ -191,7 +191,11 @@ async def run_agent( for item in collect_release_notes(evidence.get("packages", []), update_versions) ] behavior = to_jsonable( - collect_behavior_evidence(evidence.get("packages", []), update_versions) + collect_behavior_evidence( + evidence.get("packages", []), + update_versions, + inspect_candidates=options.inspect_candidates, + ) ) direction, architecture, review = await run_analysis_pipeline( selected_runtime, diff --git a/examples/sdk_evolution_agent/release_notes.py b/examples/sdk_evolution_agent/release_notes.py index 16c1345..57f5b63 100644 --- a/examples/sdk_evolution_agent/release_notes.py +++ b/examples/sdk_evolution_agent/release_notes.py @@ -247,19 +247,25 @@ def _fetch_source_text( use_github_graphql: bool, ) -> str: if use_github_graphql and source.kind == "github-discussions" and source.url: - try: - return _fetch_github_discussions_index(source.url) - except Exception: - pass + token = _github_token() + if token: + # With a token configured, a GraphQL failure (expired/insufficient + # credentials, API errors) must surface — the caller records it on + # the source — instead of silently downgrading to the + # unauthenticated HTML scrape, which would hide a broken token + # behind lower-quality evidence. Without a token, the HTML path is + # simply the tokenless mode of operation. + return _fetch_github_discussions_index(source.url, token=token) if not source.url: return "" return fetcher(source.url) -def _fetch_github_discussions_index(url: str) -> str: - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") - if not token: - raise RuntimeError("GITHUB_TOKEN or GH_TOKEN is required for GitHub Discussions GraphQL") +def _github_token() -> str | None: + return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + + +def _fetch_github_discussions_index(url: str, *, token: str) -> str: owner, repo, category_slug = _parse_github_discussion_category_url(url) query = """ query($owner: String!, $repo: String!) { @@ -384,7 +390,15 @@ def _release_note_links_for_version(source_url: str, text: str, to_version: str) window = text[match.end() : match.end() + 1200] if not any(marker in window for marker in markers): continue - links.append(urllib.parse.urljoin(source_url, match.group("path"))) + resolved = urllib.parse.urljoin(source_url, match.group("path")) + parsed = urllib.parse.urlparse(resolved) + # These hrefs come from fetched content that can carry user-generated + # markup (discussion bodies). Only follow https URLs on github.com + # itself: a protocol-relative path ("//evil.example/x/discussions/1") + # would otherwise urljoin into an off-site request. + if parsed.scheme != "https" or parsed.netloc != "github.com": + continue + links.append(resolved) return tuple(_dedupe(links)) diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py index fef8acb..f198536 100644 --- a/examples/sdk_evolution_agent/snapshots.py +++ b/examples/sdk_evolution_agent/snapshots.py @@ -97,14 +97,15 @@ def diff_snapshot_groups(snapshots: Sequence[ApiSnapshot]) -> tuple[ApiDiff, ... return tuple(diffs) -def _isolated_env(home: Path) -> dict[str, str]: +def isolated_env(home: Path) -> dict[str, str]: """A minimal environment for candidate subprocesses: PATH + a throwaway HOME. Deliberately omits the caller's credentials/config so freshly downloaded upstream code executed during inspection cannot read them. That scrub also drops proxy and CA overrides (HTTPS_PROXY, SSL_CERT_FILE, PIP_INDEX_URL...), so behind a corporate TLS-intercepting proxy or private mirror the candidate - install may fail — an accepted trade-off of the isolation. + install may fail — an accepted trade-off of the isolation. Shared with the + behavior probes, which install candidates the same way. """ env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home)} @@ -132,7 +133,7 @@ def snapshot_candidate_in_venv( # Scrub the environment for every subprocess that touches freshly downloaded # upstream code: give it a throwaway HOME and only PATH, so a malicious or # buggy candidate package cannot read the caller's credentials/config. - env = _isolated_env(Path(directory)) + env = isolated_env(Path(directory)) subprocess.run( (python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env ) diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index addb4f5..81b8817 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -30,6 +30,7 @@ from examples.sdk_evolution_agent.behavior import ( collect_behavior_evidence, diff_behavior_results, + probe_candidate_in_venv, ) from examples.sdk_evolution_agent.cli import RunOptions, _collect_snapshots, parse_args, run_agent from examples.sdk_evolution_agent.collectors import ( @@ -40,9 +41,10 @@ run_refresh_preview, ) from examples.sdk_evolution_agent.current_state import build_current_state -from examples.sdk_evolution_agent.models import ApiSnapshot, CommandResult, RunContext +from examples.sdk_evolution_agent.models import ApiSnapshot, CommandResult, RunContext, SourceRef from examples.sdk_evolution_agent.pr import build_draft_pr_body from examples.sdk_evolution_agent.release_notes import ( + _fetch_source_text, _format_github_discussions_index, collect_release_notes, ) @@ -246,6 +248,85 @@ def fetcher(url: str) -> str: assert any("Python 3.14" in summary for summary in notes[0].summaries) +def test_release_note_links_only_follow_github_https_urls() -> None: + # The discussion-index markup can carry user-generated hrefs. A + # protocol-relative path would urljoin into an off-site host and the blind + # follow-up GET would become SSRF; only https://github.com links may be + # followed. + def fetcher(url: str) -> str: + if url.endswith("/discussions/categories/announcements"): + return ( + 'v0.1.5\n' + 'v0.1.5 release notes' + ) + if "evil.example" in url: + raise AssertionError("off-site link must not be fetched") + if url.endswith("/discussions/88"): + return "## v0.1.5\n- Real release note\n" + return "no matching release note" + + notes = collect_release_notes( + [ + { + "name": "google-antigravity", + "locked_version": "0.1.4", + "installed_version": "0.1.4", + } + ], + {"google-antigravity": "0.1.5"}, + fetcher=fetcher, + ) + + assert all("evil.example" not in url for url in notes[0].checked_urls) + assert any(url.endswith("/discussions/88") for url in notes[0].checked_urls) + + +def test_fetch_source_text_surfaces_graphql_failure_when_token_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A configured token that fails must surface (the caller records it on the + # source), not silently downgrade to the unauthenticated HTML scrape — that + # would hide an expired/insufficient token behind lower-quality evidence. + monkeypatch.setenv("GITHUB_TOKEN", "configured-but-broken") + + def graphql_fails(url: str, *, token: str) -> str: + raise RuntimeError("GraphQL: Bad credentials") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.release_notes._fetch_github_discussions_index", + graphql_fails, + ) + source = SourceRef( + kind="github-discussions", + label="announcements", + url="https://github.com/o/r/discussions/categories/announcements", + ) + + with pytest.raises(RuntimeError, match="Bad credentials"): + _fetch_source_text(source, fetcher=lambda url: "html", use_github_graphql=True) + + +def test_fetch_source_text_uses_plain_fetch_without_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Tokenless operation is legitimate: without GITHUB_TOKEN/GH_TOKEN the HTML + # fetch is the normal path, not a downgrade. + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + source = SourceRef( + kind="github-discussions", + label="announcements", + url="https://github.com/o/r/discussions/categories/announcements", + ) + + text = _fetch_source_text( + source, fetcher=lambda url: "html fallback", use_github_graphql=True + ) + + assert text == "html fallback" + + def test_github_discussions_graphql_index_filters_discussion_category() -> None: index = _format_github_discussions_index( { @@ -401,6 +482,7 @@ def isolated(package: str, version: str, *, scope: str = "candidate"): } ], {"claude-agent-sdk": "0.2.106"}, + inspect_candidates=True, ) assert calls == [ @@ -410,6 +492,73 @@ def isolated(package: str, version: str, *, scope: str = "candidate"): assert behavior["diffs"][0].severity == "changed" +def test_behavior_candidate_probes_are_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + # Probing a candidate pip-installs and imports freshly downloaded upstream + # code; without --inspect-candidates that must never happen, and the gap + # must be an explicit skip record rather than a silent evidence hole or a + # phantom "breaking" diff. + def explode(package: str, version: str, *, scope: str = "candidate") -> tuple[Any, ...]: + raise AssertionError("venv probe must not run without --inspect-candidates") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_candidate_in_venv", + explode, + ) + + behavior = collect_behavior_evidence( + [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.96", + "installed_version": "0.2.106", + } + ], + {"claude-agent-sdk": "0.2.106"}, + ) + + candidate_results = [r for r in behavior["results"] if r.scope == "candidate"] + assert [r.status for r in candidate_results] == ["skip"] + assert "--inspect-candidates" in candidate_results[0].summary + # The drifted lockfile baseline falls back to the installed environment. + assert any(r.scope == "current-environment" for r in behavior["results"]) + assert behavior["summary"]["breaking_count"] == 0 + + +def test_behavior_diffs_ignore_one_sided_skip() -> None: + diffs = diff_behavior_results( + [ + _probe("claude-agent-sdk", "0.2.96", "current-environment", "pass", {"fields": ["a"]}), + _probe("claude-agent-sdk", "0.2.106", "candidate", "skip", {"reason": "opt-in"}), + ] + ) + + assert diffs == () + + +def test_candidate_behavior_probe_scrubs_subprocess_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured_envs: list[dict[str, str] | None] = [] + + def fake_run(args: Any, **kwargs: Any) -> Any: + captured_envs.append(kwargs.get("env")) + return types.SimpleNamespace(stdout="[]", returncode=0) + + monkeypatch.setenv("ARK_FAKE_SECRET", "not-for-candidates") + monkeypatch.setattr("examples.sdk_evolution_agent.behavior.subprocess.run", fake_run) + + results = probe_candidate_in_venv("claude-agent-sdk", "9.9.9") + + assert results == () + # venv create, pip install, probe script: every subprocess that touches the + # freshly downloaded candidate runs with the scrubbed environment. + assert len(captured_envs) == 3 + for env in captured_envs: + assert env is not None + assert "ARK_FAKE_SECRET" not in env + assert env.get("HOME") != os.environ.get("HOME") + + def test_behavior_probe_guard_blocks_breaking_candidate_diff() -> None: guarded = with_behavior_probe_guard( { @@ -1033,7 +1182,11 @@ def forbidden_candidate(package: str, version: str) -> ApiSnapshot: ) monkeypatch.setattr( "examples.sdk_evolution_agent.cli.collect_behavior_evidence", - lambda packages, updates: {"results": [], "diffs": [], "summary": {"status": "pass"}}, + lambda packages, updates, *, inspect_candidates=False: { + "results": [], + "diffs": [], + "summary": {"status": "pass"}, + }, ) # Default run (no inspect_candidates) must never pip-install/import upstream code. @@ -1156,7 +1309,11 @@ async def test_run_agent_autonomous_pr_path( ) monkeypatch.setattr( "examples.sdk_evolution_agent.cli.collect_behavior_evidence", - lambda packages, updates: {"results": [], "diffs": [], "summary": {"status": "pass"}}, + lambda packages, updates, *, inspect_candidates=False: { + "results": [], + "diffs": [], + "summary": {"status": "pass"}, + }, ) commands: list[tuple[str, ...]] = []