From e3bd22208a8f41accc6445bd077d886877d1b406 Mon Sep 17 00:00:00 2001 From: Eloi Date: Fri, 3 Jul 2026 15:21:17 +0200 Subject: [PATCH 1/3] Overhaul SDK evolution workflow --- .claude/commands/agent-runtime-kit/upgrade.md | 12 +- .../skills/agent-runtime-kit-upgrade/SKILL.md | 12 +- .github/workflows/sdk-evolution-report.yml | 52 ++ .sdk-evolution/README.md | 8 + docs/sdk-evolution-agent-design.md | 63 +- docs/sdk-evolution-agent.md | 66 +- examples/sdk_evolution_agent/__init__.py | 9 +- examples/sdk_evolution_agent/behavior.py | 230 +++++-- examples/sdk_evolution_agent/cli.py | 598 +++++++++++++++--- examples/sdk_evolution_agent/collectors.py | 234 ++++++- examples/sdk_evolution_agent/current_state.py | 77 +++ examples/sdk_evolution_agent/models.py | 15 +- examples/sdk_evolution_agent/pr.py | 142 ++++- examples/sdk_evolution_agent/preflight.py | 76 +++ examples/sdk_evolution_agent/release_notes.py | 32 +- examples/sdk_evolution_agent/report.py | 31 +- examples/sdk_evolution_agent/resolver.py | 248 ++++++++ examples/sdk_evolution_agent/schemas.py | 253 ++++---- examples/sdk_evolution_agent/snapshots.py | 116 +++- examples/sdk_evolution_agent/stages.py | 222 +++++-- examples/sdk_evolution_agent/status_issue.py | 247 ++++++++ pyproject.toml | 2 +- src/agent_runtime_kit/__init__.py | 3 + src/agent_runtime_kit/adapters/antigravity.py | 27 + src/agent_runtime_kit/adapters/claude.py | 18 + src/agent_runtime_kit/adapters/codex.py | 10 +- tests/test_sdk_evolution_agent.py | 86 ++- tests/test_sdk_evolution_docs.py | 78 +++ tests/test_sdk_evolution_pr_status.py | 72 +++ tests/test_sdk_evolution_preflight.py | 87 +++ tests/test_sdk_evolution_resolver.py | 133 ++++ 31 files changed, 2815 insertions(+), 444 deletions(-) create mode 100644 .github/workflows/sdk-evolution-report.yml create mode 100644 .sdk-evolution/README.md create mode 100644 examples/sdk_evolution_agent/preflight.py create mode 100644 examples/sdk_evolution_agent/resolver.py create mode 100644 examples/sdk_evolution_agent/status_issue.py create mode 100644 tests/test_sdk_evolution_docs.py create mode 100644 tests/test_sdk_evolution_pr_status.py create mode 100644 tests/test_sdk_evolution_preflight.py create mode 100644 tests/test_sdk_evolution_resolver.py diff --git a/.claude/commands/agent-runtime-kit/upgrade.md b/.claude/commands/agent-runtime-kit/upgrade.md index d19d38a..a1e63b9 100644 --- a/.claude/commands/agent-runtime-kit/upgrade.md +++ b/.claude/commands/agent-runtime-kit/upgrade.md @@ -113,7 +113,7 @@ upstream SDK releases are the point of this workflow: env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ uv run python -m examples.sdk_evolution_agent \ --runtime claude-agent-sdk \ - --refresh-preview \ + --mode report \ --package claude-agent-sdk \ --package openai-codex \ --package openai-codex-cli-bin \ @@ -159,11 +159,8 @@ BRANCH="sdk-evolution-upgrade-$(date +%Y%m%d-%H%M%S)" env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ uv run python -m examples.sdk_evolution_agent \ --runtime claude-agent-sdk \ - --refresh-preview \ - --implementation-enabled \ - --create-branch \ + --mode upgrade-pr \ --branch-name "$BRANCH" \ - --draft-pr \ --pr-base main \ --commit-message "Run SDK evolution update" \ --pr-title "Run SDK evolution update across vendor packages" \ @@ -175,6 +172,11 @@ env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ If the user chose another runtime, replace only the `--runtime` value. +Successful upgrade PRs commit the mechanical update artifacts: `uv.lock`, +`pyproject.toml` only when a cap raise was applied, and the tracked +`.sdk-evolution/` baseline. The timestamped `reports/sdk-evolution/` directory +is local and gitignored; its evidence is embedded in the PR body. + ## Verification After implementation, run or verify: diff --git a/.codex/skills/agent-runtime-kit-upgrade/SKILL.md b/.codex/skills/agent-runtime-kit-upgrade/SKILL.md index fc52cc8..7f3acfe 100644 --- a/.codex/skills/agent-runtime-kit-upgrade/SKILL.md +++ b/.codex/skills/agent-runtime-kit-upgrade/SKILL.md @@ -85,7 +85,7 @@ freshness cutoffs: env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ uv run python -m examples.sdk_evolution_agent \ --runtime codex-agent-sdk \ - --refresh-preview \ + --mode report \ --package claude-agent-sdk \ --package openai-codex \ --package openai-codex-cli-bin \ @@ -132,11 +132,8 @@ BRANCH="sdk-evolution-upgrade-$(date +%Y%m%d-%H%M%S)" env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ uv run python -m examples.sdk_evolution_agent \ --runtime codex-agent-sdk \ - --refresh-preview \ - --implementation-enabled \ - --create-branch \ + --mode upgrade-pr \ --branch-name "$BRANCH" \ - --draft-pr \ --pr-base main \ --commit-message "Run SDK evolution update" \ --pr-title "Run SDK evolution update across vendor packages" \ @@ -146,6 +143,11 @@ env -u UV_EXCLUDE_NEWER -u UV_EXCLUDE_NEWER_PACKAGE \ --package google-antigravity ``` +Successful upgrade PRs commit the mechanical update artifacts: `uv.lock`, +`pyproject.toml` only when a cap raise was applied, and the tracked +`.sdk-evolution/` baseline. The timestamped `reports/sdk-evolution/` directory +is local and gitignored; its evidence is embedded in the PR body. + ## Verification Run or verify: diff --git a/.github/workflows/sdk-evolution-report.yml b/.github/workflows/sdk-evolution-report.yml new file mode 100644 index 0000000..b5e160c --- /dev/null +++ b/.github/workflows/sdk-evolution-report.yml @@ -0,0 +1,52 @@ +name: SDK Evolution Report + +on: + schedule: + - cron: "43 5 * * 2" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v8.2.0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install + run: uv sync --locked --all-extras + + - name: Run SDK evolution report + run: | + uv run python -m examples.sdk_evolution_agent \ + --runtime fake \ + --refresh-preview \ + --inspect-candidates + + - name: Locate run directory + id: run-dir + run: | + latest="$(ls -td reports/sdk-evolution/* | head -1)" + echo "path=$latest" >> "$GITHUB_OUTPUT" + + - name: Upload report artifacts + uses: actions/upload-artifact@v4 + with: + name: sdk-evolution-report + path: ${{ steps.run-dir.outputs.path }} + retention-days: 90 + + - name: Update status issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: uv run python -m examples.sdk_evolution_agent.status_issue "${{ steps.run-dir.outputs.path }}" diff --git a/.sdk-evolution/README.md b/.sdk-evolution/README.md new file mode 100644 index 0000000..438aac1 --- /dev/null +++ b/.sdk-evolution/README.md @@ -0,0 +1,8 @@ +# SDK Evolution Baseline + +This directory is maintained by `examples.sdk_evolution_agent` after a promoted +SDK evolution run. `baseline.json` records the accepted lockfile hash, package +versions, report artifact hashes, and promoted API snapshot hashes. The +`snapshots/` files hold the current promoted API snapshot per vendor package. + +Blocked and report-only runs must not update this directory. diff --git a/docs/sdk-evolution-agent-design.md b/docs/sdk-evolution-agent-design.md index 6ad1166..5b76438 100644 --- a/docs/sdk-evolution-agent-design.md +++ b/docs/sdk-evolution-agent-design.md @@ -123,34 +123,38 @@ Step responsibilities: report with the evidence bundle, analysis, decision, reviewer output, uncertainty, blocked reasons, and the exact manual review questions. This is a valid end state, not a failed run. -- **Apply safe implementation**: Apply only the changes allowed by the accepted - architecture decision and deterministic gates. This may include lockfile - updates, adapter changes, tests, docs, examples, compatibility shims, or report - changes. It must not implement changes that were classified as - `manual_design_required`. +- **Apply safe implementation**: Apply only the bounded change vocabulary: + lockfile updates, pyproject upper-bound raises, self-adaptation inside + `examples/sdk_evolution_agent/`, and related SDK-evolution docs/tests. + Adapter source changes remain `manual_design_required`; the example records + the evidence and stops instead of editing adapter modules automatically. - **Run verification**: Run the verification commands required by the architecture decision. At minimum, this should cover formatting/linting, typing, unit tests, lock checks, report generation checks, and any available live smoke needed for the affected runtime behavior. - **Promote updated state to current baseline**: After implementation and - verification pass, save the updated lock/package/API/release-note/probe state - as the new current-state baseline for the next run. This promotion should be - explicit, atomic, and tied to the verified commit or workspace state. Failed, - blocked, or manual-design-required runs must not replace the current baseline. + verification pass, save `.sdk-evolution/baseline.json` plus + `.sdk-evolution/snapshots/.json` as the new current-state baseline + for the next run. Promotion refuses snapshots whose observed imported version + differs from the locked version. Failed, blocked, or manual-design-required + runs must not replace the current baseline. - **Write report and optional draft PR**: Write the final local report with evidence, decisions, implementation summary, baseline-promotion result, test results, uncertainty, and manual checklist. If explicitly configured and - authenticated, create or update a draft PR. This step must never auto-merge. + authenticated, create or update a draft PR. Draft PR bodies contain a + machine-readable SDK-evolution marker; matching existing PRs are reused, and + older marked PRs with different deltas are superseded. This step must never + auto-merge. Every box before direction analysis is deterministic. AI stages may interpret evidence, but they should not invent evidence that was not collected. ## Operating Modes -The default command should be report-only: +The tested report command is: ```bash -python -m examples.sdk_evolution_agent --runtime fake --refresh-preview +python -m examples.sdk_evolution_agent --mode report --runtime fake ``` This mode collects evidence, writes artifacts, runs the analysis stages through @@ -161,9 +165,9 @@ schemas, not the quality of AI reasoning. A real analysis run should select one configured runtime: ```bash -python -m examples.sdk_evolution_agent --runtime claude-agent-sdk --refresh-preview -python -m examples.sdk_evolution_agent --runtime codex-agent-sdk --refresh-preview -python -m examples.sdk_evolution_agent --runtime antigravity-agent-sdk --refresh-preview +python -m examples.sdk_evolution_agent --mode report --runtime claude-agent-sdk +python -m examples.sdk_evolution_agent --mode report --runtime codex-agent-sdk +python -m examples.sdk_evolution_agent --mode report --runtime antigravity-agent-sdk ``` Before a Codex-backed run, prepare the dedicated SDK evolution auth home: @@ -190,7 +194,7 @@ runs should inspect all tracked packages: ```bash python -m examples.sdk_evolution_agent \ --runtime antigravity-agent-sdk \ - --refresh-preview \ + --mode report \ --package claude-agent-sdk \ --package openai-codex \ --package openai-codex-cli-bin \ @@ -204,19 +208,17 @@ installs and imports freshly downloaded upstream code, so it is opt-in and runs in a credential-scrubbed environment, with explicit `skip` records when it is off. -Implementation mode should remain explicitly gated: +Implementation mode remains explicitly gated: ```bash -python -m examples.sdk_evolution_agent \ - --runtime antigravity-agent-sdk \ - --refresh-preview \ - --implementation-enabled +python -m examples.sdk_evolution_agent --mode upgrade --runtime antigravity-agent-sdk ``` -Even in implementation mode, deterministic gates decide whether edits are -allowed. Draft PR creation is separate and should only happen when the local Git -and GitHub environment is authenticated and explicitly configured with -`--draft-pr`. +`--mode upgrade` expands to refresh evidence, candidate inspection, +implementation, and guarded cap raises. Even in implementation mode, +deterministic gates decide whether edits are allowed. `--mode upgrade-pr` adds +branch creation and a draft PR, refuses to push the default branch, and only +runs after implementation and verification pass. ## Evidence Layers @@ -231,12 +233,13 @@ The agent checks: - `uv.lock` versions. - Installed distributions in the local environment. - PyPI metadata and recent releases. -- `uv lock --dry-run -P ...` output with freshness cutoffs removed. +- structured update candidates from a temporary lockfile diff with freshness + cutoff environment variables removed. -`uv lock --dry-run` is the source of truth for update candidates when it is -available. PyPI `latest` metadata is useful context, but it can be misleading -for prerelease packages. For example, a locked prerelease can be newer than the -stable value reported by package metadata. +The temporary lockfile diff is the source of truth for update candidates. PyPI +`latest` metadata is useful context, but it can be misleading for prerelease +packages. Human-facing uv stdout is stored only as auxiliary evidence and is not +parsed for pass/fail decisions. ### 2. API Shape Evidence diff --git a/docs/sdk-evolution-agent.md b/docs/sdk-evolution-agent.md index 6d5f5aa..56fe18b 100644 --- a/docs/sdk-evolution-agent.md +++ b/docs/sdk-evolution-agent.md @@ -11,16 +11,16 @@ changelog strategy, caveats, and alternatives, see Run it from the repository: ```bash -python -m examples.sdk_evolution_agent --runtime fake +python -m examples.sdk_evolution_agent --mode report --runtime fake ``` The `fake` runtime is deterministic and useful for checking the local pipeline without credentials. For real AI reasoning, select a configured runtime: ```bash -python -m examples.sdk_evolution_agent --runtime claude-agent-sdk -python -m examples.sdk_evolution_agent --runtime codex-agent-sdk -python -m examples.sdk_evolution_agent --runtime antigravity-agent-sdk +python -m examples.sdk_evolution_agent --mode report --runtime claude-agent-sdk +python -m examples.sdk_evolution_agent --mode report --runtime codex-agent-sdk +python -m examples.sdk_evolution_agent --mode report --runtime antigravity-agent-sdk ``` Every AI-backed stage is dispatched as an `AgentTask` through a runtime resolved @@ -105,29 +105,30 @@ installed distributions, then compares it with upstream package metadata for: - `openai-codex-cli-bin` - `google-antigravity` -When `--refresh-preview` is used, the targeted `uv lock --dry-run -P ...` -preview runs with freshness cutoff environment variables removed, including -`UV_EXCLUDE_NEWER`. This workflow needs fresh upstream SDK information, so local -cutoff variables must not hide candidate releases. +`--mode report` enables the refresh path. The resolver copies `pyproject.toml`, +`uv.lock`, and the minimal README metadata into a temporary directory, runs +targeted `uv lock -P ...` there with freshness cutoff environment variables +removed, then diffs the temporary lockfile against the real one. The real +workspace is not touched during candidate detection. The older dry-run stdout +preview may still be stored as auxiliary evidence, but it is not parsed for +candidate decisions. ## Candidate API Inspection The command treats `uv.lock` as the current baseline. If the active `.venv` contains a different installed version, the agent inspects the locked baseline in a temporary isolated virtualenv instead of trusting the drifted environment. -When a refresh preview is available, package update candidates come from the -resolver's `uv lock --dry-run -P ...` output, not only from PyPI's `latest` -metadata. With `--inspect-candidates`, the agent installs each resolver update -candidate in a temporary isolated virtualenv — with a credential-scrubbed -environment (throwaway `HOME`, `PATH` only) — and writes an API snapshot plus -`api_diffs.json` entry, and runs the behavior probes against the candidate the -same way. This avoids false downgrade diffs for packages whose locked -prerelease is newer than PyPI's stable latest field. Candidate inspection is -opt-in because it executes freshly downloaded upstream code; without the flag, -candidates are recorded as explicit `skip` entries rather than silently -missing evidence. - -If `uv lock --dry-run -P ...` reports an SDK update but the run cannot produce a +When refresh evidence is available, package update candidates come from the +temporary lockfile diff, not PyPI's `latest` metadata and not human-facing uv +stdout. With `--inspect-candidates`, the agent installs each resolver update +candidate and beyond-cap candidate in a temporary isolated virtualenv — with a +credential-scrubbed environment (throwaway `HOME`, `PATH` only) — then writes an +API snapshot plus `api_diffs.json` entry and runs behavior probes against the +candidate the same way. Candidate inspection is opt-in because it executes +freshly downloaded upstream code; without the flag, candidates are recorded as +explicit `skip` entries rather than silently missing evidence. + +If the structured resolver reports an SDK update but the run cannot produce a candidate-version API diff for that package, implementation is blocked and the architecture decision is marked `manual_design_required`. An empty added / removed / changed diff is valid; a missing diff object is not. @@ -143,9 +144,13 @@ and API diffs, but only breaking adapter-contract diffs block implementation. Report-only mode is the default. To allow the implementation stage, pass: ```bash -python -m examples.sdk_evolution_agent --runtime claude-agent-sdk --implementation-enabled +python -m examples.sdk_evolution_agent --mode upgrade --runtime claude-agent-sdk ``` +`--mode upgrade` expands to report + candidate inspection + implementation + +guarded cap raises. It refuses to run from a dirty worktree unless `--allow-dirty` +is set for local development. + Implementation is still blocked when: - the architecture decision sets `manual_design_required`, @@ -169,18 +174,15 @@ design review. Draft PR creation is opt-in: ```bash -python -m examples.sdk_evolution_agent \ - --runtime claude-agent-sdk \ - --implementation-enabled \ - --create-branch \ - --branch-name sdk-evolution-update \ - --pr-base main \ - --draft-pr +python -m examples.sdk_evolution_agent --mode upgrade-pr --runtime claude-agent-sdk ``` -When `--draft-pr` is set, the agent stages `uv.lock` and the run report -directory, commits them with `--commit-message`, pushes the branch, and opens a -draft PR with `gh`. It never auto-merges. +`--mode upgrade-pr` creates a `sdk-evolution/` branch when one is not +supplied, refuses to push the default branch, stages `uv.lock`, +`pyproject.toml` when a cap raise was applied, and the tracked `.sdk-evolution/` +baseline. The report directory remains local and gitignored; the draft PR body +embeds report evidence plus a machine-readable SDK-evolution marker. The PR +flow runs only after implementation is applied and verification passes. The command uses local Git and `gh` authentication. It never auto-merges, auto-publishes, or scrapes unsupported credentials. diff --git a/examples/sdk_evolution_agent/__init__.py b/examples/sdk_evolution_agent/__init__.py index 71738f3..d437b3a 100644 --- a/examples/sdk_evolution_agent/__init__.py +++ b/examples/sdk_evolution_agent/__init__.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any _REPO_SRC = Path(__file__).resolve().parents[2] / "src" if _REPO_SRC.exists() and str(_REPO_SRC) not in sys.path: @@ -12,12 +12,17 @@ __all__ = ["RunOptions", "run_agent"] +if TYPE_CHECKING: + from examples.sdk_evolution_agent.cli import run_agent + from examples.sdk_evolution_agent.models import RunOptions + def __getattr__(name: str) -> Any: """Lazily expose CLI helpers without importing every submodule.""" if name in __all__: - from examples.sdk_evolution_agent.cli import RunOptions, run_agent + from examples.sdk_evolution_agent.cli import run_agent + from examples.sdk_evolution_agent.models import RunOptions exports = {"RunOptions": RunOptions, "run_agent": run_agent} return exports[name] diff --git a/examples/sdk_evolution_agent/behavior.py b/examples/sdk_evolution_agent/behavior.py index 96317b4..a1e57e0 100644 --- a/examples/sdk_evolution_agent/behavior.py +++ b/examples/sdk_evolution_agent/behavior.py @@ -14,8 +14,17 @@ from pathlib import Path from typing import Any +from agent_runtime_kit.adapters.antigravity import VENDOR_CONTRACT as ANTIGRAVITY_CONTRACT +from agent_runtime_kit.adapters.claude import VENDOR_CONTRACT as CLAUDE_CONTRACT +from agent_runtime_kit.adapters.codex import VENDOR_CONTRACT as CODEX_CONTRACT from examples.sdk_evolution_agent.models import BehaviorDiff, BehaviorProbeResult -from examples.sdk_evolution_agent.snapshots import DEFAULT_MODULES, isolated_env +from examples.sdk_evolution_agent.snapshots import isolated_env + +_CONTRACTS = { + "claude-agent-sdk": {key: sorted(value) for key, value in CLAUDE_CONTRACT.items()}, + "openai-codex": {key: sorted(value) for key, value in CODEX_CONTRACT.items()}, + "google-antigravity": {key: sorted(value) for key, value in ANTIGRAVITY_CONTRACT.items()}, +} def collect_behavior_evidence( @@ -92,25 +101,48 @@ def probe_candidate_in_venv( # throwaway HOME and only PATH, so a malicious or buggy candidate # package cannot read the caller's credentials/config. env = isolated_env(Path(directory)) - subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env) + venv_result = _run_probe_subprocess( + (python, "-m", "venv", str(venv)), env=env, timeout=timeout + ) + if venv_result is not None: + return (venv_result(package, version, scope, "candidate-install"),) bin_dir = "Scripts" if sys.platform == "win32" else "bin" venv_python = venv / bin_dir / "python" - subprocess.run( + install_result = _run_probe_subprocess( (str(venv_python), "-m", "pip", "install", f"{package}=={version}"), - check=True, text=True, capture_output=True, timeout=timeout, env=env, ) + if install_result is not None: + return (install_result(package, version, scope, "candidate-install"),) completed = subprocess.run( - (str(venv_python), "-c", _PROBE_SCRIPT, package, version, scope), - check=True, + ( + str(venv_python), + "-c", + _PROBE_SCRIPT, + package, + version, + scope, + json.dumps(_CONTRACTS, sort_keys=True), + ), text=True, capture_output=True, timeout=timeout, + check=False, env=env, ) + if completed.returncode != 0: + return ( + _install_failed_probe( + package, + version, + scope, + "candidate-probe", + _tail(completed.stderr or completed.stdout), + ), + ) raw = json.loads(completed.stdout) return tuple(BehaviorProbeResult(**item) for item in raw) @@ -199,25 +231,14 @@ def _probe_package( def _probe_claude(*, version: str | None, scope: str) -> BehaviorProbeResult: package = "claude-agent-sdk" + observed = _observed_version(package) try: module = importlib.import_module("claude_agent_sdk") options_cls = module.ClaudeAgentOptions except Exception as exc: - return _failed(package, version, scope, "adapter-contract", exc) + return _failed(package, version, scope, "adapter-contract", exc, observed_version=observed) fields = _fields(options_cls) - expected = { - "model", - "allowed_tools", - "disallowed_tools", - "permission_mode", - "system_prompt", - "cwd", - "mcp_servers", - "resume", - "env", - "max_budget_usd", - "output_format", - } + expected = set(CLAUDE_CONTRACT["options_fields"]) missing = sorted(expected - fields) return BehaviorProbeResult( package=package, @@ -231,19 +252,23 @@ def _probe_claude(*, version: str | None, scope: str) -> BehaviorProbeResult: else "ClaudeAgentOptions is missing required adapter fields." ), details={"fields": sorted(fields), "required_fields": sorted(expected), "missing": missing}, + requested_version=version, + observed_version=observed, + provenance=_provenance(version, observed), ) def _probe_codex(*, version: str | None, scope: str) -> BehaviorProbeResult: package = "openai-codex" + observed = _observed_version(package) try: module = importlib.import_module("openai_codex") run_params = set(inspect.signature(module.AsyncThread.run).parameters) start_params = set(inspect.signature(module.AsyncCodex.thread_start).parameters) except Exception as exc: - return _failed(package, version, scope, "adapter-contract", exc) - expected_run = {"cwd", "model", "approval_mode", "sandbox", "output_schema", "effort"} - expected_start = {"developer_instructions", "cwd", "model", "approval_mode", "sandbox"} + return _failed(package, version, scope, "adapter-contract", exc, observed_version=observed) + expected_run = set(CODEX_CONTRACT["run_params"]) + expected_start = set(CODEX_CONTRACT["thread_start_params"]) missing_run = sorted(expected_run - run_params) missing_start = sorted(expected_start - start_params) missing = missing_run + [f"thread_start.{item}" for item in missing_start] @@ -265,6 +290,9 @@ def _probe_codex(*, version: str | None, scope: str) -> BehaviorProbeResult: "required_start_params": sorted(expected_start), "missing": missing, }, + requested_version=version, + observed_version=observed, + provenance=_provenance(version, observed), ) @@ -282,39 +310,26 @@ def _probe_codex_cli_bin(*, version: str | None, scope: str) -> BehaviorProbeRes status="pass", summary="Codex CLI binary distribution metadata is available.", details={"installed_version": installed}, + requested_version=version, + observed_version=installed, + provenance=_provenance(version, installed), ) def _probe_antigravity(*, version: str | None, scope: str) -> BehaviorProbeResult: package = "google-antigravity" + observed = _observed_version(package) try: - importlib.import_module(DEFAULT_MODULES[package]) - importlib.import_module("google.antigravity.types") - importlib.import_module("google.antigravity.agent") - importlib.import_module("google.antigravity.hooks.policy") + for module_name in ANTIGRAVITY_CONTRACT["required_imports"]: + importlib.import_module(module_name) config_module = importlib.import_module( "google.antigravity.connections.local.local_connection_config" ) config_cls = config_module.LocalAgentConfig except Exception as exc: - return _failed(package, version, scope, "adapter-contract", exc) + return _failed(package, version, scope, "adapter-contract", exc, observed_version=observed) fields = _fields(config_cls) - expected = { - "model", - "api_key", - "vertex", - "project", - "location", - "system_instructions", - "capabilities", - "policies", - "workspaces", - "conversation_id", - "save_dir", - "app_data_dir", - "response_schema", - "mcp_servers", - } + expected = set(ANTIGRAVITY_CONTRACT["config_fields"]) missing = sorted(expected - fields) return BehaviorProbeResult( package=package, @@ -328,6 +343,9 @@ def _probe_antigravity(*, version: str | None, scope: str) -> BehaviorProbeResul else "Antigravity LocalAgentConfig is missing required adapter fields." ), details={"fields": sorted(fields), "required_fields": sorted(expected), "missing": missing}, + requested_version=version, + observed_version=observed, + provenance=_provenance(version, observed), ) @@ -364,6 +382,7 @@ def _failed( scope: str, probe: str, exc: Exception, + observed_version: str | None = None, ) -> BehaviorProbeResult: return BehaviorProbeResult( package=package, @@ -373,6 +392,9 @@ def _failed( status="fail", summary=str(exc), details={"error": str(exc)}, + requested_version=version, + observed_version=observed_version, + provenance=_provenance(version, observed_version), ) @@ -389,6 +411,9 @@ def _skipped_candidate_probe(package: str, version: str) -> BehaviorProbeResult: "with --inspect-candidates to collect this evidence." ), details={"reason": "candidate installs are opt-in (--inspect-candidates)"}, + requested_version=version, + observed_version=None, + provenance="not-observed", ) @@ -399,6 +424,78 @@ def _string_or_none(value: object) -> str | None: return text or None +def _observed_version(package: str) -> str | None: + try: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + return None + + +def _provenance(requested: str | None, observed: str | None) -> str: + if requested and observed and requested != observed: + return "mismatched" + if observed is None: + return "not-observed" + return "observed" + + +def _tail(text: str, *, limit: int = 500) -> str: + return text[-limit:] + + +def _install_failed_probe( + package: str, + version: str, + scope: str, + probe: str, + summary: str, +) -> BehaviorProbeResult: + return BehaviorProbeResult( + package=package, + version=version, + scope=scope, + probe=probe, + status="fail", + summary=summary, + details={"stderr": summary}, + requested_version=version, + observed_version=None, + provenance="not-observed", + ) + + +def _run_probe_subprocess( + command: tuple[str, ...], + *, + env: dict[str, str], + timeout: int, + text: bool = True, + capture_output: bool = True, +) -> Any: + try: + completed = subprocess.run( + command, + text=text, + capture_output=capture_output, + timeout=timeout, + check=False, + env=env, + ) + except subprocess.TimeoutExpired: + return lambda package, version, scope, probe: _install_failed_probe( + package, version, scope, probe, f"timed out after {timeout}s" + ) + if completed.returncode == 0: + return None + return lambda package, version, scope, probe: _install_failed_probe( + package, + version, + scope, + probe, + _tail(completed.stderr or completed.stdout), + ) + + _PROBE_SCRIPT = textwrap.dedent( """ import importlib @@ -408,6 +505,7 @@ def _string_or_none(value: object) -> str | None: import sys package, version, scope = sys.argv[1:4] + contracts = json.loads(sys.argv[4]) def fields(cls): if hasattr(cls, "model_fields"): @@ -420,6 +518,7 @@ def fields(cls): return set() def failed(probe, exc): + observed = observed_version() return { "package": package, "version": version, @@ -428,9 +527,26 @@ def failed(probe, exc): "status": "fail", "summary": str(exc), "details": {"error": str(exc)}, + "requested_version": version, + "observed_version": observed, + "provenance": provenance(observed), } + def observed_version(): + try: + return importlib.metadata.version(package) + except Exception: + return None + + def provenance(observed): + if version and observed and version != observed: + return "mismatched" + if observed is None: + return "not-observed" + return "observed" + def result(probe, status, summary, details): + observed = observed_version() return { "package": package, "version": version, @@ -439,17 +555,16 @@ def result(probe, status, summary, details): "status": status, "summary": summary, "details": details, + "requested_version": version, + "observed_version": observed, + "provenance": provenance(observed), } try: if package == "claude-agent-sdk": module = importlib.import_module("claude_agent_sdk") option_fields = fields(getattr(module, "ClaudeAgentOptions")) - expected = { - "model", "allowed_tools", "disallowed_tools", "permission_mode", - "system_prompt", "cwd", "mcp_servers", "resume", "env", - "max_budget_usd", "output_format", - } + expected = set(contracts[package]["options_fields"]) missing = sorted(expected - option_fields) payload = [result( "adapter-contract", @@ -466,8 +581,8 @@ def result(probe, status, summary, details): module = importlib.import_module("openai_codex") run_params = set(inspect.signature(module.AsyncThread.run).parameters) start_params = set(inspect.signature(module.AsyncCodex.thread_start).parameters) - expected_run = {"cwd", "model", "approval_mode", "sandbox", "output_schema", "effort"} - expected_start = {"developer_instructions", "cwd", "model", "approval_mode", "sandbox"} + expected_run = set(contracts[package]["run_params"]) + expected_start = set(contracts[package]["thread_start_params"]) missing_run = sorted(expected_run - run_params) missing_start = sorted(expected_start - start_params) missing = missing_run + [f"thread_start.{item}" for item in missing_start] @@ -493,20 +608,13 @@ def result(probe, status, summary, details): {"installed_version": installed}, )] elif package == "google-antigravity": - importlib.import_module("google.antigravity") - importlib.import_module("google.antigravity.types") - importlib.import_module("google.antigravity.agent") - importlib.import_module("google.antigravity.hooks.policy") + for module_name in contracts[package]["required_imports"]: + importlib.import_module(module_name) config_module = importlib.import_module( "google.antigravity.connections.local.local_connection_config" ) config_fields = fields(getattr(config_module, "LocalAgentConfig")) - expected = { - "model", "api_key", "vertex", "project", "location", - "system_instructions", "capabilities", "policies", "workspaces", - "conversation_id", "save_dir", "app_data_dir", "response_schema", - "mcp_servers", - } + expected = set(contracts[package]["config_fields"]) missing = sorted(expected - config_fields) payload = [result( "adapter-contract", diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py index b776339..b136b7e 100644 --- a/examples/sdk_evolution_agent/cli.py +++ b/examples/sdk_evolution_agent/cli.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -import re +import sys from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -17,24 +17,37 @@ run_lock_update, run_verification_commands, ) -from examples.sdk_evolution_agent.current_state import build_current_state +from examples.sdk_evolution_agent.current_state import ( + build_current_state, + load_baseline, + promote_baseline, +) from examples.sdk_evolution_agent.events import JsonlEventSink from examples.sdk_evolution_agent.models import ( DEFAULT_PACKAGES, + CommandResult, RunContext, RunOptions, to_jsonable, ) from examples.sdk_evolution_agent.pr import ( + add_label_to_pr, build_draft_pr_body, + close_pr, + comment_pr, commit_staged, create_branch, create_draft_pr, + ensure_label, + list_open_sdk_evolution_prs, + parse_pr_marker, push_branch, stage_paths, ) +from examples.sdk_evolution_agent.preflight import PreflightError, validate_run_plan from examples.sdk_evolution_agent.release_notes import collect_release_notes from examples.sdk_evolution_agent.report import write_run_report +from examples.sdk_evolution_agent.resolver import candidate_map, raise_package_cap_in_workspace from examples.sdk_evolution_agent.snapshots import ( diff_snapshot_groups, snapshot_candidate_in_venv, @@ -44,6 +57,9 @@ maybe_run_implementation, resolve_runtime, run_analysis_pipeline, + run_implementation_review_stage, + run_implementation_stage, + with_cap_raise_guard, ) DEFAULT_VERIFICATION_COMMANDS = ( @@ -58,7 +74,11 @@ async def main(argv: list[str] | None = None) -> int: """Parse CLI args and run the agent.""" options = parse_args(argv) - report_path = await run_agent(options) + try: + report_path = await run_agent(options) + except PreflightError as exc: + print(str(exc), file=sys.stderr) + return 2 print(f"SDK evolution report: {report_path}") return 0 @@ -67,6 +87,11 @@ def parse_args(argv: list[str] | None = None) -> RunOptions: """Parse command-line options.""" parser = argparse.ArgumentParser(description="Run the local SDK evolution agent.") + parser.add_argument( + "--mode", + choices=("report", "upgrade", "upgrade-pr"), + help="Apply a tested flag bundle: report, upgrade, or upgrade-pr.", + ) parser.add_argument( "--runtime", default="fake", @@ -109,6 +134,16 @@ def parse_args(argv: list[str] | None = None) -> RunOptions: parser.add_argument("--branch-name", help="Branch name for optional branch creation.") parser.add_argument("--draft-pr", action="store_true", help="Create a draft PR with gh.") parser.add_argument("--pr-base", help="Base branch for optional draft PR creation.") + parser.add_argument( + "--allow-dirty", + action="store_true", + help="Dev-only escape hatch: allow implementation from a dirty worktree.", + ) + parser.add_argument( + "--allow-cap-raise", + action="store_true", + help="Allow guarded pyproject upper-bound raises for beyond-cap candidates.", + ) parser.add_argument( "--commit-message", default="Run SDK evolution update", @@ -120,18 +155,32 @@ def parse_args(argv: list[str] | None = None) -> RunOptions: help="Draft PR title.", ) args = parser.parse_args(argv) + run_id = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") + mode = args.mode + refresh_preview = args.refresh_preview or mode in {"report", "upgrade", "upgrade-pr"} + inspect_candidates = args.inspect_candidates or mode in {"upgrade", "upgrade-pr"} + implementation_enabled = args.implementation_enabled or mode in {"upgrade", "upgrade-pr"} + allow_cap_raise = args.allow_cap_raise or mode in {"upgrade", "upgrade-pr"} + create_branch = args.create_branch or mode == "upgrade-pr" + draft_pr = args.draft_pr or mode == "upgrade-pr" + branch_name = args.branch_name + if mode == "upgrade-pr" and not branch_name: + branch_name = f"sdk-evolution/{run_id}" return RunOptions( workspace=Path.cwd(), runtime=args.runtime, + mode=mode, packages=tuple(args.packages or DEFAULT_PACKAGES), report_dir=args.report_dir, - implementation_enabled=args.implementation_enabled, - refresh_preview=args.refresh_preview, - inspect_candidates=args.inspect_candidates, - create_branch=args.create_branch, - branch_name=args.branch_name, - draft_pr=args.draft_pr, - pr_base=args.pr_base, + implementation_enabled=implementation_enabled, + refresh_preview=refresh_preview, + inspect_candidates=inspect_candidates, + create_branch=create_branch, + branch_name=branch_name, + draft_pr=draft_pr, + pr_base=args.pr_base or ("main" if mode == "upgrade-pr" else None), + allow_dirty=args.allow_dirty, + allow_cap_raise=allow_cap_raise, commit_message=args.commit_message, pr_title=args.pr_title, ) @@ -148,6 +197,9 @@ async def run_agent( """Run the full local SDK evolution workflow.""" run_id = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") + violations = validate_run_plan(options, options.workspace, command_runner=command_runner) + if violations: + raise PreflightError(violations) report_root = (options.workspace / options.report_dir / run_id).resolve() event_log_path = report_root / "events.jsonl" event_sink = JsonlEventSink(event_log_path) @@ -183,17 +235,21 @@ async def run_agent( pypi_client=pypi_client, command_runner=command_runner, ) - update_versions = _refresh_update_versions(evidence) + evidence["baseline"] = load_baseline(options.workspace) + update_versions = candidate_map(evidence.get("update_candidates", [])) + beyond_versions = candidate_map(evidence.get("update_candidates_beyond_cap", [])) + candidate_versions = {**update_versions, **beyond_versions} snapshots = _collect_snapshots(evidence, inspect_candidates=options.inspect_candidates) + _record_snapshot_uncertainty(evidence, snapshots) api_diffs = [to_jsonable(diff) for diff in diff_snapshot_groups(snapshots)] release_notes = [ to_jsonable(item) - for item in collect_release_notes(evidence.get("packages", []), update_versions) + for item in collect_release_notes(evidence.get("packages", []), candidate_versions) ] behavior = to_jsonable( collect_behavior_evidence( evidence.get("packages", []), - update_versions, + candidate_versions, inspect_candidates=options.inspect_candidates, ) ) @@ -222,16 +278,25 @@ async def run_agent( review=review, context=context, ) - implementation.setdefault("verification_results", []).extend(pre_run_results) + implementation.setdefault("setup_results", []).extend(pre_run_results) config = to_jsonable(options) config["run_id"] = run_id config["event_log_path"] = str(context.event_log_path) if options.implementation_enabled and implementation.get("allowed"): - implementation = _run_local_sdk_update( + implementation = await _run_local_sdk_update( options, update_versions=update_versions, + beyond_candidates=evidence.get("update_candidates_beyond_cap", []), implementation=implementation, + runtime=selected_runtime, + context=context, + evidence=evidence, + architecture=architecture, + review=review, + api_diffs=api_diffs, + release_notes=release_notes, + behavior=behavior, command_runner=command_runner, ) @@ -256,6 +321,7 @@ async def run_agent( implementation=implementation, review=review, ) + snapshots_json = [to_jsonable(snapshot) for snapshot in snapshots] current_state = build_current_state( context, promoted=promoted, @@ -266,11 +332,25 @@ async def run_agent( ), implementation=implementation, ) + if promoted: + promotion = promote_baseline( + context, + snapshots=snapshots_json, + current_state=current_state, + ) + current_state["promotion"] = { + **current_state.get("promotion", {}), + **promotion, + } + if not promotion.get("promoted"): + promoted = False + implementation["applied"] = False + implementation["blocked_reason"] = str(promotion.get("blocked_reason") or "") report_path = _write_full_report( context, config=config, evidence=evidence, - snapshots=[to_jsonable(snapshot) for snapshot in snapshots], + snapshots=snapshots_json, api_diffs=api_diffs, release_notes=release_notes, behavior=behavior, @@ -281,14 +361,23 @@ async def run_agent( review=review, ) - if options.draft_pr: + if ( + options.draft_pr + and implementation.get("applied") + and _verification_passed(implementation) + ): git_results = _create_autonomous_pr( options.workspace, report_path=report_path, options=options, + package_versions={**update_versions, **_applied_cap_raise_versions(implementation)}, + run_id=run_id, command_runner=command_runner, ) - implementation.setdefault("verification_results", []).extend(git_results) + implementation.setdefault("pr_results", []).extend(git_results) + for item in git_results: + if isinstance(item, dict) and item.get("pr_skipped_reason"): + implementation["pr_skipped_reason"] = item["pr_skipped_reason"] report_path = _write_full_report( context, config=config, @@ -309,6 +398,24 @@ async def run_agent( options=options, command_runner=command_runner, ) + elif options.draft_pr: + implementation["pr_skipped_reason"] = ( + "implementation was not applied or verification did not pass" + ) + report_path = _write_full_report( + context, + config=config, + evidence=evidence, + snapshots=[to_jsonable(snapshot) for snapshot in snapshots], + api_diffs=api_diffs, + release_notes=release_notes, + behavior=behavior, + current_state=current_state, + direction=direction, + architecture=architecture, + implementation=implementation, + review=review, + ) return report_path finally: if close_owned_runtime: @@ -323,45 +430,318 @@ async def _close_runtime(runtime: AgentRuntime) -> None: await result -def _run_local_sdk_update( +async def _run_local_sdk_update( options: RunOptions, *, update_versions: dict[str, str], + beyond_candidates: Any, implementation: dict[str, Any], + runtime: AgentRuntime, + context: RunContext, + evidence: dict[str, Any], + architecture: dict[str, Any], + review: dict[str, Any], + api_diffs: list[dict[str, Any]], + release_notes: list[dict[str, Any]], + behavior: dict[str, Any], command_runner: CommandRunner | None, ) -> dict[str, Any]: - packages = tuple(sorted(update_versions)) + root = options.workspace + transaction_files = _snapshot_transaction_files(root) + allowed_file_snapshot = _snapshot_allowed_files(root) + apply_results = list(implementation.get("apply_results") or []) + verification_results: list[dict[str, Any]] = [] + changes = list(implementation.get("changes") or []) + cap_raise_versions: dict[str, str] = {} + packages = set(update_versions) + + if options.allow_cap_raise: + for candidate in beyond_candidates if isinstance(beyond_candidates, list) else []: + if not isinstance(candidate, dict): + continue + gate = with_cap_raise_guard( + architecture, + candidate=candidate, + api_diffs=api_diffs, + release_notes=release_notes, + behavior=behavior, + review=review, + ) + if not gate.allowed: + continue + package = str(candidate["package"]) + version = str(candidate["to_version"]) + raised = raise_package_cap_in_workspace(root, package, version) + if raised is not None: + cap_raise_versions[package] = version + packages.add(package) + changes.append( + f"Raised {package} upper bound {raised.current} -> {raised.replacement}" + ) + if not packages: return { **implementation, "applied": False, "blocked_reason": "no resolver-selected SDK updates", } - update_result = run_lock_update( - options.workspace, - packages, - command_runner=command_runner, - ) - results = list(implementation.get("verification_results") or []) - results.append(to_jsonable(update_result)) - applied = update_result.returncode == 0 - changes = list(implementation.get("changes") or []) - if applied: - changes.append("Updated uv.lock for resolver-selected SDK packages: " + ", ".join(packages)) - verification_commands = tuple(DEFAULT_VERIFICATION_COMMANDS) - verification_results = run_verification_commands( - options.workspace, - verification_commands, - command_runner=command_runner, + try: + update_result = run_lock_update( + root, tuple(sorted(packages)), command_runner=command_runner ) - results.extend(to_jsonable(verification_results)) - return { - **implementation, - "applied": applied, - "changes": changes, - "verification_results": results, - "blocked_reason": "" if applied else update_result.stderr or update_result.stdout, - } + apply_results.append(to_jsonable(update_result)) + if update_result.returncode != 0: + _restore_transaction(root, transaction_files, allowed_file_snapshot) + return { + **implementation, + "applied": False, + "rolled_back": True, + "changes": changes, + "apply_results": apply_results, + "verification_results": verification_results, + "cap_raise_versions": cap_raise_versions, + "blocked_reason": update_result.stderr or update_result.stdout, + } + + changes.append( + "Updated uv.lock for resolver-selected SDK packages: " + ", ".join(sorted(packages)) + ) + if _needs_ai_implementation(architecture): + stage_output = await run_implementation_stage( + runtime, + payload={ + "evidence": evidence, + "architecture_decision": architecture, + "review": review, + "allowed_roots": sorted(_ALLOWED_IMPLEMENTATION_PREFIXES), + }, + context=context, + ) + changes.extend(str(item) for item in stage_output.get("changes", [])) + if stage_output.get("blocked_reason"): + _restore_transaction(root, transaction_files, allowed_file_snapshot) + return { + **implementation, + "applied": False, + "rolled_back": True, + "changes": changes, + "apply_results": apply_results, + "verification_results": verification_results, + "cap_raise_versions": cap_raise_versions, + "blocked_reason": str(stage_output["blocked_reason"]), + } + + status = _git_status(root, command_runner=command_runner) + changed_paths = _changed_paths(status) + out_of_scope = [ + path for path in changed_paths if not _implementation_path_allowed(path) + ] + if out_of_scope: + _restore_transaction(root, transaction_files, allowed_file_snapshot) + return { + **implementation, + "applied": False, + "rolled_back": True, + "changes": changes, + "apply_results": apply_results, + "verification_results": verification_results, + "cap_raise_versions": cap_raise_versions, + "blocked_reason": "implementation changed files outside allowed scope: " + + ", ".join(out_of_scope), + } + + verification_commands = _verification_commands(architecture) + try: + verification = run_verification_commands( + root, + verification_commands, + command_runner=command_runner, + ) + verification_results.extend(to_jsonable(verification)) + except Exception as exc: + verification_results.append( + to_jsonable( + CommandResult( + command=("verification",), + returncode=1, + stderr=str(exc), + ) + ) + ) + candidate = { + **implementation, + "applied": True, + "rolled_back": False, + "changes": changes, + "apply_results": apply_results, + "verification_results": verification_results, + "cap_raise_versions": cap_raise_versions, + "blocked_reason": "", + } + if not _verification_passed(candidate): + _restore_transaction(root, transaction_files, allowed_file_snapshot) + return { + **candidate, + "applied": False, + "rolled_back": True, + "blocked_reason": "verification failed; workspace restored", + } + + diff = _git_diff(root, command_runner=command_runner) + implementation_review = await run_implementation_review_stage( + runtime, + payload={ + "changed_files": changed_paths, + "diff": diff[:65536], + "verification_results": verification_results, + }, + context=context, + ) + candidate["implementation_review"] = implementation_review + if implementation_review.get("status") != "pass": + _restore_transaction(root, transaction_files, allowed_file_snapshot) + return { + **candidate, + "applied": False, + "rolled_back": True, + "blocked_reason": "implementation reviewer rejected the diff", + } + return candidate + except Exception as exc: + _restore_transaction(root, transaction_files, allowed_file_snapshot) + return { + **implementation, + "applied": False, + "rolled_back": True, + "changes": changes, + "apply_results": apply_results, + "verification_results": verification_results, + "cap_raise_versions": cap_raise_versions, + "blocked_reason": f"implementation failed; workspace restored: {exc}", + } + + +_ALLOWED_IMPLEMENTATION_PREFIXES = ( + "examples/sdk_evolution_agent/", + "tests/test_sdk_evolution_", + "docs/sdk-evolution-agent", + ".sdk-evolution/", +) +_ALLOWED_IMPLEMENTATION_FILES = {"uv.lock", "pyproject.toml"} + + +def _verification_commands(architecture: dict[str, Any]) -> tuple[str, ...]: + commands = list(DEFAULT_VERIFICATION_COMMANDS) + for command in architecture.get("verification_commands") or []: + if isinstance(command, str) and command not in commands: + commands.append(command) + return tuple(commands) + + +def _snapshot_transaction_files(root: Path) -> dict[str, bytes | None]: + snapshot: dict[str, bytes | None] = {} + for name in ("uv.lock", "pyproject.toml"): + path = root / name + snapshot[name] = path.read_bytes() if path.exists() else None + return snapshot + + +def _snapshot_allowed_files(root: Path) -> dict[str, bytes]: + snapshot: dict[str, bytes] = {} + for base in (root / "examples" / "sdk_evolution_agent", root / "docs", root / "tests"): + if not base.exists(): + continue + for path in base.rglob("*"): + if path.is_file(): + rel = _relative_path(root, path) + if _implementation_path_allowed(rel): + snapshot[rel] = path.read_bytes() + return snapshot + + +def _restore_transaction( + root: Path, + transaction_files: dict[str, bytes | None], + allowed_file_snapshot: dict[str, bytes], +) -> None: + for relative, data in transaction_files.items(): + path = root / relative + if data is None: + path.unlink(missing_ok=True) + else: + path.write_bytes(data) + for relative, data in allowed_file_snapshot.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + for relative in _changed_paths(_git_status(root, command_runner=None)): + if relative.startswith("reports/"): + continue + if relative in transaction_files or relative in allowed_file_snapshot: + continue + path = root / relative + if path.exists() and _implementation_path_allowed(relative): + if path.is_dir(): + for child in sorted(path.rglob("*"), reverse=True): + if child.is_file() or child.is_symlink(): + child.unlink(missing_ok=True) + elif child.is_dir(): + child.rmdir() + path.rmdir() + else: + path.unlink(missing_ok=True) + + +def _git_status(root: Path, *, command_runner: CommandRunner | None) -> str: + runner = command_runner + if runner is None: + from examples.sdk_evolution_agent.collectors import run_command + + runner = run_command + result = runner(("git", "status", "--porcelain"), cwd=root) + return result.stdout if result.returncode == 0 else "" + + +def _git_diff(root: Path, *, command_runner: CommandRunner | None) -> str: + runner = command_runner + if runner is None: + from examples.sdk_evolution_agent.collectors import run_command + + runner = run_command + result = runner(("git", "diff", "--", "."), cwd=root) + return result.stdout if result.returncode == 0 else "" + + +def _changed_paths(status: str) -> list[str]: + paths: list[str] = [] + for line in status.splitlines(): + if len(line) < 4: + continue + path = line[3:].strip() + if " -> " in path: + path = path.rsplit(" -> ", 1)[1] + if path: + paths.append(path) + return paths + + +def _implementation_path_allowed(path: str) -> bool: + return path in _ALLOWED_IMPLEMENTATION_FILES or any( + path.startswith(prefix) for prefix in _ALLOWED_IMPLEMENTATION_PREFIXES + ) + + +def _needs_ai_implementation(architecture: dict[str, Any]) -> bool: + items = list(architecture.get("self_adaptation_plan") or []) + items.extend(architecture.get("docs_test_changes") or []) + return any(_implementation_path_allowed(str(item)) for item in items) + + +def _applied_cap_raise_versions(implementation: dict[str, Any]) -> dict[str, str]: + versions = implementation.get("cap_raise_versions") + if not isinstance(versions, dict): + return {} + return {str(package): str(version) for package, version in versions.items()} def _write_full_report( @@ -412,43 +792,98 @@ def _create_autonomous_pr( *, report_path: Path, options: RunOptions, + package_versions: dict[str, str], + run_id: str, command_runner: CommandRunner | None, ) -> list[dict[str, Any]]: branch_name = options.branch_name or _current_branch(root, command_runner=command_runner) - body = build_draft_pr_body(report_path.read_text(encoding="utf-8")) - # Only stage the lockfile. The report lives under the gitignored default - # report dir, so `git add`-ing it returned rc=1 and previously broke the - # --draft-pr flow; its content is already embedded in the PR body above. - paths = ("uv.lock",) - results = [ - to_jsonable(stage_paths(root, paths, command_runner=command_runner)), - to_jsonable( - commit_staged( + body = build_draft_pr_body( + report_path.read_text(encoding="utf-8"), + package_versions=package_versions, + run_id=run_id, + ) + results: list[dict[str, Any]] = [] + superseded: list[int] = [] + for item in list_open_sdk_evolution_prs(root, command_runner=command_runner): + marker = parse_pr_marker(str(item.get("body") or "")) + if marker is None: + continue + number = int(item.get("number") or 0) + if marker.get("packages") == package_versions and number: + comment = comment_pr( root, - message=options.commit_message, + number, + f"New SDK evolution run `{run_id}` produced the same package delta. " + f"Report: {report_path}", command_runner=command_runner, ) - ), - ] + results.append(to_jsonable(comment)) + _raise_on_failed_results(results, "comment on identical SDK evolution PR") + results.append({"pr_skipped_reason": f"identical delta: #{number}"}) + return results + if number: + superseded.append(number) + + paths = ("uv.lock", "pyproject.toml", ".sdk-evolution") + results.extend( + [ + to_jsonable(stage_paths(root, paths, command_runner=command_runner)), + to_jsonable( + commit_staged( + root, + message=options.commit_message, + command_runner=command_runner, + ) + ), + ] + ) if branch_name: results.append( to_jsonable(push_branch(root, branch_name=branch_name, command_runner=command_runner)) ) - results.append( - to_jsonable( - create_draft_pr( - root, - title=options.pr_title, - body=body, - base=options.pr_base, - head=branch_name, - command_runner=command_runner, + results.append(to_jsonable(ensure_label(root, "sdk-evolution", command_runner=command_runner))) + create_result = create_draft_pr( + root, + title=options.pr_title, + body=body, + base=options.pr_base, + head=branch_name, + command_runner=command_runner, + ) + results.append(to_jsonable(create_result)) + _raise_on_failed_results(results, "create SDK evolution draft PR") + pr_ref = (create_result.stdout.strip().splitlines() or [branch_name or ""])[0] + if pr_ref: + results.append( + to_jsonable( + add_label_to_pr(root, pr_ref, "sdk-evolution", command_runner=command_runner) ) ) - ) + _raise_on_failed_results(results, "label SDK evolution draft PR") + for number in superseded: + results.append( + to_jsonable( + close_pr( + root, + number, + f"Superseded by SDK evolution run `{run_id}` ({pr_ref}).", + command_runner=command_runner, + ) + ) + ) + _raise_on_failed_results(results, "supersede old SDK evolution PRs") return results +def _raise_on_failed_results(results: list[dict[str, Any]], step: str) -> None: + for result in results: + if "returncode" not in result: + continue + if int(result.get("returncode", 1)) != 0: + detail = result.get("stderr") or result.get("stdout") or result.get("command") + raise RuntimeError(f"failed to {step}: {detail}") + + def _commit_final_autonomous_pr_report( root: Path, *, @@ -517,8 +952,10 @@ def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = F # off, every snapshot uses the already-installed version via snapshot_current_api, # which imports nothing new. snapshots = [] - update_versions = _refresh_update_versions(evidence) - refresh_preview_seen = evidence.get("refresh_preview") is not None + update_versions = { + **candidate_map(evidence.get("update_candidates", [])), + **candidate_map(evidence.get("update_candidates_beyond_cap", [])), + } for package in evidence.get("packages", []): if not isinstance(package, dict): continue @@ -533,24 +970,19 @@ def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = F if not inspect_candidates: continue candidate = update_versions.get(name) - if candidate is None and not refresh_preview_seen: - latest = package.get("latest_version") - if latest and latest != baseline: - candidate = str(latest) if candidate: snapshots.append(snapshot_candidate_in_venv(name, candidate)) return snapshots -def _refresh_update_versions(evidence: dict[str, Any]) -> dict[str, str]: - preview = evidence.get("refresh_preview") - if not isinstance(preview, dict): - return {} - text = f"{preview.get('stdout') or ''}\n{preview.get('stderr') or ''}" - return { - package: version - for package, version in re.findall( - r"Update\s+([A-Za-z0-9_.-]+)\s+v\S+\s+->\s+v(\S+)", - text, +def _record_snapshot_uncertainty(evidence: dict[str, Any], snapshots: list[Any]) -> None: + mismatches = [] + for snapshot in snapshots: + requested = getattr(snapshot, "requested_version", None) + observed = getattr(snapshot, "observed_version", None) + if requested and observed and requested != observed: + mismatches.append(f"{snapshot.package}: requested {requested}, observed {observed}") + if mismatches: + evidence.setdefault("uncertainty", []).append( + "Snapshot provenance mismatches: " + "; ".join(mismatches) ) - } diff --git a/examples/sdk_evolution_agent/collectors.py b/examples/sdk_evolution_agent/collectors.py index 8dc135c..15d6a48 100644 --- a/examples/sdk_evolution_agent/collectors.py +++ b/examples/sdk_evolution_agent/collectors.py @@ -2,15 +2,19 @@ from __future__ import annotations +import importlib import importlib.metadata import json import os import re import shlex import subprocess +import sys import urllib.request from collections.abc import Callable, Mapping, Sequence +from datetime import datetime, timezone from pathlib import Path +from types import ModuleType from typing import Any from examples.sdk_evolution_agent.models import ( @@ -21,6 +25,12 @@ to_jsonable, ) +tomllib: ModuleType | None +if sys.version_info >= (3, 11): # pragma: no cover - exercised on modern runtimes + tomllib = importlib.import_module("tomllib") +else: # pragma: no cover - Python 3.10 fallback + tomllib = None + FRESHNESS_CUTOFF_ENV_VARS = ("UV_EXCLUDE_NEWER",) PACKAGE_SOURCE_HINTS: dict[str, tuple[SourceRef, ...]] = { @@ -83,14 +93,36 @@ def collect_evidence( "packages": [to_jsonable(item) for item in package_states], "adapter_sources": [to_jsonable(item) for item in adapter_source_refs(root)], "refresh_preview": None, + "update_candidates": [], + "update_candidates_beyond_cap": [], "facts": [ "pyproject.toml dependency declarations are lower bounds or constraints.", "uv.lock is the tested local dependency state.", ], } if include_refresh_preview: + from examples.sdk_evolution_agent.resolver import ( + resolve_constraint_horizon_candidates, + resolve_update_candidates, + ) + + candidates, resolver_result = resolve_update_candidates( + root, packages, command_runner=command_runner + ) preview = run_refresh_preview(root, packages, command_runner=command_runner) evidence["refresh_preview"] = to_jsonable(preview) + evidence["resolver_result"] = to_jsonable(resolver_result) + evidence["update_candidates"] = [to_jsonable(candidate) for candidate in candidates] + evidence["update_candidates_beyond_cap"] = [ + to_jsonable(candidate) + for candidate in resolve_constraint_horizon_candidates( + root, + packages, + adoptable=candidates, + pypi_metadata={package: _safe_pypi(pypi_client, package) for package in packages}, + command_runner=command_runner, + ) + ] return evidence @@ -99,8 +131,26 @@ def read_pyproject_dependency_specs(path: Path) -> dict[str, str]: if not path.exists(): return {} - text = path.read_text(encoding="utf-8") + data = _load_toml(path) specs: dict[str, str] = {} + if data is not None: + optional = data.get("project", {}).get("optional-dependencies", {}) + if isinstance(optional, dict): + for dependencies in optional.values(): + if not isinstance(dependencies, list): + continue + for dependency in dependencies: + if not isinstance(dependency, str): + continue + package = _dependency_name(dependency) + if package in DEFAULT_PACKAGES: + specs[package] = dependency + return specs + + text = path.read_text(encoding="utf-8") + specs.update(_read_optional_dependency_specs_from_text(text)) + if specs: + return specs for package in DEFAULT_PACKAGES: match = re.search(rf"{re.escape(package)}[^\"'\],]*", text) if match: @@ -109,18 +159,32 @@ def read_pyproject_dependency_specs(path: Path) -> dict[str, str]: def read_uv_lock_versions(path: Path) -> dict[str, str]: - """Read package versions from uv.lock without depending on a TOML parser.""" + """Read package versions from uv.lock.""" if not path.exists(): return {} - versions: dict[str, str] = {} + data = _load_toml(path) + if data is not None: + packages = data.get("package", []) + if isinstance(packages, list): + toml_versions: dict[str, str] = {} + for package in packages: + if not isinstance(package, dict): + continue + name = package.get("name") + version = package.get("version") + if isinstance(name, str) and isinstance(version, str): + toml_versions[name] = version + return toml_versions + + regex_versions: dict[str, str] = {} text = path.read_text(encoding="utf-8") for block in re.split(r"\n\[\[package\]\]\n", text): name_match = re.search(r'^name = "([^"]+)"', block, re.MULTILINE) version_match = re.search(r'^version = "([^"]+)"', block, re.MULTILINE) if name_match and version_match: - versions[name_match.group(1)] = version_match.group(1) - return versions + regex_versions[name_match.group(1)] = version_match.group(1) + return regex_versions def adapter_source_refs(root: Path) -> tuple[SourceRef, ...]: @@ -201,7 +265,8 @@ def fetch_pypi_metadata(package: str) -> Mapping[str, Any]: url = f"https://pypi.org/pypi/{package}/json" request = urllib.request.Request(url, headers={"User-Agent": "agent-runtime-kit-sdk-evolution"}) with urllib.request.urlopen(request, timeout=20) as response: - return json.loads(response.read().decode("utf-8")) + payload = json.loads(response.read().decode("utf-8")) + return payload if isinstance(payload, Mapping) else {} def installed_version(package: str) -> str | None: @@ -258,7 +323,7 @@ def run_refresh_preview( command_runner = command_runner or run_command env, removed = cutoff_free_env() command = build_refresh_preview_command(packages) - result = command_runner(command, cwd=root, env=env) + result = _call_runner(command_runner, command, cwd=root, env=env, timeout=300) return CommandResult( command=result.command, returncode=result.returncode, @@ -281,7 +346,7 @@ def run_lock_update( command = ["uv", "lock"] for package in packages: command.extend(("-P", package)) - result = command_runner(tuple(command), cwd=root, env=env) + result = _call_runner(command_runner, tuple(command), cwd=root, env=env, timeout=300) return CommandResult( command=result.command, returncode=result.returncode, @@ -302,7 +367,9 @@ def run_verification_commands( command_runner = command_runner or run_command results: list[CommandResult] = [] for command in commands: - results.append(command_runner(tuple(shlex.split(command)), cwd=root)) + results.append( + _call_runner(command_runner, tuple(shlex.split(command)), cwd=root, timeout=900) + ) return tuple(results) @@ -315,15 +382,23 @@ def run_command( ) -> CommandResult: """Run a local command and capture output.""" - completed = subprocess.run( - tuple(command), - cwd=cwd, - env=dict(env) if env is not None else None, - text=True, - capture_output=True, - timeout=timeout, - check=False, - ) + try: + completed = subprocess.run( + tuple(command), + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + return CommandResult( + command=tuple(command), + returncode=124, + stdout=str(exc.stdout or ""), + stderr=f"timed out after {timeout}s", + ) return CommandResult( command=tuple(command), returncode=completed.returncode, @@ -332,8 +407,123 @@ def run_command( ) +def _call_runner( + command_runner: CommandRunner, + command: Sequence[str], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + timeout: int | None = None, +) -> CommandResult: + kwargs: dict[str, Any] = {"cwd": cwd} + if env is not None: + kwargs["env"] = env + if timeout is not None: + kwargs["timeout"] = timeout + try: + return command_runner(tuple(command), **kwargs) + except TypeError as exc: + if "timeout" not in str(exc): + raise + kwargs.pop("timeout", None) + return command_runner(tuple(command), **kwargs) + + def _version_key(version: str) -> tuple[Any, ...]: - parts: list[Any] = [] - for part in re.split(r"[.\-+_]", version): - parts.append((0, int(part)) if part.isdigit() else (1, part)) - return tuple(parts) + """Return a small dependency-free PEP 440-ish ordering key.""" + + text = version.strip().lower().replace("_", ".").replace("-", ".") + public = text.split("+", 1)[0] + epoch = 0 + if "!" in public: + raw_epoch, public = public.split("!", 1) + epoch = int(raw_epoch or "0") if raw_epoch.isdigit() else 0 + + dev_number = _suffix_number(public, "dev") + post_number = _suffix_number(public, "post") + pre_match = re.search(r"(a|b|rc)(\d*)", public) + release_text = re.split(r"(?:a|b|rc|post|dev)", public, maxsplit=1)[0].strip(".") + release = tuple(int(part) for part in release_text.split(".") if part.isdigit()) + release = release + (0,) * (6 - len(release)) + + if dev_number is not None and pre_match is None: + phase = (0, dev_number) + elif pre_match is not None: + phase_order = {"a": 1, "b": 2, "rc": 3} + phase = (phase_order[pre_match.group(1)], int(pre_match.group(2) or "0")) + elif post_number is not None: + phase = (5, post_number) + else: + phase = (4, 0) + return (epoch, release, phase) + + +def _suffix_number(version: str, label: str) -> int | None: + match = re.search(rf"(?:^|[.]){label}(\d*)", version) + if not match: + return None + return int(match.group(1) or "0") + + +def _load_toml(path: Path) -> dict[str, Any] | None: + if tomllib is None: + return None + try: + return dict(tomllib.loads(path.read_text(encoding="utf-8"))) + except Exception: + return None + + +def _dependency_name(spec: str) -> str: + match = re.match(r"\s*([A-Za-z0-9_.-]+)", spec) + return (match.group(1) if match else "").replace("_", "-").lower() + + +def _read_optional_dependency_specs_from_text(text: str) -> dict[str, str]: + specs: dict[str, str] = {} + in_optional = False + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.startswith("[") and stripped.endswith("]"): + in_optional = stripped == "[project.optional-dependencies]" + continue + if not in_optional: + continue + for dependency in re.findall(r'["\']([^"\']+)["\']', stripped): + package = _dependency_name(dependency) + if package in DEFAULT_PACKAGES: + specs[package] = dependency + return specs + + +def _safe_pypi(pypi_client: PypiClient, package: str) -> Mapping[str, Any]: + try: + return pypi_client(package) + except Exception: + return {} + + +def upload_time_for_version(metadata: Mapping[str, Any], version: str) -> datetime | None: + """Return the first PyPI upload timestamp for a version, when present.""" + + releases = metadata.get("releases", {}) + if not isinstance(releases, Mapping): + return None + files = releases.get(version) + if not isinstance(files, list) or not files: + return None + for file_info in files: + if not isinstance(file_info, Mapping): + continue + raw = file_info.get("upload_time_iso_8601") or file_info.get("upload_time") + if not isinstance(raw, str) or not raw: + continue + normalized = raw.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + continue + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + return None diff --git a/examples/sdk_evolution_agent/current_state.py b/examples/sdk_evolution_agent/current_state.py index 9f37bae..f9d2b0f 100644 --- a/examples/sdk_evolution_agent/current_state.py +++ b/examples/sdk_evolution_agent/current_state.py @@ -3,14 +3,17 @@ from __future__ import annotations import hashlib +import json import subprocess from pathlib import Path from typing import Any from examples.sdk_evolution_agent.collectors import read_uv_lock_versions from examples.sdk_evolution_agent.models import RunContext +from examples.sdk_evolution_agent.report import write_json CURRENT_STATE_SCHEMA_VERSION = "1" +BASELINE_DIR = ".sdk-evolution" def build_current_state( @@ -41,6 +44,80 @@ def build_current_state( } +def load_baseline(workspace: Path) -> dict[str, Any]: + """Load and classify the tracked SDK evolution baseline.""" + + path = workspace / BASELINE_DIR / "baseline.json" + if not path.exists(): + return {"status": "missing", "path": str(path)} + try: + baseline = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + return {"status": "stale", "path": str(path), "reason": f"cannot read baseline: {exc}"} + lockfile = workspace / "uv.lock" + packages = read_uv_lock_versions(lockfile) + lock_hash = _sha256(lockfile) + if baseline.get("lockfile_sha256") == lock_hash and baseline.get("packages") == packages: + return {"status": "current", "path": str(path), "baseline": baseline} + return { + "status": "stale", + "path": str(path), + "baseline": baseline, + "reason": "lockfile hash or package versions differ", + } + + +def promote_baseline( + context: RunContext, + *, + snapshots: list[dict[str, Any]], + current_state: dict[str, Any], +) -> dict[str, Any]: + """Write the tracked baseline and promoted API snapshots.""" + + workspace = context.workspace + versions = read_uv_lock_versions(workspace / "uv.lock") + mismatched = [ + snapshot + for snapshot in snapshots + if snapshot.get("package") in versions + and snapshot.get("observed_version") + and snapshot.get("observed_version") != versions[snapshot["package"]] + ] + if mismatched: + return { + "promoted": False, + "status": "baseline-promotion-refused", + "blocked_reason": "snapshot observed_version does not match locked version", + } + + root = workspace / BASELINE_DIR + snapshots_dir = root / "snapshots" + snapshots_dir.mkdir(parents=True, exist_ok=True) + package_snapshots: dict[str, str] = {} + for snapshot in snapshots: + package = str(snapshot.get("package") or "") + if not package or snapshot.get("import_error"): + continue + if package not in versions: + continue + path = snapshots_dir / f"{package}.json" + write_json(path, snapshot) + package_snapshots[package] = _sha256(path) + + baseline = { + "schema_version": CURRENT_STATE_SCHEMA_VERSION, + "source_run_id": context.run_id, + "commit": _git_output(workspace, ("git", "rev-parse", "HEAD")), + "lockfile_sha256": _sha256(workspace / "uv.lock"), + "packages": versions, + "artifacts": current_state.get("artifacts", {}), + "snapshot_sha256s": package_snapshots, + } + write_json(root / "baseline.json", baseline) + return {"promoted": True, "status": "promoted", "baseline": baseline} + + def _artifact_refs(report_root: Path, *, workspace: Path) -> dict[str, dict[str, str]]: names = ( "evidence.json", diff --git a/examples/sdk_evolution_agent/models.py b/examples/sdk_evolution_agent/models.py index 9ce477e..126863c 100644 --- a/examples/sdk_evolution_agent/models.py +++ b/examples/sdk_evolution_agent/models.py @@ -4,7 +4,7 @@ from dataclasses import asdict, dataclass, field, is_dataclass from pathlib import Path -from typing import Any +from typing import Any, cast DEFAULT_PACKAGES = ( "claude-agent-sdk", @@ -80,6 +80,9 @@ class ApiSnapshot: package: str version: str | None module: str + requested_version: str | None = None + observed_version: str | None = None + provenance: str = "observed" members: tuple[ApiMember, ...] = () import_error: str | None = None source: str = "current-environment" @@ -122,6 +125,9 @@ class BehaviorProbeResult: status: str summary: str details: dict[str, Any] = field(default_factory=dict) + requested_version: str | None = None + observed_version: str | None = None + provenance: str = "observed" @dataclass(frozen=True) @@ -144,6 +150,7 @@ class RunOptions: workspace: Path runtime: str = "fake" + mode: str | None = None packages: tuple[str, ...] = DEFAULT_PACKAGES report_dir: Path = Path("reports/sdk-evolution") implementation_enabled: bool = False @@ -155,6 +162,8 @@ class RunOptions: branch_name: str | None = None draft_pr: bool = False pr_base: str | None = None + allow_dirty: bool = False + allow_cap_raise: bool = False commit_message: str = "Run SDK evolution update" pr_title: str = "Adapt agent-runtime-kit to upstream SDK evolution" @@ -184,8 +193,8 @@ class GateResult: def to_jsonable(value: Any) -> Any: """Convert dataclasses and paths into JSON-compatible values.""" - if is_dataclass(value): - return to_jsonable(asdict(value)) + if is_dataclass(value) and not isinstance(value, type): + return to_jsonable(asdict(cast(Any, value))) if isinstance(value, Path): return str(value) if isinstance(value, dict): diff --git a/examples/sdk_evolution_agent/pr.py b/examples/sdk_evolution_agent/pr.py index 6d94100..3fcb154 100644 --- a/examples/sdk_evolution_agent/pr.py +++ b/examples/sdk_evolution_agent/pr.py @@ -2,17 +2,30 @@ from __future__ import annotations +import json from pathlib import Path +from typing import Any -from examples.sdk_evolution_agent.collectors import CommandRunner, run_command +from examples.sdk_evolution_agent.collectors import CommandRunner, _call_runner, run_command from examples.sdk_evolution_agent.models import CommandResult +SDK_EVOLUTION_MARKER_PREFIX = "" -def build_draft_pr_body(report_markdown: str) -> str: + +def build_draft_pr_body( + report_markdown: str, + *, + package_versions: dict[str, str] | None = None, + run_id: str | None = None, +) -> str: """Build a draft PR body from a local report.""" + marker = build_pr_marker(package_versions or {}, run_id=run_id or "") return "\n".join( [ + marker, + "", "## SDK Evolution Report", "", report_markdown, @@ -26,6 +39,27 @@ def build_draft_pr_body(report_markdown: str) -> str: ) +def build_pr_marker(package_versions: dict[str, str], *, run_id: str) -> str: + payload = {"packages": dict(sorted(package_versions.items())), "run_id": run_id} + marker_json = json.dumps(payload, sort_keys=True) + return f"{SDK_EVOLUTION_MARKER_PREFIX} {marker_json} {SDK_EVOLUTION_MARKER_SUFFIX}" + + +def parse_pr_marker(body: str) -> dict[str, Any] | None: + start = body.find(SDK_EVOLUTION_MARKER_PREFIX) + if start < 0: + return None + start += len(SDK_EVOLUTION_MARKER_PREFIX) + end = body.find(SDK_EVOLUTION_MARKER_SUFFIX, start) + if end < 0: + return None + try: + payload = json.loads(body[start:end].strip()) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + def create_branch( root: Path, branch_name: str, @@ -35,7 +69,7 @@ def create_branch( """Create or switch to a local branch.""" command_runner = command_runner or run_command - return command_runner(("git", "switch", "-c", branch_name), cwd=root) + return _call_runner(command_runner, ("git", "switch", "-c", branch_name), cwd=root, timeout=120) def stage_paths( @@ -47,7 +81,7 @@ def stage_paths( """Stage paths for an autonomous SDK update PR.""" command_runner = command_runner or run_command - return command_runner(("git", "add", *paths), cwd=root) + return _call_runner(command_runner, ("git", "add", *paths), cwd=root, timeout=120) def commit_staged( @@ -59,7 +93,7 @@ def commit_staged( """Commit staged SDK update artifacts.""" command_runner = command_runner or run_command - return command_runner(("git", "commit", "-m", message), cwd=root) + return _call_runner(command_runner, ("git", "commit", "-m", message), cwd=root, timeout=120) def push_branch( @@ -71,7 +105,12 @@ def push_branch( """Push the current SDK update branch.""" command_runner = command_runner or run_command - return command_runner(("git", "push", "-u", "origin", branch_name), cwd=root) + return _call_runner( + command_runner, + ("git", "push", "-u", "origin", branch_name), + cwd=root, + timeout=120, + ) def create_draft_pr( @@ -91,4 +130,93 @@ def create_draft_pr( command.extend(("--base", base)) if head: command.extend(("--head", head)) - return command_runner(tuple(command), cwd=root) + return _call_runner(command_runner, tuple(command), cwd=root, timeout=120) + + +def list_open_sdk_evolution_prs( + root: Path, *, command_runner: CommandRunner | None = None +) -> tuple[dict[str, Any], ...]: + """List open SDK evolution PRs visible to gh.""" + + command_runner = command_runner or run_command + result = _call_runner( + command_runner, + ( + "gh", + "pr", + "list", + "--state", + "open", + "--json", + "number,headRefName,body,labels", + ), + cwd=root, + timeout=120, + ) + if result.returncode != 0: + return () + try: + payload = json.loads(result.stdout or "[]") + except json.JSONDecodeError: + return () + if not isinstance(payload, list): + return () + return tuple(item for item in payload if isinstance(item, dict) and _is_sdk_evolution_pr(item)) + + +def ensure_label( + root: Path, label: str, *, command_runner: CommandRunner | None = None +) -> CommandResult: + """Create a label if it does not exist.""" + + command_runner = command_runner or run_command + return _call_runner( + command_runner, + ("gh", "label", "create", label, "--force"), + cwd=root, + timeout=120, + ) + + +def comment_pr( + root: Path, number: int, body: str, *, command_runner: CommandRunner | None = None +) -> CommandResult: + command_runner = command_runner or run_command + return _call_runner( + command_runner, + ("gh", "pr", "comment", str(number), "--body", body), + cwd=root, + timeout=120, + ) + + +def close_pr( + root: Path, number: int, body: str, *, command_runner: CommandRunner | None = None +) -> CommandResult: + command_runner = command_runner or run_command + return _call_runner( + command_runner, + ("gh", "pr", "close", str(number), "--comment", body), + cwd=root, + timeout=120, + ) + + +def add_label_to_pr( + root: Path, number_or_url: str, label: str, *, command_runner: CommandRunner | None = None +) -> CommandResult: + command_runner = command_runner or run_command + return _call_runner( + command_runner, + ("gh", "pr", "edit", number_or_url, "--add-label", label), + cwd=root, + timeout=120, + ) + + +def _is_sdk_evolution_pr(item: dict[str, Any]) -> bool: + head = str(item.get("headRefName") or "") + if head.startswith("sdk-evolution"): + return True + labels = item.get("labels") or [] + return any(isinstance(label, dict) and label.get("name") == "sdk-evolution" for label in labels) diff --git a/examples/sdk_evolution_agent/preflight.py b/examples/sdk_evolution_agent/preflight.py new file mode 100644 index 0000000..04086d3 --- /dev/null +++ b/examples/sdk_evolution_agent/preflight.py @@ -0,0 +1,76 @@ +"""Preflight validation for SDK evolution runs.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from examples.sdk_evolution_agent.collectors import CommandRunner, run_command +from examples.sdk_evolution_agent.models import RunOptions + + +class PreflightError(RuntimeError): + """Raised when a run configuration is unsafe before any side effect.""" + + def __init__(self, violations: tuple[str, ...]) -> None: + self.violations = violations + message = "SDK evolution preflight failed:\n" + "\n".join( + f"- {violation}" for violation in violations + ) + super().__init__(message) + + +def validate_run_plan( + options: RunOptions, + workspace: Path, + *, + command_runner: CommandRunner | None = None, +) -> tuple[str, ...]: + """Return human-readable run-plan violations.""" + + runner = command_runner or run_command + violations: list[str] = [] + + if options.draft_pr and not options.implementation_enabled: + violations.append("--draft-pr requires --implementation-enabled") + if options.implementation_enabled and not options.refresh_preview: + violations.append("--implementation-enabled requires --refresh-preview") + if options.create_branch and not options.branch_name: + violations.append("--create-branch requires --branch-name") + if options.allow_cap_raise and not options.implementation_enabled: + violations.append("--allow-cap-raise requires --implementation-enabled") + if options.allow_cap_raise and not options.inspect_candidates: + violations.append("--allow-cap-raise requires --inspect-candidates") + + if options.draft_pr: + default_branch = _default_branch(workspace, runner) + head_branch = ( + options.branch_name + if options.create_branch + else _git_stdout(workspace, runner, ("git", "branch", "--show-current")) + ) + if not head_branch: + violations.append("--draft-pr requires a non-detached head branch") + elif head_branch == default_branch: + violations.append(f"--draft-pr refuses to push the default branch ({default_branch})") + + if options.implementation_enabled and not options.allow_dirty: + status = _git_stdout(workspace, runner, ("git", "status", "--porcelain")) + if status: + violations.append("--implementation-enabled requires a clean git worktree") + + return tuple(violations) + + +def _default_branch(root: Path, runner: Callable[..., object]) -> str: + raw = _git_stdout(root, runner, ("git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD")) + if raw.startswith("origin/"): + return raw.removeprefix("origin/") + return raw or "main" + + +def _git_stdout(root: Path, runner: Callable[..., object], command: tuple[str, ...]) -> str: + result = runner(command, cwd=root) + if getattr(result, "returncode", 1) != 0: + return "" + return str(getattr(result, "stdout", "")).strip() diff --git a/examples/sdk_evolution_agent/release_notes.py b/examples/sdk_evolution_agent/release_notes.py index 57f5b63..ede2bc9 100644 --- a/examples/sdk_evolution_agent/release_notes.py +++ b/examples/sdk_evolution_agent/release_notes.py @@ -150,6 +150,13 @@ def collect_release_notes( url=source.url, version=to_version, available=True, + note=( + "html-fallback (degraded)" + if source.kind == "github-discussions" + and use_default_fetcher + and not _github_token() + else "" + ), ) ) if source.kind != "github-discussions": @@ -237,7 +244,7 @@ def fetch_url_text(url: str) -> str: raw = response.read() if raw.startswith(b"\x1f\x8b"): raw = gzip.decompress(raw) - return raw.decode("utf-8", errors="replace") + return str(raw.decode("utf-8", errors="replace")) def _fetch_source_text( @@ -320,7 +327,10 @@ def _format_github_discussions_index(payload: Mapping[str, object], *, category_ repository = _mapping_or_empty(_mapping_or_empty(payload.get("data")).get("repository")) discussions = _mapping_or_empty(repository.get("discussions")) lines: list[str] = [] - for item in discussions.get("nodes") or (): + nodes = discussions.get("nodes") + if not isinstance(nodes, list): + return "" + for item in nodes: if not isinstance(item, Mapping): continue category = _mapping_or_empty(item.get("category")) @@ -346,14 +356,14 @@ def _summaries_for_interval( ) -> list[str]: text = _prefer_github_discussion_body(text) lines = [line.strip() for line in text.splitlines()] - version_patterns = [to_version] + version_patterns = [_version_pattern(to_version)] if from_version: - version_patterns.append(from_version) + version_patterns.append(_version_pattern(from_version)) matches: list[str] = [] for index, line in enumerate(lines): if not line: continue - if any(pattern and pattern in line for pattern in version_patterns): + if any(pattern.search(line) for pattern in version_patterns): matches.append(_clean_summary(line)) for nearby in lines[index + 1 : index + 9]: cleaned = _clean_summary(nearby) @@ -361,14 +371,18 @@ def _summaries_for_interval( matches.append(cleaned) if not matches and to_version: compact = re.sub(r"\s+", " ", text) - version_index = compact.find(to_version) - if version_index >= 0: - start = max(0, version_index - 160) - end = min(len(compact), version_index + 320) + match = _version_pattern(to_version).search(compact) + if match: + start = max(0, match.start() - 160) + end = min(len(compact), match.end() + 320) matches.append(_clean_summary(compact[start:end])) return [match for match in matches if match] +def _version_pattern(version: str) -> re.Pattern[str]: + return re.compile(rf"(? str: match = re.search( r"]*comment-body)(?=[^>]*markdown-body)[^>]*>(?P.*?)", diff --git a/examples/sdk_evolution_agent/report.py b/examples/sdk_evolution_agent/report.py index ec26afa..c6329e6 100644 --- a/examples/sdk_evolution_agent/report.py +++ b/examples/sdk_evolution_agent/report.py @@ -115,9 +115,29 @@ def render_markdown_report( for item in release_notes if isinstance(item, dict) and item.get("to_version") ] - behavior_summary = behavior.get("summary") if isinstance(behavior, dict) else {} + raw_behavior_summary = behavior.get("summary") if isinstance(behavior, dict) else {} + behavior_summary = raw_behavior_summary if isinstance(raw_behavior_summary, dict) else {} behavior_diffs = behavior.get("diffs", []) if isinstance(behavior, dict) else [] promotion = current_state.get("promotion", {}) if isinstance(current_state, dict) else {} + baseline = evidence.get("baseline", {}) if isinstance(evidence, dict) else {} + candidates = evidence.get("update_candidates", []) if isinstance(evidence, dict) else [] + beyond_cap = ( + evidence.get("update_candidates_beyond_cap", []) if isinstance(evidence, dict) else [] + ) + candidate_lines = [ + "- {package}: {from_version} -> {to_version}".format(**item) + for item in candidates + if isinstance(item, dict) + ] + beyond_lines = [ + ( + "- {package}: {from_version} -> {to_version} " + "(blocked_by_cap={blocked_by_cap}, " + "cutoff_delayed_until={cutoff_delayed_until})" + ).format(**item) + for item in beyond_cap + if isinstance(item, dict) + ] return "\n".join( [ "# SDK Evolution Agent Report", @@ -132,6 +152,14 @@ def render_markdown_report( "", *(package_lines or ["- No package evidence collected."]), "", + "## Resolver Candidates", + "", + *(candidate_lines or ["- No adoptable update candidates."]), + "", + "## Beyond-Cap Candidates", + "", + *(beyond_lines or ["- No beyond-cap candidates."]), + "", "## API Diffs", "", f"- Diff count: `{len(api_diffs)}`", @@ -171,6 +199,7 @@ def render_markdown_report( "", "## Current State Baseline", "", + f"- Baseline status: `{baseline.get('status')}`", f"- Promotion status: `{promotion.get('status')}`", f"- Promoted: `{promotion.get('promoted')}`", "", diff --git a/examples/sdk_evolution_agent/resolver.py b/examples/sdk_evolution_agent/resolver.py new file mode 100644 index 0000000..13c4db3 --- /dev/null +++ b/examples/sdk_evolution_agent/resolver.py @@ -0,0 +1,248 @@ +"""Lockfile-diff resolver evidence for SDK evolution runs. + +The resolver copies the minimal project metadata that uv needs into a temporary +directory, runs targeted ``uv lock -P`` there, and diffs the before/after +lockfiles. The real workspace is never mutated by candidate detection. +""" + +from __future__ import annotations + +import re +import shutil +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path + +from examples.sdk_evolution_agent.collectors import ( + CommandRunner, + _call_runner, + cutoff_free_env, + read_pyproject_dependency_specs, + read_uv_lock_versions, + run_command, + upload_time_for_version, +) +from examples.sdk_evolution_agent.models import CommandResult + + +@dataclass(frozen=True) +class UpdateCandidate: + package: str + from_version: str + to_version: str + blocked_by_cap: str | None = None + cutoff_delayed_until: str | None = None + + +def resolve_update_candidates( + root: Path, + packages: Sequence[str], + *, + command_runner: CommandRunner | None = None, +) -> tuple[tuple[UpdateCandidate, ...], CommandResult]: + """Resolve adoptable update candidates by diffing a temp lockfile.""" + + return _resolve_in_temp(root, packages, command_runner=command_runner) + + +def resolve_constraint_horizon_candidates( + root: Path, + packages: Sequence[str], + *, + adoptable: Sequence[UpdateCandidate] = (), + pypi_metadata: Mapping[str, Mapping[str, object]] | None = None, + command_runner: CommandRunner | None = None, +) -> tuple[UpdateCandidate, ...]: + """Resolve candidates hidden by upper caps or the freshness cutoff.""" + + pyproject = root / "pyproject.toml" + if not pyproject.exists(): + return () + original_specs = read_pyproject_dependency_specs(pyproject) + raised_text, raised_caps = raise_upper_bounds_in_pyproject_text( + pyproject.read_text(encoding="utf-8"), packages + ) + candidates, result = _resolve_in_temp( + root, + packages, + pyproject_text=raised_text, + strip_project_cutoff=True, + command_runner=command_runner, + ) + if result.returncode != 0: + return () + + adoptable_keys = {(candidate.package, candidate.to_version) for candidate in adoptable} + metadata = pypi_metadata or {} + horizon: list[UpdateCandidate] = [] + for candidate in candidates: + if (candidate.package, candidate.to_version) in adoptable_keys: + continue + cap = raised_caps.get(candidate.package) + blocked_by_cap = cap.current if cap else _upper_bound(original_specs.get(candidate.package)) + cutoff_delayed_until = None + if blocked_by_cap is None: + upload = upload_time_for_version( + metadata.get(candidate.package, {}), candidate.to_version + ) + if upload is not None: + cutoff_delayed_until = (upload + timedelta(days=8)).date().isoformat() + horizon.append( + UpdateCandidate( + package=candidate.package, + from_version=candidate.from_version, + to_version=candidate.to_version, + blocked_by_cap=blocked_by_cap, + cutoff_delayed_until=cutoff_delayed_until, + ) + ) + return tuple(horizon) + + +@dataclass(frozen=True) +class RaisedCap: + package: str + current: str + replacement: str + + +def raise_upper_bounds_in_pyproject_text( + text: str, packages: Sequence[str], *, versions: Mapping[str, str] | None = None +) -> tuple[str, dict[str, RaisedCap]]: + """Raise only tracked package upper bounds enough to admit candidate versions.""" + + result = text + raised: dict[str, RaisedCap] = {} + for package in packages: + pattern = re.compile( + rf"(?P{re.escape(package)}[^\"']*?,)(?P<\s*\d+(?:\.\d+)*)" + ) + match = pattern.search(result) + if not match: + continue + current = match.group("cap").replace(" ", "") + replacement = _raised_upper_bound(current, versions.get(package) if versions else None) + if replacement == current: + continue + result = result[: match.start("cap")] + replacement + result[match.end("cap") :] + raised[package] = RaisedCap(package=package, current=current, replacement=replacement) + result = _remove_tool_uv_exclude_newer(result) + return result, raised + + +def raise_package_cap_in_workspace(root: Path, package: str, version: str) -> RaisedCap | None: + """Raise one package cap in the real pyproject, preserving floors.""" + + path = root / "pyproject.toml" + text = path.read_text(encoding="utf-8") + updated, raised = raise_upper_bounds_in_pyproject_text( + text, (package,), versions={package: version} + ) + cap = raised.get(package) + if cap is None: + return None + path.write_text(updated, encoding="utf-8") + return cap + + +def candidate_map( + candidates: Sequence[Mapping[str, object]] | Sequence[UpdateCandidate], +) -> dict[str, str]: + """Return package -> candidate version for serialized or typed candidates.""" + + mapped: dict[str, str] = {} + for candidate in candidates: + if isinstance(candidate, UpdateCandidate): + mapped[candidate.package] = candidate.to_version + elif isinstance(candidate, Mapping): + package = candidate.get("package") + version = candidate.get("to_version") + if package and version: + mapped[str(package)] = str(version) + return mapped + + +def _resolve_in_temp( + root: Path, + packages: Sequence[str], + *, + pyproject_text: str | None = None, + strip_project_cutoff: bool = False, + command_runner: CommandRunner | None = None, +) -> tuple[tuple[UpdateCandidate, ...], CommandResult]: + runner = command_runner or run_command + baseline = read_uv_lock_versions(root / "uv.lock") + with tempfile.TemporaryDirectory(prefix="ark-sdk-resolver-") as directory: + temp = Path(directory) + _copy_minimal_project(root, temp) + if pyproject_text is not None: + text = ( + _remove_tool_uv_exclude_newer(pyproject_text) + if strip_project_cutoff + else pyproject_text + ) + (temp / "pyproject.toml").write_text(text, encoding="utf-8") + env, removed = cutoff_free_env() + command = ["uv", "lock"] + for package in packages: + command.extend(("-P", package)) + result = _call_runner(runner, tuple(command), cwd=temp, env=env, timeout=300) + result = CommandResult( + command=result.command, + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + removed_env=removed, + ) + if result.returncode != 0: + return (), result + resolved = read_uv_lock_versions(temp / "uv.lock") + candidates = tuple( + UpdateCandidate(package=package, from_version=from_version, to_version=to_version) + for package in packages + if (from_version := baseline.get(package)) + and (to_version := resolved.get(package)) + and to_version != from_version + ) + return candidates, result + + +def _copy_minimal_project(root: Path, temp: Path) -> None: + for name in ("pyproject.toml", "uv.lock", "README.md"): + source = root / name + target = temp / name + if source.exists(): + shutil.copy2(source, target) + elif name == "README.md": + target.write_text("# temporary SDK evolution resolver project\n", encoding="utf-8") + + +def _raised_upper_bound(current: str, version: str | None) -> str: + numbers = [int(part) for part in current.removeprefix("<").split(".") if part.isdigit()] + if version: + release = [int(part) for part in re.split(r"[^\d]+", version) if part.isdigit()] + while len(release) < 2: + release.append(0) + if len(numbers) >= 2 and release[:2] < numbers[:2]: + return current + if len(numbers) >= 2: + return f"<{release[0]}.{release[1] + 1}" + return f"<{release[0] + 1}" + if len(numbers) >= 2: + return f"<{numbers[0]}.{numbers[1] + 1}" + if numbers: + return f"<{numbers[0] + 1}" + return current + + +def _upper_bound(spec: str | None) -> str | None: + if not spec: + return None + match = re.search(r"<\s*\d+(?:\.\d+)*", spec) + return match.group(0).replace(" ", "") if match else None + + +def _remove_tool_uv_exclude_newer(text: str) -> str: + return re.sub(r'(?m)^\s*exclude-newer\s*=\s*["\'][^"\']+["\']\s*\n?', "", text) diff --git a/examples/sdk_evolution_agent/schemas.py b/examples/sdk_evolution_agent/schemas.py index 1c73b83..e8d518d 100644 --- a/examples/sdk_evolution_agent/schemas.py +++ b/examples/sdk_evolution_agent/schemas.py @@ -2,8 +2,10 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import Any +from dataclasses import asdict, dataclass, field, is_dataclass +from typing import Any, Literal, cast + +from agent_runtime_kit import OutputTypeError, json_schema_for, parse_as JsonSchema = dict[str, Any] @@ -12,116 +14,155 @@ class SchemaValidationError(ValueError): """Raised when a runtime output does not satisfy the required shape.""" -DIRECTION_ANALYSIS_SCHEMA: JsonSchema = { - "type": "object", - "required": ["packages", "themes", "uncertainty"], - "additionalProperties": False, - "properties": { - "packages": { - "type": "array", - "items": { - "type": "object", - "required": ["name", "direction", "evidence"], - "additionalProperties": False, - "properties": { - "name": {"type": "string"}, - "direction": {"type": "string"}, - "evidence": {"type": "array", "items": {"type": "string"}}, - }, - }, - }, - "themes": { - "type": "array", - "items": { - "type": "object", - "required": ["name", "summary"], - "additionalProperties": False, - "properties": { - "name": {"type": "string"}, - "summary": {"type": "string"}, - }, - }, - }, - "uncertainty": {"type": "array", "items": {"type": "string"}}, - }, -} +@dataclass(frozen=True) +class DirectionPackage: + name: str + direction: str + evidence: list[str] -ARCHITECTURE_DECISION_SCHEMA: JsonSchema = { - "type": "object", - "required": [ - "findings", - "safe_to_implement", - "manual_design_required", - "recursive_self_adaptation_impact", - "self_adaptation_plan", - "verification_commands", - "uncertainty", - ], - "additionalProperties": False, - "properties": { - "findings": { - "type": "array", - "items": { - "type": "object", - "required": ["classification", "summary", "evidence"], - "additionalProperties": False, - "properties": { - "classification": {"type": "string"}, - "summary": {"type": "string"}, - "evidence": {"type": "array", "items": {"type": "string"}}, - }, - }, - }, - "safe_to_implement": {"type": "boolean"}, - "manual_design_required": {"type": "boolean"}, - "recursive_self_adaptation_impact": {"type": "boolean"}, - "self_adaptation_plan": {"type": "array", "items": {"type": "string"}}, - "verification_commands": {"type": "array", "items": {"type": "string"}}, - "uncertainty": {"type": "array", "items": {"type": "string"}}, - }, -} -IMPLEMENTATION_SUMMARY_SCHEMA: JsonSchema = { - "type": "object", - "required": ["applied", "changes", "verification_results", "blocked_reason"], - "additionalProperties": False, - "properties": { - "applied": {"type": "boolean"}, - "changes": {"type": "array", "items": {"type": "string"}}, - "verification_results": {"type": "array", "items": {"type": "string"}}, - "blocked_reason": {"type": "string"}, - }, -} +@dataclass(frozen=True) +class DirectionTheme: + name: str + summary: str + + +@dataclass(frozen=True) +class DirectionAnalysis: + packages: list[DirectionPackage] + themes: list[DirectionTheme] + uncertainty: list[str] + + +@dataclass(frozen=True) +class ArchitectureFinding: + classification: str + summary: str + evidence: list[str] + + +@dataclass(frozen=True) +class ArchitectureDecision: + findings: list[ArchitectureFinding] + safe_to_implement: bool + manual_design_required: bool + recursive_self_adaptation_impact: bool + self_adaptation_plan: list[str] + verification_commands: list[str] + uncertainty: list[str] + docs_test_changes: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class ReviewerOutput: + status: Literal["pass", "reject"] + reasons: list[str] + required_changes: list[str] -REVIEWER_OUTPUT_SCHEMA: JsonSchema = { - "type": "object", - "required": ["status", "reasons", "required_changes"], - "additionalProperties": False, - "properties": { - "status": {"type": "string", "enum": ["pass", "reject"]}, - "reasons": {"type": "array", "items": {"type": "string"}}, - "required_changes": {"type": "array", "items": {"type": "string"}}, - }, + +@dataclass(frozen=True) +class ImplementationSummary: + applied: bool + changes: list[str] + blocked_reason: str + + +DIRECTION_ANALYSIS_SCHEMA: JsonSchema = json_schema_for(DirectionAnalysis) +ARCHITECTURE_DECISION_SCHEMA: JsonSchema = json_schema_for(ArchitectureDecision) +IMPLEMENTATION_SUMMARY_SCHEMA: JsonSchema = json_schema_for(ImplementationSummary) +REVIEWER_OUTPUT_SCHEMA: JsonSchema = json_schema_for(ReviewerOutput) + +_STAGE_OUTPUT_TYPES: dict[str, Any] = { + "direction-analysis": DirectionAnalysis, + "architecture-decision": ArchitectureDecision, + "review": ReviewerOutput, + "implementation": ImplementationSummary, + "review-implementation": ReviewerOutput, } +_SCHEMA_OUTPUT_TYPES: tuple[tuple[JsonSchema, Any], ...] = ( + (DIRECTION_ANALYSIS_SCHEMA, DirectionAnalysis), + (ARCHITECTURE_DECISION_SCHEMA, ArchitectureDecision), + (REVIEWER_OUTPUT_SCHEMA, ReviewerOutput), + (IMPLEMENTATION_SUMMARY_SCHEMA, ImplementationSummary), +) + + +def parse_stage_output(stage: str, data: Any) -> dict[str, Any]: + """Validate a stage payload through the kit's typed bridge.""" + output_type = _STAGE_OUTPUT_TYPES.get(stage) + if output_type is None: + if not isinstance(data, dict): + raise SchemaValidationError(f"{stage} must be an object") + return dict(data) + try: + parsed = parse_as(output_type, data) + except OutputTypeError as exc: + raise SchemaValidationError(str(exc)) from exc + return _dataclass_to_dict(parsed) -def validate_mapping(data: Any, schema: Mapping[str, Any], *, name: str) -> dict[str, Any]: - """Validate the small schema subset used by this example.""" +def validate_mapping(data: Any, schema: JsonSchema, *, name: str) -> dict[str, Any]: + """Validate the small schema subset used by this example. + + The runtime stages use :func:`parse_stage_output`; this wrapper remains for + direct tests and for any future caller that only has a JSON schema object. + """ + + for known_schema, output_type in _SCHEMA_OUTPUT_TYPES: + if schema == known_schema: + try: + parsed = parse_as(output_type, data) + except OutputTypeError as exc: + raise SchemaValidationError(str(exc)) from exc + return _dataclass_to_dict(parsed) + _validate_schema_value(data, schema, path=name) if not isinstance(data, dict): raise SchemaValidationError(f"{name} must be an object") - for field in schema.get("required", ()): - if field not in data: - raise SchemaValidationError(f"{name} missing required field {field!r}") - properties = schema.get("properties", {}) - for field, field_schema in properties.items(): - if field not in data: - continue - expected = field_schema.get("type") - if expected == "array" and not isinstance(data[field], list): - raise SchemaValidationError(f"{name}.{field} must be an array") - if expected == "boolean" and not isinstance(data[field], bool): - raise SchemaValidationError(f"{name}.{field} must be a boolean") - if expected == "string" and not isinstance(data[field], str): - raise SchemaValidationError(f"{name}.{field} must be a string") - return data + return dict(data) + + +def _dataclass_to_dict(value: Any) -> dict[str, Any]: + if not is_dataclass(value): + raise SchemaValidationError("validated stage output did not produce a dataclass") + return asdict(cast(Any, value)) + + +def _validate_schema_value(value: Any, schema: JsonSchema, *, path: str) -> None: + if "enum" in schema and value not in schema["enum"]: + raise SchemaValidationError(f"{path} must be one of {schema['enum']!r}") + expected = schema.get("type") + if expected == "object": + if not isinstance(value, dict): + raise SchemaValidationError(f"{path} must be an object") + properties = schema.get("properties", {}) + required = schema.get("required", ()) + for field in required: + if field not in value: + raise SchemaValidationError(f"{path} missing required field {field!r}") + if schema.get("additionalProperties") is False: + extra = sorted(set(value) - set(properties)) + if extra: + raise SchemaValidationError(f"{path} has unknown field(s): {', '.join(extra)}") + for field, item in value.items(): + if field in properties: + _validate_schema_value(item, properties[field], path=f"{path}.{field}") + return + if expected == "array": + if not isinstance(value, list): + raise SchemaValidationError(f"{path} must be an array") + item_schema = schema.get("items") + if isinstance(item_schema, dict): + for index, item in enumerate(value): + _validate_schema_value(item, item_schema, path=f"{path}[{index}]") + return + if expected == "boolean" and not isinstance(value, bool): + raise SchemaValidationError(f"{path} must be a boolean") + if expected == "string" and not isinstance(value, str): + raise SchemaValidationError(f"{path} must be a string") + if expected == "integer" and (not isinstance(value, int) or isinstance(value, bool)): + raise SchemaValidationError(f"{path} must be an integer") + if expected == "number" and ( + not isinstance(value, int | float) or isinstance(value, bool) + ): + raise SchemaValidationError(f"{path} must be a number") diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py index f198536..6e136f6 100644 --- a/examples/sdk_evolution_agent/snapshots.py +++ b/examples/sdk_evolution_agent/snapshots.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib +import importlib.metadata import inspect import json import os @@ -28,6 +29,7 @@ def snapshot_current_api(package: str, *, version: str | None = None) -> ApiSnap """Capture public API for a package importable in the current environment.""" module_name = DEFAULT_MODULES.get(package, package.replace("-", "_")) + observed = _observed_version(package) try: module = importlib.import_module(module_name) except Exception as exc: @@ -35,6 +37,9 @@ def snapshot_current_api(package: str, *, version: str | None = None) -> ApiSnap package=package, version=version, module=module_name, + requested_version=version, + observed_version=observed, + provenance=_provenance(version, observed), import_error=str(exc), ) members: list[ApiMember] = [] @@ -51,8 +56,11 @@ def snapshot_current_api(package: str, *, version: str | None = None) -> ApiSnap ) return ApiSnapshot( package=package, - version=version or str(getattr(module, "__version__", "") or ""), + version=version or observed or str(getattr(module, "__version__", "") or ""), module=module_name, + requested_version=version, + observed_version=observed, + provenance=_provenance(version, observed), members=tuple(sorted(members, key=lambda item: item.name)), ) @@ -134,19 +142,26 @@ def snapshot_candidate_in_venv( # upstream code: give it a throwaway HOME and only PATH, so a malicious or # buggy candidate package cannot read the caller's credentials/config. env = isolated_env(Path(directory)) - subprocess.run( - (python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env + venv_result = _run_snapshot_subprocess( + (python, "-m", "venv", str(venv)), env=env, timeout=timeout ) + if venv_result is not None: + return _failed_snapshot( + package, version, module_name, venv_result, "venv creation failed" + ) bin_dir = "Scripts" if sys.platform == "win32" else "bin" venv_python = venv / bin_dir / "python" - subprocess.run( + install_result = _run_snapshot_subprocess( (str(venv_python), "-m", "pip", "install", f"{package}=={version}"), - check=True, text=True, capture_output=True, timeout=timeout, env=env, ) + if install_result is not None: + return _failed_snapshot( + package, version, module_name, install_result, "pip install failed" + ) completed = subprocess.run( ( str(venv_python), @@ -156,17 +171,28 @@ def snapshot_candidate_in_venv( version, module_name, ), - check=True, text=True, capture_output=True, timeout=timeout, + check=False, env=env, ) + if completed.returncode != 0: + return _failed_snapshot( + package, + version, + module_name, + completed.stderr or completed.stdout, + "snapshot probe failed", + ) raw = json.loads(completed.stdout) return ApiSnapshot( package=raw["package"], version=raw["version"], module=raw["module"], + requested_version=raw.get("requested_version"), + observed_version=raw.get("observed_version"), + provenance=raw.get("provenance", "observed"), members=tuple(ApiMember(**item) for item in raw.get("members", ())), import_error=raw.get("import_error"), source="isolated-venv", @@ -190,14 +216,86 @@ def _signature(value: Any) -> str: return "" +def _observed_version(package: str) -> str | None: + try: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + return None + + +def _provenance(requested: str | None, observed: str | None) -> str: + if requested and observed and requested != observed: + return "mismatched" + if observed is None: + return "not-observed" + return "observed" + + +def _tail(text: str, *, limit: int = 500) -> str: + return text[-limit:] + + +def _failed_snapshot( + package: str, + version: str, + module_name: str, + stderr: str, + reason: str, +) -> ApiSnapshot: + return ApiSnapshot( + package=package, + version=version, + module=module_name, + requested_version=version, + observed_version=None, + provenance="not-observed", + import_error=f"{reason}: {_tail(stderr)}", + source="isolated-venv", + ) + + +def _run_snapshot_subprocess( + command: tuple[str, ...], + *, + env: dict[str, str], + timeout: int, + text: bool = True, + capture_output: bool = True, +) -> str | None: + try: + completed = subprocess.run( + command, + text=text, + capture_output=capture_output, + timeout=timeout, + check=False, + env=env, + ) + except subprocess.TimeoutExpired: + return f"timed out after {timeout}s" + if completed.returncode == 0: + return None + return _tail(completed.stderr or completed.stdout) + + _SNAPSHOT_SCRIPT = textwrap.dedent( """ import importlib + import importlib.metadata import inspect import json import sys package, version, module_name = sys.argv[1:4] + try: + observed_version = importlib.metadata.version(package) + except Exception: + observed_version = None + provenance = ( + "mismatched" if version and observed_version and version != observed_version + else "not-observed" if observed_version is None + else "observed" + ) try: module = importlib.import_module(module_name) members = [] @@ -225,6 +323,9 @@ def _signature(value: Any) -> str: payload = { "package": package, "version": version, + "requested_version": version, + "observed_version": observed_version, + "provenance": provenance, "module": module_name, "members": sorted(members, key=lambda item: item["name"]), "import_error": None, @@ -233,6 +334,9 @@ def _signature(value: Any) -> str: payload = { "package": package, "version": version, + "requested_version": version, + "observed_version": observed_version, + "provenance": provenance, "module": module_name, "members": [], "import_error": str(exc), diff --git a/examples/sdk_evolution_agent/stages.py b/examples/sdk_evolution_agent/stages.py index 15c9c53..3546e1a 100644 --- a/examples/sdk_evolution_agent/stages.py +++ b/examples/sdk_evolution_agent/stages.py @@ -2,8 +2,9 @@ from __future__ import annotations +import importlib.metadata import json -import re +import os from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any @@ -39,10 +40,11 @@ from examples.sdk_evolution_agent.schemas import ( ARCHITECTURE_DECISION_SCHEMA, DIRECTION_ANALYSIS_SCHEMA, + IMPLEMENTATION_SUMMARY_SCHEMA, REVIEWER_OUTPUT_SCHEMA, JsonSchema, SchemaValidationError, - validate_mapping, + parse_stage_output, ) @@ -53,6 +55,7 @@ class StageExecutionError(RuntimeError): SDK_EVOLUTION_CODEX_HOME = Path("~/.codex_agent_runtime_sdk").expanduser() SDK_EVOLUTION_CODEX_MODEL = "gpt-5.5" SDK_EVOLUTION_CODEX_REASONING_EFFORT = "xhigh" +ADAPTED_FOR_KIT_VERSION = "0.4.0" class FixtureEvolutionRuntime: @@ -196,7 +199,8 @@ async def run_stage( except json.JSONDecodeError as exc: raise StageExecutionError(f"{stage} returned no valid structured output") from exc try: - return validate_mapping(data, schema, name=stage) + del schema + return parse_stage_output(stage, data) except SchemaValidationError as exc: raise StageExecutionError(f"{stage} returned invalid structured output: {exc}") from exc @@ -239,7 +243,12 @@ async def run_analysis_pipeline( schema=ARCHITECTURE_DECISION_SCHEMA, context=context, ) - architecture = with_recursive_impact(architecture, api_diffs) + architecture = with_kit_version_impact(architecture) + architecture = with_recursive_impact( + architecture, + api_diffs, + implementation_enabled=context.implementation_enabled, + ) architecture = with_candidate_api_diff_guard(architecture, evidence, api_diffs) architecture = with_release_note_guard(architecture, release_notes) architecture = with_behavior_probe_guard(architecture, behavior) @@ -297,6 +306,41 @@ async def maybe_run_implementation( } +async def run_implementation_stage( + runtime: AgentRuntime, + *, + payload: Mapping[str, Any], + context: RunContext, +) -> dict[str, Any]: + """Run the bounded write-enabled implementation stage.""" + + return await run_stage( + runtime, + stage="implementation", + payload=payload, + schema=IMPLEMENTATION_SUMMARY_SCHEMA, + context=context, + write_enabled=True, + ) + + +async def run_implementation_review_stage( + runtime: AgentRuntime, + *, + payload: Mapping[str, Any], + context: RunContext, +) -> dict[str, Any]: + """Review the implementation diff after deterministic verification.""" + + return await run_stage( + runtime, + stage="review-implementation", + payload=payload, + schema=REVIEWER_OUTPUT_SCHEMA, + context=context, + ) + + def evaluate_implementation_gate( architecture: Mapping[str, Any], review: Mapping[str, Any], @@ -321,8 +365,7 @@ def evaluate_implementation_gate( def _review_passed(review: Mapping[str, Any]) -> bool: - status = str(review.get("status", "")).strip().lower() - return status in {"pass", "passed", "approve", "approved", "accepted"} + return review.get("status") == "pass" def detects_recursive_impact(api_diffs: Sequence[Mapping[str, Any] | ApiDiff]) -> bool: @@ -347,6 +390,8 @@ def detects_recursive_impact(api_diffs: Sequence[Mapping[str, Any] | ApiDiff]) - def with_recursive_impact( architecture: Mapping[str, Any], api_diffs: Sequence[Mapping[str, Any] | ApiDiff], + *, + implementation_enabled: bool = False, ) -> dict[str, Any]: """Ensure recursive runtime-contract impacts are explicit.""" @@ -354,13 +399,12 @@ def with_recursive_impact( if not detects_recursive_impact(api_diffs): return result result["recursive_self_adaptation_impact"] = True - result.setdefault( - "self_adaptation_plan", - [ - "Update examples/sdk_evolution_agent runtime usage, schemas, tests, and docs " - "in the same scoped change.", - ], + plan = [str(item) for item in result.get("self_adaptation_plan") or []] + has_executable_plan = implementation_enabled and any( + item.startswith("examples/sdk_evolution_agent/") for item in plan ) + if not has_executable_plan: + result["manual_design_required"] = True # relaxed in T3.3 when plan is executable. findings = list(result.get("findings") or []) findings.append( { @@ -373,6 +417,32 @@ def with_recursive_impact( return result +def with_kit_version_impact(architecture: Mapping[str, Any]) -> dict[str, Any]: + """Flag the example when the installed kit version moves past its adaptation marker.""" + + try: + current = importlib.metadata.version("agent-runtime-kit") + except importlib.metadata.PackageNotFoundError: + current = ADAPTED_FOR_KIT_VERSION + if current == ADAPTED_FOR_KIT_VERSION: + return dict(architecture) + result = dict(architecture) + result["recursive_self_adaptation_impact"] = True + findings = list(result.get("findings") or []) + findings.append( + { + "classification": "manual-design-required", + "summary": ( + "SDK evolution example adaptation marker does not match " + f"agent-runtime-kit {current}." + ), + "evidence": ["agent-runtime-kit version"], + } + ) + result["findings"] = findings + return result + + def with_candidate_api_diff_guard( architecture: Mapping[str, Any], evidence: Mapping[str, Any], @@ -380,7 +450,13 @@ def with_candidate_api_diff_guard( ) -> dict[str, Any]: """Block SDK update implementation when candidate API evidence is missing.""" - update_packages = _refresh_update_packages(evidence) + update_packages = tuple( + sorted( + str(item.get("package")) + for item in evidence.get("update_candidates", []) + if isinstance(item, Mapping) and item.get("package") + ) + ) if not update_packages: return dict(architecture) diff_packages = { @@ -498,14 +574,63 @@ def with_manual_design_gate(architecture: Mapping[str, Any]) -> dict[str, Any]: return result -def _refresh_update_packages(evidence: Mapping[str, Any]) -> tuple[str, ...]: - preview = evidence.get("refresh_preview") - if not isinstance(preview, Mapping): - return () - text = f"{preview.get('stdout') or ''}\n{preview.get('stderr') or ''}" - return tuple( - sorted(set(re.findall(r"Update\s+([A-Za-z0-9_.-]+)\s+v\S+\s+->\s+v\S+", text))) - ) +def with_cap_raise_guard( + architecture: Mapping[str, Any], + *, + candidate: Mapping[str, Any], + api_diffs: Sequence[Mapping[str, Any] | ApiDiff], + release_notes: Sequence[Mapping[str, Any]], + behavior: Mapping[str, Any], + review: Mapping[str, Any], +) -> GateResult: + """Gate a beyond-cap candidate before pyproject mutation.""" + + del architecture + package = str(candidate.get("package") or "") + version = str(candidate.get("to_version") or "") + if not package or not version: + return GateResult(False, "cap raise candidate is incomplete") + if not _review_passed(review): + return GateResult(False, "reviewer did not pass the proposal") + if not any(_diff_matches(item, package, version) for item in api_diffs): + return GateResult(False, f"missing candidate API diff for {package} {version}") + if not _release_notes_ok(release_notes, package, version): + return GateResult(False, f"release-note evidence unavailable for {package} {version}") + if not _behavior_probe_passed(behavior, package, version): + return GateResult( + False, f"candidate behavior probe missing or failed for {package} {version}" + ) + return GateResult(True, "cap raise gates passed") + + +def _diff_matches(item: Mapping[str, Any] | ApiDiff, package: str, version: str) -> bool: + if isinstance(item, ApiDiff): + return item.package == package and item.to_version == version + return str(item.get("package")) == package and str(item.get("to_version")) == version + + +def _release_notes_ok( + release_notes: Sequence[Mapping[str, Any]], package: str, version: str +) -> bool: + for item in release_notes: + if str(item.get("package")) != package or str(item.get("to_version")) != version: + continue + return str(item.get("status")) in {"found", "no-matching-version"} + return False + + +def _behavior_probe_passed(behavior: Mapping[str, Any], package: str, version: str) -> bool: + results = behavior.get("results") + if not isinstance(results, list): + return False + for item in results: + if not isinstance(item, Mapping): + continue + if str(item.get("package")) != package or str(item.get("version")) != version: + continue + if str(item.get("scope")) == "candidate" and str(item.get("status")) == "pass": + return True + return False def _compact_stage_output(value: Mapping[str, Any]) -> dict[str, Any]: @@ -559,10 +684,20 @@ def _stage_system_prompt(stage: str, schema: JsonSchema) -> str: "reviewer-identified unsupported vendor behavior, or recursive " "runtime-contract impact remain hard blockers. Release-note status found " "is direct release-note evidence. Status no-matching-version is source " - "coverage with explicit uncertainty, not unavailable evidence." + "coverage with explicit uncertainty, not unavailable evidence. " + "Adoptable update candidates are dependency changes allowed by the " + "current pyproject caps. Beyond-cap candidates are future releases " + "visible only after a bounded cap-horizon probe. Use classification " + "capability-opportunity for non-blocking upstream features worth " + "tracking separately." + ) + if stage == "implementation": + prompt += ( + " You may edit only files under examples/sdk_evolution_agent/, " + "tests/test_sdk_evolution_*.py, and docs/sdk-evolution-agent*.md. " + "Do not run uv, git, gh, or verification commands; deterministic " + "mechanical steps and verification are owned by the CLI." ) - if stage == "review": - prompt += " The review status must be exactly pass or reject." return prompt @@ -600,6 +735,7 @@ def _fixture_payload(stage: str, task: AgentTask) -> dict[str, Any]: source = json.loads(task.goal) except json.JSONDecodeError: source = {} + permissive = os.environ.get("SDK_EVOLUTION_FIXTURE_PROFILE") == "permissive" if stage == "direction-analysis": packages = [ { @@ -621,18 +757,31 @@ def _fixture_payload(stage: str, task: AgentTask) -> dict[str, Any]: "uncertainty": ["fake runtime cannot infer real upstream product direction"], } if stage == "architecture-decision": - return { - "findings": [], - "safe_to_implement": False, - "manual_design_required": False, - "recursive_self_adaptation_impact": detects_recursive_impact( - source.get("api_diffs", []) - ), - "self_adaptation_plan": [], - "verification_commands": ["uv run pytest tests/test_sdk_evolution_agent.py"], - "uncertainty": ["fixture decision is conservative"], - } - if stage == "review": + recursive = detects_recursive_impact(source.get("api_diffs", [])) + return ( + { + "findings": [], + "safe_to_implement": True, + "manual_design_required": False, + "recursive_self_adaptation_impact": recursive, + "self_adaptation_plan": [], + "verification_commands": [], + "uncertainty": [], + "docs_test_changes": [], + } + if permissive + else { + "findings": [], + "safe_to_implement": False, + "manual_design_required": False, + "recursive_self_adaptation_impact": recursive, + "self_adaptation_plan": [], + "verification_commands": ["uv run pytest tests/test_sdk_evolution_agent.py"], + "uncertainty": ["fixture decision is conservative"], + "docs_test_changes": [], + } + ) + if stage in {"review", "review-implementation"}: return { "status": "pass", "reasons": ["fixture review only verifies the pipeline shape"], @@ -642,7 +791,6 @@ def _fixture_payload(stage: str, task: AgentTask) -> dict[str, Any]: return { "applied": False, "changes": [], - "verification_results": [], "blocked_reason": "fixture runtime does not edit files", } return {"status": "unknown", "reasons": [], "required_changes": []} diff --git a/examples/sdk_evolution_agent/status_issue.py b/examples/sdk_evolution_agent/status_issue.py new file mode 100644 index 0000000..5687cd0 --- /dev/null +++ b/examples/sdk_evolution_agent/status_issue.py @@ -0,0 +1,247 @@ +"""Create or update the rolling SDK evolution status issue.""" + +from __future__ import annotations + +import argparse +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from examples.sdk_evolution_agent.collectors import CommandRunner, _call_runner, run_command +from examples.sdk_evolution_agent.models import CommandResult + +STATUS_TITLE = "SDK evolution status" +STATUS_LABEL = "sdk-evolution" +CAPABILITY_LABEL = "sdk-evolution-capability" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("run_dir", type=Path) + args = parser.parse_args(argv) + update_status_issue(args.run_dir) + return 0 + + +def update_status_issue( + run_dir: Path, *, command_runner: CommandRunner | None = None +) -> tuple[CommandResult, ...]: + """Update the rolling status issue from one run directory.""" + + runner = command_runner or run_command + evidence = _load_json(run_dir / "evidence.json", {}) + architecture = _load_json(run_dir / "architecture_decision.json", {}) + body = render_status_body(run_dir, evidence=evidence, architecture=architecture) + findings = _status_findings(evidence, architecture) + results: list[CommandResult] = [ + _gh(runner, ("gh", "label", "create", STATUS_LABEL, "--force"), cwd=run_dir), + _gh(runner, ("gh", "label", "create", CAPABILITY_LABEL, "--force"), cwd=run_dir), + ] + existing = _find_issue(STATUS_TITLE, runner=runner, cwd=run_dir) + if findings: + if existing: + results.append( + _gh( + runner, + ("gh", "issue", "edit", str(existing["number"]), "--body", body), + cwd=run_dir, + ) + ) + if existing.get("state") == "CLOSED": + results.append( + _gh(runner, ("gh", "issue", "reopen", str(existing["number"])), cwd=run_dir) + ) + else: + results.append( + _gh( + runner, + ( + "gh", + "issue", + "create", + "--title", + STATUS_TITLE, + "--label", + STATUS_LABEL, + "--body", + body, + ), + cwd=run_dir, + ) + ) + elif existing and existing.get("state") != "CLOSED": + results.append( + _gh( + runner, + ( + "gh", + "issue", + "close", + str(existing["number"]), + "--comment", + f"All current as of {_timestamp()}.", + ), + cwd=run_dir, + ) + ) + results.extend(_file_capability_issues(architecture, runner=runner, cwd=run_dir)) + return tuple(results) + + +def render_status_body( + run_dir: Path, *, evidence: dict[str, Any], architecture: dict[str, Any] +) -> str: + candidates = evidence.get("update_candidates") or [] + beyond = evidence.get("update_candidates_beyond_cap") or [] + blockers = _blockers(architecture) + artifact = _workflow_artifact_url() + lines = [ + f"Updated: {_timestamp()}", + "", + "| Type | Package | From | To | Note |", + "| --- | --- | --- | --- | --- |", + ] + for item in candidates: + if isinstance(item, dict): + lines.append(_candidate_row("adoptable", item, "")) + for item in beyond: + if isinstance(item, dict): + note = str(item.get("blocked_by_cap") or item.get("cutoff_delayed_until") or "") + lines.append(_candidate_row("beyond-cap", item, note)) + if not candidates and not beyond: + lines.append("| none | - | - | - | - |") + lines.extend(["", "## Gate Blockers", ""]) + lines.extend([f"- {blocker}" for blocker in blockers] or ["- None"]) + lines.extend( + [ + "", + "## Artifacts", + "", + f"- Run directory: `{run_dir}`", + f"- Workflow artifact: {artifact or 'not available'}", + ] + ) + return "\n".join(lines) + "\n" + + +def _candidate_row(kind: str, item: dict[str, Any], note: str) -> str: + return "| {kind} | {package} | {from_version} | {to_version} | {note} |".format( + kind=kind, + package=item.get("package", ""), + from_version=item.get("from_version", ""), + to_version=item.get("to_version", ""), + note=note, + ) + + +def _status_findings(evidence: dict[str, Any], architecture: dict[str, Any]) -> list[str]: + findings: list[str] = [] + findings.extend(str(item) for item in evidence.get("update_candidates") or []) + findings.extend(str(item) for item in evidence.get("update_candidates_beyond_cap") or []) + findings.extend(_blockers(architecture)) + return findings + + +def _blockers(architecture: dict[str, Any]) -> list[str]: + blockers: list[str] = [] + if architecture.get("manual_design_required"): + blockers.append("manual_design_required") + if architecture.get("safe_to_implement") is False: + blockers.append("safe_to_implement=false") + blockers.extend(str(item) for item in architecture.get("uncertainty") or []) + return blockers + + +def _file_capability_issues( + architecture: dict[str, Any], *, runner: CommandRunner, cwd: Path +) -> tuple[CommandResult, ...]: + results: list[CommandResult] = [] + for finding in architecture.get("findings") or []: + if ( + not isinstance(finding, dict) + or finding.get("classification") != "capability-opportunity" + ): + continue + summary = str(finding.get("summary") or "SDK evolution capability opportunity")[:80] + title = f"[sdk-evolution] {summary}" + if _find_issue(title, runner=runner, cwd=cwd): + continue + body = json.dumps(finding, indent=2, sort_keys=True) + results.append( + _gh( + runner, + ( + "gh", + "issue", + "create", + "--title", + title, + "--label", + CAPABILITY_LABEL, + "--body", + body, + ), + cwd=cwd, + ) + ) + return tuple(results) + + +def _find_issue(title: str, *, runner: CommandRunner, cwd: Path) -> dict[str, Any] | None: + result = _gh( + runner, + ( + "gh", + "issue", + "list", + "--state", + "all", + "--search", + f"{title} in:title", + "--json", + "number,state,title", + ), + cwd=cwd, + ) + if result.returncode != 0: + return None + try: + payload = json.loads(result.stdout or "[]") + except json.JSONDecodeError: + return None + if not isinstance(payload, list): + return None + for item in payload: + if isinstance(item, dict) and item.get("title") == title: + return item + return None + + +def _gh(runner: CommandRunner, command: tuple[str, ...], *, cwd: Path) -> CommandResult: + return _call_runner(runner, command, cwd=cwd, timeout=120) + + +def _load_json(path: Path, default: Any) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return default + + +def _timestamp() -> str: + return datetime.now(tz=timezone.utc).isoformat(timespec="seconds") + + +def _workflow_artifact_url() -> str: + server = os.environ.get("GITHUB_SERVER_URL") + repo = os.environ.get("GITHUB_REPOSITORY") + run_id = os.environ.get("GITHUB_RUN_ID") + if not (server and repo and run_id): + return "" + return f"{server}/{repo}/actions/runs/{run_id}" + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 9d36756..1d0cb66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,7 @@ select = ["E", "F", "I", "UP", "B"] [tool.mypy] python_version = "3.10" -packages = ["agent_runtime_kit"] +packages = ["agent_runtime_kit", "examples.sdk_evolution_agent"] strict = true warn_unused_configs = true diff --git a/src/agent_runtime_kit/__init__.py b/src/agent_runtime_kit/__init__.py index 3a98d79..0a259f8 100644 --- a/src/agent_runtime_kit/__init__.py +++ b/src/agent_runtime_kit/__init__.py @@ -9,6 +9,7 @@ ) from agent_runtime_kit._kit import KIND_ALIASES, AgentKit from agent_runtime_kit._runtime import FakeAgentRuntime +from agent_runtime_kit._schema import json_schema_for, parse_as from agent_runtime_kit._types import ( AgentCapabilities, AgentResult, @@ -71,6 +72,8 @@ "UnsupportedTaskInputError", "Usage", "create_default_registry", + "json_schema_for", + "parse_as", "runtime_kind_value", "output_delta_event", "safe_emit", diff --git a/src/agent_runtime_kit/adapters/antigravity.py b/src/agent_runtime_kit/adapters/antigravity.py index acb705f..e182648 100644 --- a/src/agent_runtime_kit/adapters/antigravity.py +++ b/src/agent_runtime_kit/adapters/antigravity.py @@ -53,6 +53,33 @@ logger = logging.getLogger(__name__) +VENDOR_CONTRACT = { + "config_fields": frozenset( + { + "model", + "api_key", + "vertex", + "project", + "location", + "system_instructions", + "capabilities", + "policies", + "workspaces", + "conversation_id", + "save_dir", + "app_data_dir", + "response_schema", + "mcp_servers", + } + ), + "required_imports": ( + "google.antigravity", + "google.antigravity.types", + "google.antigravity.agent", + "google.antigravity.hooks.policy", + ), +} + class AntigravityAgentRuntime: """Run tasks through Google's ``google-antigravity`` SDK.""" diff --git a/src/agent_runtime_kit/adapters/claude.py b/src/agent_runtime_kit/adapters/claude.py index 3a5c716..f4d6578 100644 --- a/src/agent_runtime_kit/adapters/claude.py +++ b/src/agent_runtime_kit/adapters/claude.py @@ -49,6 +49,24 @@ logger = logging.getLogger(__name__) +VENDOR_CONTRACT = { + "options_fields": frozenset( + { + "model", + "allowed_tools", + "disallowed_tools", + "permission_mode", + "system_prompt", + "cwd", + "mcp_servers", + "resume", + "env", + "max_budget_usd", + "output_format", + } + ) +} + class ClaudeAgentRuntime: """Run tasks through ``claude-agent-sdk`` using the shared runtime API.""" diff --git a/src/agent_runtime_kit/adapters/codex.py b/src/agent_runtime_kit/adapters/codex.py index fd18548..a3e8828 100644 --- a/src/agent_runtime_kit/adapters/codex.py +++ b/src/agent_runtime_kit/adapters/codex.py @@ -57,6 +57,15 @@ logger = logging.getLogger(__name__) +VENDOR_CONTRACT = { + "run_params": frozenset( + {"cwd", "model", "approval_mode", "sandbox", "output_schema", "effort"} + ), + "thread_start_params": frozenset( + {"developer_instructions", "cwd", "model", "approval_mode", "sandbox"} + ), +} + class CodexAgentRuntime: """Run tasks through the official ``openai_codex`` Python SDK.""" @@ -737,4 +746,3 @@ def _uses_bedrock_provider(config_overrides: tuple[str, ...]) -> bool: if normalized_key == "model_provider" and normalized_value == "amazon-bedrock": return True return False - diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 81b8817..445bacc 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -694,8 +694,8 @@ def candidate_snapshot(package: str, version: str) -> ApiSnapshot: inspect_candidates=True, ) - assert len(snapshots) == 2 - assert calls == [("candidate", "0.1.2"), ("candidate", "0.1.4")] + assert len(snapshots) == 1 + assert calls == [("candidate", "0.1.2")] def test_collect_snapshots_uses_refresh_preview_update_targets( @@ -741,10 +741,13 @@ def candidate_snapshot(package: str, version: str) -> ApiSnapshot: "latest_version": "0.136.0", }, ], - "refresh_preview": { - "stdout": "", - "stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n", - }, + "update_candidates": [ + { + "package": "claude-agent-sdk", + "from_version": "0.2.96", + "to_version": "0.2.106", + } + ], }, inspect_candidates=True, ) @@ -789,10 +792,13 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot: "latest_version": "0.2.106", }, ], - "refresh_preview": { - "stdout": "", - "stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n", - }, + "update_candidates": [ + { + "package": "claude-agent-sdk", + "from_version": "0.2.96", + "to_version": "0.2.106", + } + ], }, inspect_candidates=True, ) @@ -859,10 +865,13 @@ def test_candidate_api_diff_guard_blocks_missing_update_diff() -> None: "self_adaptation_plan": [], }, { - "refresh_preview": { - "stdout": "", - "stderr": "Update google-antigravity v0.1.2 -> v0.1.4\n", - } + "update_candidates": [ + { + "package": "google-antigravity", + "from_version": "0.1.2", + "to_version": "0.1.4", + } + ] }, [], ) @@ -880,10 +889,13 @@ def test_candidate_api_diff_guard_accepts_empty_update_diff() -> None: "manual_design_required": False, }, { - "refresh_preview": { - "stdout": "", - "stderr": "Update google-antigravity v0.1.2 -> v0.1.4\n", - } + "update_candidates": [ + { + "package": "google-antigravity", + "from_version": "0.1.2", + "to_version": "0.1.4", + } + ] }, [ { @@ -1041,7 +1053,7 @@ def test_recursive_self_adaptation_detection_and_gates() -> None: [{"removed": ["AgentResult"]}], ) assert architecture["recursive_self_adaptation_impact"] is True - assert architecture["self_adaptation_plan"] + assert architecture["manual_design_required"] is True blocked = evaluate_implementation_gate( { @@ -1060,7 +1072,7 @@ def test_recursive_self_adaptation_detection_and_gates() -> None: {"status": "pass"}, implementation_enabled=True, ) - assert allowed.allowed is True + assert allowed.allowed is False def test_reviewer_rejection_blocks_implementation() -> None: @@ -1078,7 +1090,7 @@ def test_reviewer_rejection_blocks_implementation() -> None: assert "reviewer" in gate.reason -def test_reviewer_approved_status_allows_implementation() -> None: +def test_only_exact_reviewer_pass_allows_implementation() -> None: gate = evaluate_implementation_gate( { "safe_to_implement": True, @@ -1089,7 +1101,7 @@ def test_reviewer_approved_status_allows_implementation() -> None: implementation_enabled=True, ) - assert gate.allowed is True + assert gate.allowed is False @pytest.mark.asyncio @@ -1148,9 +1160,7 @@ async def test_run_agent_report_only_generates_artifacts( assert (report_path.parent / "implementation_summary.json").exists() assert (report_path.parent / "review.json").exists() assert (report_path.parent / "events.jsonl").exists() - assert '"package": "claude-agent-sdk"' in (report_path.parent / "api_diffs.json").read_text( - encoding="utf-8" - ) + assert (report_path.parent / "api_diffs.json").read_text(encoding="utf-8") == "[]\n" assert "Recursive self-adaptation impact" in report_path.read_text(encoding="utf-8") @@ -1322,16 +1332,26 @@ def runner( *, cwd: Path | None = None, env: dict[str, str] | None = None, + timeout: int | None = None, ) -> CommandResult: - del cwd, env + del env, timeout commands.append(command) - if command[:3] == ("uv", "lock", "--dry-run"): - return CommandResult( - command=command, - returncode=0, - stderr="Update claude-agent-sdk v0.2.1 -> v0.3.0\n", - ) + if command == ("git", "status", "--porcelain"): + return CommandResult(command=command, returncode=0, stdout="") + if command == ("git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD"): + return CommandResult(command=command, returncode=0, stdout="origin/main\n") + if command[:3] == ("gh", "pr", "list"): + return CommandResult(command=command, returncode=0, stdout="[]") if command[:2] == ("uv", "lock"): + if cwd is not None: + (cwd / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.3.0" +""", + encoding="utf-8", + ) return CommandResult(command=command, returncode=0, stdout="updated") if command[:2] == ("git", "check-ignore"): # Report dir is tracked in this scenario -> not ignored (exit 1). @@ -1690,7 +1710,7 @@ async def run(self, task: AgentTask) -> AgentResult: "verification_commands": [], "uncertainty": [], } - elif stage == "review": + elif stage in {"review", "review-implementation"}: payload = {"status": "pass", "reasons": [], "required_changes": []} else: payload = { diff --git a/tests/test_sdk_evolution_docs.py b/tests/test_sdk_evolution_docs.py new file mode 100644 index 0000000..6147bc2 --- /dev/null +++ b/tests/test_sdk_evolution_docs.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import shlex +from pathlib import Path + +from examples.sdk_evolution_agent.cli import parse_args +from examples.sdk_evolution_agent.models import CommandResult +from examples.sdk_evolution_agent.preflight import validate_run_plan + +DOCS_WITH_COMMANDS = ( + Path("docs/sdk-evolution-agent.md"), + Path(".claude/commands/agent-runtime-kit/upgrade.md"), + Path(".codex/skills/agent-runtime-kit-upgrade/SKILL.md"), +) + + +def test_documented_sdk_evolution_commands_parse_and_preflight(tmp_path: Path) -> None: + for command in _documented_agent_commands(): + argv = _agent_argv(command) + options = parse_args(argv) + options = _with_workspace(options, tmp_path) + assert validate_run_plan(options, tmp_path, command_runner=_runner) == () + + +def _documented_agent_commands() -> list[str]: + commands: list[str] = [] + for path in DOCS_WITH_COMMANDS: + in_bash = False + pending: list[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip() == "```bash": + in_bash = True + pending = [] + continue + if line.strip() == "```": + in_bash = False + pending = [] + continue + if not in_bash: + continue + stripped = line.strip() + if not stripped or stripped.startswith("env "): + continue + if "python -m examples.sdk_evolution_agent" in stripped and ".auth" not in stripped: + if "--help" in stripped: + continue + pending = [stripped.rstrip("\\").strip()] + if not stripped.endswith("\\"): + commands.append(" ".join(pending)) + pending = [] + continue + if pending: + pending.append(stripped.rstrip("\\").strip()) + if not stripped.endswith("\\"): + commands.append(" ".join(pending)) + pending = [] + return commands + + +def _agent_argv(command: str) -> list[str]: + parts = shlex.split(command) + index = parts.index("examples.sdk_evolution_agent") + return parts[index + 1 :] + + +def _with_workspace(options: object, workspace: Path): + return type(options)(**{**options.__dict__, "workspace": workspace}) + + +def _runner(command: tuple[str, ...], *, cwd: Path | None = None) -> CommandResult: + del cwd + if command == ("git", "branch", "--show-current"): + return CommandResult(command=command, returncode=0, stdout="sdk-evolution/test\n") + if command == ("git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD"): + return CommandResult(command=command, returncode=0, stdout="origin/main\n") + if command == ("git", "status", "--porcelain"): + return CommandResult(command=command, returncode=0, stdout="") + return CommandResult(command=command, returncode=0) diff --git a/tests/test_sdk_evolution_pr_status.py b/tests/test_sdk_evolution_pr_status.py new file mode 100644 index 0000000..a00f7e2 --- /dev/null +++ b/tests/test_sdk_evolution_pr_status.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path + +from examples.sdk_evolution_agent.models import CommandResult +from examples.sdk_evolution_agent.pr import build_pr_marker, parse_pr_marker +from examples.sdk_evolution_agent.status_issue import render_status_body, update_status_issue + + +def test_pr_marker_round_trip() -> None: + marker = build_pr_marker({"claude-agent-sdk": "0.2.110"}, run_id="run-1") + + assert parse_pr_marker(marker) == { + "packages": {"claude-agent-sdk": "0.2.110"}, + "run_id": "run-1", + } + assert parse_pr_marker("no marker") is None + + +def test_status_body_renders_candidates_and_blockers(tmp_path: Path) -> None: + body = render_status_body( + tmp_path / "reports" / "run-1", + evidence={ + "update_candidates": [ + { + "package": "claude-agent-sdk", + "from_version": "0.2.1", + "to_version": "0.2.2", + } + ], + "update_candidates_beyond_cap": [], + }, + architecture={"manual_design_required": True, "safe_to_implement": False}, + ) + + assert "claude-agent-sdk" in body + assert "manual_design_required" in body + assert "safe_to_implement=false" in body + + +def test_status_issue_updates_existing_issue(tmp_path: Path) -> None: + run_dir = tmp_path / "reports" / "run-1" + run_dir.mkdir(parents=True) + (run_dir / "evidence.json").write_text( + '{"update_candidates":[{"package":"claude-agent-sdk","from_version":"1","to_version":"2"}]}', + encoding="utf-8", + ) + (run_dir / "architecture_decision.json").write_text( + '{"manual_design_required":false,"safe_to_implement":true}', + encoding="utf-8", + ) + commands: list[tuple[str, ...]] = [] + + def runner( + command: tuple[str, ...], + *, + cwd: Path | None = None, + timeout: int | None = None, + ) -> CommandResult: + del cwd, timeout + commands.append(command) + if command[:3] == ("gh", "issue", "list"): + return CommandResult( + command=command, + returncode=0, + stdout='[{"number":7,"state":"OPEN","title":"SDK evolution status"}]', + ) + return CommandResult(command=command, returncode=0) + + update_status_issue(run_dir, command_runner=runner) + + assert ("gh", "issue", "edit", "7", "--body") == commands[3][:5] diff --git a/tests/test_sdk_evolution_preflight.py b/tests/test_sdk_evolution_preflight.py new file mode 100644 index 0000000..f630761 --- /dev/null +++ b/tests/test_sdk_evolution_preflight.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from pathlib import Path + +from examples.sdk_evolution_agent.models import CommandResult, RunOptions +from examples.sdk_evolution_agent.preflight import validate_run_plan + + +def _runner( + command: tuple[str, ...], + *, + cwd: Path | None = None, +) -> CommandResult: + del cwd + if command == ("git", "branch", "--show-current"): + return CommandResult(command=command, returncode=0, stdout="main\n") + if command == ("git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD"): + return CommandResult(command=command, returncode=0, stdout="origin/main\n") + if command == ("git", "status", "--porcelain"): + return CommandResult(command=command, returncode=0, stdout="") + return CommandResult(command=command, returncode=0) + + +def test_preflight_reports_all_violations_at_once(tmp_path: Path) -> None: + violations = validate_run_plan( + RunOptions( + workspace=tmp_path, + draft_pr=True, + implementation_enabled=False, + create_branch=True, + ), + tmp_path, + command_runner=_runner, + ) + + assert "--draft-pr requires --implementation-enabled" in violations + assert "--create-branch requires --branch-name" in violations + assert "--draft-pr requires a non-detached head branch" in violations + + +def test_draft_pr_refused_on_default_branch(tmp_path: Path) -> None: + violations = validate_run_plan( + RunOptions( + workspace=tmp_path, + draft_pr=True, + implementation_enabled=True, + refresh_preview=True, + ), + tmp_path, + command_runner=_runner, + ) + + assert "--draft-pr refuses to push the default branch (main)" in violations + + +def test_allow_dirty_overrides_clean_check(tmp_path: Path) -> None: + def dirty_runner(command: tuple[str, ...], *, cwd: Path | None = None) -> CommandResult: + del cwd + if command == ("git", "status", "--porcelain"): + return CommandResult(command=command, returncode=0, stdout=" M uv.lock\n") + return _runner(command) + + assert validate_run_plan( + RunOptions( + workspace=tmp_path, + implementation_enabled=True, + refresh_preview=True, + allow_dirty=True, + ), + tmp_path, + command_runner=dirty_runner, + ) == () + + +def test_allow_cap_raise_requires_candidate_inspection(tmp_path: Path) -> None: + violations = validate_run_plan( + RunOptions( + workspace=tmp_path, + implementation_enabled=True, + refresh_preview=True, + allow_cap_raise=True, + ), + tmp_path, + command_runner=_runner, + ) + + assert "--allow-cap-raise requires --inspect-candidates" in violations diff --git a/tests/test_sdk_evolution_resolver.py b/tests/test_sdk_evolution_resolver.py new file mode 100644 index 0000000..f31f3a7 --- /dev/null +++ b/tests/test_sdk_evolution_resolver.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from pathlib import Path + +from examples.sdk_evolution_agent.collectors import ( + read_pyproject_dependency_specs, + read_uv_lock_versions, + select_recent_versions, +) +from examples.sdk_evolution_agent.models import CommandResult +from examples.sdk_evolution_agent.resolver import ( + raise_upper_bounds_in_pyproject_text, + resolve_update_candidates, +) + + +def test_read_pyproject_specs_from_optional_dependencies(tmp_path: Path) -> None: + path = tmp_path / "pyproject.toml" + path.write_text( + """ +[project.optional-dependencies] +# claude-agent-sdk>=9.9 is only a comment. +claude = ["claude-agent-sdk>=0.2.87,<0.3"] +all = ["openai-codex>=0.1.0b3,<0.2"] +""", + encoding="utf-8", + ) + + assert read_pyproject_dependency_specs(path) == { + "claude-agent-sdk": "claude-agent-sdk>=0.2.87,<0.3", + "openai-codex": "openai-codex>=0.1.0b3,<0.2", + } + + +def test_read_uv_lock_versions_toml(tmp_path: Path) -> None: + path = tmp_path / "uv.lock" + path.write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.2.110" +""", + encoding="utf-8", + ) + + assert read_uv_lock_versions(path) == {"claude-agent-sdk": "0.2.110"} + + +def test_version_key_pep440_ordering() -> None: + releases = { + "0.1.0b3": [{}], + "0.1.0": [{}], + "0.134.0a1": [{}], + "0.134.0": [{}], + "1.0.0": [{}], + "1.0.0.post1": [{}], + "1.0.0.dev1": [{}], + "1.0.0a1": [{}], + "0.2.87": [{}], + "0.2.110": [{}], + } + + ordered = select_recent_versions(releases, limit=len(releases)) + + assert ordered.index("0.1.0") < ordered.index("0.1.0b3") + assert ordered.index("0.134.0") < ordered.index("0.134.0a1") + assert ordered.index("1.0.0.post1") < ordered.index("1.0.0") + assert ordered.index("1.0.0a1") < ordered.index("1.0.0.dev1") + assert ordered.index("0.2.110") < ordered.index("0.2.87") + + +def test_resolver_never_mutates_workspace(tmp_path: Path) -> None: + (tmp_path / "README.md").write_text("# x\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """ +[project] +name = "x" +version = "0.0.0" +[project.optional-dependencies] +claude = ["claude-agent-sdk>=0.2,<0.3"] +""", + encoding="utf-8", + ) + lock = tmp_path / "uv.lock" + lock.write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.2.1" +""", + encoding="utf-8", + ) + before = lock.read_text(encoding="utf-8") + + def runner( + command: tuple[str, ...], + *, + cwd: Path, + env: dict[str, str], + timeout: int, + ) -> CommandResult: + del env, timeout + assert command == ("uv", "lock", "-P", "claude-agent-sdk") + (cwd / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.2.2" +""", + encoding="utf-8", + ) + return CommandResult(command=command, returncode=0) + + candidates, result = resolve_update_candidates( + tmp_path, ("claude-agent-sdk",), command_runner=runner + ) + + assert result.returncode == 0 + assert lock.read_text(encoding="utf-8") == before + assert [(item.package, item.from_version, item.to_version) for item in candidates] == [ + ("claude-agent-sdk", "0.2.1", "0.2.2") + ] + + +def test_cap_rewrite_touches_only_upper_bound() -> None: + text = 'claude = ["claude-agent-sdk>=0.2.87,<0.3"]\n' + + updated, raised = raise_upper_bounds_in_pyproject_text( + text, ("claude-agent-sdk",), versions={"claude-agent-sdk": "0.3.0"} + ) + + assert updated == 'claude = ["claude-agent-sdk>=0.2.87,<0.4"]\n' + assert raised["claude-agent-sdk"].current == "<0.3" From 41745756be3126cd23f6bf63e7ece26350511fe8 Mon Sep 17 00:00:00 2001 From: Eloi Date: Fri, 3 Jul 2026 17:20:52 +0200 Subject: [PATCH 2/3] Address SDK evolution review feedback --- .github/workflows/sdk-evolution-report.yml | 4 +- examples/sdk_evolution_agent/cli.py | 158 ++++++-- examples/sdk_evolution_agent/current_state.py | 31 +- examples/sdk_evolution_agent/models.py | 1 + examples/sdk_evolution_agent/pr.py | 41 +- examples/sdk_evolution_agent/resolver.py | 48 ++- examples/sdk_evolution_agent/stages.py | 17 +- tests/test_sdk_evolution_agent.py | 380 +++++++++++++++++- tests/test_sdk_evolution_resolver.py | 88 +++- 9 files changed, 686 insertions(+), 82 deletions(-) diff --git a/.github/workflows/sdk-evolution-report.yml b/.github/workflows/sdk-evolution-report.yml index b5e160c..07796ec 100644 --- a/.github/workflows/sdk-evolution-report.yml +++ b/.github/workflows/sdk-evolution-report.yml @@ -13,13 +13,13 @@ jobs: report: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up uv uses: astral-sh/setup-uv@v8.2.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py index b136b7e..51bb165 100644 --- a/examples/sdk_evolution_agent/cli.py +++ b/examples/sdk_evolution_agent/cli.py @@ -14,6 +14,7 @@ CommandRunner, PypiClient, collect_evidence, + read_uv_lock_versions, run_lock_update, run_verification_commands, ) @@ -183,6 +184,7 @@ def parse_args(argv: list[str] | None = None) -> RunOptions: allow_cap_raise=allow_cap_raise, commit_message=args.commit_message, pr_title=args.pr_title, + run_id=run_id, ) @@ -196,7 +198,7 @@ async def run_agent( ) -> Path: """Run the full local SDK evolution workflow.""" - run_id = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") + run_id = options.run_id or datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") violations = validate_run_plan(options, options.workspace, command_runner=command_runner) if violations: raise PreflightError(violations) @@ -344,8 +346,9 @@ async def run_agent( } if not promotion.get("promoted"): promoted = False - implementation["applied"] = False - implementation["blocked_reason"] = str(promotion.get("blocked_reason") or "") + implementation["promotion_blocked_reason"] = str( + promotion.get("blocked_reason") or "" + ) report_path = _write_full_report( context, config=config, @@ -363,6 +366,7 @@ async def run_agent( if ( options.draft_pr + and promoted and implementation.get("applied") and _verification_passed(implementation) ): @@ -371,6 +375,7 @@ async def run_agent( report_path=report_path, options=options, package_versions={**update_versions, **_applied_cap_raise_versions(implementation)}, + changed_paths=_applied_changed_paths(implementation), run_id=run_id, command_runner=command_runner, ) @@ -399,9 +404,12 @@ async def run_agent( command_runner=command_runner, ) elif options.draft_pr: - implementation["pr_skipped_reason"] = ( - "implementation was not applied or verification did not pass" - ) + if implementation.get("applied") and _verification_passed(implementation): + implementation["pr_skipped_reason"] = "baseline promotion did not pass" + else: + implementation["pr_skipped_reason"] = ( + "implementation was not applied or verification did not pass" + ) report_path = _write_full_report( context, config=config, @@ -448,7 +456,7 @@ async def _run_local_sdk_update( ) -> dict[str, Any]: root = options.workspace transaction_files = _snapshot_transaction_files(root) - allowed_file_snapshot = _snapshot_allowed_files(root) + rollback_file_snapshot = _snapshot_rollback_files(root) apply_results = list(implementation.get("apply_results") or []) verification_results: list[dict[str, Any]] = [] changes = list(implementation.get("changes") or []) @@ -472,12 +480,22 @@ async def _run_local_sdk_update( package = str(candidate["package"]) version = str(candidate["to_version"]) raised = raise_package_cap_in_workspace(root, package, version) - if raised is not None: - cap_raise_versions[package] = version - packages.add(package) - changes.append( - f"Raised {package} upper bound {raised.current} -> {raised.replacement}" + if raised is None: + apply_results.append( + to_jsonable( + CommandResult( + command=("raise-cap", package, version), + returncode=1, + stderr="cap raise gate passed but no upper bound was changed", + ) + ) ) + continue + cap_raise_versions[package] = version + packages.add(package) + changes.append( + f"Raised {package} upper bound {raised.current} -> {raised.replacement}" + ) if not packages: return { @@ -491,7 +509,12 @@ async def _run_local_sdk_update( ) apply_results.append(to_jsonable(update_result)) if update_result.returncode != 0: - _restore_transaction(root, transaction_files, allowed_file_snapshot) + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) return { **implementation, "applied": False, @@ -502,6 +525,27 @@ async def _run_local_sdk_update( "cap_raise_versions": cap_raise_versions, "blocked_reason": update_result.stderr or update_result.stdout, } + cap_raise_mismatches = _cap_raise_lock_mismatches(root, cap_raise_versions) + if cap_raise_mismatches: + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) + return { + **implementation, + "applied": False, + "rolled_back": True, + "changes": changes, + "apply_results": apply_results, + "verification_results": verification_results, + "cap_raise_versions": cap_raise_versions, + "blocked_reason": ( + "cap raise did not resolve requested versions: " + + ", ".join(cap_raise_mismatches) + ), + } changes.append( "Updated uv.lock for resolver-selected SDK packages: " + ", ".join(sorted(packages)) @@ -519,7 +563,12 @@ async def _run_local_sdk_update( ) changes.extend(str(item) for item in stage_output.get("changes", [])) if stage_output.get("blocked_reason"): - _restore_transaction(root, transaction_files, allowed_file_snapshot) + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) return { **implementation, "applied": False, @@ -537,7 +586,12 @@ async def _run_local_sdk_update( path for path in changed_paths if not _implementation_path_allowed(path) ] if out_of_scope: - _restore_transaction(root, transaction_files, allowed_file_snapshot) + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) return { **implementation, "applied": False, @@ -576,10 +630,16 @@ async def _run_local_sdk_update( "apply_results": apply_results, "verification_results": verification_results, "cap_raise_versions": cap_raise_versions, + "changed_paths": changed_paths, "blocked_reason": "", } if not _verification_passed(candidate): - _restore_transaction(root, transaction_files, allowed_file_snapshot) + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) return { **candidate, "applied": False, @@ -599,7 +659,12 @@ async def _run_local_sdk_update( ) candidate["implementation_review"] = implementation_review if implementation_review.get("status") != "pass": - _restore_transaction(root, transaction_files, allowed_file_snapshot) + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) return { **candidate, "applied": False, @@ -608,7 +673,12 @@ async def _run_local_sdk_update( } return candidate except Exception as exc: - _restore_transaction(root, transaction_files, allowed_file_snapshot) + _restore_transaction( + root, + transaction_files, + rollback_file_snapshot, + command_runner=command_runner, + ) return { **implementation, "applied": False, @@ -625,9 +695,9 @@ async def _run_local_sdk_update( "examples/sdk_evolution_agent/", "tests/test_sdk_evolution_", "docs/sdk-evolution-agent", - ".sdk-evolution/", ) _ALLOWED_IMPLEMENTATION_FILES = {"uv.lock", "pyproject.toml"} +_ROLLBACK_PROTECTED_PREFIXES = ("reports/", ".sdk-evolution/") def _verification_commands(architecture: dict[str, Any]) -> tuple[str, ...]: @@ -646,15 +716,20 @@ def _snapshot_transaction_files(root: Path) -> dict[str, bytes | None]: return snapshot -def _snapshot_allowed_files(root: Path) -> dict[str, bytes]: +def _snapshot_rollback_files(root: Path) -> dict[str, bytes]: snapshot: dict[str, bytes] = {} - for base in (root / "examples" / "sdk_evolution_agent", root / "docs", root / "tests"): + for base in ( + root / "examples" / "sdk_evolution_agent", + root / "docs", + root / "tests", + root / ".sdk-evolution", + ): if not base.exists(): continue for path in base.rglob("*"): if path.is_file(): rel = _relative_path(root, path) - if _implementation_path_allowed(rel): + if _implementation_path_allowed(rel) or rel.startswith(".sdk-evolution/"): snapshot[rel] = path.read_bytes() return snapshot @@ -662,7 +737,9 @@ def _snapshot_allowed_files(root: Path) -> dict[str, bytes]: def _restore_transaction( root: Path, transaction_files: dict[str, bytes | None], - allowed_file_snapshot: dict[str, bytes], + rollback_file_snapshot: dict[str, bytes], + *, + command_runner: CommandRunner | None, ) -> None: for relative, data in transaction_files.items(): path = root / relative @@ -670,17 +747,17 @@ def _restore_transaction( path.unlink(missing_ok=True) else: path.write_bytes(data) - for relative, data in allowed_file_snapshot.items(): + for relative, data in rollback_file_snapshot.items(): path = root / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) - for relative in _changed_paths(_git_status(root, command_runner=None)): - if relative.startswith("reports/"): + for relative in _changed_paths(_git_status(root, command_runner=command_runner)): + if _rollback_delete_protected(relative): continue - if relative in transaction_files or relative in allowed_file_snapshot: + if relative in transaction_files or relative in rollback_file_snapshot: continue path = root / relative - if path.exists() and _implementation_path_allowed(relative): + if path.exists(): if path.is_dir(): for child in sorted(path.rglob("*"), reverse=True): if child.is_file() or child.is_symlink(): @@ -692,6 +769,10 @@ def _restore_transaction( path.unlink(missing_ok=True) +def _rollback_delete_protected(relative: str) -> bool: + return any(relative.startswith(prefix) for prefix in _ROLLBACK_PROTECTED_PREFIXES) + + def _git_status(root: Path, *, command_runner: CommandRunner | None) -> str: runner = command_runner if runner is None: @@ -744,6 +825,24 @@ def _applied_cap_raise_versions(implementation: dict[str, Any]) -> dict[str, str return {str(package): str(version) for package, version in versions.items()} +def _applied_changed_paths(implementation: dict[str, Any]) -> tuple[str, ...]: + paths = implementation.get("changed_paths") + if not isinstance(paths, list | tuple): + return () + return tuple(str(path) for path in paths if path) + + +def _cap_raise_lock_mismatches(root: Path, cap_raise_versions: dict[str, str]) -> tuple[str, ...]: + if not cap_raise_versions: + return () + locked = read_uv_lock_versions(root / "uv.lock") + return tuple( + f"{package} locked {locked.get(package) or 'missing'} != requested {version}" + for package, version in sorted(cap_raise_versions.items()) + if locked.get(package) != version + ) + + def _write_full_report( context: RunContext, *, @@ -793,6 +892,7 @@ def _create_autonomous_pr( report_path: Path, options: RunOptions, package_versions: dict[str, str], + changed_paths: tuple[str, ...] = (), run_id: str, command_runner: CommandRunner | None, ) -> list[dict[str, Any]]: @@ -824,7 +924,7 @@ def _create_autonomous_pr( if number: superseded.append(number) - paths = ("uv.lock", "pyproject.toml", ".sdk-evolution") + paths = tuple(dict.fromkeys(("uv.lock", "pyproject.toml", ".sdk-evolution", *changed_paths))) results.extend( [ to_jsonable(stage_paths(root, paths, command_runner=command_runner)), diff --git a/examples/sdk_evolution_agent/current_state.py b/examples/sdk_evolution_agent/current_state.py index f9d2b0f..2678102 100644 --- a/examples/sdk_evolution_agent/current_state.py +++ b/examples/sdk_evolution_agent/current_state.py @@ -81,26 +81,41 @@ def promote_baseline( snapshot for snapshot in snapshots if snapshot.get("package") in versions + and snapshot.get("requested_version") and snapshot.get("observed_version") - and snapshot.get("observed_version") != versions[snapshot["package"]] + and snapshot.get("observed_version") != snapshot.get("requested_version") ] if mismatched: return { "promoted": False, "status": "baseline-promotion-refused", - "blocked_reason": "snapshot observed_version does not match locked version", + "blocked_reason": "snapshot observed_version does not match requested_version", + } + selected_snapshots: dict[str, dict[str, Any]] = {} + snapshot_packages: set[str] = set() + for snapshot in snapshots: + package = str(snapshot.get("package") or "") + if not package or package not in versions or snapshot.get("import_error"): + continue + snapshot_packages.add(package) + if snapshot.get("observed_version") == versions[package]: + selected_snapshots[package] = snapshot + missing_promoted = sorted(snapshot_packages - set(selected_snapshots)) + if missing_promoted: + return { + "promoted": False, + "status": "baseline-promotion-refused", + "blocked_reason": ( + "no snapshot observed_version matches locked version for " + + ", ".join(missing_promoted) + ), } root = workspace / BASELINE_DIR snapshots_dir = root / "snapshots" snapshots_dir.mkdir(parents=True, exist_ok=True) package_snapshots: dict[str, str] = {} - for snapshot in snapshots: - package = str(snapshot.get("package") or "") - if not package or snapshot.get("import_error"): - continue - if package not in versions: - continue + for package, snapshot in selected_snapshots.items(): path = snapshots_dir / f"{package}.json" write_json(path, snapshot) package_snapshots[package] = _sha256(path) diff --git a/examples/sdk_evolution_agent/models.py b/examples/sdk_evolution_agent/models.py index 126863c..9217b10 100644 --- a/examples/sdk_evolution_agent/models.py +++ b/examples/sdk_evolution_agent/models.py @@ -166,6 +166,7 @@ class RunOptions: allow_cap_raise: bool = False commit_message: str = "Run SDK evolution update" pr_title: str = "Adapt agent-runtime-kit to upstream SDK evolution" + run_id: str | None = None @dataclass(frozen=True) diff --git a/examples/sdk_evolution_agent/pr.py b/examples/sdk_evolution_agent/pr.py index 3fcb154..ab5a7bf 100644 --- a/examples/sdk_evolution_agent/pr.py +++ b/examples/sdk_evolution_agent/pr.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import tempfile from pathlib import Path from typing import Any @@ -125,12 +126,25 @@ def create_draft_pr( """Open a draft PR with gh when authenticated.""" command_runner = command_runner or run_command - command = ["gh", "pr", "create", "--draft", "--title", title, "--body", body] - if base: - command.extend(("--base", base)) - if head: - command.extend(("--head", head)) - return _call_runner(command_runner, tuple(command), cwd=root, timeout=120) + body_path = _write_temp_pr_body(root, body) + try: + command = [ + "gh", + "pr", + "create", + "--draft", + "--title", + title, + "--body-file", + str(body_path), + ] + if base: + command.extend(("--base", base)) + if head: + command.extend(("--head", head)) + return _call_runner(command_runner, tuple(command), cwd=root, timeout=120) + finally: + body_path.unlink(missing_ok=True) def list_open_sdk_evolution_prs( @@ -154,7 +168,7 @@ def list_open_sdk_evolution_prs( timeout=120, ) if result.returncode != 0: - return () + raise RuntimeError(result.stderr or result.stdout or "failed to list SDK evolution PRs") try: payload = json.loads(result.stdout or "[]") except json.JSONDecodeError: @@ -220,3 +234,16 @@ def _is_sdk_evolution_pr(item: dict[str, Any]) -> bool: return True labels = item.get("labels") or [] return any(isinstance(label, dict) and label.get("name") == "sdk-evolution" for label in labels) + + +def _write_temp_pr_body(root: Path, body: str) -> Path: + del root + with tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + prefix=".sdk-evolution-pr-", + suffix=".md", + delete=False, + ) as handle: + handle.write(body) + return Path(handle.name) diff --git a/examples/sdk_evolution_agent/resolver.py b/examples/sdk_evolution_agent/resolver.py index 13c4db3..9951f17 100644 --- a/examples/sdk_evolution_agent/resolver.py +++ b/examples/sdk_evolution_agent/resolver.py @@ -18,6 +18,7 @@ from examples.sdk_evolution_agent.collectors import ( CommandRunner, _call_runner, + _version_key, cutoff_free_env, read_pyproject_dependency_specs, read_uv_lock_versions, @@ -81,7 +82,12 @@ def resolve_constraint_horizon_candidates( if (candidate.package, candidate.to_version) in adoptable_keys: continue cap = raised_caps.get(candidate.package) - blocked_by_cap = cap.current if cap else _upper_bound(original_specs.get(candidate.package)) + original_cap = cap.current if cap else _upper_bound(original_specs.get(candidate.package)) + blocked_by_cap = ( + original_cap + if original_cap and _violates_upper_bound(candidate.to_version, original_cap) + else None + ) cutoff_delayed_until = None if blocked_by_cap is None: upload = upload_time_for_version( @@ -109,7 +115,11 @@ class RaisedCap: def raise_upper_bounds_in_pyproject_text( - text: str, packages: Sequence[str], *, versions: Mapping[str, str] | None = None + text: str, + packages: Sequence[str], + *, + versions: Mapping[str, str] | None = None, + remove_exclude_newer: bool = False, ) -> tuple[str, dict[str, RaisedCap]]: """Raise only tracked package upper bounds enough to admit candidate versions.""" @@ -119,16 +129,26 @@ def raise_upper_bounds_in_pyproject_text( pattern = re.compile( rf"(?P{re.escape(package)}[^\"']*?,)(?P<\s*\d+(?:\.\d+)*)" ) - match = pattern.search(result) - if not match: - continue - current = match.group("cap").replace(" ", "") - replacement = _raised_upper_bound(current, versions.get(package) if versions else None) - if replacement == current: - continue - result = result[: match.start("cap")] + replacement + result[match.end("cap") :] - raised[package] = RaisedCap(package=package, current=current, replacement=replacement) - result = _remove_tool_uv_exclude_newer(result) + target_version = versions.get(package) if versions else None + + def replace( + match: re.Match[str], + package: str = package, + target_version: str | None = target_version, + ) -> str: + current = match.group("cap").replace(" ", "") + replacement = _raised_upper_bound(current, target_version) + if replacement == current: + return match.group(0) + raised.setdefault( + package, + RaisedCap(package=package, current=current, replacement=replacement), + ) + return match.group("prefix") + replacement + + result = pattern.sub(replace, result) + if remove_exclude_newer: + result = _remove_tool_uv_exclude_newer(result) return result, raised @@ -244,5 +264,9 @@ def _upper_bound(spec: str | None) -> str | None: return match.group(0).replace(" ", "") if match else None +def _violates_upper_bound(version: str, upper_bound: str) -> bool: + return _version_key(version) >= _version_key(upper_bound.removeprefix("<")) + + def _remove_tool_uv_exclude_newer(text: str) -> str: return re.sub(r'(?m)^\s*exclude-newer\s*=\s*["\'][^"\']+["\']\s*\n?', "", text) diff --git a/examples/sdk_evolution_agent/stages.py b/examples/sdk_evolution_agent/stages.py index 3546e1a..621fe95 100644 --- a/examples/sdk_evolution_agent/stages.py +++ b/examples/sdk_evolution_agent/stages.py @@ -243,7 +243,10 @@ async def run_analysis_pipeline( schema=ARCHITECTURE_DECISION_SCHEMA, context=context, ) - architecture = with_kit_version_impact(architecture) + architecture = with_kit_version_impact( + architecture, + implementation_enabled=context.implementation_enabled, + ) architecture = with_recursive_impact( architecture, api_diffs, @@ -417,7 +420,11 @@ def with_recursive_impact( return result -def with_kit_version_impact(architecture: Mapping[str, Any]) -> dict[str, Any]: +def with_kit_version_impact( + architecture: Mapping[str, Any], + *, + implementation_enabled: bool = False, +) -> dict[str, Any]: """Flag the example when the installed kit version moves past its adaptation marker.""" try: @@ -428,6 +435,12 @@ def with_kit_version_impact(architecture: Mapping[str, Any]) -> dict[str, Any]: return dict(architecture) result = dict(architecture) result["recursive_self_adaptation_impact"] = True + plan = [str(item) for item in result.get("self_adaptation_plan") or []] + has_executable_plan = implementation_enabled and any( + item.startswith("examples/sdk_evolution_agent/") for item in plan + ) + if not has_executable_plan: + result["manual_design_required"] = True findings = list(result.get("findings") or []) findings.append( { diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 445bacc..8441b78 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import stat import sys @@ -32,7 +33,15 @@ diff_behavior_results, probe_candidate_in_venv, ) -from examples.sdk_evolution_agent.cli import RunOptions, _collect_snapshots, parse_args, run_agent +from examples.sdk_evolution_agent.cli import ( + RunOptions, + _cap_raise_lock_mismatches, + _collect_snapshots, + _create_autonomous_pr, + _restore_transaction, + parse_args, + run_agent, +) from examples.sdk_evolution_agent.collectors import ( build_refresh_preview_command, collect_evidence, @@ -40,7 +49,11 @@ run_lock_update, run_refresh_preview, ) -from examples.sdk_evolution_agent.current_state import build_current_state +from examples.sdk_evolution_agent.current_state import ( + build_current_state, + load_baseline, + promote_baseline, +) from examples.sdk_evolution_agent.models import ApiSnapshot, CommandResult, RunContext, SourceRef from examples.sdk_evolution_agent.pr import build_draft_pr_body from examples.sdk_evolution_agent.release_notes import ( @@ -66,6 +79,7 @@ run_stage, with_behavior_probe_guard, with_candidate_api_diff_guard, + with_kit_version_impact, with_manual_design_gate, with_recursive_impact, with_release_note_guard, @@ -622,6 +636,167 @@ def test_current_state_artifact_paths_are_repo_relative(tmp_path: Path) -> None: assert all("/private/tmp" not in path and "/tmp/" not in path for path in paths) +def test_promote_baseline_selects_snapshot_matching_updated_lock( + tmp_path: Path, +) -> None: + (tmp_path / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.3.0" +""", + encoding="utf-8", + ) + report_root = tmp_path / "reports" / "run-1" + report_root.mkdir(parents=True) + context = RunContext( + run_id="run-1", + workspace=tmp_path, + report_root=report_root, + runtime="fake", + event_log_path=report_root / "events.jsonl", + implementation_enabled=True, + draft_pr=False, + ) + + promotion = promote_baseline( + context, + snapshots=[ + { + "package": "claude-agent-sdk", + "requested_version": "0.2.1", + "observed_version": "0.2.1", + "members": [], + }, + { + "package": "claude-agent-sdk", + "requested_version": "0.3.0", + "observed_version": "0.3.0", + "members": [], + }, + ], + current_state={"artifacts": {"report.md": {"path": "reports/run-1/report.md"}}}, + ) + + assert promotion["promoted"] is True + snapshot = json.loads( + (tmp_path / ".sdk-evolution" / "snapshots" / "claude-agent-sdk.json").read_text( + encoding="utf-8" + ) + ) + assert snapshot["observed_version"] == "0.3.0" + assert load_baseline(tmp_path)["status"] == "current" + + +def test_promote_baseline_refuses_snapshot_provenance_mismatch(tmp_path: Path) -> None: + (tmp_path / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.3.0" +""", + encoding="utf-8", + ) + context = RunContext( + run_id="run-1", + workspace=tmp_path, + report_root=tmp_path / "reports", + runtime="fake", + event_log_path=tmp_path / "events.jsonl", + implementation_enabled=True, + draft_pr=False, + ) + + promotion = promote_baseline( + context, + snapshots=[ + { + "package": "claude-agent-sdk", + "requested_version": "0.3.0", + "observed_version": "0.2.1", + } + ], + current_state={}, + ) + + assert promotion["promoted"] is False + assert "requested_version" in promotion["blocked_reason"] + + +def test_restore_transaction_deletes_out_of_scope_strays_with_injected_status( + tmp_path: Path, +) -> None: + (tmp_path / "uv.lock").write_text("new lock", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text("new project", encoding="utf-8") + allowed = tmp_path / "examples" / "sdk_evolution_agent" / "feature.py" + allowed.parent.mkdir(parents=True) + allowed.write_text("new feature", encoding="utf-8") + injected = tmp_path / "src" / "agent_runtime_kit" / "injected.py" + injected.parent.mkdir(parents=True) + injected.write_text("stray", encoding="utf-8") + baseline = tmp_path / ".sdk-evolution" / "README.md" + baseline.parent.mkdir(parents=True) + baseline.write_text("mutated baseline", encoding="utf-8") + generated_baseline = tmp_path / ".sdk-evolution" / "generated.json" + generated_baseline.write_text("new baseline artifact", encoding="utf-8") + report = tmp_path / "reports" / "sdk-evolution" / "run" / "report.md" + report.parent.mkdir(parents=True) + report.write_text("report", encoding="utf-8") + commands: list[tuple[str, ...]] = [] + + def runner(command: tuple[str, ...], *, cwd: Path | None = None) -> CommandResult: + del cwd + commands.append(command) + assert command == ("git", "status", "--porcelain") + return CommandResult( + command=command, + returncode=0, + stdout="\n".join( + [ + " M uv.lock", + " M examples/sdk_evolution_agent/feature.py", + " M .sdk-evolution/README.md", + "?? .sdk-evolution/generated.json", + "?? reports/sdk-evolution/run/report.md", + "?? src/agent_runtime_kit/injected.py", + ] + ), + ) + + _restore_transaction( + tmp_path, + {"uv.lock": b"old lock", "pyproject.toml": b"old project"}, + { + "examples/sdk_evolution_agent/feature.py": b"old feature", + ".sdk-evolution/README.md": b"baseline", + }, + command_runner=runner, + ) + + assert commands == [("git", "status", "--porcelain")] + assert (tmp_path / "uv.lock").read_text(encoding="utf-8") == "old lock" + assert allowed.read_text(encoding="utf-8") == "old feature" + assert baseline.read_text(encoding="utf-8") == "baseline" + assert not injected.exists() + assert report.exists() + assert generated_baseline.exists() + + +def test_cap_raise_postcondition_detects_unapplied_lock_version(tmp_path: Path) -> None: + (tmp_path / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.2.1" +""", + encoding="utf-8", + ) + + assert _cap_raise_lock_mismatches(tmp_path, {"claude-agent-sdk": "0.3.0"}) == ( + "claude-agent-sdk locked 0.2.1 != requested 0.3.0", + ) + + def test_snapshot_and_diff_public_api(monkeypatch: pytest.MonkeyPatch) -> None: module = types.ModuleType("fake_sdk") @@ -855,6 +1030,51 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot: assert calls == [("current", "claude-agent-sdk", "0.2.96")] +def test_collect_snapshots_ignores_refresh_preview_text_without_structured_candidate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str, str | None]] = [] + + def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot: + calls.append(("current", package, version)) + return ApiSnapshot(package=package, version=version, module=package.replace("-", "_")) + + def isolated_snapshot(package: str, version: str) -> ApiSnapshot: + raise AssertionError( + f"refresh preview text must not trigger candidate install: {package} {version}" + ) + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + current_snapshot, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", + isolated_snapshot, + ) + + snapshots = _collect_snapshots( + { + "packages": [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.96", + "installed_version": "0.2.96", + "latest_version": "0.2.106", + } + ], + "refresh_preview": { + "stdout": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n", + "stderr": "", + }, + }, + inspect_candidates=True, + ) + + assert len(snapshots) == 1 + assert calls == [("current", "claude-agent-sdk", "0.2.96")] + + def test_candidate_api_diff_guard_blocks_missing_update_diff() -> None: guarded = with_candidate_api_diff_guard( { @@ -1075,6 +1295,38 @@ def test_recursive_self_adaptation_detection_and_gates() -> None: assert allowed.allowed is False +def test_kit_version_impact_requires_executable_self_adaptation_plan( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "examples.sdk_evolution_agent.stages.importlib.metadata.version", + lambda package: "9.0.0", + ) + + prose_only = with_kit_version_impact( + { + "findings": [], + "safe_to_implement": True, + "manual_design_required": False, + "self_adaptation_plan": ["Update the runtime usage to match the new kit."], + }, + implementation_enabled=True, + ) + executable = with_kit_version_impact( + { + "findings": [], + "safe_to_implement": True, + "manual_design_required": False, + "self_adaptation_plan": ["examples/sdk_evolution_agent/cli.py"], + }, + implementation_enabled=True, + ) + + assert prose_only["recursive_self_adaptation_impact"] is True + assert prose_only["manual_design_required"] is True + assert executable["manual_design_required"] is False + + def test_reviewer_rejection_blocks_implementation() -> None: gate = evaluate_implementation_gate( { @@ -1117,22 +1369,32 @@ async def test_run_agent_report_only_generates_artifacts( encoding="utf-8", ) (tmp_path / "uv.lock").write_text("", encoding="utf-8") - monkeypatch.setattr( - "examples.sdk_evolution_agent.cli.snapshot_current_api", - lambda package, *, version=None: ApiSnapshot( + def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot: + return ApiSnapshot( package=package, version=version, module=package.replace("-", "_"), - ), - ) - monkeypatch.setattr( - "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", - lambda package, version: ApiSnapshot( + requested_version=version, + observed_version=version, + ) + + def candidate_snapshot(package: str, version: str) -> ApiSnapshot: + return ApiSnapshot( package=package, version=version, module=package.replace("-", "_"), + requested_version=version, + observed_version=version, source="isolated-venv", - ), + ) + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + current_snapshot, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", + candidate_snapshot, ) report_path = await run_agent( @@ -1276,6 +1538,61 @@ def runner( assert not any(command[:2] == ("git", "add") for command in commands) +def test_create_autonomous_pr_stages_dynamic_in_scope_changes_and_body_file( + tmp_path: Path, +) -> None: + report_path = tmp_path / "reports" / "run" / "report.md" + report_path.parent.mkdir(parents=True) + report_path.write_text("# report\n", encoding="utf-8") + commands: list[tuple[str, ...]] = [] + + def runner( + command: tuple[str, ...], + *, + cwd: Path | None = None, + timeout: int | None = None, + ) -> CommandResult: + del cwd, timeout + commands.append(command) + if command[:3] == ("gh", "pr", "list"): + return CommandResult(command=command, returncode=0, stdout="[]") + if command[:3] == ("gh", "pr", "create"): + assert "--body-file" in command + assert "--body" not in command + body_path = Path(command[command.index("--body-file") + 1]) + assert "# report" in body_path.read_text(encoding="utf-8") + return CommandResult( + command=command, + returncode=0, + stdout="https://github.com/ebarti/agent-runtime-kit/pull/99\n", + ) + return CommandResult(command=command, returncode=0, stdout="ok") + + _create_autonomous_pr( + tmp_path, + report_path=report_path, + options=RunOptions( + workspace=tmp_path, + runtime="fake", + branch_name="sdk-update-test", + pr_base="main", + ), + package_versions={"claude-agent-sdk": "0.3.0"}, + changed_paths=( + "examples/sdk_evolution_agent/cli.py", + "tests/test_sdk_evolution_agent.py", + "docs/sdk-evolution-agent.md", + ), + run_id="run-1", + command_runner=runner, + ) + + add_command = next(command for command in commands if command[:2] == ("git", "add")) + assert "examples/sdk_evolution_agent/cli.py" in add_command + assert "tests/test_sdk_evolution_agent.py" in add_command + assert "docs/sdk-evolution-agent.md" in add_command + + @pytest.mark.asyncio async def test_run_agent_autonomous_pr_path( tmp_path: Path, @@ -1296,22 +1613,32 @@ async def test_run_agent_autonomous_pr_path( """, encoding="utf-8", ) - monkeypatch.setattr( - "examples.sdk_evolution_agent.cli.snapshot_current_api", - lambda package, *, version=None: ApiSnapshot( + def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot: + return ApiSnapshot( package=package, version=version, module=package.replace("-", "_"), - ), - ) - monkeypatch.setattr( - "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", - lambda package, version: ApiSnapshot( + requested_version=version, + observed_version=version, + ) + + def candidate_snapshot(package: str, version: str) -> ApiSnapshot: + return ApiSnapshot( package=package, version=version, module=package.replace("-", "_"), + requested_version=version, + observed_version=version, source="isolated-venv", - ), + ) + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + current_snapshot, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", + candidate_snapshot, ) monkeypatch.setattr( "examples.sdk_evolution_agent.cli.collect_release_notes", @@ -1378,6 +1705,19 @@ def runner( ) assert report_path.exists() + current_state = json.loads((report_path.parent / "current_state.json").read_text()) + implementation = json.loads((report_path.parent / "implementation_summary.json").read_text()) + assert current_state["promotion"]["promoted"] is True + assert current_state["packages"]["claude-agent-sdk"] == "0.3.0" + assert implementation["applied"] is True + assert implementation["pr_results"] + assert load_baseline(tmp_path)["status"] == "current" + promoted_snapshot = json.loads( + (tmp_path / ".sdk-evolution" / "snapshots" / "claude-agent-sdk.json").read_text( + encoding="utf-8" + ) + ) + assert promoted_snapshot["observed_version"] == "0.3.0" assert ("git", "switch", "-c", "sdk-update-test") in commands assert ("uv", "lock", "-P", "claude-agent-sdk") in commands assert any(command[:3] == ("git", "commit", "-m") for command in commands) diff --git a/tests/test_sdk_evolution_resolver.py b/tests/test_sdk_evolution_resolver.py index f31f3a7..642e726 100644 --- a/tests/test_sdk_evolution_resolver.py +++ b/tests/test_sdk_evolution_resolver.py @@ -10,6 +10,7 @@ from examples.sdk_evolution_agent.models import CommandResult from examples.sdk_evolution_agent.resolver import ( raise_upper_bounds_in_pyproject_text, + resolve_constraint_horizon_candidates, resolve_update_candidates, ) @@ -123,11 +124,94 @@ def runner( def test_cap_rewrite_touches_only_upper_bound() -> None: - text = 'claude = ["claude-agent-sdk>=0.2.87,<0.3"]\n' + text = ( + '[project.optional-dependencies]\n' + 'claude = ["claude-agent-sdk>=0.2.87,<0.3"]\n' + 'all = ["claude-agent-sdk>=0.2.87,<0.3"]\n' + '[tool.uv]\n' + 'exclude-newer = "2026-01-01T00:00:00Z"\n' + ) updated, raised = raise_upper_bounds_in_pyproject_text( text, ("claude-agent-sdk",), versions={"claude-agent-sdk": "0.3.0"} ) - assert updated == 'claude = ["claude-agent-sdk>=0.2.87,<0.4"]\n' + assert updated.count("claude-agent-sdk>=0.2.87,<0.4") == 2 + assert "exclude-newer" in updated assert raised["claude-agent-sdk"].current == "<0.3" + + +def test_horizon_candidates_distinguish_cap_blocked_from_cutoff_delayed( + tmp_path: Path, +) -> None: + (tmp_path / "README.md").write_text("# x\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """ +[project] +name = "x" +version = "0.0.0" +[project.optional-dependencies] +claude = ["claude-agent-sdk>=0.2,<0.3"] +codex = ["openai-codex>=0.1.0b3,<0.2"] +[tool.uv] +exclude-newer = "2026-01-01T00:00:00Z" +""", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.2.1" + +[[package]] +name = "openai-codex" +version = "0.1.0b3" +""", + encoding="utf-8", + ) + + def runner( + command: tuple[str, ...], + *, + cwd: Path, + env: dict[str, str], + timeout: int, + ) -> CommandResult: + del command, env, timeout + (cwd / "uv.lock").write_text( + """ +[[package]] +name = "claude-agent-sdk" +version = "0.3.0" + +[[package]] +name = "openai-codex" +version = "0.1.1" +""", + encoding="utf-8", + ) + return CommandResult(command=("uv", "lock"), returncode=0) + + candidates = resolve_constraint_horizon_candidates( + tmp_path, + ("claude-agent-sdk", "openai-codex"), + pypi_metadata={ + "openai-codex": { + "releases": { + "0.1.1": [ + { + "upload_time_iso_8601": "2026-01-03T00:00:00.000Z", + } + ] + } + } + }, + command_runner=runner, + ) + + by_package = {candidate.package: candidate for candidate in candidates} + assert by_package["claude-agent-sdk"].blocked_by_cap == "<0.3" + assert by_package["claude-agent-sdk"].cutoff_delayed_until is None + assert by_package["openai-codex"].blocked_by_cap is None + assert by_package["openai-codex"].cutoff_delayed_until == "2026-01-11" From 476dc0c789b4596eacab9a10e32820c670c340e6 Mon Sep 17 00:00:00 2001 From: Eloi Date: Fri, 3 Jul 2026 20:35:02 +0200 Subject: [PATCH 3/3] Remove SDK evolution report workflow --- .github/workflows/sdk-evolution-report.yml | 52 ---------------------- 1 file changed, 52 deletions(-) delete mode 100644 .github/workflows/sdk-evolution-report.yml diff --git a/.github/workflows/sdk-evolution-report.yml b/.github/workflows/sdk-evolution-report.yml deleted file mode 100644 index 07796ec..0000000 --- a/.github/workflows/sdk-evolution-report.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: SDK Evolution Report - -on: - schedule: - - cron: "43 5 * * 2" - workflow_dispatch: - -permissions: - contents: read - issues: write - -jobs: - report: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: Set up uv - uses: astral-sh/setup-uv@v8.2.0 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install - run: uv sync --locked --all-extras - - - name: Run SDK evolution report - run: | - uv run python -m examples.sdk_evolution_agent \ - --runtime fake \ - --refresh-preview \ - --inspect-candidates - - - name: Locate run directory - id: run-dir - run: | - latest="$(ls -td reports/sdk-evolution/* | head -1)" - echo "path=$latest" >> "$GITHUB_OUTPUT" - - - name: Upload report artifacts - uses: actions/upload-artifact@v4 - with: - name: sdk-evolution-report - path: ${{ steps.run-dir.outputs.path }} - retention-days: 90 - - - name: Update status issue - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: uv run python -m examples.sdk_evolution_agent.status_issue "${{ steps.run-dir.outputs.path }}"