diff --git a/CHANGELOG.md b/CHANGELOG.md
index 130f2e2..47077ea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -118,8 +118,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
metadata cannot abort a run.
- SDK evolution example: inspecting candidate SDK versions (which pip-installs
and imports freshly downloaded upstream code) is now opt-in via
- `--inspect-candidates` and runs in a credential-scrubbed environment, and
- `--draft-pr` no longer fails when the report directory is gitignored.
+ `--inspect-candidates` and runs in a credential-scrubbed environment — and
+ the behavior probes now honor the same gate and scrub instead of installing
+ candidates unconditionally with the caller's full environment (skipped
+ candidates are recorded explicitly). `--draft-pr` no longer fails when the
+ report directory is gitignored.
+- SDK evolution example: release-note links extracted from fetched pages are
+ only followed when they resolve to `https://github.com` (a protocol-relative
+ href in user-generated discussion markup could otherwise trigger an off-site
+ request), and a configured-but-failing GitHub GraphQL token now surfaces on
+ the source instead of silently downgrading to the unauthenticated scrape.
## 0.2.0 - 2026-06-23
diff --git a/docs/sdk-evolution-agent-design.md b/docs/sdk-evolution-agent-design.md
index 986ab3c..6ad1166 100644
--- a/docs/sdk-evolution-agent-design.md
+++ b/docs/sdk-evolution-agent-design.md
@@ -197,9 +197,12 @@ python -m examples.sdk_evolution_agent \
--package google-antigravity
```
-`--inspect-candidates` should be effectively always on. The CLI can keep the
-flag for compatibility, but update candidates without candidate API snapshots
-are not actionable.
+`--inspect-candidates` was originally envisioned as effectively always on,
+since update candidates without candidate API snapshots are not actionable.
+The shipped behavior deliberately inverts that default: candidate inspection
+installs and imports freshly downloaded upstream code, so it is opt-in and
+runs in a credential-scrubbed environment, with explicit `skip` records when
+it is off.
Implementation mode should remain explicitly gated:
diff --git a/docs/sdk-evolution-agent.md b/docs/sdk-evolution-agent.md
index b8b46f2..6d5f5aa 100644
--- a/docs/sdk-evolution-agent.md
+++ b/docs/sdk-evolution-agent.md
@@ -117,12 +117,15 @@ contains a different installed version, the agent inspects the locked baseline
in a temporary isolated virtualenv instead of trusting the drifted environment.
When a refresh preview is available, package update candidates come from the
resolver's `uv lock --dry-run -P ...` output, not only from PyPI's `latest`
-metadata. For each resolver update candidate, the agent installs the target
-version in a temporary isolated virtualenv and writes an API snapshot plus
-`api_diffs.json` entry. This avoids false downgrade diffs for packages whose
-locked prerelease is newer than PyPI's stable latest field. Candidate inspection
-is always enabled for update candidates; `--inspect-candidates` remains accepted
-only for CLI compatibility.
+metadata. With `--inspect-candidates`, the agent installs each resolver update
+candidate in a temporary isolated virtualenv — with a credential-scrubbed
+environment (throwaway `HOME`, `PATH` only) — and writes an API snapshot plus
+`api_diffs.json` entry, and runs the behavior probes against the candidate the
+same way. This avoids false downgrade diffs for packages whose locked
+prerelease is newer than PyPI's stable latest field. Candidate inspection is
+opt-in because it executes freshly downloaded upstream code; without the flag,
+candidates are recorded as explicit `skip` entries rather than silently
+missing evidence.
If `uv lock --dry-run -P ...` reports an SDK update but the run cannot produce a
candidate-version API diff for that package, implementation is blocked and the
diff --git a/examples/run_same_task.py b/examples/run_same_task.py
index c33968d..1b33963 100644
--- a/examples/run_same_task.py
+++ b/examples/run_same_task.py
@@ -4,7 +4,7 @@
import asyncio
-from agent_runtime_kit import AgentTask, create_default_registry
+from agent_runtime_kit import AgentTask, create_default_registry, runtime_kind_value
from agent_runtime_kit.adapters import register_adapters
@@ -14,15 +14,17 @@ async def main() -> None:
task = AgentTask(goal="Summarize this repository in one paragraph.")
for kind in registry.kinds():
- if kind.value == "fake":
+ # runtime_kind_value: kinds may be enum members or namespaced strings.
+ label = runtime_kind_value(kind)
+ if label == "fake":
continue
runtime = registry.resolve(kind)
diagnostic = runtime.availability()
if not diagnostic.available:
- print(f"{kind.value}: unavailable - {diagnostic.message}")
+ print(f"{label}: unavailable - {diagnostic.message}")
continue
result = await runtime.run(task)
- print(f"{kind.value}: {result.output}")
+ print(f"{label}: {result.output}")
if __name__ == "__main__":
diff --git a/examples/sdk_evolution_agent/behavior.py b/examples/sdk_evolution_agent/behavior.py
index 2847111..96317b4 100644
--- a/examples/sdk_evolution_agent/behavior.py
+++ b/examples/sdk_evolution_agent/behavior.py
@@ -15,14 +15,24 @@
from typing import Any
from examples.sdk_evolution_agent.models import BehaviorDiff, BehaviorProbeResult
-from examples.sdk_evolution_agent.snapshots import DEFAULT_MODULES
+from examples.sdk_evolution_agent.snapshots import DEFAULT_MODULES, isolated_env
def collect_behavior_evidence(
packages: Sequence[Mapping[str, object]],
update_versions: Mapping[str, str],
+ *,
+ inspect_candidates: bool = False,
) -> dict[str, Any]:
- """Collect current/candidate behavior probes and compare them."""
+ """Collect current/candidate behavior probes and compare them.
+
+ Probing a version that is not installed means pip-installing and importing
+ freshly downloaded upstream code, so those venv probes run only when the
+ caller opted in via ``inspect_candidates`` (the ``--inspect-candidates`` CLI
+ flag) — the same gate the API snapshots use. Without the opt-in, candidates
+ get an explicit ``skip`` record instead of a silent evidence hole, and a
+ drifted lockfile baseline falls back to probing the installed environment.
+ """
results: list[BehaviorProbeResult] = []
for package in packages:
@@ -32,13 +42,21 @@ def collect_behavior_evidence(
locked_version = _string_or_none(package.get("locked_version"))
installed_version = _string_or_none(package.get("installed_version"))
current_version = locked_version or installed_version
- if locked_version and installed_version and locked_version != installed_version:
+ if (
+ inspect_candidates
+ and locked_version
+ and installed_version
+ and locked_version != installed_version
+ ):
results.extend(probe_candidate_in_venv(name, locked_version, scope="current-baseline"))
else:
results.extend(probe_current_package(name, version=current_version))
candidate = update_versions.get(name)
if candidate:
- results.extend(probe_candidate_in_venv(name, candidate, scope="candidate"))
+ if inspect_candidates:
+ results.extend(probe_candidate_in_venv(name, candidate, scope="candidate"))
+ else:
+ results.append(_skipped_candidate_probe(name, candidate))
diffs = diff_behavior_results(results)
return {
"results": [result for result in results],
@@ -69,7 +87,12 @@ def probe_candidate_in_venv(
with tempfile.TemporaryDirectory(prefix="ark-sdk-behavior-") as directory:
venv = Path(directory) / ".venv"
- subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout)
+ # Scrub the environment for every subprocess that touches freshly
+ # downloaded upstream code (same scrub as the API snapshots): a
+ # throwaway HOME and only PATH, so a malicious or buggy candidate
+ # package cannot read the caller's credentials/config.
+ env = isolated_env(Path(directory))
+ subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env)
bin_dir = "Scripts" if sys.platform == "win32" else "bin"
venv_python = venv / bin_dir / "python"
subprocess.run(
@@ -78,6 +101,7 @@ def probe_candidate_in_venv(
text=True,
capture_output=True,
timeout=timeout,
+ env=env,
)
completed = subprocess.run(
(str(venv_python), "-c", _PROBE_SCRIPT, package, version, scope),
@@ -85,6 +109,7 @@ def probe_candidate_in_venv(
text=True,
capture_output=True,
timeout=timeout,
+ env=env,
)
raw = json.loads(completed.stdout)
return tuple(BehaviorProbeResult(**item) for item in raw)
@@ -102,6 +127,10 @@ def diff_behavior_results(results: Sequence[BehaviorProbeResult]) -> tuple[Behav
after = scopes.get("candidate") or scopes.get("isolated-venv")
if before is None or after is None:
continue
+ if (before.status == "skip") != (after.status == "skip"):
+ # One side was not probed (candidate installs are opt-in): absence
+ # of evidence is not a behavior change and must not read as one.
+ continue
if before.status == after.status and _contract_details(before) == _contract_details(after):
severity = "none"
summary = "No behavior contract difference detected."
@@ -347,6 +376,22 @@ def _failed(
)
+def _skipped_candidate_probe(package: str, version: str) -> BehaviorProbeResult:
+ return BehaviorProbeResult(
+ package=package,
+ version=version,
+ scope="candidate",
+ probe="adapter-contract",
+ status="skip",
+ summary=(
+ "Candidate behavior probe skipped: probing a candidate installs and "
+ "imports freshly downloaded upstream code, which is opt-in. Rerun "
+ "with --inspect-candidates to collect this evidence."
+ ),
+ details={"reason": "candidate installs are opt-in (--inspect-candidates)"},
+ )
+
+
def _string_or_none(value: object) -> str | None:
if value is None:
return None
diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py
index 93a25bf..b776339 100644
--- a/examples/sdk_evolution_agent/cli.py
+++ b/examples/sdk_evolution_agent/cli.py
@@ -4,7 +4,6 @@
import argparse
import re
-from dataclasses import replace
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
@@ -98,10 +97,12 @@ def parse_args(argv: list[str] | None = None) -> RunOptions:
parser.add_argument(
"--inspect-candidates",
action="store_true",
- default=True,
+ default=False,
help=(
- "Inspect latest candidate SDK versions in temporary virtualenvs. "
- "Always enabled for update candidates; accepted for compatibility."
+ "Opt in to inspecting candidate SDK versions by pip-installing and "
+ "importing them in a temporary, credential-scrubbed virtualenv. This "
+ "executes freshly downloaded upstream code, so it is OFF by default; "
+ "without it, snapshots use the already-installed versions only."
),
)
parser.add_argument("--create-branch", action="store_true", help="Create a local branch first.")
@@ -146,7 +147,6 @@ async def run_agent(
) -> Path:
"""Run the full local SDK evolution workflow."""
- options = replace(options, inspect_candidates=True)
run_id = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ")
report_root = (options.workspace / options.report_dir / run_id).resolve()
event_log_path = report_root / "events.jsonl"
@@ -184,14 +184,18 @@ async def run_agent(
command_runner=command_runner,
)
update_versions = _refresh_update_versions(evidence)
- snapshots = _collect_snapshots(evidence)
+ snapshots = _collect_snapshots(evidence, inspect_candidates=options.inspect_candidates)
api_diffs = [to_jsonable(diff) for diff in diff_snapshot_groups(snapshots)]
release_notes = [
to_jsonable(item)
for item in collect_release_notes(evidence.get("packages", []), update_versions)
]
behavior = to_jsonable(
- collect_behavior_evidence(evidence.get("packages", []), update_versions)
+ collect_behavior_evidence(
+ evidence.get("packages", []),
+ update_versions,
+ inspect_candidates=options.inspect_candidates,
+ )
)
direction, architecture, review = await run_analysis_pipeline(
selected_runtime,
@@ -412,8 +416,10 @@ def _create_autonomous_pr(
) -> list[dict[str, Any]]:
branch_name = options.branch_name or _current_branch(root, command_runner=command_runner)
body = build_draft_pr_body(report_path.read_text(encoding="utf-8"))
- relative_report = _relative_path(root, report_path.parent)
- paths = ("uv.lock", relative_report)
+ # Only stage the lockfile. The report lives under the gitignored default
+ # report dir, so `git add`-ing it returned rc=1 and previously broke the
+ # --draft-pr flow; its content is already embedded in the PR body above.
+ paths = ("uv.lock",)
results = [
to_jsonable(stage_paths(root, paths, command_runner=command_runner)),
to_jsonable(
@@ -452,6 +458,12 @@ def _commit_final_autonomous_pr_report(
) -> None:
branch_name = options.branch_name or _current_branch(root, command_runner=command_runner)
relative_report = _relative_path(root, report_path.parent)
+ if not _report_dir_committable(root, relative_report, command_runner=command_runner):
+ # Skipped when the report dir is gitignored (the default location) or
+ # outside the repository entirely: `git add` would fail on either, and
+ # the report content is already embedded in the draft PR body, so there
+ # is nothing that must be committed.
+ return
results = [
stage_paths(root, (relative_report,), command_runner=command_runner),
commit_staged(
@@ -468,6 +480,20 @@ def _commit_final_autonomous_pr_report(
raise RuntimeError(f"failed to commit final autonomous PR report: {detail}")
+def _report_dir_committable(
+ root: Path, path: str, *, command_runner: CommandRunner | None
+) -> bool:
+ runner = command_runner
+ if runner is None:
+ from examples.sdk_evolution_agent.collectors import run_command
+
+ runner = run_command
+ # `git check-ignore` exits 0 when the path is ignored, 1 when it is not
+ # ignored, and 128 when it cannot judge it (e.g. the path lies outside the
+ # repository). Only a definitively not-ignored path can be staged.
+ return runner(("git", "check-ignore", path), cwd=root).returncode == 1
+
+
def _current_branch(root: Path, *, command_runner: CommandRunner | None) -> str:
runner = command_runner or None
if runner is None:
@@ -485,8 +511,11 @@ def _relative_path(root: Path, path: Path) -> str:
return str(path)
-def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = True) -> list[Any]:
- del inspect_candidates # Candidate inspection is mandatory for update candidates.
+def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = False) -> list[Any]:
+ # Candidate inspection pip-installs and imports freshly downloaded upstream
+ # code, so it only runs when explicitly opted in (--inspect-candidates). When
+ # off, every snapshot uses the already-installed version via snapshot_current_api,
+ # which imports nothing new.
snapshots = []
update_versions = _refresh_update_versions(evidence)
refresh_preview_seen = evidence.get("refresh_preview") is not None
@@ -497,10 +526,12 @@ def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = T
locked = package.get("locked_version")
installed = package.get("installed_version")
baseline = locked or installed
- if locked and installed and locked != installed:
+ if inspect_candidates and locked and installed and locked != installed:
snapshots.append(snapshot_candidate_in_venv(name, str(locked)))
else:
snapshots.append(snapshot_current_api(name, version=baseline))
+ if not inspect_candidates:
+ continue
candidate = update_versions.get(name)
if candidate is None and not refresh_preview_seen:
latest = package.get("latest_version")
diff --git a/examples/sdk_evolution_agent/models.py b/examples/sdk_evolution_agent/models.py
index 8878709..9ce477e 100644
--- a/examples/sdk_evolution_agent/models.py
+++ b/examples/sdk_evolution_agent/models.py
@@ -148,7 +148,9 @@ class RunOptions:
report_dir: Path = Path("reports/sdk-evolution")
implementation_enabled: bool = False
refresh_preview: bool = False
- inspect_candidates: bool = True
+ # Off by default: candidate inspection pip-installs and imports freshly
+ # downloaded upstream code (see _collect_snapshots / --inspect-candidates).
+ inspect_candidates: bool = False
create_branch: bool = False
branch_name: str | None = None
draft_pr: bool = False
diff --git a/examples/sdk_evolution_agent/release_notes.py b/examples/sdk_evolution_agent/release_notes.py
index 16c1345..57f5b63 100644
--- a/examples/sdk_evolution_agent/release_notes.py
+++ b/examples/sdk_evolution_agent/release_notes.py
@@ -247,19 +247,25 @@ def _fetch_source_text(
use_github_graphql: bool,
) -> str:
if use_github_graphql and source.kind == "github-discussions" and source.url:
- try:
- return _fetch_github_discussions_index(source.url)
- except Exception:
- pass
+ token = _github_token()
+ if token:
+ # With a token configured, a GraphQL failure (expired/insufficient
+ # credentials, API errors) must surface — the caller records it on
+ # the source — instead of silently downgrading to the
+ # unauthenticated HTML scrape, which would hide a broken token
+ # behind lower-quality evidence. Without a token, the HTML path is
+ # simply the tokenless mode of operation.
+ return _fetch_github_discussions_index(source.url, token=token)
if not source.url:
return ""
return fetcher(source.url)
-def _fetch_github_discussions_index(url: str) -> str:
- token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
- if not token:
- raise RuntimeError("GITHUB_TOKEN or GH_TOKEN is required for GitHub Discussions GraphQL")
+def _github_token() -> str | None:
+ return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+
+
+def _fetch_github_discussions_index(url: str, *, token: str) -> str:
owner, repo, category_slug = _parse_github_discussion_category_url(url)
query = """
query($owner: String!, $repo: String!) {
@@ -384,7 +390,15 @@ def _release_note_links_for_version(source_url: str, text: str, to_version: str)
window = text[match.end() : match.end() + 1200]
if not any(marker in window for marker in markers):
continue
- links.append(urllib.parse.urljoin(source_url, match.group("path")))
+ resolved = urllib.parse.urljoin(source_url, match.group("path"))
+ parsed = urllib.parse.urlparse(resolved)
+ # These hrefs come from fetched content that can carry user-generated
+ # markup (discussion bodies). Only follow https URLs on github.com
+ # itself: a protocol-relative path ("//evil.example/x/discussions/1")
+ # would otherwise urljoin into an off-site request.
+ if parsed.scheme != "https" or parsed.netloc != "github.com":
+ continue
+ links.append(resolved)
return tuple(_dedupe(links))
diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py
index 548d8a3..f198536 100644
--- a/examples/sdk_evolution_agent/snapshots.py
+++ b/examples/sdk_evolution_agent/snapshots.py
@@ -5,6 +5,7 @@
import importlib
import inspect
import json
+import os
import subprocess
import sys
import tempfile
@@ -96,6 +97,27 @@ def diff_snapshot_groups(snapshots: Sequence[ApiSnapshot]) -> tuple[ApiDiff, ...
return tuple(diffs)
+def isolated_env(home: Path) -> dict[str, str]:
+ """A minimal environment for candidate subprocesses: PATH + a throwaway HOME.
+
+ Deliberately omits the caller's credentials/config so freshly downloaded
+ upstream code executed during inspection cannot read them. That scrub also
+ drops proxy and CA overrides (HTTPS_PROXY, SSL_CERT_FILE, PIP_INDEX_URL...),
+ so behind a corporate TLS-intercepting proxy or private mirror the candidate
+ install may fail — an accepted trade-off of the isolation. Shared with the
+ behavior probes, which install candidates the same way.
+ """
+
+ env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home)}
+ if sys.platform == "win32":
+ env["USERPROFILE"] = str(home)
+ for key in ("SYSTEMROOT", "SystemRoot", "COMSPEC", "PATHEXT", "TEMP", "TMP"):
+ value = os.environ.get(key)
+ if value:
+ env[key] = value
+ return env
+
+
def snapshot_candidate_in_venv(
package: str,
version: str,
@@ -108,7 +130,13 @@ def snapshot_candidate_in_venv(
module_name = DEFAULT_MODULES.get(package, package.replace("-", "_"))
with tempfile.TemporaryDirectory(prefix="ark-sdk-snapshot-") as directory:
venv = Path(directory) / ".venv"
- subprocess.run((python, "-m", "venv", str(venv)), check=True, timeout=timeout)
+ # Scrub the environment for every subprocess that touches freshly downloaded
+ # upstream code: give it a throwaway HOME and only PATH, so a malicious or
+ # buggy candidate package cannot read the caller's credentials/config.
+ env = isolated_env(Path(directory))
+ subprocess.run(
+ (python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env
+ )
bin_dir = "Scripts" if sys.platform == "win32" else "bin"
venv_python = venv / bin_dir / "python"
subprocess.run(
@@ -117,6 +145,7 @@ def snapshot_candidate_in_venv(
text=True,
capture_output=True,
timeout=timeout,
+ env=env,
)
completed = subprocess.run(
(
@@ -131,6 +160,7 @@ def snapshot_candidate_in_venv(
text=True,
capture_output=True,
timeout=timeout,
+ env=env,
)
raw = json.loads(completed.stdout)
return ApiSnapshot(
diff --git a/examples/sdk_evolution_agent/stages.py b/examples/sdk_evolution_agent/stages.py
index 843a468..15c9c53 100644
--- a/examples/sdk_evolution_agent/stages.py
+++ b/examples/sdk_evolution_agent/stages.py
@@ -19,6 +19,7 @@
PermissionProfile,
RuntimeAvailability,
RuntimeRegistry,
+ runtime_kind_value,
)
from agent_runtime_kit.adapters import (
AntigravityAgentRuntime,
@@ -93,6 +94,15 @@ async def cancel(self, task_id: str) -> None:
del task_id
+ async def aclose(self) -> None:
+ """Fixture runtime holds no resources."""
+
+ async def __aenter__(self) -> FixtureEvolutionRuntime:
+ return self
+
+ async def __aexit__(self, exc_type: object, exc: object, tb: object, /) -> None:
+ await self.aclose()
+
def build_registry() -> RuntimeRegistry:
"""Build the runtime registry used by the SDK evolution agent."""
@@ -159,10 +169,12 @@ async def run_stage(
) -> dict[str, Any]:
"""Run one AI stage through agent-runtime-kit and validate its output."""
+ # runtime_kind_value: third-party runtimes may use plain string kinds.
+ kind_label = runtime_kind_value(runtime.kind)
if not runtime.capabilities.structured_output:
- raise StageExecutionError(f"{runtime.kind.value} cannot honor required output_schema")
+ raise StageExecutionError(f"{kind_label} cannot honor required output_schema")
if not runtime.capabilities.working_directory:
- raise StageExecutionError(f"{runtime.kind.value} cannot honor required working_directory")
+ raise StageExecutionError(f"{kind_label} cannot honor required working_directory")
permissions = _stage_permissions(runtime, write_enabled=write_enabled)
task = AgentTask(
goal=json.dumps(payload, sort_keys=True, default=str),
@@ -176,7 +188,7 @@ async def run_stage(
try:
result = await runtime.run(task)
except Exception as exc:
- raise StageExecutionError(f"{stage} failed through {runtime.kind.value}: {exc}") from exc
+ raise StageExecutionError(f"{stage} failed through {kind_label}: {exc}") from exc
data = result.parsed_output
if data is None:
try:
diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py
index 5d2ef33..81b8817 100644
--- a/tests/test_sdk_evolution_agent.py
+++ b/tests/test_sdk_evolution_agent.py
@@ -30,6 +30,7 @@
from examples.sdk_evolution_agent.behavior import (
collect_behavior_evidence,
diff_behavior_results,
+ probe_candidate_in_venv,
)
from examples.sdk_evolution_agent.cli import RunOptions, _collect_snapshots, parse_args, run_agent
from examples.sdk_evolution_agent.collectors import (
@@ -40,9 +41,10 @@
run_refresh_preview,
)
from examples.sdk_evolution_agent.current_state import build_current_state
-from examples.sdk_evolution_agent.models import ApiSnapshot, CommandResult, RunContext
+from examples.sdk_evolution_agent.models import ApiSnapshot, CommandResult, RunContext, SourceRef
from examples.sdk_evolution_agent.pr import build_draft_pr_body
from examples.sdk_evolution_agent.release_notes import (
+ _fetch_source_text,
_format_github_discussions_index,
collect_release_notes,
)
@@ -246,6 +248,85 @@ def fetcher(url: str) -> str:
assert any("Python 3.14" in summary for summary in notes[0].summaries)
+def test_release_note_links_only_follow_github_https_urls() -> None:
+ # The discussion-index markup can carry user-generated hrefs. A
+ # protocol-relative path would urljoin into an off-site host and the blind
+ # follow-up GET would become SSRF; only https://github.com links may be
+ # followed.
+ def fetcher(url: str) -> str:
+ if url.endswith("/discussions/categories/announcements"):
+ return (
+ 'v0.1.5\n'
+ 'v0.1.5 release notes'
+ )
+ if "evil.example" in url:
+ raise AssertionError("off-site link must not be fetched")
+ if url.endswith("/discussions/88"):
+ return "## v0.1.5\n- Real release note\n"
+ return "no matching release note"
+
+ notes = collect_release_notes(
+ [
+ {
+ "name": "google-antigravity",
+ "locked_version": "0.1.4",
+ "installed_version": "0.1.4",
+ }
+ ],
+ {"google-antigravity": "0.1.5"},
+ fetcher=fetcher,
+ )
+
+ assert all("evil.example" not in url for url in notes[0].checked_urls)
+ assert any(url.endswith("/discussions/88") for url in notes[0].checked_urls)
+
+
+def test_fetch_source_text_surfaces_graphql_failure_when_token_configured(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A configured token that fails must surface (the caller records it on the
+ # source), not silently downgrade to the unauthenticated HTML scrape — that
+ # would hide an expired/insufficient token behind lower-quality evidence.
+ monkeypatch.setenv("GITHUB_TOKEN", "configured-but-broken")
+
+ def graphql_fails(url: str, *, token: str) -> str:
+ raise RuntimeError("GraphQL: Bad credentials")
+
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.release_notes._fetch_github_discussions_index",
+ graphql_fails,
+ )
+ source = SourceRef(
+ kind="github-discussions",
+ label="announcements",
+ url="https://github.com/o/r/discussions/categories/announcements",
+ )
+
+ with pytest.raises(RuntimeError, match="Bad credentials"):
+ _fetch_source_text(source, fetcher=lambda url: "html", use_github_graphql=True)
+
+
+def test_fetch_source_text_uses_plain_fetch_without_token(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Tokenless operation is legitimate: without GITHUB_TOKEN/GH_TOKEN the HTML
+ # fetch is the normal path, not a downgrade.
+ monkeypatch.delenv("GITHUB_TOKEN", raising=False)
+ monkeypatch.delenv("GH_TOKEN", raising=False)
+ source = SourceRef(
+ kind="github-discussions",
+ label="announcements",
+ url="https://github.com/o/r/discussions/categories/announcements",
+ )
+
+ text = _fetch_source_text(
+ source, fetcher=lambda url: "html fallback", use_github_graphql=True
+ )
+
+ assert text == "html fallback"
+
+
def test_github_discussions_graphql_index_filters_discussion_category() -> None:
index = _format_github_discussions_index(
{
@@ -401,6 +482,7 @@ def isolated(package: str, version: str, *, scope: str = "candidate"):
}
],
{"claude-agent-sdk": "0.2.106"},
+ inspect_candidates=True,
)
assert calls == [
@@ -410,6 +492,73 @@ def isolated(package: str, version: str, *, scope: str = "candidate"):
assert behavior["diffs"][0].severity == "changed"
+def test_behavior_candidate_probes_are_opt_in(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Probing a candidate pip-installs and imports freshly downloaded upstream
+ # code; without --inspect-candidates that must never happen, and the gap
+ # must be an explicit skip record rather than a silent evidence hole or a
+ # phantom "breaking" diff.
+ def explode(package: str, version: str, *, scope: str = "candidate") -> tuple[Any, ...]:
+ raise AssertionError("venv probe must not run without --inspect-candidates")
+
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.behavior.probe_candidate_in_venv",
+ explode,
+ )
+
+ behavior = collect_behavior_evidence(
+ [
+ {
+ "name": "claude-agent-sdk",
+ "locked_version": "0.2.96",
+ "installed_version": "0.2.106",
+ }
+ ],
+ {"claude-agent-sdk": "0.2.106"},
+ )
+
+ candidate_results = [r for r in behavior["results"] if r.scope == "candidate"]
+ assert [r.status for r in candidate_results] == ["skip"]
+ assert "--inspect-candidates" in candidate_results[0].summary
+ # The drifted lockfile baseline falls back to the installed environment.
+ assert any(r.scope == "current-environment" for r in behavior["results"])
+ assert behavior["summary"]["breaking_count"] == 0
+
+
+def test_behavior_diffs_ignore_one_sided_skip() -> None:
+ diffs = diff_behavior_results(
+ [
+ _probe("claude-agent-sdk", "0.2.96", "current-environment", "pass", {"fields": ["a"]}),
+ _probe("claude-agent-sdk", "0.2.106", "candidate", "skip", {"reason": "opt-in"}),
+ ]
+ )
+
+ assert diffs == ()
+
+
+def test_candidate_behavior_probe_scrubs_subprocess_env(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured_envs: list[dict[str, str] | None] = []
+
+ def fake_run(args: Any, **kwargs: Any) -> Any:
+ captured_envs.append(kwargs.get("env"))
+ return types.SimpleNamespace(stdout="[]", returncode=0)
+
+ monkeypatch.setenv("ARK_FAKE_SECRET", "not-for-candidates")
+ monkeypatch.setattr("examples.sdk_evolution_agent.behavior.subprocess.run", fake_run)
+
+ results = probe_candidate_in_venv("claude-agent-sdk", "9.9.9")
+
+ assert results == ()
+ # venv create, pip install, probe script: every subprocess that touches the
+ # freshly downloaded candidate runs with the scrubbed environment.
+ assert len(captured_envs) == 3
+ for env in captured_envs:
+ assert env is not None
+ assert "ARK_FAKE_SECRET" not in env
+ assert env.get("HOME") != os.environ.get("HOME")
+
+
def test_behavior_probe_guard_blocks_breaking_candidate_diff() -> None:
guarded = with_behavior_probe_guard(
{
@@ -497,10 +646,11 @@ def run_new(value: str, *, verbose: bool = False) -> str:
assert diff.changed == ("run",)
-def test_parse_args_inspects_candidates_by_default() -> None:
- options = parse_args(["--runtime", "fake"])
-
- assert options.inspect_candidates is True
+def test_parse_args_candidate_inspection_is_opt_in() -> None:
+ # Candidate inspection pip-installs+imports upstream code, so it is off by
+ # default and only enabled with the explicit flag.
+ assert parse_args(["--runtime", "fake"]).inspect_candidates is False
+ assert parse_args(["--runtime", "fake", "--inspect-candidates"]).inspect_candidates is True
def test_collect_snapshots_uses_lockfile_baseline_for_candidates(
@@ -541,7 +691,7 @@ def candidate_snapshot(package: str, version: str) -> ApiSnapshot:
}
]
},
- inspect_candidates=False,
+ inspect_candidates=True,
)
assert len(snapshots) == 2
@@ -596,6 +746,7 @@ def candidate_snapshot(package: str, version: str) -> ApiSnapshot:
"stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n",
},
},
+ inspect_candidates=True,
)
assert len(snapshots) == 3
@@ -643,6 +794,7 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot:
"stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n",
},
},
+ inspect_candidates=True,
)
assert calls == [
@@ -651,6 +803,52 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot:
]
+def test_collect_snapshots_without_opt_in_never_installs_even_when_lock_drifted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The security default: a drifted lock (locked != installed) plus pending
+ # updates must NOT trigger isolated-venv installs unless --inspect-candidates
+ # was passed; everything snapshots from the already-installed environment.
+ calls: list[tuple[str, str, str | None]] = []
+
+ def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot:
+ calls.append(("current", package, version))
+ return ApiSnapshot(package=package, version=version, module=package.replace("-", "_"))
+
+ def isolated_snapshot(package: str, version: str) -> ApiSnapshot:
+ calls.append(("isolated", package, version))
+ return ApiSnapshot(package=package, version=version, module=package.replace("-", "_"))
+
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.cli.snapshot_current_api",
+ current_snapshot,
+ )
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv",
+ isolated_snapshot,
+ )
+
+ _collect_snapshots(
+ {
+ "packages": [
+ {
+ "name": "claude-agent-sdk",
+ "locked_version": "0.2.96",
+ "installed_version": "0.2.106",
+ "latest_version": "0.2.106",
+ },
+ ],
+ "refresh_preview": {
+ "stdout": "",
+ "stderr": "Update claude-agent-sdk v0.2.96 -> v0.2.106\n",
+ },
+ },
+ inspect_candidates=False,
+ )
+
+ assert calls == [("current", "claude-agent-sdk", "0.2.96")]
+
+
def test_candidate_api_diff_guard_blocks_missing_update_diff() -> None:
guarded = with_candidate_api_diff_guard(
{
@@ -932,7 +1130,7 @@ async def test_run_agent_report_only_generates_artifacts(
packages=("claude-agent-sdk",),
report_dir=Path("reports"),
implementation_enabled=False,
- inspect_candidates=False,
+ inspect_candidates=True,
),
pypi_client=_fake_pypi,
runtime=FixtureEvolutionRuntime(),
@@ -956,6 +1154,118 @@ async def test_run_agent_report_only_generates_artifacts(
assert "Recursive self-adaptation impact" in report_path.read_text(encoding="utf-8")
+@pytest.mark.asyncio
+async def test_run_agent_does_not_install_candidates_by_default(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ (tmp_path / "pyproject.toml").write_text(
+ '[project.optional-dependencies]\nclaude = ["claude-agent-sdk>=0.2"]\n',
+ encoding="utf-8",
+ )
+ (tmp_path / "uv.lock").write_text("", encoding="utf-8")
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.cli.snapshot_current_api",
+ lambda package, *, version=None: ApiSnapshot(
+ package=package, version=version, module=package.replace("-", "_")
+ ),
+ )
+
+ def forbidden_candidate(package: str, version: str) -> ApiSnapshot:
+ raise AssertionError("candidate install must not run without --inspect-candidates")
+
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", forbidden_candidate
+ )
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.cli.collect_release_notes", lambda packages, updates: []
+ )
+ monkeypatch.setattr(
+ "examples.sdk_evolution_agent.cli.collect_behavior_evidence",
+ lambda packages, updates, *, inspect_candidates=False: {
+ "results": [],
+ "diffs": [],
+ "summary": {"status": "pass"},
+ },
+ )
+
+ # Default run (no inspect_candidates) must never pip-install/import upstream code.
+ report_path = await run_agent(
+ RunOptions(workspace=tmp_path, runtime="fake", packages=("claude-agent-sdk",)),
+ pypi_client=_fake_pypi,
+ runtime=FixtureEvolutionRuntime(),
+ )
+
+ assert report_path.exists()
+
+
+def test_finalize_report_skips_when_report_dir_is_gitignored(tmp_path: Path) -> None:
+ from examples.sdk_evolution_agent.cli import _commit_final_autonomous_pr_report
+
+ commands: list[tuple[str, ...]] = []
+
+ def runner(
+ command: tuple[str, ...],
+ *,
+ cwd: Path | None = None,
+ env: dict[str, str] | None = None,
+ ) -> CommandResult:
+ del cwd, env
+ commands.append(command)
+ # check-ignore exits 0 => path IS ignored (the default report dir).
+ return CommandResult(command=command, returncode=0)
+
+ report_path = tmp_path / "reports" / "sdk-evolution" / "run" / "report.md"
+ report_path.parent.mkdir(parents=True)
+ report_path.write_text("x", encoding="utf-8")
+
+ # Must skip cleanly (no add/commit of a gitignored path) instead of raising.
+ _commit_final_autonomous_pr_report(
+ tmp_path,
+ report_path=report_path,
+ options=RunOptions(workspace=tmp_path, runtime="fake"),
+ command_runner=runner,
+ )
+
+ assert not any(command[:2] == ("git", "commit") for command in commands)
+ assert not any(command[:2] == ("git", "add") for command in commands)
+
+
+def test_finalize_report_skips_when_report_dir_is_outside_the_repo(tmp_path: Path) -> None:
+ from examples.sdk_evolution_agent.cli import _commit_final_autonomous_pr_report
+
+ commands: list[tuple[str, ...]] = []
+
+ def runner(
+ command: tuple[str, ...],
+ *,
+ cwd: Path | None = None,
+ env: dict[str, str] | None = None,
+ ) -> CommandResult:
+ del cwd, env
+ commands.append(command)
+ # check-ignore exits 128 when it cannot judge the path (outside the repo).
+ if command[:2] == ("git", "check-ignore"):
+ return CommandResult(command=command, returncode=128, stderr="fatal: outside repo")
+ return CommandResult(command=command, returncode=0)
+
+ report_path = tmp_path / "elsewhere" / "run" / "report.md"
+ report_path.parent.mkdir(parents=True)
+ report_path.write_text("x", encoding="utf-8")
+
+ # A --report-dir outside the workspace must skip cleanly, not stage a
+ # doomed out-of-repo path and raise.
+ _commit_final_autonomous_pr_report(
+ tmp_path / "repo",
+ report_path=report_path,
+ options=RunOptions(workspace=tmp_path / "repo", runtime="fake"),
+ command_runner=runner,
+ )
+
+ assert not any(command[:2] == ("git", "commit") for command in commands)
+ assert not any(command[:2] == ("git", "add") for command in commands)
+
+
@pytest.mark.asyncio
async def test_run_agent_autonomous_pr_path(
tmp_path: Path,
@@ -999,7 +1309,11 @@ async def test_run_agent_autonomous_pr_path(
)
monkeypatch.setattr(
"examples.sdk_evolution_agent.cli.collect_behavior_evidence",
- lambda packages, updates: {"results": [], "diffs": [], "summary": {"status": "pass"}},
+ lambda packages, updates, *, inspect_candidates=False: {
+ "results": [],
+ "diffs": [],
+ "summary": {"status": "pass"},
+ },
)
commands: list[tuple[str, ...]] = []
@@ -1019,6 +1333,9 @@ def runner(
)
if command[:2] == ("uv", "lock"):
return CommandResult(command=command, returncode=0, stdout="updated")
+ if command[:2] == ("git", "check-ignore"):
+ # Report dir is tracked in this scenario -> not ignored (exit 1).
+ return CommandResult(command=command, returncode=1, stdout="")
return CommandResult(command=command, returncode=0, stdout="ok")
report_path = await run_agent(
@@ -1027,6 +1344,7 @@ def runner(
runtime="fake",
packages=("claude-agent-sdk",),
report_dir=Path("reports"),
+ inspect_candidates=True,
implementation_enabled=True,
refresh_preview=True,
create_branch=True,
@@ -1130,7 +1448,7 @@ def test_parse_args_and_pr_body() -> None:
assert options.runtime == "claude-agent-sdk"
assert options.packages == ("claude-agent-sdk",)
- assert options.inspect_candidates is True
+ assert options.inspect_candidates is False
assert options.implementation_enabled is True
assert options.draft_pr is True
assert "No auto-merge" in body
@@ -1346,6 +1664,15 @@ async def run(self, task: AgentTask) -> AgentResult:
async def cancel(self, task_id: str) -> None:
del task_id
+ async def aclose(self) -> None:
+ return None
+
+ async def __aenter__(self) -> RecordingRuntime:
+ return self
+
+ async def __aexit__(self, exc_type: object, exc: object, tb: object, /) -> None:
+ await self.aclose()
+
class PermissiveRuntime(RecordingRuntime):
async def run(self, task: AgentTask) -> AgentResult: