From 48bf2603f46223d5a157918c028e364e1d124784 Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 12:11:30 -0400 Subject: [PATCH 1/7] docs: design verified public provenance boundary --- .../2026-08-09-public-provenance-boundary.md | 181 ++++++++++++++++++ ...08-09-public-provenance-boundary-design.md | 64 +++++++ 2 files changed, 245 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-09-public-provenance-boundary.md create mode 100644 docs/superpowers/specs/2026-08-09-public-provenance-boundary-design.md diff --git a/docs/superpowers/plans/2026-08-09-public-provenance-boundary.md b/docs/superpowers/plans/2026-08-09-public-provenance-boundary.md new file mode 100644 index 0000000..26fd4dd --- /dev/null +++ b/docs/superpowers/plans/2026-08-09-public-provenance-boundary.md @@ -0,0 +1,181 @@ +# Verified Public Provenance Boundary Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ensure only pull requests with verified public GitHub repository metadata can be labeled live-public or counted as genuine alpha evidence. + +**Architecture:** GitHub ingestion produces the only trusted verified-public visibility fact. Shared Pydantic models persist that fact through snapshots, reviews, alpha cases, storage, and exports; CLI and Streamlit copy it only through guarded review construction, and alpha transitions independently require it. + +**Tech Stack:** Python 3.11+, Pydantic v2, httpx, Typer CLI, Streamlit, pytest + +## Global Constraints + +- ScopeProof remains an evidence assistant, not a correctness oracle. +- Never execute target-repository code. +- `verified_public` requires unambiguous GitHub metadata; missing or contradictory metadata fails closed. +- Tokens remain session-only and absent from records, errors, logs, and exports. +- Legacy records without the fact remain readable as `unverified` but Stage 1-ineligible. +- Every persisted or exported object remains Pydantic-validated. +- Use named-file staging only and preserve `.coverage 2` untouched. + +--- + +### Task 1: GitHub ingestion establishes verified visibility + +**Files:** +- Modify: `scopeproof_core/schemas/models.py` +- Modify: `scopeproof_core/github/client.py` +- Test: `tests/github/test_client.py` + +**Interfaces:** +- Produces: `RepositoryVisibility.UNVERIFIED` and `RepositoryVisibility.VERIFIED_PUBLIC` +- Produces: `PullRequestSnapshot.repository_visibility: RepositoryVisibility` +- Produces: `RepositoryVisibilityUnverified`, a bounded ingestion error + +- [ ] **Step 1: Write failing public/private/malformed/mismatched metadata tests** + +Add literal GitHub response fixtures asserting that only matching `private: false` plus +`visibility: public` returns a snapshot with `verified_public`; explicit private and +all ambiguous variants raise bounded errors before secondary fetches. + +- [ ] **Step 2: Run the focused tests and confirm the intended failures** + +Run: `pytest tests/github/test_client.py -q` + +Expected: failures because the visibility enum, field, and fail-closed validation do +not exist. + +- [ ] **Step 3: Implement the minimal typed ingestion boundary** + +Add the enum and snapshot field. Validate `base.repo.full_name`, `private`, and +`visibility` immediately after the PR response, before fetching files, commits, or +checks. Return `verified_public` only for the unambiguous public case. + +- [ ] **Step 4: Run the focused tests and confirm they pass** + +Run: `pytest tests/github/test_client.py -q` + +- [ ] **Step 5: Commit the ingestion contract with named files** + +Stage only the schema, client, and GitHub client test files. + +### Task 2: Review construction, persistence, and exports retain the fact + +**Files:** +- Modify: `scopeproof_core/schemas/models.py` +- Modify: `scopeproof_core/cli.py` +- Modify: `apps/web/app.py` +- Test: `tests/schemas/test_models.py` +- Test: `tests/cli/test_cli.py` +- Test: `tests/apps/test_streamlit_app.py` +- Test: `tests/storage/test_json_store.py` +- Test: `tests/reporting/test_exporters.py` + +**Interfaces:** +- Produces: `Review.repository_visibility: RepositoryVisibility` +- Consumes: `PullRequestSnapshot.repository_visibility` +- Enforces: current construction cannot pair `LIVE_PUBLIC_GITHUB` with `unverified` + +- [ ] **Step 1: Write failing schema, CLI, Streamlit, reopen, and export regressions** + +Assert live review construction rejects unverified snapshots, verified snapshots keep +the fact across CLI/web analysis, save/reopen, and JSON export, and historical payloads +without the field load as `unverified` without receiving invented evidence. + +- [ ] **Step 2: Run the focused tests and confirm the intended failures** + +Run the named test modules with `pytest -q` and verify each failure names the absent +guard or field. + +- [ ] **Step 3: Implement minimal propagation and guards** + +Copy snapshot visibility into every review constructor. Guard shared CLI bundle +construction and Streamlit analysis before assigning `LIVE_PUBLIC_GITHUB`. Preserve +legacy readability through the enum default. + +- [ ] **Step 4: Run all focused modules and confirm they pass** + +Run: `pytest tests/schemas/test_models.py tests/cli/test_cli.py tests/apps/test_streamlit_app.py tests/storage/test_json_store.py tests/reporting/test_exporters.py -q` + +- [ ] **Step 5: Commit the propagation contract with named files** + +Stage only the touched production and test files. + +### Task 3: Genuine alpha requires verified public provenance + +**Files:** +- Modify: `scopeproof_core/alpha/models.py` +- Modify: `scopeproof_core/alpha/service.py` +- Modify: `scopeproof_core/alpha/__init__.py` +- Modify: `apps/web/app.py` +- Test: `tests/alpha/test_models.py` +- Test: `tests/alpha/test_service.py` +- Test: `tests/alpha/test_storage.py` +- Test: `tests/apps/test_streamlit_app.py` + +**Interfaces:** +- Produces: `AlphaQualificationInput` for pre-fetch session validation +- Produces: `AlphaQualification.repository_visibility == verified_public` +- Produces: `AlphaCaseRecord.repository_visibility`, defaulting legacy records to `unverified` +- Consumes: the loaded snapshot's verified visibility during alpha-case creation + +- [ ] **Step 1: Write failing qualification, legacy, outcome, and UI regressions** + +Assert verified visibility is required to create a new alpha case, legacy cases remain +readable but cannot record outcomes, unverified reviews cannot record outcomes, and +the Streamlit alpha flow binds the qualification to the loaded verified snapshot. + +- [ ] **Step 2: Run the focused tests and confirm the intended failures** + +Run: `pytest tests/alpha/test_models.py tests/alpha/test_service.py tests/alpha/test_storage.py tests/apps/test_streamlit_app.py -q` + +- [ ] **Step 3: Implement the minimal alpha boundary** + +Separate session intake from verified qualification, persist the verified fact in new +alpha cases, add both case and review checks to outcome and public-summary transitions, +and pass the loaded snapshot fact through Streamlit. + +- [ ] **Step 4: Run the focused tests and confirm they pass** + +Run the same named modules and confirm no token or private metadata enters output. + +- [ ] **Step 5: Commit the alpha contract with named files** + +Stage only the touched alpha, app, and test files. + +### Task 4: Full verification and owner-ready pull request + +**Files:** +- Modify if needed: authoritative trust/status documentation directly affected by this repair +- Test: all repository checks and installed-wheel browser workflow + +**Interfaces:** +- Consumes: the completed verified-public implementation +- Produces: a ready-for-review PR with all available checks resolved or classified + +- [ ] **Step 1: Run Ruff and the complete suite with combined coverage** + +Require at least 95% combined coverage and preserve intentional skip classifications. + +- [ ] **Step 2: Run repository contracts and both deterministic benchmarks** + +Require zero mismatches, zero must-have False Ready outcomes, zero false blockers, and +zero unexecuted declared categories. + +- [ ] **Step 3: Build two wheels and compare SHA-256 values** + +Require byte-identical wheel hashes, validate clean dependencies, installed version +equality, both CLI versions, installed benchmarks, and exact loopback workbench health. + +- [ ] **Step 4: Run the explicit installed-wheel browser regression** + +Require loopback-only networking and zero console or page errors. + +- [ ] **Step 5: Audit the diff and request independent review** + +Resolve every actionable Critical or Important finding without unrelated refactoring. + +- [ ] **Step 6: Commit remaining intentional files, push, and open a ready PR** + +Monitor CI, CodeQL, Pages, and every available check. Do not merge; stop only at the +clean owner merge decision. diff --git a/docs/superpowers/specs/2026-08-09-public-provenance-boundary-design.md b/docs/superpowers/specs/2026-08-09-public-provenance-boundary-design.md new file mode 100644 index 0000000..1f69ca4 --- /dev/null +++ b/docs/superpowers/specs/2026-08-09-public-provenance-boundary-design.md @@ -0,0 +1,64 @@ +# Verified Public Provenance Boundary + +## Problem + +ScopeProof currently treats any successful non-fixture GitHub pull-request fetch as +`live_public_github`. An optional token can make a private repository readable, so a +successful response is not evidence that the repository is public. That unsupported +label can flow into saved reviews, exports, and genuine-alpha outcomes. + +## Decision + +Introduce a typed `RepositoryVisibility` fact with two persisted states: + +- `verified_public`: GitHub returned an unambiguous public visibility assertion for + the repository that owns the requested pull request. +- `unverified`: no current verified-public assertion exists. This is the default for + historical records and local/demo fixtures. + +GitHub ingestion is the only adapter allowed to produce `verified_public`. It must +require a complete repository object whose `full_name` matches the requested +repository, whose `private` field is exactly `false`, and whose `visibility` field is +exactly `public`. An explicit private/internal value is rejected as private or +inaccessible. Missing, malformed, mismatched, or contradictory metadata is rejected +with a bounded visibility-unverified error. No snapshot is returned on either path. + +## Data flow + +`PullRequestSnapshot.repository_visibility` carries the ingestion result. Review +construction copies it to `Review.repository_visibility`. A review may be labeled +`LIVE_PUBLIC_GITHUB` only when the snapshot is `verified_public`; current CLI and web +paths fail closed otherwise. The field is included automatically in validated saved +records and JSON exports. + +Historical saved reviews remain readable because missing fields validate as +`unverified`. They are not silently migrated to `verified_public`. A fresh GitHub +fetch and normal criteria reconfirmation are required to create a newly verified +review. + +Genuine-alpha qualification is split into a session-only intake and a verified +qualification. `AlphaQualificationInput` validates the pre-fetch human inputs. +`AlphaQualification` adds a required `verified_public` fact obtained from the loaded +snapshot. `AlphaCaseRecord` persists that fact; legacy cases default to `unverified` +and cannot record outcomes or produce public summaries. Outcome recording requires +both the case and the matching review to be verified public. + +## Security and privacy invariants + +- Tokens remain HTTP-client state only and are never added to a model, exception, + log, saved record, or export. +- Private, internal, missing, malformed, contradictory, and repository-mismatched + visibility responses fail closed before files, commits, or checks are fetched. +- Local fixtures and the constructed demo remain usable but never become live-public + evidence. +- All persisted and exported data remains Pydantic-validated. +- Legacy records remain inspectable while carrying an explicit `unverified` fact. +- No target-repository code is executed. + +## Verification + +Regression coverage spans GitHub ingestion, schema validation, CLI review, Streamlit +loading and analysis, alpha qualification and outcomes, local save/reopen, and JSON +export. Focused tests are run red before the implementation. The final branch must +also pass the full repository verification, reproducible wheel, installed-package, +workbench, browser, and GitHub check gates defined by the owner objective. From 9b7727c75315b3b365197134456797238b864f8f Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 12:11:30 -0400 Subject: [PATCH 2/7] fix: verify public repository provenance at ingestion --- scopeproof_core/github/client.py | 43 +++++++++++++ scopeproof_core/schemas/models.py | 8 +++ tests/github/test_client.py | 100 +++++++++++++++++++++++++++++- 3 files changed, 150 insertions(+), 1 deletion(-) diff --git a/scopeproof_core/github/client.py b/scopeproof_core/github/client.py index b36110d..a48a1c7 100644 --- a/scopeproof_core/github/client.py +++ b/scopeproof_core/github/client.py @@ -19,6 +19,7 @@ IngestionState, LineChangeType, PullRequestSnapshot, + RepositoryVisibility, RetrievedFile, ) @@ -44,6 +45,10 @@ class PrivateOrInaccessibleRepository(GitHubIngestionError): pass +class RepositoryVisibilityUnverified(GitHubIngestionError): + pass + + class GitHubRateLimited(GitHubIngestionError): def __init__(self, reset_at: str | None = None) -> None: self.reset_at = reset_at @@ -192,6 +197,39 @@ def _raise_for_pr(response: httpx.Response) -> None: except httpx.HTTPStatusError as error: raise GitHubIngestionError(f"GitHub returned HTTP {response.status_code}.") from error + @staticmethod + def _verified_repository_visibility( + pr_data: object, + *, + expected_repository: str, + ) -> RepositoryVisibility: + """Return verified-public only for complete, consistent GitHub metadata.""" + + base = pr_data.get("base") if isinstance(pr_data, dict) else None + repository = base.get("repo") if isinstance(base, dict) else None + if not isinstance(repository, dict): + raise RepositoryVisibilityUnverified( + "GitHub did not provide enough metadata to verify public repository visibility." + ) + + full_name = repository.get("full_name") + private = repository.get("private") + visibility = repository.get("visibility") + if private is True or visibility in {"private", "internal"}: + raise PrivateOrInaccessibleRepository( + "ScopeProof accepts only a verified public GitHub repository." + ) + if ( + not isinstance(full_name, str) + or full_name.casefold() != expected_repository.casefold() + or private is not False + or visibility != "public" + ): + raise RepositoryVisibilityUnverified( + "GitHub did not provide enough metadata to verify public repository visibility." + ) + return RepositoryVisibility.VERIFIED_PUBLIC + def _get_all(self, path: str) -> list[dict]: """Follow GitHub pagination while retaining normal HTTP error handling.""" response = self._get(path) @@ -478,6 +516,10 @@ def fetch_pull_request(self, url: str) -> PullRequestSnapshot: pr_response = self._get(f"{root}/pulls/{pr_number}") self._raise_for_pr(pr_response) pr_data = pr_response.json() + repository_visibility = self._verified_repository_visibility( + pr_data, + expected_repository=f"{owner}/{repository}", + ) raw_files = self._get_all(f"{root}/pulls/{pr_number}/files?per_page=100") raw_commits = self._get_all(f"{root}/pulls/{pr_number}/commits?per_page=100") @@ -570,6 +612,7 @@ def fetch_pull_request(self, url: str) -> PullRequestSnapshot: ) return PullRequestSnapshot( repository=f"{owner}/{repository}", + repository_visibility=repository_visibility, pr_number=pr_number, title=pr_data.get("title", ""), description=pr_data.get("body") or "", diff --git a/scopeproof_core/schemas/models.py b/scopeproof_core/schemas/models.py index 7e7da0c..90a46ca 100644 --- a/scopeproof_core/schemas/models.py +++ b/scopeproof_core/schemas/models.py @@ -82,6 +82,13 @@ class ReviewInputOrigin(StringEnum): LEGACY_UNKNOWN = "legacy_unknown" +class RepositoryVisibility(StringEnum): + """Whether the owning repository has current verified-public provenance.""" + + VERIFIED_PUBLIC = "verified_public" + UNVERIFIED = "unverified" + + _SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") CONSTRUCTED_DEMO_CRITERIA_SOURCE_URI = ( "scopeproof://constructed-demo/acceptance-criteria" @@ -731,6 +738,7 @@ class RetrievedFile(BaseModel): class PullRequestSnapshot(BaseModel): repository: str = Field(pattern=GITHUB_REPOSITORY_PATTERN) + repository_visibility: RepositoryVisibility = RepositoryVisibility.UNVERIFIED pr_number: int = Field(gt=0) title: str description: str = "" diff --git a/tests/github/test_client.py b/tests/github/test_client.py index 26ade33..6ae0c1a 100644 --- a/tests/github/test_client.py +++ b/tests/github/test_client.py @@ -21,6 +21,8 @@ LineChangeType, ) +_DEFAULT_PULL_REPOSITORY = object() + def _response(status: int, data: object, headers: dict[str, str] | None = None) -> httpx.Response: return httpx.Response(status, json=data, headers=headers) @@ -37,6 +39,7 @@ def fixture_transport( status_data: dict | None = None, status_status: int = 200, requested_urls: list[httpx.URL] | None = None, + pull_repository: object = _DEFAULT_PULL_REPOSITORY, ) -> httpx.MockTransport: default_files = [ { @@ -61,6 +64,15 @@ def handler(request: httpx.Request) -> httpx.Response: if path == "/repos/acme/widget/pulls/42": if pull_status != 200: return _response(pull_status, {"message": "request failed"}, pull_headers) + base = {"sha": "base123"} + if pull_repository is _DEFAULT_PULL_REPOSITORY: + base["repo"] = { + "full_name": "acme/widget", + "private": False, + "visibility": "public", + } + elif pull_repository is not None: + base["repo"] = pull_repository return _response( 200, { @@ -68,7 +80,7 @@ def handler(request: httpx.Request) -> httpx.Response: "title": "Export CSV", "body": "Adds export", "html_url": "https://github.com/acme/widget/pull/42", - "base": {"sha": "base123"}, + "base": base, "head": {"sha": "head123"}, }, ) @@ -117,10 +129,96 @@ def test_client_uses_optional_token_without_placing_it_in_snapshot() -> None: client = GitHubClient(token="secret", transport=fixture_transport()) snapshot = client.fetch_pull_request("https://github.com/acme/widget/pull/42") assert snapshot.repository == "acme/widget" + assert snapshot.repository_visibility.value == "verified_public" assert "secret" not in snapshot.model_dump_json() assert client.last_request_authorized is True +@pytest.mark.parametrize( + "repository_metadata", + [ + pytest.param( + {"full_name": "acme/widget", "private": True, "visibility": "private"}, + id="private", + ), + pytest.param( + {"full_name": "acme/widget", "private": False, "visibility": "internal"}, + id="internal", + ), + pytest.param( + {"full_name": "acme/widget", "private": True, "visibility": "public"}, + id="contradictory-private", + ), + ], +) +def test_client_rejects_non_public_repository_before_secondary_fetches( + repository_metadata: dict, +) -> None: + requested_urls: list[httpx.URL] = [] + + with pytest.raises(GitHubIngestionError, match="verified public"): + GitHubClient( + token="session-token", + transport=fixture_transport( + pull_repository=repository_metadata, + requested_urls=requested_urls, + ), + ).fetch_pull_request("https://github.com/acme/widget/pull/42") + + assert [url.path for url in requested_urls] == ["/repos/acme/widget/pulls/42"] + + +@pytest.mark.parametrize( + "repository_metadata", + [ + pytest.param(None, id="missing"), + pytest.param([], id="non-object"), + pytest.param( + {"private": False, "visibility": "public"}, + id="missing-name", + ), + pytest.param( + { + "full_name": "acme/other", + "private": False, + "visibility": "public", + }, + id="wrong-name", + ), + pytest.param( + {"full_name": "acme/widget", "visibility": "public"}, + id="missing-private", + ), + pytest.param( + {"full_name": "acme/widget", "private": 0, "visibility": "public"}, + id="non-boolean-private", + ), + pytest.param( + {"full_name": "acme/widget", "private": False}, + id="missing-visibility", + ), + pytest.param( + {"full_name": "acme/widget", "private": False, "visibility": "PUBLIC"}, + id="malformed-visibility", + ), + ], +) +def test_client_rejects_ambiguous_repository_visibility_before_secondary_fetches( + repository_metadata: object, +) -> None: + requested_urls: list[httpx.URL] = [] + + with pytest.raises(GitHubIngestionError, match="verify public repository visibility"): + GitHubClient( + transport=fixture_transport( + pull_repository=repository_metadata, + requested_urls=requested_urls, + ) + ).fetch_pull_request("https://github.com/acme/widget/pull/42") + + assert [url.path for url in requested_urls] == ["/repos/acme/widget/pulls/42"] + + def test_client_maps_patch_lines_and_keeps_removed_lines_distinct() -> None: snapshot = GitHubClient(transport=fixture_transport()).fetch_pull_request( "https://github.com/acme/widget/pull/42" From b72bf626b356965f603c728586ccdc2ff2392a7a Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 12:19:29 -0400 Subject: [PATCH 3/7] fix: preserve verified provenance through reviews --- apps/web/app.py | 14 ++++-- scopeproof_core/cli.py | 3 ++ scopeproof_core/demo.py | 1 + scopeproof_core/schemas/models.py | 17 +++++++ tests/apps/test_streamlit_app.py | 42 ++++++++++++++-- tests/cli/test_cli.py | 82 +++++++++++++++++++++++++++++++ tests/reporting/test_exporters.py | 15 ++++++ tests/schemas/test_models.py | 8 +++ tests/storage/test_json_store.py | 25 ++++++++++ 9 files changed, 199 insertions(+), 8 deletions(-) diff --git a/apps/web/app.py b/apps/web/app.py index 2a0cde0..57c90dc 100644 --- a/apps/web/app.py +++ b/apps/web/app.py @@ -91,6 +91,7 @@ ReviewInputOrigin, ReviewState, RuntimeEvidence, + require_verified_public_origin, ) from scopeproof_core.storage.json_store import ( JsonReviewStore, @@ -675,8 +676,15 @@ def _hydrate_reopened_review(state: ReviewState) -> None: def _analyze() -> ReviewBundle: snapshot = st.session_state["snapshot"] criteria = st.session_state["criteria"] + input_origin = ( + ReviewInputOrigin.CONSTRUCTED_DEMO + if st.session_state["criteria_source_mode"] == "demo" + else ReviewInputOrigin.LIVE_PUBLIC_GITHUB + ) + require_verified_public_origin(snapshot.repository_visibility, input_origin) review = Review( repository=snapshot.repository, + repository_visibility=snapshot.repository_visibility, pr_number=snapshot.pr_number, base_sha=snapshot.base_sha, head_sha=snapshot.head_sha, @@ -687,11 +695,7 @@ def _analyze() -> ReviewBundle: ingestion_state=snapshot.ingestion_state, ingestion_warnings=snapshot.warnings, skipped_files=snapshot.skipped_files, - input_origin=( - ReviewInputOrigin.CONSTRUCTED_DEMO - if st.session_state["criteria_source_mode"] == "demo" - else ReviewInputOrigin.LIVE_PUBLIC_GITHUB - ), + input_origin=input_origin, ) retrieval_result = retrieve_evidence_with_diagnostics( snapshot, criteria, unchanged_files=st.session_state["candidate_files"] diff --git a/scopeproof_core/cli.py b/scopeproof_core/cli.py index f049f07..7cfa94a 100644 --- a/scopeproof_core/cli.py +++ b/scopeproof_core/cli.py @@ -62,6 +62,7 @@ RuntimeEvidence, SavedReviewListing, normalize_public_https_source_uri, + require_verified_public_origin, ) from scopeproof_core.storage.json_store import JsonReviewStore from scopeproof_core.verification.service import build_findings @@ -118,8 +119,10 @@ def _build_bundle( research_case_id: str | None = None, input_origin: ReviewInputOrigin = ReviewInputOrigin.LEGACY_UNKNOWN, ) -> ReviewBundle: + require_verified_public_origin(snapshot.repository_visibility, input_origin) review = Review( repository=snapshot.repository, + repository_visibility=snapshot.repository_visibility, pr_number=snapshot.pr_number, base_sha=snapshot.base_sha, head_sha=snapshot.head_sha, diff --git a/scopeproof_core/demo.py b/scopeproof_core/demo.py index e9bda05..aa2e3fe 100644 --- a/scopeproof_core/demo.py +++ b/scopeproof_core/demo.py @@ -50,6 +50,7 @@ def build_review(snapshot: PullRequestSnapshot, labels: dict) -> ReviewBundle: ) review = Review( repository=snapshot.repository, + repository_visibility=snapshot.repository_visibility, pr_number=snapshot.pr_number, base_sha=snapshot.base_sha, head_sha=snapshot.head_sha, diff --git a/scopeproof_core/schemas/models.py b/scopeproof_core/schemas/models.py index 90a46ca..80cfbf1 100644 --- a/scopeproof_core/schemas/models.py +++ b/scopeproof_core/schemas/models.py @@ -89,6 +89,22 @@ class RepositoryVisibility(StringEnum): UNVERIFIED = "unverified" +def require_verified_public_origin( + repository_visibility: RepositoryVisibility, + input_origin: ReviewInputOrigin, +) -> None: + """Reject current live-public labeling without verified repository provenance.""" + + if ( + input_origin is ReviewInputOrigin.LIVE_PUBLIC_GITHUB + and repository_visibility is not RepositoryVisibility.VERIFIED_PUBLIC + ): + raise ValueError( + "live public GitHub review construction requires verified public " + "repository visibility" + ) + + _SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") CONSTRUCTED_DEMO_CRITERIA_SOURCE_URI = ( "scopeproof://constructed-demo/acceptance-criteria" @@ -793,6 +809,7 @@ def limitations_require_noncomplete_ingestion(self) -> PullRequestSnapshot: class Review(BaseModel): review_id: str = Field(default_factory=lambda: str(uuid4())) repository: str = Field(pattern=GITHUB_REPOSITORY_PATTERN) + repository_visibility: RepositoryVisibility = RepositoryVisibility.UNVERIFIED pr_number: int = Field(gt=0) base_sha: str = Field(min_length=1) head_sha: str = Field(min_length=1) diff --git a/tests/apps/test_streamlit_app.py b/tests/apps/test_streamlit_app.py index ce30715..000f4ab 100644 --- a/tests/apps/test_streamlit_app.py +++ b/tests/apps/test_streamlit_app.py @@ -33,6 +33,7 @@ HumanDecision, IngestionState, Priority, + RepositoryVisibility, ResearchContext, ResolutionEvent, ReviewState, @@ -65,6 +66,9 @@ def analyzed_demo(app: AppTest) -> AppTest: def analyzed_standard_demo(app: AppTest) -> AppTest: app = app.button(key="load_demo").click().run() + app.session_state["snapshot"] = app.session_state["snapshot"].model_copy( + update={"repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC} + ) app.session_state["criteria_source_mode"] = "standard" app = app.run() app = app.text_input(key="criteria_source_reference").set_value( @@ -168,7 +172,12 @@ def qualified_alpha_analyzed_app(app: AppTest) -> AppTest: app = app.checkbox(key="source_owner_confirmed").check().run() app = app.checkbox(key="no_confidential_information").check().run() snapshot = load_demo_snapshot().model_copy( - update={"repository": "acme/repo", "pr_number": 7, "head_sha": "a" * 40} + update={ + "repository": "acme/repo", + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + "pr_number": 7, + "head_sha": "a" * 40, + } ) with patch( "scopeproof_core.github.client.GitHubClient.fetch_pull_request", @@ -227,6 +236,24 @@ def test_constructed_demo_disclosure_is_not_shown_for_standard_review() -> None: assert disclosure not in [item.value for item in standard.caption] +def test_standard_analysis_rejects_unverified_repository_snapshot() -> None: + app = load_demo(new_app()) + app.session_state["criteria_source_mode"] = "standard" + app = app.run() + app = app.text_input(key="criteria_source_reference").set_value( + "https://github.com/acme/repo/issues/6" + ).run() + app = app.text_input(key="criteria_source_confirmer").set_value( + "Product owner" + ).run() + app = app.button(key="confirm_criteria").click().run() + + app = app.button(key="run_analysis").click().run() + + assert app.session_state["review_state"] is None + assert any("could not be completed" in item.value for item in app.error) + + def test_alpha_outcome_is_ready_after_authoritative_review_autosaves( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -3279,7 +3306,12 @@ def test_reanalysis_shows_previous_and_current_head_sha( monkeypatch.setenv("HOME", str(tmp_path)) saved, review_id = saved_demo_review(new_app()) previous_head = saved.session_state["review_state"].review.head_sha - changed_snapshot = load_demo_snapshot().model_copy(update={"head_sha": "b" * 40}) + changed_snapshot = load_demo_snapshot().model_copy( + update={ + "head_sha": "b" * 40, + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + } + ) fresh = new_app() fresh = select_saved_review(fresh, review_id) @@ -3366,7 +3398,11 @@ def test_rereview_comparison_shows_modified_candidate_excerpt( ) files[0] = files[0].model_copy(update={"lines": lines}) changed_snapshot = original.model_copy( - update={"head_sha": "c" * 40, "files": files} + update={ + "head_sha": "c" * 40, + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + "files": files, + } ) fresh = new_app() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 8721aae..386c032 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -31,6 +31,7 @@ HumanDecision, LifecycleMutationMetadata, PullRequestSnapshot, + RepositoryVisibility, ResolutionEvent, ReviewInputOrigin, RuntimeEvidence, @@ -263,6 +264,86 @@ def test_fixture_review_saves_validated_local_record(tmp_path: Path, capsys) -> assert diagnostic.accepted_candidate_count == len(state.bundle.evidence) +def test_live_review_rejects_unverified_snapshot_without_saving( + tmp_path: Path, capsys, monkeypatch: pytest.MonkeyPatch +) -> None: + requirements = tmp_path / "requirements.txt" + requirements.write_text("Export CSV\n", encoding="utf-8") + snapshot = PullRequestSnapshot( + repository="acme/repo", + pr_number=7, + title="Export CSV", + html_url="https://github.com/acme/repo/pull/7", + base_sha="b" * 40, + head_sha="a" * 40, + ) + monkeypatch.setattr( + "scopeproof_core.cli.GitHubClient.fetch_pull_request", + lambda _self, _pr: snapshot, + ) + storage = tmp_path / "reviews" + + with pytest.raises(SystemExit) as error: + main( + [ + "review", + "--pr", + "https://github.com/acme/repo/pull/7", + "--requirements", + str(requirements), + "--confirmation", + str(write_requirements_confirmation(requirements)), + "--storage-dir", + str(storage), + ] + ) + + assert error.value.code == 2 + assert "verified public" in capsys.readouterr().err + assert not storage.exists() + + +def test_live_review_persists_verified_public_snapshot_provenance( + tmp_path: Path, capsys, monkeypatch: pytest.MonkeyPatch +) -> None: + requirements = tmp_path / "requirements.txt" + requirements.write_text("Export CSV\n", encoding="utf-8") + snapshot = PullRequestSnapshot( + repository="acme/repo", + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, + pr_number=7, + title="Export CSV", + html_url="https://github.com/acme/repo/pull/7", + base_sha="b" * 40, + head_sha="a" * 40, + ) + monkeypatch.setattr( + "scopeproof_core.cli.GitHubClient.fetch_pull_request", + lambda _self, _pr: snapshot, + ) + storage = tmp_path / "reviews" + + assert main( + [ + "review", + "--pr", + "https://github.com/acme/repo/pull/7", + "--requirements", + str(requirements), + "--confirmation", + str(write_requirements_confirmation(requirements)), + "--storage-dir", + str(storage), + ] + ) == 0 + + review_id = json.loads(capsys.readouterr().out)["review_id"] + state = JsonReviewStore(storage).load(review_id) + assert state.review.repository_visibility is RepositoryVisibility.VERIFIED_PUBLIC + assert state.bundle is not None + assert state.bundle.review.repository_visibility is RepositoryVisibility.VERIFIED_PUBLIC + + def test_fixture_review_preserves_exact_crlf_requirements_digest( tmp_path: Path, capsys ) -> None: @@ -1950,6 +2031,7 @@ def _initialize_alpha_case(tmp_path: Path, capsys) -> tuple[Path, str, Path, str ) snapshot = PullRequestSnapshot( repository="acme/repo", + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, pr_number=7, title="Export CSV", html_url="https://github.com/acme/repo/pull/7", diff --git a/tests/reporting/test_exporters.py b/tests/reporting/test_exporters.py index 25638d3..763fc02 100644 --- a/tests/reporting/test_exporters.py +++ b/tests/reporting/test_exporters.py @@ -31,6 +31,7 @@ HumanDecision, HumanResolution, IngestionState, + RepositoryVisibility, ResearchContext, ResolutionEvent, RetrievalOutcome, @@ -638,6 +639,20 @@ def test_exports_preserve_tool_and_ruleset_provenance() -> None: assert bundle.review.ruleset_version in output +def test_json_export_preserves_verified_public_repository_provenance() -> None: + bundle = example_bundle() + bundle.review = Review.model_validate( + { + **bundle.review.model_dump(mode="python"), + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + } + ) + + payload = json.loads(export_json(bundle)) + + assert payload["review"]["repository_visibility"] == "verified_public" + + def test_exports_preserve_ingestion_limitations_and_escape_html() -> None: bundle = example_bundle() bundle.review.ingestion_state = IngestionState.PARTIAL diff --git a/tests/schemas/test_models.py b/tests/schemas/test_models.py index d53c23b..be2ddf8 100644 --- a/tests/schemas/test_models.py +++ b/tests/schemas/test_models.py @@ -18,6 +18,7 @@ IngestionState, Priority, PullRequestSnapshot, + RepositoryVisibility, Review, SavedReviewListing, ) @@ -241,6 +242,13 @@ def test_review_identity_rejects_whitespace_only_shas(model, field_name, blank) model.model_validate(review_identity_payload(model, **{field_name: blank})) +@pytest.mark.parametrize("model", [PullRequestSnapshot, Review]) +def test_historical_review_models_default_repository_visibility_to_unverified(model) -> None: + value = model.model_validate(review_identity_payload(model)) + + assert value.repository_visibility is RepositoryVisibility.UNVERIFIED + + @pytest.mark.parametrize("model", [PullRequestSnapshot, Review]) @pytest.mark.parametrize("repository", [" / ", "acme/ ", " acme/widgets", "acme/widgets/extra"]) def test_review_identity_rejects_malformed_repositories(model, repository) -> None: diff --git a/tests/storage/test_json_store.py b/tests/storage/test_json_store.py index fcde38d..97d12d3 100644 --- a/tests/storage/test_json_store.py +++ b/tests/storage/test_json_store.py @@ -34,6 +34,7 @@ HumanDecision, HumanResolution, PullRequestSnapshot, + RepositoryVisibility, ResolutionEvent, RuntimeEvidence, ) @@ -294,6 +295,30 @@ def test_saved_review_round_trips_without_token(tmp_path: Path) -> None: assert "authorization" not in path.read_text(encoding="utf-8").lower() +def test_legacy_saved_review_without_visibility_reopens_as_unverified( + tmp_path: Path, +) -> None: + store = JsonReviewStore(tmp_path) + state = review_state() + path = store.save(state) + payload = json.loads(path.read_text(encoding="utf-8")) + payload["state"]["review"].pop("repository_visibility", None) + payload["state"]["bundle"]["review"].pop("repository_visibility", None) + for historical in payload["state"]["analysis_history"]: + historical["review"].pop("repository_visibility", None) + path.write_text(json.dumps(payload), encoding="utf-8") + + loaded = store.load(state.review.review_id) + + assert loaded.review.repository_visibility is RepositoryVisibility.UNVERIFIED + assert loaded.bundle is not None + assert loaded.bundle.review.repository_visibility is RepositoryVisibility.UNVERIFIED + assert all( + bundle.review.repository_visibility is RepositoryVisibility.UNVERIFIED + for bundle in loaded.analysis_history + ) + + def test_mutation_lock_filename_does_not_embed_the_review_id(tmp_path: Path) -> None: review_id = "user-controlled-review-id" From 51269a11620fac71b1bdb389207e004d9a3fd0bc Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 12:26:19 -0400 Subject: [PATCH 4/7] fix: require verified provenance for alpha evidence --- apps/web/app.py | 22 ++++++- scopeproof_core/alpha/__init__.py | 2 + scopeproof_core/alpha/models.py | 13 +++- scopeproof_core/alpha/service.py | 32 +++++++++- scopeproof_core/alpha/storage.py | 1 + scopeproof_core/cli.py | 5 ++ tests/alpha/test_models.py | 48 +++++++++++++- tests/alpha/test_service.py | 64 +++++++++++++++++++ tests/alpha/test_storage.py | 5 ++ tests/apps/test_streamlit_app.py | 34 ++++++++++ tests/cli/test_cli.py | 103 ++++++++++++++++++++++++------ 11 files changed, 304 insertions(+), 25 deletions(-) diff --git a/apps/web/app.py b/apps/web/app.py index 57c90dc..5168f60 100644 --- a/apps/web/app.py +++ b/apps/web/app.py @@ -17,6 +17,7 @@ AlphaFrictionStage, AlphaOutcome, AlphaQualification, + AlphaQualificationInput, ParticipantRole, ) from scopeproof_core.alpha.service import ensure_alpha_case, record_alpha_outcome @@ -1205,6 +1206,7 @@ def _render_ingestion_limitations(source: PullRequestSnapshot | Review | None) - key="no_confidential_information", ) alpha_qualification_ready = True +alpha_qualification_input: AlphaQualificationInput | None = None alpha_qualification: AlphaQualification | None = None if alpha_feedback_mode: alpha_qualification_ready = False @@ -1215,7 +1217,7 @@ def _render_ingestion_limitations(source: PullRequestSnapshot | Review | None) - and no_confidential_information ): try: - alpha_qualification = AlphaQualification( + alpha_qualification_input = AlphaQualificationInput( public_pr_url=pr_url, requirements_source_url=requirements_source_url, participant_role=ParticipantRole(participant_role), @@ -1226,6 +1228,23 @@ def _render_ingestion_limitations(source: PullRequestSnapshot | Review | None) - st.warning("Use a public HTTPS requirements source and a canonical public PR URL.") else: alpha_qualification_ready = True + loaded_for_alpha = st.session_state["snapshot"] + if alpha_qualification_input is not None and loaded_for_alpha is not None: + try: + alpha_owner, alpha_repository, alpha_pr_number = parse_pr_url( + alpha_qualification_input.public_pr_url + ) + if ( + f"{alpha_owner}/{alpha_repository}" != loaded_for_alpha.repository + or alpha_pr_number != loaded_for_alpha.pr_number + ): + raise ValueError("alpha qualification must match the loaded public PR") + alpha_qualification = AlphaQualification( + **alpha_qualification_input.model_dump(mode="python"), + repository_visibility=loaded_for_alpha.repository_visibility, + ) + except ValueError: + alpha_qualification = None else: st.caption("Standard review mode does not create participant research records.") @@ -1760,6 +1779,7 @@ def _render_ingestion_limitations(source: PullRequestSnapshot | Review | None) - confirmed_criteria=[item.text for item in edited_criteria], confirmed_criterion_snapshot=edited_criteria, criteria_source_provenance=provenance, + repository_visibility=alpha_qualification.repository_visibility, ) except ValueError: st.error( diff --git a/scopeproof_core/alpha/__init__.py b/scopeproof_core/alpha/__init__.py index bf9674c..6231d44 100644 --- a/scopeproof_core/alpha/__init__.py +++ b/scopeproof_core/alpha/__init__.py @@ -7,6 +7,7 @@ AlphaOutcome, AlphaPublicationConsent, AlphaQualification, + AlphaQualificationInput, ParticipantRole, ) from scopeproof_core.alpha.rehearsal import ( @@ -35,6 +36,7 @@ "AlphaOutcome", "AlphaPublicationConsent", "AlphaQualification", + "AlphaQualificationInput", "AlphaRehearsalInput", "AlphaRehearsalRecord", "JsonAlphaCaseStore", diff --git a/scopeproof_core/alpha/models.py b/scopeproof_core/alpha/models.py index 329b763..06c76dd 100644 --- a/scopeproof_core/alpha/models.py +++ b/scopeproof_core/alpha/models.py @@ -13,6 +13,7 @@ CriteriaSourceProvenance, Criterion, LocalReviewId, + RepositoryVisibility, normalize_public_https_source_uri, normalized_criteria_sha256, ) @@ -43,8 +44,8 @@ class AlphaFrictionStage(StrEnum): INTEGRATION = "integration" -class AlphaQualification(BaseModel): - """Session-safe preflight contract for a genuine public-alpha review.""" +class AlphaQualificationInput(BaseModel): + """Session-safe human inputs collected before repository verification.""" model_config = ConfigDict(extra="forbid") @@ -61,6 +62,12 @@ def require_https_source(cls, value: HttpUrl) -> HttpUrl: return value +class AlphaQualification(AlphaQualificationInput): + """Genuine public-alpha qualification bound to verified repository visibility.""" + + repository_visibility: Literal[RepositoryVisibility.VERIFIED_PUBLIC] + + class AlphaPublicationConsent(BaseModel): """Separate permission boundaries for reports and quotations.""" @@ -80,6 +87,7 @@ class AlphaCaseRecord(BaseModel): pattern=r"^alpha-[0-9a-f]{32}$", ) public_pr_url: str = Field(pattern=PUBLIC_PR_PATTERN) + repository_visibility: RepositoryVisibility = RepositoryVisibility.UNVERIFIED requirements_source_url: HttpUrl participant_role: ParticipantRole source_owner_confirmed: Literal[True] @@ -182,6 +190,7 @@ class AlphaCasePublicSummary(BaseModel): case_id: str public_pr_url: str = Field(pattern=PUBLIC_PR_PATTERN) + repository_visibility: Literal[RepositoryVisibility.VERIFIED_PUBLIC] requirements_source_url: HttpUrl participant_role: ParticipantRole reviewed_head_sha: str = Field(pattern=r"^[0-9a-f]{40}$") diff --git a/scopeproof_core/alpha/service.py b/scopeproof_core/alpha/service.py index dd624db..6c3197b 100644 --- a/scopeproof_core/alpha/service.py +++ b/scopeproof_core/alpha/service.py @@ -10,6 +10,7 @@ AlphaFrictionStage, AlphaOutcome, AlphaPublicationConsent, + AlphaQualification, ParticipantRole, ) from scopeproof_core.alpha.storage import JsonAlphaCaseStore @@ -18,6 +19,7 @@ from scopeproof_core.schemas.models import ( CriteriaSourceProvenance, Criterion, + RepositoryVisibility, ReviewInputOrigin, ReviewState, normalize_public_https_source_uri, @@ -33,6 +35,12 @@ _LEGACY_CRITERIA_SNAPSHOT_REQUIRED = ( "legacy alpha case must reconfirm a criteria snapshot before recording an outcome" ) +_VERIFIED_PUBLIC_QUALIFICATION_REQUIRED = ( + "alpha qualification requires verified public repository visibility" +) +_LEGACY_VISIBILITY_REQUIRED = ( + "legacy alpha case must be re-fetched before recording an outcome" +) def _require_genuine_alpha_case_record(record: object) -> AlphaCaseRecord: @@ -52,13 +60,16 @@ def initialize_alpha_case( confirmed_criteria: list[str], confirmed_criterion_snapshot: list[Criterion] | None = None, criteria_source_provenance: CriteriaSourceProvenance | None = None, + repository_visibility: RepositoryVisibility = RepositoryVisibility.UNVERIFIED, ) -> AlphaCaseRecord: """Create a qualified local case without claiming an outcome.""" if criteria_source_provenance is None: raise ValueError(_NEW_CASE_PROVENANCE_REQUIRED) if confirmed_criterion_snapshot is None: raise ValueError(_NEW_CASE_CRITERIA_SNAPSHOT_REQUIRED) - return AlphaCaseRecord( + if repository_visibility is not RepositoryVisibility.VERIFIED_PUBLIC: + raise ValueError(_VERIFIED_PUBLIC_QUALIFICATION_REQUIRED) + qualification = AlphaQualification( public_pr_url=public_pr_url, requirements_source_url=normalize_public_https_source_uri( requirements_source_url @@ -66,6 +77,10 @@ def initialize_alpha_case( participant_role=participant_role, source_owner_confirmed=source_owner_confirmed, no_confidential_information=no_confidential_information, + repository_visibility=repository_visibility, + ) + return AlphaCaseRecord( + **qualification.model_dump(mode="python"), confirmed_criteria=confirmed_criteria, confirmed_criterion_snapshot=confirmed_criterion_snapshot, criteria_source_provenance=criteria_source_provenance, @@ -83,6 +98,7 @@ def ensure_alpha_case( confirmed_criteria: list[str], confirmed_criterion_snapshot: list[Criterion] | None = None, criteria_source_provenance: CriteriaSourceProvenance | None = None, + repository_visibility: RepositoryVisibility = RepositoryVisibility.UNVERIFIED, case_id: str | None = None, ) -> AlphaCaseRecord: """Create one validated case or return the matching case already named by the caller.""" @@ -96,6 +112,7 @@ def ensure_alpha_case( confirmed_criteria=confirmed_criteria, confirmed_criterion_snapshot=confirmed_criterion_snapshot, criteria_source_provenance=criteria_source_provenance, + repository_visibility=repository_visibility, ) if case_id is None: store.save(candidate) @@ -104,6 +121,7 @@ def ensure_alpha_case( existing = store.load(case_id) comparable_fields = ( "public_pr_url", + "repository_visibility", "requirements_source_url", "participant_role", "source_owner_confirmed", @@ -131,6 +149,8 @@ def record_alpha_outcome( record = _require_genuine_alpha_case_record(record) if record.outcome is not None: raise ValueError("alpha outcome may be recorded only once") + if record.repository_visibility is not RepositoryVisibility.VERIFIED_PUBLIC: + raise ValueError(_LEGACY_VISIBILITY_REQUIRED) if record.criteria_source_provenance is None: raise ValueError(_LEGACY_RECONFIRMATION_REQUIRED) if record.confirmed_criterion_snapshot is None: @@ -142,6 +162,13 @@ def record_alpha_outcome( raise ValueError("engineering research reviews cannot record alpha outcomes") if review_state.review.input_origin is not ReviewInputOrigin.LIVE_PUBLIC_GITHUB: raise ValueError("alpha outcome requires live public GitHub ingestion") + if ( + review_state.review.repository_visibility + is not RepositoryVisibility.VERIFIED_PUBLIC + ): + raise ValueError( + "alpha outcome requires verified public repository visibility" + ) owner, repository, pr_number = parse_pr_url(record.public_pr_url) if ( review_state.review.repository != f"{owner}/{repository}" @@ -186,6 +213,8 @@ def public_alpha_summary(record: AlphaCaseRecord) -> AlphaCasePublicSummary: record = _require_genuine_alpha_case_record(record) if not record.publication_consent.report: raise ValueError("public summary requires report publication consent") + if record.repository_visibility is not RepositoryVisibility.VERIFIED_PUBLIC: + raise ValueError("public summary requires verified public repository visibility") if record.outcome is None or record.reviewed_head_sha is None or record.completed_at is None: raise ValueError("public summary requires a completed alpha outcome") if ( @@ -199,6 +228,7 @@ def public_alpha_summary(record: AlphaCaseRecord) -> AlphaCasePublicSummary: return AlphaCasePublicSummary( case_id=record.case_id, public_pr_url=record.public_pr_url, + repository_visibility=record.repository_visibility, requirements_source_url=requirements_source_url, participant_role=record.participant_role, reviewed_head_sha=record.reviewed_head_sha, diff --git a/scopeproof_core/alpha/storage.py b/scopeproof_core/alpha/storage.py index 6d9051d..f718368 100644 --- a/scopeproof_core/alpha/storage.py +++ b/scopeproof_core/alpha/storage.py @@ -78,6 +78,7 @@ def update(self, record: AlphaCaseRecord) -> Path: raise ValueError("alpha-case update must preserve criteria source provenance") immutable_fields = ( "public_pr_url", + "repository_visibility", "requirements_source_url", "participant_role", "source_owner_confirmed", diff --git a/scopeproof_core/cli.py b/scopeproof_core/cli.py index 7cfa94a..e7ef60b 100644 --- a/scopeproof_core/cli.py +++ b/scopeproof_core/cli.py @@ -510,6 +510,7 @@ def _alpha_init(args: argparse.Namespace) -> int: source_text=source_text, criteria=criteria, ) + snapshot = GitHubClient(token=args.token or None).fetch_pull_request(args.pr) record = initialize_alpha_case( public_pr_url=args.pr, requirements_source_url=args.requirements_source, @@ -519,6 +520,7 @@ def _alpha_init(args: argparse.Namespace) -> int: confirmed_criteria=[criterion.text for criterion in criteria], confirmed_criterion_snapshot=criteria, criteria_source_provenance=provenance, + repository_visibility=snapshot.repository_visibility, ) path = JsonAlphaCaseStore(Path(args.storage_dir)).save(record) payload = record.model_dump(mode="json") @@ -754,6 +756,9 @@ def _parser() -> argparse.ArgumentParser: help="Confirm the case contains no private or confidential information", ) alpha_init.add_argument("--storage-dir", default=".scopeproof/alpha-cases") + alpha_init.add_argument( + "--token", help="Optional GitHub token; never persisted or printed" + ) alpha_init.set_defaults(handler=_alpha_init) alpha_outcome = alpha_commands.add_parser( "outcome", help="Record one bounded alpha outcome" diff --git a/tests/alpha/test_models.py b/tests/alpha/test_models.py index c0d8501..f1dbad4 100644 --- a/tests/alpha/test_models.py +++ b/tests/alpha/test_models.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError +import scopeproof_core.alpha.models as alpha_models from scopeproof_core.alpha.models import ( AlphaCaseRecord, AlphaFrictionStage, @@ -12,7 +13,7 @@ ParticipantRole, ) from scopeproof_core.criteria.confirmation import build_criteria_source_provenance -from scopeproof_core.schemas.models import Criterion +from scopeproof_core.schemas.models import Criterion, RepositoryVisibility def criteria_source_provenance(*, source_uri: str = "https://github.com/acme/repo/issues/6"): @@ -53,6 +54,7 @@ def test_alpha_qualification_accepts_only_confirmed_public_safe_inputs() -> None participant_role=ParticipantRole.PRODUCT_MANAGER, source_owner_confirmed=True, no_confidential_information=True, + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) assert qualification.source_owner_confirmed is True @@ -78,6 +80,7 @@ def test_alpha_qualification_rejects_unqualified_inputs(field: str, value: objec "participant_role": ParticipantRole.QA, "source_owner_confirmed": True, "no_confidential_information": True, + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, } data[field] = value @@ -85,6 +88,39 @@ def test_alpha_qualification_rejects_unqualified_inputs(field: str, value: objec AlphaQualification(**data) +def test_alpha_qualification_requires_verified_public_repository_visibility() -> None: + inputs: dict[str, object] = { + "public_pr_url": "https://github.com/acme/repo/pull/7", + "requirements_source_url": "https://github.com/acme/repo/issues/6", + "participant_role": ParticipantRole.QA, + "source_owner_confirmed": True, + "no_confidential_information": True, + } + + with pytest.raises(ValidationError): + AlphaQualification(**inputs) + with pytest.raises(ValidationError): + AlphaQualification( + **inputs, + repository_visibility=RepositoryVisibility.UNVERIFIED, + ) + + +def test_session_preflight_model_does_not_claim_verified_visibility() -> None: + intake_model = getattr(alpha_models, "AlphaQualificationInput", None) + + assert intake_model is not None + intake = intake_model( + public_pr_url="https://github.com/acme/repo/pull/7", + requirements_source_url="https://github.com/acme/repo/issues/6", + participant_role=ParticipantRole.QA, + source_owner_confirmed=True, + no_confidential_information=True, + ) + + assert "repository_visibility" not in intake.model_dump(mode="json") + + def test_alpha_record_has_privacy_safe_defaults() -> None: record = AlphaCaseRecord(**valid_record_data()) @@ -93,6 +129,7 @@ def test_alpha_record_has_privacy_safe_defaults() -> None: assert record.outcome is None assert record.completed_at is None assert record.criteria_source_provenance == criteria_source_provenance() + assert record.repository_visibility is RepositoryVisibility.UNVERIFIED assert [item.text for item in record.confirmed_criterion_snapshot or []] == ( record.confirmed_criteria ) @@ -101,6 +138,15 @@ def test_alpha_record_has_privacy_safe_defaults() -> None: assert all(prohibited not in field for field in schema_fields) +def test_alpha_record_persists_verified_public_repository_visibility() -> None: + record = AlphaCaseRecord( + **valid_record_data(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, + ) + + assert record.repository_visibility is RepositoryVisibility.VERIFIED_PUBLIC + + @pytest.mark.parametrize( ("field", "value"), [ diff --git a/tests/alpha/test_service.py b/tests/alpha/test_service.py index 26351fa..e333091 100644 --- a/tests/alpha/test_service.py +++ b/tests/alpha/test_service.py @@ -23,6 +23,7 @@ from scopeproof_core.schemas.models import ( Criterion, PullRequestSnapshot, + RepositoryVisibility, ReviewInputOrigin, ) @@ -52,6 +53,7 @@ def initialized_case(): confirmed_criteria=["Export CSV"], confirmed_criterion_snapshot=confirmed_criterion_snapshot(), criteria_source_provenance=criteria_source_provenance(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) @@ -66,6 +68,7 @@ def matching_review_state( provenance = criteria_source_provenance() snapshot = PullRequestSnapshot( repository=repository, + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, pr_number=pr_number, title="Export CSV", html_url=f"https://github.com/{repository}/pull/{pr_number}", @@ -84,6 +87,21 @@ def matching_review_state( ) +def test_initialize_alpha_case_requires_verified_public_visibility() -> None: + with pytest.raises( + ValueError, + match="alpha qualification requires verified public repository visibility", + ): + initialize_alpha_case( + public_pr_url="https://github.com/acme/repo/pull/7", + requirements_source_url="https://github.com/acme/repo/issues/6", + participant_role=ParticipantRole.QA, + source_owner_confirmed=True, + no_confidential_information=True, + confirmed_criteria=["Export CSV"], + confirmed_criterion_snapshot=confirmed_criterion_snapshot(), + criteria_source_provenance=criteria_source_provenance(), + ) def test_initialize_alpha_case_is_qualified_and_unpublished() -> None: record = initialized_case() @@ -104,6 +122,7 @@ def test_ensure_alpha_case_creates_once_and_returns_matching_existing(tmp_path) "confirmed_criteria": ["Export CSV"], "confirmed_criterion_snapshot": confirmed_criterion_snapshot(), "criteria_source_provenance": criteria_source_provenance(), + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, } created = ensure_alpha_case(store=store, **inputs) @@ -125,6 +144,7 @@ def test_ensure_alpha_case_rejects_reusing_id_for_different_case(tmp_path) -> No confirmed_criteria=["Export CSV"], confirmed_criterion_snapshot=confirmed_criterion_snapshot(), criteria_source_provenance=criteria_source_provenance(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) with pytest.raises(ValueError, match="does not match"): @@ -139,6 +159,7 @@ def test_ensure_alpha_case_rejects_reusing_id_for_different_case(tmp_path) -> No confirmed_criteria=["Export CSV"], confirmed_criterion_snapshot=confirmed_criterion_snapshot(), criteria_source_provenance=criteria_source_provenance(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) @@ -159,6 +180,7 @@ def test_initialize_alpha_case_requires_explicit_safe_confirmations( confirmed_criteria=["Export CSV"], confirmed_criterion_snapshot=confirmed_criterion_snapshot(), criteria_source_provenance=criteria_source_provenance(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) @@ -216,6 +238,7 @@ def test_initialize_alpha_case_rejects_criteria_provenance_mismatch() -> None: Criterion(criterion_id="AC-01", text="Delete production data") ], criteria_source_provenance=criteria_source_provenance(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) @@ -297,6 +320,46 @@ def test_record_alpha_outcome_rejects_engineering_only_review_sources( ) +def test_record_alpha_outcome_rejects_unverified_legacy_review() -> None: + state = matching_review_state() + assert state.bundle is not None + unverified_review = state.review.model_copy( + update={"repository_visibility": RepositoryVisibility.UNVERIFIED} + ) + state = state.model_copy( + update={ + "review": unverified_review, + "bundle": state.bundle.model_copy(update={"review": unverified_review}), + } + ) + + with pytest.raises( + ValueError, + match="alpha outcome requires verified public repository visibility", + ): + record_alpha_outcome( + initialized_case(), + review_state=state, + outcome=AlphaOutcome.FOUND_USEFUL_GAP, + ) + + +def test_record_alpha_outcome_rejects_legacy_case_without_verified_visibility() -> None: + payload = initialized_case().model_dump(mode="python") + payload["repository_visibility"] = RepositoryVisibility.UNVERIFIED + legacy = AlphaCaseRecord.model_validate(payload) + + with pytest.raises( + ValueError, + match="legacy alpha case must be re-fetched before recording an outcome", + ): + record_alpha_outcome( + legacy, + review_state=matching_review_state(), + outcome=AlphaOutcome.FOUND_USEFUL_GAP, + ) + + def test_public_summary_requires_report_consent() -> None: completed = record_alpha_outcome( initialized_case(), @@ -326,6 +389,7 @@ def test_public_summary_omits_local_notes_and_consent_fields() -> None: payload = public_alpha_summary(completed).model_dump(mode="json") assert payload["outcome"] == "showed_only_known_information" + assert payload["repository_visibility"] == "verified_public" assert "outcome_notes" not in payload assert "publication_consent" not in payload assert "criteria_source_provenance" not in payload diff --git a/tests/alpha/test_storage.py b/tests/alpha/test_storage.py index b40e587..661928a 100644 --- a/tests/alpha/test_storage.py +++ b/tests/alpha/test_storage.py @@ -14,6 +14,7 @@ from scopeproof_core.schemas.models import ( Criterion, PullRequestSnapshot, + RepositoryVisibility, ReviewInputOrigin, ) @@ -41,6 +42,7 @@ def alpha_case(): Criterion(criterion_id="AC-01", text="Export CSV") ], criteria_source_provenance=criteria_source_provenance(), + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, ) @@ -48,6 +50,7 @@ def matching_review_state(): criteria = [Criterion(criterion_id="AC-01", text="Export CSV")] snapshot = PullRequestSnapshot( repository="acme/repo", + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, pr_number=7, title="Export CSV", html_url="https://github.com/acme/repo/pull/7", @@ -151,11 +154,13 @@ def test_alpha_store_reads_legacy_record_without_inventing_provenance( path = store.save(record) payload = json.loads(path.read_text(encoding="utf-8")) payload.pop("criteria_source_provenance") + payload.pop("repository_visibility") path.write_text(json.dumps(payload), encoding="utf-8") loaded = store.load(record.case_id) assert loaded.criteria_source_provenance is None + assert loaded.repository_visibility is RepositoryVisibility.UNVERIFIED @pytest.mark.parametrize("case_id", ["../escape", "alpha-not-a-uuid", "/tmp/case"]) diff --git a/tests/apps/test_streamlit_app.py b/tests/apps/test_streamlit_app.py index 000f4ab..0f0285e 100644 --- a/tests/apps/test_streamlit_app.py +++ b/tests/apps/test_streamlit_app.py @@ -610,11 +610,45 @@ def test_alpha_mode_creates_case_after_confirming_criteria( record = JsonAlphaCaseStore(default_alpha_case_directory()).load( app.session_state["alpha_case_id"] ) + assert record.repository_visibility is RepositoryVisibility.VERIFIED_PUBLIC assert record.outcome is AlphaOutcome.FOUND_USEFUL_GAP assert record.publication_consent.report is False assert record.publication_consent.quote is False +def test_alpha_mode_does_not_qualify_an_unverified_loaded_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + app = new_app().checkbox(key="alpha_feedback_mode").check().run() + app = app.text_input(key="pr_url").set_value( + "https://github.com/acme/repo/pull/7" + ).run() + app = app.text_input(key="requirements_source_url").set_value( + "https://github.com/acme/repo/issues/6" + ).run() + app = app.checkbox(key="source_owner_confirmed").check().run() + app = app.checkbox(key="no_confidential_information").check().run() + snapshot = load_demo_snapshot().model_copy( + update={"repository": "acme/repo", "pr_number": 7, "head_sha": "a" * 40} + ) + with patch( + "scopeproof_core.github.client.GitHubClient.fetch_pull_request", + return_value=snapshot, + ): + app = app.button(key="fetch_pr").click().run() + app = app.text_area(key="requirements_input").set_value("Export CSV").run() + app = app.button(key="prepare_criteria").click().run() + app = app.text_input(key="criteria_source_confirmer").set_value( + "Alpha source owner" + ).run() + + app = app.button(key="confirm_criteria").click().run() + + assert app.session_state["alpha_case_id"] is None + assert any("could not be confirmed" in item.value for item in app.error) + + @pytest.mark.parametrize( ("widget_kind", "widget_key", "value"), [ diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 386c032..00a66bf 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -4,6 +4,7 @@ from hashlib import sha256 from pathlib import Path from threading import Event +from unittest.mock import patch import pytest @@ -2001,26 +2002,39 @@ def _initialize_alpha_case(tmp_path: Path, capsys) -> tuple[Path, str, Path, str requirements, source_uri="https://github.com/acme/repo/issues/6" ) store = tmp_path / "alpha-cases" - assert main( - [ - "alpha", - "init", - "--pr", - "https://github.com/acme/repo/pull/7", - "--requirements-source", - "https://github.com/acme/repo/issues/6", - "--participant-role", - "qa", - "--requirements", - str(requirements), - "--confirmation", - str(confirmation), - "--source-owner-confirmed", - "--confirmed-no-confidential-information", - "--storage-dir", - str(store), - ] - ) == 0 + verified_snapshot = PullRequestSnapshot( + repository="acme/repo", + repository_visibility=RepositoryVisibility.VERIFIED_PUBLIC, + pr_number=7, + title="Export CSV", + html_url="https://github.com/acme/repo/pull/7", + base_sha="b" * 40, + head_sha="a" * 40, + ) + with patch( + "scopeproof_core.github.client.GitHubClient.fetch_pull_request", + return_value=verified_snapshot, + ): + assert main( + [ + "alpha", + "init", + "--pr", + "https://github.com/acme/repo/pull/7", + "--requirements-source", + "https://github.com/acme/repo/issues/6", + "--participant-role", + "qa", + "--requirements", + str(requirements), + "--confirmation", + str(confirmation), + "--source-owner-confirmed", + "--confirmed-no-confidential-information", + "--storage-dir", + str(store), + ] + ) == 0 case_id = json.loads(capsys.readouterr().out)["case_id"] criteria = [ Criterion(criterion_id=draft.criterion_id, text=draft.text) @@ -2052,6 +2066,54 @@ def _initialize_alpha_case(tmp_path: Path, capsys) -> tuple[Path, str, Path, str return store, case_id, review_store, review_state.review.review_id +def test_alpha_init_rejects_unverified_repository_without_saving( + tmp_path: Path, capsys, monkeypatch: pytest.MonkeyPatch +) -> None: + requirements = tmp_path / "alpha-requirements.txt" + requirements.write_text("Export CSV\n", encoding="utf-8") + confirmation = write_requirements_confirmation( + requirements, source_uri="https://github.com/acme/repo/issues/6" + ) + monkeypatch.setattr( + "scopeproof_core.cli.GitHubClient.fetch_pull_request", + lambda _self, _pr: PullRequestSnapshot( + repository="acme/repo", + pr_number=7, + title="Export CSV", + html_url="https://github.com/acme/repo/pull/7", + base_sha="b" * 40, + head_sha="a" * 40, + ), + ) + storage = tmp_path / "alpha-cases" + + with pytest.raises(SystemExit) as error: + main( + [ + "alpha", + "init", + "--pr", + "https://github.com/acme/repo/pull/7", + "--requirements-source", + "https://github.com/acme/repo/issues/6", + "--participant-role", + "qa", + "--requirements", + str(requirements), + "--confirmation", + str(confirmation), + "--source-owner-confirmed", + "--confirmed-no-confidential-information", + "--storage-dir", + str(storage), + ] + ) + + assert error.value.code == 2 + assert "verified public" in capsys.readouterr().err + assert not storage.exists() + + def test_alpha_init_creates_validated_local_record(tmp_path: Path, capsys) -> None: store_dir, case_id, _, _ = _initialize_alpha_case(tmp_path, capsys) @@ -2060,6 +2122,7 @@ def test_alpha_init_creates_validated_local_record(tmp_path: Path, capsys) -> No assert record.confirmed_criteria == ["Export CSV", "Show an error state"] assert record.source_owner_confirmed is True assert record.no_confidential_information is True + assert record.repository_visibility is RepositoryVisibility.VERIFIED_PUBLIC assert record.criteria_source_provenance is not None assert record.criteria_source_provenance.source_uri == ( "https://github.com/acme/repo/issues/6" From 35a443932c99a1336abbf08f6abed14b332fc12f Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 12:49:08 -0400 Subject: [PATCH 5/7] docs: explain verified public alpha provenance --- README.md | 8 +++++++- ROADMAP.md | 2 ++ docs/alpha/outcome-form.md | 6 ++++-- docs/alpha/participant-quickstart.md | 3 ++- tests/github/test_pagination_and_candidates.py | 9 ++++++++- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cb221dd..133e6e4 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,10 @@ reviewer-confirmed criteria against a real public PR. The commands below describe the `0.2.3` source workflow. From a current source checkout, the CLI provides the same read-only public-PR ingestion and deterministic core without starting Streamlit. +Live ingestion labels a review public only when GitHub returns matching, unambiguous repository +metadata (`private: false` and `visibility: public`). A token-accessible private repository and +missing, malformed, or contradictory visibility metadata fail closed before review data is saved. +Historical records without that verified fact remain readable but cannot qualify for Stage 1. First create `requirements.txt` with one atomic criterion per line. A human requirements owner or authorized role must inspect and approve the exact file before creating the confirmation record. `--confirmed-by` is that human's @@ -314,7 +318,9 @@ is a symbolic link or another existing non-directory. This app-owned local direc browser input from selecting arbitrary file paths. Records preserve the review SHAs, criteria revisions, evidence, findings, resolution history, and gate decision. They never contain the optional GitHub token. A reopened review prepares its public PR URL and bounded unchanged-candidate -paths for a one-click current-head check rather than silently reusing old evidence. After a new +paths for a one-click current-head check rather than silently reusing old evidence. Records also +preserve whether public repository visibility was verified; legacy records without that fact +remain explicitly unverified. After a new analysis, the workbench compares previous and current heads, candidates, finding states, reviewer decisions, and review status without mutating either bundle. Candidate evidence is classified as **Unchanged**, **Relocated**, **Modified**, **Added**, or **Removed**. diff --git a/ROADMAP.md b/ROADMAP.md index edb35f7..54a75f7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -42,6 +42,8 @@ advances Stage 1, which remains at zero. - [x] Acceptance-coverage vocabulary separates candidate strength from reviewer decisions. - [x] Standard flow is public PR → confirmed criteria → coverage → decisions → export. +- [x] Live reviews and genuine-alpha records require an explicit verified-public repository fact; + private, ambiguous, and legacy-unverified sources fail closed for Stage 1. - [x] Enforce at the core boundary that manual verification records runtime evidence and its decision atomically, and reject unpaired reconstructed bundles or states at every trusted boundary. diff --git a/docs/alpha/outcome-form.md b/docs/alpha/outcome-form.md index f3290c3..155f803 100644 --- a/docs/alpha/outcome-form.md +++ b/docs/alpha/outcome-form.md @@ -19,8 +19,10 @@ scopeproof alpha outcome CASE_ID \ ScopeProof loads and revalidates that saved review, then derives its exact head SHA. The outcome is rejected unless the review came from live public GitHub ingestion and matches the qualified -public PR, ordered criteria snapshot, and source provenance. Fixture, demo, research, and legacy -unknown review origins cannot complete genuine alpha evidence. +public PR, verified-public repository fact, ordered criteria snapshot, and source provenance. +The alpha case itself must preserve the same verified-public fact. Fixture, demo, research, and legacy +or unverified review origins cannot complete genuine alpha evidence; a historical record +without visibility evidence must be re-fetched and reconfirmed rather than silently migrated. Report consent and quotation consent are independent. Both default to no. Add `--report-consent` only to allow the reduced public summary; add `--quote-consent` only to permit a quotation. The public summary excludes local notes and permission fields. diff --git a/docs/alpha/participant-quickstart.md b/docs/alpha/participant-quickstart.md index 2cc7f40..5c907f2 100644 --- a/docs/alpha/participant-quickstart.md +++ b/docs/alpha/participant-quickstart.md @@ -40,7 +40,8 @@ They do not establish correctness, customer validation, or target-repository run 3. **Minute 3 — enter public sources.** Paste the canonical public PR URL and public HTTPS requirements source. Select only your role; ScopeProof does not collect names or contact data. 4. **Minute 4 — load the PR.** Confirm source authority and no confidential information, then fetch - the PR. A session-only token is optional under Advanced source options. + the PR. ScopeProof confirms public visibility from GitHub metadata; a session-only token is + optional under Advanced source options but cannot make a private repository eligible. 5. **Minute 5 — confirm criteria.** Prepare one independently judgeable behavior per line, review the normalized set, and explicitly confirm it. This creates one validated local alpha case. 6. **Minute 6 — review coverage.** Run analysis and inspect every Strong candidate, Weak candidate, diff --git a/tests/github/test_pagination_and_candidates.py b/tests/github/test_pagination_and_candidates.py index af2c1a6..5bcf5fa 100644 --- a/tests/github/test_pagination_and_candidates.py +++ b/tests/github/test_pagination_and_candidates.py @@ -24,7 +24,14 @@ def handler(request: httpx.Request) -> httpx.Response: "title": "Paged export", "body": "", "html_url": "https://github.com/acme/widget/pull/42", - "base": {"sha": "base"}, + "base": { + "sha": "base", + "repo": { + "full_name": "acme/widget", + "private": False, + "visibility": "public", + }, + }, "head": {"sha": "head"}, } ) From 59d154730f99908e702f913ca3f04337f609da8a Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 13:04:27 -0400 Subject: [PATCH 6/7] fix: preserve provenance in legacy exports --- scopeproof_core/alpha/service.py | 4 ++-- scopeproof_core/reporting/exporters.py | 6 +++++ scopeproof_core/schemas/models.py | 33 +++++++++++++++++++------- tests/reporting/test_exporters.py | 18 ++++++++++++++ tests/storage/test_json_store.py | 10 ++++++++ 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/scopeproof_core/alpha/service.py b/scopeproof_core/alpha/service.py index 6c3197b..186337c 100644 --- a/scopeproof_core/alpha/service.py +++ b/scopeproof_core/alpha/service.py @@ -160,8 +160,6 @@ def record_alpha_outcome( raise ValueError("alpha outcome requires a completed review analysis") if review_state.bundle.research_context is not None: raise ValueError("engineering research reviews cannot record alpha outcomes") - if review_state.review.input_origin is not ReviewInputOrigin.LIVE_PUBLIC_GITHUB: - raise ValueError("alpha outcome requires live public GitHub ingestion") if ( review_state.review.repository_visibility is not RepositoryVisibility.VERIFIED_PUBLIC @@ -169,6 +167,8 @@ def record_alpha_outcome( raise ValueError( "alpha outcome requires verified public repository visibility" ) + if review_state.review.input_origin is not ReviewInputOrigin.LIVE_PUBLIC_GITHUB: + raise ValueError("alpha outcome requires live public GitHub ingestion") owner, repository, pr_number = parse_pr_url(record.public_pr_url) if ( review_state.review.repository != f"{owner}/{repository}" diff --git a/scopeproof_core/reporting/exporters.py b/scopeproof_core/reporting/exporters.py index 905d9cf..08e5e6b 100644 --- a/scopeproof_core/reporting/exporters.py +++ b/scopeproof_core/reporting/exporters.py @@ -355,6 +355,8 @@ def export_markdown(bundle: ExportableReview) -> str: f"**Review status:** {review_status}", f"**Review ID:** {_render_markdown_code(bundle.review.review_id)}", f"**Repository:** {_render_markdown_code(bundle.review.repository)}", + "**Repository visibility:** " + f"{_render_markdown_code(bundle.review.repository_visibility.value)}", f"**Pull request:** #{bundle.review.pr_number}", f"**Base SHA:** {_render_markdown_code(bundle.review.base_sha)}", f"**Head SHA:** {_render_markdown_code(bundle.review.head_sha)}", @@ -683,6 +685,7 @@ def export_csv(bundle: ExportableReview) -> str: fieldnames = [ "review_id", "repository", + "repository_visibility", "pr_number", "base_sha", "head_sha", @@ -763,6 +766,7 @@ def export_csv(bundle: ExportableReview) -> str: { "review_id": _csv_text(bundle.review.review_id), "repository": _csv_text(bundle.review.repository), + "repository_visibility": bundle.review.repository_visibility.value, "pr_number": bundle.review.pr_number, "base_sha": _csv_text(bundle.review.base_sha), "head_sha": _csv_text(bundle.review.head_sha), @@ -1007,6 +1011,8 @@ def export_html(value: ExportableReview) -> str: f"

Review status: {review_status}

", f"

Review ID: {html.escape(bundle.review.review_id)} · " f"Repository: {html.escape(bundle.review.repository)} · " + "Repository visibility: " + f"{html.escape(bundle.review.repository_visibility.value)} · " f"PR #{bundle.review.pr_number} · Base SHA " f"{html.escape(bundle.review.base_sha)} · Head SHA " f"{html.escape(bundle.review.head_sha)} · " diff --git a/scopeproof_core/schemas/models.py b/scopeproof_core/schemas/models.py index 80cfbf1..995917e 100644 --- a/scopeproof_core/schemas/models.py +++ b/scopeproof_core/schemas/models.py @@ -839,17 +839,34 @@ def require_non_blank_review_identity(cls, value: str) -> str: @model_validator(mode="before") @classmethod - def preserve_historical_ci_state(cls, value: object) -> object: - if isinstance(value, dict) and "check_state" in value: - if "ci_observation" not in value: + def preserve_historical_review_state(cls, value: object) -> object: + if not isinstance(value, dict): + return value + + migrated = value + if ( + value.get("input_origin") + in {ReviewInputOrigin.LIVE_PUBLIC_GITHUB, ReviewInputOrigin.LIVE_PUBLIC_GITHUB.value} + and value.get( + "repository_visibility", RepositoryVisibility.UNVERIFIED + ) + not in { + RepositoryVisibility.VERIFIED_PUBLIC, + RepositoryVisibility.VERIFIED_PUBLIC.value, + } + ): + migrated = {**migrated, "input_origin": ReviewInputOrigin.LEGACY_UNKNOWN} + + if "check_state" in migrated: + if "ci_observation" not in migrated: return { - **value, + **migrated, "check_state": CheckState.UNAVAILABLE, - "ci_observation": _historical_ci_observation(value.get("check_state")), + "ci_observation": _historical_ci_observation(migrated.get("check_state")), } - if _requires_historical_ci_fail_closed_migration(value): - return {**value, "check_state": CheckState.UNAVAILABLE} - return value + if _requires_historical_ci_fail_closed_migration(migrated): + return {**migrated, "check_state": CheckState.UNAVAILABLE} + return migrated @model_validator(mode="after") def limitations_require_noncomplete_ingestion(self) -> Review: diff --git a/tests/reporting/test_exporters.py b/tests/reporting/test_exporters.py index 763fc02..d25a03c 100644 --- a/tests/reporting/test_exporters.py +++ b/tests/reporting/test_exporters.py @@ -653,6 +653,24 @@ def test_json_export_preserves_verified_public_repository_provenance() -> None: assert payload["review"]["repository_visibility"] == "verified_public" +def test_human_readable_exports_preserve_verified_public_repository_provenance() -> None: + bundle = example_bundle() + bundle.review = Review.model_validate( + { + **bundle.review.model_dump(mode="python"), + "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + } + ) + + markdown_report = export_markdown(bundle) + csv_row = next(csv.DictReader(io.StringIO(export_csv(bundle)))) + html_report = export_html(bundle) + + assert "**Repository visibility:** verified_public" in markdown_report + assert csv_row["repository_visibility"] == "verified_public" + assert "Repository visibility: verified_public" in html_report + + def test_exports_preserve_ingestion_limitations_and_escape_html() -> None: bundle = example_bundle() bundle.review.ingestion_state = IngestionState.PARTIAL diff --git a/tests/storage/test_json_store.py b/tests/storage/test_json_store.py index 97d12d3..24e5a7e 100644 --- a/tests/storage/test_json_store.py +++ b/tests/storage/test_json_store.py @@ -36,6 +36,7 @@ PullRequestSnapshot, RepositoryVisibility, ResolutionEvent, + ReviewInputOrigin, RuntimeEvidence, ) from scopeproof_core.storage.json_store import ( @@ -302,21 +303,30 @@ def test_legacy_saved_review_without_visibility_reopens_as_unverified( state = review_state() path = store.save(state) payload = json.loads(path.read_text(encoding="utf-8")) + payload["state"]["review"]["input_origin"] = "live_public_github" payload["state"]["review"].pop("repository_visibility", None) + payload["state"]["bundle"]["review"]["input_origin"] = "live_public_github" payload["state"]["bundle"]["review"].pop("repository_visibility", None) for historical in payload["state"]["analysis_history"]: + historical["review"]["input_origin"] = "live_public_github" historical["review"].pop("repository_visibility", None) path.write_text(json.dumps(payload), encoding="utf-8") loaded = store.load(state.review.review_id) assert loaded.review.repository_visibility is RepositoryVisibility.UNVERIFIED + assert loaded.review.input_origin is ReviewInputOrigin.LEGACY_UNKNOWN assert loaded.bundle is not None assert loaded.bundle.review.repository_visibility is RepositoryVisibility.UNVERIFIED + assert loaded.bundle.review.input_origin is ReviewInputOrigin.LEGACY_UNKNOWN assert all( bundle.review.repository_visibility is RepositoryVisibility.UNVERIFIED for bundle in loaded.analysis_history ) + assert all( + bundle.review.input_origin is ReviewInputOrigin.LEGACY_UNKNOWN + for bundle in loaded.analysis_history + ) def test_mutation_lock_filename_does_not_embed_the_review_id(tmp_path: Path) -> None: From 12c7c1b920d2afb154afbaa7942a15e7bf70230a Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sun, 9 Aug 2026 13:51:58 -0400 Subject: [PATCH 7/7] fix: reject self-asserted public provenance --- scopeproof_core/alpha/service.py | 5 +++++ scopeproof_core/github/client.py | 4 +++- scopeproof_core/schemas/models.py | 16 +++++++++++++++ tests/cli/test_cli.py | 34 +++++++++++++++++++++++++++++++ tests/github/test_client.py | 8 ++++++++ tests/reporting/test_exporters.py | 3 +++ 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/scopeproof_core/alpha/service.py b/scopeproof_core/alpha/service.py index 186337c..d756628 100644 --- a/scopeproof_core/alpha/service.py +++ b/scopeproof_core/alpha/service.py @@ -160,6 +160,11 @@ def record_alpha_outcome( raise ValueError("alpha outcome requires a completed review analysis") if review_state.bundle.research_context is not None: raise ValueError("engineering research reviews cannot record alpha outcomes") + if review_state.review.input_origin in { + ReviewInputOrigin.LOCAL_FIXTURE, + ReviewInputOrigin.CONSTRUCTED_DEMO, + }: + raise ValueError("alpha outcome requires live public GitHub ingestion") if ( review_state.review.repository_visibility is not RepositoryVisibility.VERIFIED_PUBLIC diff --git a/scopeproof_core/github/client.py b/scopeproof_core/github/client.py index a48a1c7..708bb6c 100644 --- a/scopeproof_core/github/client.py +++ b/scopeproof_core/github/client.py @@ -215,7 +215,9 @@ def _verified_repository_visibility( full_name = repository.get("full_name") private = repository.get("private") visibility = repository.get("visibility") - if private is True or visibility in {"private", "internal"}: + if private is True or ( + isinstance(visibility, str) and visibility in {"private", "internal"} + ): raise PrivateOrInaccessibleRepository( "ScopeProof accepts only a verified public GitHub repository." ) diff --git a/scopeproof_core/schemas/models.py b/scopeproof_core/schemas/models.py index 995917e..91ea37e 100644 --- a/scopeproof_core/schemas/models.py +++ b/scopeproof_core/schemas/models.py @@ -856,6 +856,22 @@ def preserve_historical_review_state(cls, value: object) -> object: } ): migrated = {**migrated, "input_origin": ReviewInputOrigin.LEGACY_UNKNOWN} + elif ( + value.get("input_origin", ReviewInputOrigin.LEGACY_UNKNOWN) + not in { + ReviewInputOrigin.LIVE_PUBLIC_GITHUB, + ReviewInputOrigin.LIVE_PUBLIC_GITHUB.value, + } + and value.get("repository_visibility") + in { + RepositoryVisibility.VERIFIED_PUBLIC, + RepositoryVisibility.VERIFIED_PUBLIC.value, + } + ): + migrated = { + **migrated, + "repository_visibility": RepositoryVisibility.UNVERIFIED, + } if "check_state" in migrated: if "ci_observation" not in migrated: diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 00a66bf..1ab1862 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -423,6 +423,40 @@ def test_fixture_review_metadata_reports_validated_ci_observation(tmp_path: Path assert "stage1_credit" not in metadata +def test_fixture_review_cannot_self_assert_verified_public_visibility( + tmp_path: Path, capsys +) -> None: + requirements = tmp_path / "requirements.txt" + requirements.write_text("Export CSV\n", encoding="utf-8") + fixture_payload = json.loads( + Path("evals/fixtures/complete_implementation_pr.json").read_text(encoding="utf-8") + ) + fixture_payload["repository_visibility"] = "verified_public" + fixture = tmp_path / "self-asserted-public.json" + fixture.write_text(json.dumps(fixture_payload), encoding="utf-8") + storage = tmp_path / "reviews" + + assert main( + [ + "review", + "--fixture", + str(fixture), + "--requirements", + str(requirements), + "--confirmation", + str(write_requirements_confirmation(requirements)), + "--storage-dir", + str(storage), + ] + ) == 0 + + review_id = json.loads(capsys.readouterr().out)["review_id"] + state = JsonReviewStore(storage).load(review_id) + assert state.review.repository_visibility is RepositoryVisibility.UNVERIFIED + assert state.bundle is not None + assert state.bundle.review.repository_visibility is RepositoryVisibility.UNVERIFIED + + def test_fixture_review_metadata_reports_ci_collection_notes(tmp_path: Path, capsys) -> None: requirements = tmp_path / "requirements.txt" requirements.write_text("Export CSV\n", encoding="utf-8") diff --git a/tests/github/test_client.py b/tests/github/test_client.py index 6ae0c1a..a5b5ffa 100644 --- a/tests/github/test_client.py +++ b/tests/github/test_client.py @@ -201,6 +201,14 @@ def test_client_rejects_non_public_repository_before_secondary_fetches( {"full_name": "acme/widget", "private": False, "visibility": "PUBLIC"}, id="malformed-visibility", ), + pytest.param( + {"full_name": "acme/widget", "private": False, "visibility": []}, + id="unhashable-list-visibility", + ), + pytest.param( + {"full_name": "acme/widget", "private": False, "visibility": {}}, + id="unhashable-object-visibility", + ), ], ) def test_client_rejects_ambiguous_repository_visibility_before_secondary_fetches( diff --git a/tests/reporting/test_exporters.py b/tests/reporting/test_exporters.py index d25a03c..f94e99a 100644 --- a/tests/reporting/test_exporters.py +++ b/tests/reporting/test_exporters.py @@ -37,6 +37,7 @@ RetrievalOutcome, Review, ReviewBundle, + ReviewInputOrigin, RuntimeEvidence, ) @@ -645,6 +646,7 @@ def test_json_export_preserves_verified_public_repository_provenance() -> None: { **bundle.review.model_dump(mode="python"), "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + "input_origin": ReviewInputOrigin.LIVE_PUBLIC_GITHUB, } ) @@ -659,6 +661,7 @@ def test_human_readable_exports_preserve_verified_public_repository_provenance() { **bundle.review.model_dump(mode="python"), "repository_visibility": RepositoryVisibility.VERIFIED_PUBLIC, + "input_origin": ReviewInputOrigin.LIVE_PUBLIC_GITHUB, } )