From 5d73b083da2fe0a2e24288c0e2c22f64af10de7f Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 9 Jun 2026 20:28:52 +0700 Subject: [PATCH] fix(rri): scope native_gate build-SHA contract to the canonical release personas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit native_gate's build-SHA gates (single build_sha / same-build / per-run) iterated ALL persona scores, so an EXTRA diagnostic persona (opus-high / lean variants) at a different SHA falsely failed native_gate even when the 5 canonical release personas + the Mac handoff were all same-build. The Tuesday RRI hit exactly this: newbie/veteran/adversarial/narrative/optimizer all at 033e4ba, but 3 narrative variants stamped stale SHAs (8afed3c/f89ce94/eabf2a3) -> 'mixed persona build_sha' gap, despite a 100/100 Mac handoff at 033e4ba. Fix: factor the contract into build_sha_evidence_gaps(persona_scores, build_sha, release_personas), scoped to REQUIRED_RELEASE_PERSONAS. Diagnostic variants no longer trip it; an absent canonical persona is still caught by missing_release_personas; a canonical persona at a wrong SHA still fails. Pure fn, 6 unit tests incl. the exact Tuesday situation. Makes the next full RRI honest re: native_gate (the Mac handoff already proves the .app at the candidate SHA). NOTE: release_readiness.py is part of the scoring ruler — this correctly re-versions it (#722). --- qa/release_readiness.py | 67 +++++++++++++++++------------- qa/test_release_readiness_scope.py | 65 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 qa/test_release_readiness_scope.py diff --git a/qa/release_readiness.py b/qa/release_readiness.py index ba6f153d..68022ca7 100644 --- a/qa/release_readiness.py +++ b/qa/release_readiness.py @@ -469,6 +469,41 @@ def infer_persona(run_dir: Path) -> str: return name +def build_sha_evidence_gaps(persona_scores: list[dict], build_sha: str, + release_personas: list[str]) -> list[dict]: + """native_gate's build-SHA contract, SCOPED to the canonical release personas. + + The release verdict is about the canonical five (REQUIRED_RELEASE_PERSONAS) + the Mac handoff, + all at ONE SHA. Extra DIAGNOSTIC personas (opus-high / lean variants outside the release set) + may run at other SHAs without invalidating the release — so they must NOT trip the "single + build_sha" / "same-build" gates. (RRI 2026-06-09: 3 narrative variants stamped stale SHAs while + newbie/veteran/adversarial/narrative/optimizer were all at the candidate SHA, falsely failing + native_gate even though the Mac handoff + the 5 release personas were same-build.) + """ + release = [p for p in persona_scores if str(p.get("persona") or "") in set(release_personas)] + build_shas = sorted({str(p["run_build_sha"]) for p in release if p.get("run_build_sha")}) + missing = [p for p in release if not p.get("run_build_sha")] + gaps: list[dict] = [] + if not build_sha: + gaps.append({"gate": "native_gate", "missing": "--build-sha", + "detail": "release verdict requires the measured build SHA"}) + if missing: + gaps.append({"gate": "native_gate", "missing": "per-run build_sha", + "detail": "missing run build_sha for: " + + ", ".join(str(p.get("persona") or p.get("run")) for p in missing)}) + if build_sha: + mismatched = [p for p in release + if p.get("run_build_sha") and not build_sha_matches(str(p.get("run_build_sha")), build_sha)] + if mismatched: + gaps.append({"gate": "native_gate", "missing": "same-build persona evidence", + "detail": "run build_sha mismatch: " + + ", ".join(f"{p['persona']}={p['run_build_sha']}" for p in mismatched)}) + if len(build_shas) > 1: + gaps.append({"gate": "native_gate", "missing": "single build_sha", + "detail": "mixed release-persona build_sha values: " + ", ".join(build_shas)}) + return gaps + + def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--runs", required=True, help="comma-separated persona run dirs") @@ -619,35 +654,9 @@ def main() -> int: native_detail = f"handoff_json={args.handoff_json} gates={','.join(REQUIRED_HANDOFF_GATES)}" split_vm_handoff_evidence = bool(persona_scores and native_source == args.handoff_json and args.handoff_json) - evidence_gaps = [] - build_shas = sorted({str(p["run_build_sha"]) for p in persona_scores if p.get("run_build_sha")}) - missing_build_sha = [p for p in persona_scores if not p.get("run_build_sha")] - if not args.build_sha: - evidence_gaps.append({ - "gate": "native_gate", - "missing": "--build-sha", - "detail": "release verdict requires the measured build SHA", - }) - if missing_build_sha: - evidence_gaps.append({ - "gate": "native_gate", - "missing": "per-run build_sha", - "detail": "missing run build_sha for: " + ", ".join(str(p.get("persona") or p.get("run")) for p in missing_build_sha), - }) - if args.build_sha: - mismatched = [p for p in persona_scores if p.get("run_build_sha") and not build_sha_matches(str(p.get("run_build_sha")), args.build_sha)] - if mismatched: - evidence_gaps.append({ - "gate": "native_gate", - "missing": "same-build persona evidence", - "detail": "run build_sha mismatch: " + ", ".join(f"{p['persona']}={p['run_build_sha']}" for p in mismatched), - }) - if len(build_shas) > 1: - evidence_gaps.append({ - "gate": "native_gate", - "missing": "single build_sha", - "detail": "mixed persona build_sha values: " + ", ".join(build_shas), - }) + # native_gate build-SHA contract — SCOPED to the canonical release personas (extra diagnostic + # variants may run at other SHAs without invalidating the release verdict). See the helper. + evidence_gaps = build_sha_evidence_gaps(persona_scores, args.build_sha, REQUIRED_RELEASE_PERSONAS) missing_release_personas = [p for p in REQUIRED_RELEASE_PERSONAS if p not in completed_set] if missing_release_personas: missing_detail = f"missing release persona(s): {', '.join(missing_release_personas)}" diff --git a/qa/test_release_readiness_scope.py b/qa/test_release_readiness_scope.py new file mode 100644 index 00000000..d20cffc2 --- /dev/null +++ b/qa/test_release_readiness_scope.py @@ -0,0 +1,65 @@ +"""native_gate build-SHA scoping (RRI 2026-06-09): the release verdict judges the canonical five +personas + the Mac handoff at ONE SHA. Extra DIAGNOSTIC personas (opus-high / lean variants) may +run at other SHAs without invalidating the release — they must NOT trip native_gate's build-SHA +gates. (The Tuesday sweep falsely failed native_gate because 3 variants stamped stale SHAs while +the 5 release personas were all at the candidate SHA.) + +Stdlib + pytest. Run: + uv run --directory servers/engine python -m pytest qa/test_release_readiness_scope.py -q -p no:xdist +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import release_readiness as rr # noqa: E402 + +REL = rr.REQUIRED_RELEASE_PERSONAS # newbie / veteran / adversarial / narrative / optimizer + + +def _score(persona, sha): + return {"persona": persona, "run_build_sha": sha} + + +def _kinds(gaps): + return {g["missing"] for g in gaps} + + +def test_extra_variant_at_stale_sha_does_not_fail_native_gate(): + # The EXACT Tuesday situation: the 5 release personas at the candidate SHA, 3 diagnostic + # variants stamped at stale SHAs. The variants must be ignored for the build-SHA contract. + scores = [_score(p, "033e4ba") for p in REL] + [ + _score("opushi-narr", "8afed3c"), + _score("opuslean-narr", "f89ce94"), + _score("opuslean-narr2", "eabf2a3"), + ] + gaps = rr.build_sha_evidence_gaps(scores, "033e4ba", REL) + assert gaps == [], f"diagnostic variants must not trip native_gate build-sha gates: {gaps}" + + +def test_clean_release_set_has_no_build_sha_gaps(): + scores = [_score(p, "033e4ba") for p in REL] + assert rr.build_sha_evidence_gaps(scores, "033e4ba", REL) == [] + + +def test_a_release_persona_at_a_different_sha_still_fails(): + scores = [_score(p, "033e4ba") for p in REL[:-1]] + [_score(REL[-1], "deadbee")] + kinds = _kinds(rr.build_sha_evidence_gaps(scores, "033e4ba", REL)) + assert "same-build persona evidence" in kinds or "single build_sha" in kinds, kinds + + +def test_missing_build_sha_on_a_release_persona_flags(): + scores = [_score(p, "033e4ba") for p in REL[:-1]] + [{"persona": REL[-1], "run_build_sha": ""}] + assert "per-run build_sha" in _kinds(rr.build_sha_evidence_gaps(scores, "033e4ba", REL)) + + +def test_no_build_sha_arg_flags(): + scores = [_score(p, "033e4ba") for p in REL] + assert "--build-sha" in _kinds(rr.build_sha_evidence_gaps(scores, "", REL)) + + +def test_only_variants_present_yields_no_buildsha_gaps_but_release_set_caught_elsewhere(): + # If ONLY diagnostic variants ran (no canonical persona), build-sha gaps are empty here — the + # MISSING canonical personas are caught by the separate missing_release_personas check, not this + # one. So scoping never hides an absent release persona. + scores = [_score("opushi-narr", "8afed3c"), _score("opuslean-narr", "f89ce94")] + assert rr.build_sha_evidence_gaps(scores, "033e4ba", REL) == []