From c56dd615e7e4334c72732f063554ee5b6b7d0599 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 1 Jun 2026 15:32:34 +0700 Subject: [PATCH] Harden RRI handoff evidence bridge --- WorldOS-GUI-RUNBOOK.md | 36 ++- WorldOS-OPERATING-GOAL.md | 78 +++--- WorldOS-RUNBOOK.md | 23 +- qa/SCORECARD.md | 1 + qa/release_readiness.py | 289 +++++++++++++++++++-- qa/test_release_readiness.py | 471 +++++++++++++++++++++++++++++++++++ 6 files changed, 836 insertions(+), 62 deletions(-) diff --git a/WorldOS-GUI-RUNBOOK.md b/WorldOS-GUI-RUNBOOK.md index 7d1c6e84..c0188b75 100644 --- a/WorldOS-GUI-RUNBOOK.md +++ b/WorldOS-GUI-RUNBOOK.md @@ -8,7 +8,7 @@ > `qa/release_readiness.py` (the RRI scorer), `qa/SCORECARD.md` (the ledger). > > Takeover routing, 2026-06-01: `/Users/lume/ClawDnD-val` is the synced local app/private-art checkout -> (`f7ab6d7 == origin/main` after #475/#494/#495/#496/#498/#499/#500) and the default place to build/run/test the GUI and native app. +> (`4a0efe1 == origin/main` after #475/#494/#495/#496/#498/#499/#500/#501/#504) and the default place to build/run/test the GUI and native app. > Lexar is for evidence/snapshots/logs, not the default runtime tree, because macOS permission prompts > can break AI/browser tests when assets live on the external drive. For tracked GUI edits, prefer a > same-disk local worktree; use Lexar worktrees only for non-GUI slices that will not launch against art. @@ -64,6 +64,14 @@ accessibility review showed the chronicle with one opening narration row and one follow-up narration row, not duplicate chat/event prose. This closes the #479 diagnostic blocker, but release still requires #466's full non-partial RRI gate. +- The post-#504 merged-main handoff gate + (`/Volumes/LEXAR/Codex/worldos-agent-grade-app-testability/handoff-20260601T081016Z-4a0efe1/`, build + `4a0efe1`) is the current fastest GUI trust proof. It scored `handoff_score=100` with web scripted smoke + 5 moves, built `dist/WorldOS.app` scripted smoke 5 moves, and built `dist/WorldOS.app` Codex-provider + playtest 1 move. All three evidence manifests passed with zero gaps, private art present, screenshots, + app-status/session-surface snapshots, move logs, provider trace, console/network/action logs, and failure + bucket fields. The Codex trace summary reported `trace_exists=true`, `line_count=175`, and + `failed_or_error_count=0`. This is the fast GUI velocity gate, not the release verdict. ## Agent-facing app contract @@ -74,11 +82,12 @@ campaign, can the player act, where is the move sink, and is private art configured?" - `qa/ui_playtest_app.sh` captures launcher and minted-provider `app-status` JSON into the native evidence folder. A built-app proof that cannot produce this status object is a harness/product observability failure. -- Agent-grade testing progress as of `f7ab6d7`: #481 app-status is closed, #482 deterministic scripted - provider is merged, #483 failure buckets are merged, and #484 stable accessibility/DOM hooks are merged. - #485 evidence bundle completion and #486 gate-split follow-through remain active. A scripted `:8899` - harness surface can prove app observability, but it is not built-app release proof unless it came from - `dist/WorldOS.app` / `qa/ui_playtest_app.sh`. +- Agent-grade testing progress as of `4a0efe1`: #481 app-status is closed, #482 deterministic scripted + provider is merged, #483 failure buckets are merged, #484 stable accessibility/DOM hooks are merged, + and #504's hybrid handoff gate is merged and green on `main`. #485 evidence bundle completion and + #486 gate-split follow-through remain active. A scripted `:8899` harness surface can prove app + observability, but it is not built-app release proof unless it came from `dist/WorldOS.app` / + `qa/ui_playtest_app.sh`. ## Stand up the iteration surface (8799, playable, from canonical) ```bash @@ -183,10 +192,25 @@ release truth still requires `qa/ui_playtest_app.sh` Part A+B and the full RRI s auth/profile status, `uv`, Node/npm/Playwright availability, private-art availability or explicit backend-only/no-art classification, env vars, budget/concurrency cap, teardown commands, and the artifact return path under `/Volumes/LEXAR/Codex`. +- Current preflight status (2026-06-01): this local Codex Desktop session can parse an SSH alias for + `support-vm-1`, but `ssh -o BatchMode=yes support-vm-1 ...` failed DNS resolution (`nodename nor servname + provided`). Do not start #466 there until operator routing is restored and the preflight fields above are + written to Lexar evidence. +- Read-only VM scout (2026-06-01): an operator-only endpoint note can reach `evaos-support` without printing + the endpoint. Capacity/tooling look suitable for heavy sweeps: ~32 GB RAM, 16 CPUs, ~537 GB free disk, `git`, + `python3`, `uv 0.11.17`, Node `v22.22.1`, npm `10.9.4`, `codex-cli 0.120.0`, Playwright modules, and private + art. The VM WorldOS checkout at `/root/worldos-qa/WorldOS` is clean but stale at `4524b3e`, 19 commits behind + current `main@4a0efe1`; Codex auth/config is not proven; `/Volumes/LEXAR/Codex` does not exist on the VM. + Before #466, approve/sync the VM checkout, prove Codex auth, set a remote staging path, and copy artifacts + back to local Lexar. - RRI rollup rule: Mac/local evidence supplies native Part A and built-app screenshots; VM artifacts can supply persona, behavior, image/network, palette-live, and score evidence only when `run.json`, `score.json`, `session_surface.final.json`, `network.ndjson`, and build SHA are present. Missing or mixed-SHA artifacts must remain `partial` / `harness_contaminated`. +- Split Mac/VM rollup command shape: pass the Mac proof into RRI as + `--handoff-json /Volumes/LEXAR/Codex/worldos-agent-grade-app-testability//handoff.json` + alongside the VM persona run dirs. RRI should then satisfy the native gate from the Mac handoff bundle + only if all required handoff gates and manifests are same-SHA, clean, private-art-present, and gap-free. ## Release (when RRI = 10/10 on a fresh .app build) Bump `.claude-plugin/plugin.json` → 1.0.4, tag `v1.0.4`, GitHub release + CHANGELOG. Then MAINTAIN: diff --git a/WorldOS-OPERATING-GOAL.md b/WorldOS-OPERATING-GOAL.md index 5ace5865..c84c24e5 100644 --- a/WorldOS-OPERATING-GOAL.md +++ b/WorldOS-OPERATING-GOAL.md @@ -5,10 +5,10 @@ Post-compaction agents: this 6-line block is ground truth. Do NOT reconstruct state from scattered docs or old plans; trust this, verify the sha, then act. ────────────────────────────────────────────────────────────────────────── - AS OF: 2026-06-01T08:05:48+07:00 #479 trace-clean merged-main proof archived + AS OF: 2026-06-01T15:35:00+07:00 #504 handoff gate merged + post-merge fast gate passed MAIN BASELINE: - f7ab6d7 (PRs #475, #494, #495, #496, #498, #499, #500 merged; verified - `/Users/lume/ClawDnD-val` was fast-forwarded after #500). + 4a0efe1 (PRs #475, #494, #495, #496, #498, #499, #500, #501, #504 + merged; verified `/Users/lume/ClawDnD-val` was fast-forwarded after #504). Re-verify current `origin/main` before acting. CANONICAL: /Users/lume/ClawDnD-val is now the synced local app/private-art checkout and the default place to build/run/test the Mac app. Keep GUI/runtime tests on this @@ -20,34 +20,41 @@ in local operator-only evidence/runbooks, not tracked repo docs. Use it for heavy backend/persona sweeps only after Codex/config/credentials are intentionally installed. Mac-built `.app` smoke/play proof stays on this Mac or macOS CI. + Current local preflight note: `ssh -G support-vm-1` resolves only to the alias; + `ssh -o BatchMode=yes support-vm-1 ...` could not resolve the hostname in this + Codex Desktop session. A read-only operator-endpoint scout reached `evaos-support` + (~32 GB RAM, 16 CPUs) with WorldOS at `/root/worldos-qa/WorldOS`, but that checkout + was stale (`4524b3e`, 19 commits behind `4a0efe1`) and Codex auth/config was not + proven. Restore/verify operator routing, fast-forward the VM repo, verify Codex auth, + and define artifact return before running #466 there. LAST MEASURED GATE BUILD: f5500ac produced qa/RRI.json = 2.7/10, but this is PARTIAL / HARNESS-CONTAMINATED evidence: only newbie wrote score.json; the other personas failed around port/backend harness setup; behavioral/UI/palette/image evidence was not a valid five-persona release verdict. LAST BUILT-APP PLAY PROOF: - Last real-provider built-app proof on merged main is `f7ab6d7` - (`codex-main-f7ab6d7-proof-20260601T010058Z`): private BG art loaded, - Codex provider minted the live session, Alfira seated, visible DM - narration, five enabled actions, and one `/move` resolved. It is - trace-clean merged-main diagnostic evidence: no app-evidence gaps, - provider trace summary records 0 parse errors and 0 failed/error tool - calls, chat roles reached `dm, player, dm`, `/session-surface` stayed - actionable, and native accessibility review showed one opening narration - row plus one post-move narration row, not duplicate chat/event prose. - Latest deterministic built-app smoke proof is PR #495 app-code - `698ed65` (`ui-hooks-app-proof-20260601T054934`): scripted provider, - private art, Abby seated, five enabled actions, writable `/move`, one - deterministic follow-up, and real image bytes. A current-session - `080497e` scripted harness proof exists at `scripted-smoke-20260531T233022Z-080497e`, - but ad-hoc harness ports are transient and are not built `.app` release proof. + Last merged-main handoff proof is `4a0efe1` + (`/Volumes/LEXAR/Codex/worldos-agent-grade-app-testability/handoff-20260601T081016Z-4a0efe1/`): + `qa/app_handoff_gate.py` scored `handoff_score=100` with web scripted smoke + 5 moves, built `dist/WorldOS.app` scripted smoke 5 moves, and built + `dist/WorldOS.app` Codex-provider playtest 1 move. Private BG art was present, + Alfira was seated, visible narration and five enabled actions were present, + `/move` resolved, all three manifests had zero evidence gaps, and the Codex + provider trace reported `trace_exists=true`, `line_count=175`, and + `failed_or_error_count=0`. + Prior trace-clean real-provider built-app proof on merged main is `f7ab6d7` + (`codex-main-f7ab6d7-proof-20260601T010058Z`); it remains useful diagnostic + #479 evidence but is superseded as the latest merged-main app proof by the + `4a0efe1` handoff gate. LAST VALID RELEASE GATE: none after the RRI contract hardening. A release verdict requires expected persona count, disk-backed palette/image/behavioral evidence, and built .app play. - NEXT ACTION: Close/update #479 against the merged-main proof, then run #466 for a trustworthy - clean RRI failure list/result: Mac/local or macOS CI supplies built - `.app` proof, while the 32GB support VM runs heavy backend/persona - sweeps after explicit VM preflight/auth setup. Continue #485/#486 for + NEXT ACTION: #479 is closed and #504 gives a 100/100 merged-main fast GUI velocity gate. + Do not claim release. Run #466 for a trustworthy clean RRI failure list/result: + Mac/local or macOS CI supplies built `.app` proof, while the 32GB support VM + runs heavy backend/persona sweeps after explicit VM routing/auth/config + preflight. If the VM route is still unavailable, record that as the blocker and + file/fix repo-side RRI harness gaps only if found. Continue #485/#486 for evidence export and gate split follow-through; #481/#482/#483/#484 are closed. Keep sprint work UX-first (#467): first-turn playability, clickability/chrome, launcher clarity, live-response feel, and CRPG depth before more hardening/proxy/security work. @@ -139,7 +146,8 @@ by an average). **RRI 10/10 = every gate holds on one fresh build:** > The RRI output contract also records `required_release_personas`, `expected_personas`, > `completed_personas`, `missing_personas`, `missing_release_personas`, `partial`, > `harness_contaminated`, explicit `evidence_gaps`, image source/denominator, behavioral evidence path, -> UI-audit log, palette-live source, per-run Part B pass status, and per-run build SHA. Smoke-sized +> UI-audit log, palette-live source, optional `--handoff-json` Mac app proof, per-run Part B pass status, +> and per-run build SHA. Smoke-sized > persona sets, mixed-build evidence, failed app-persona loops, missing persona scores, or missing > artifact denominators can never silently produce a release-ready result. @@ -188,7 +196,7 @@ verifier; can revert the goal to "fix" anytime. --- -## 9. CURRENT STATUS (2026-06-01T08:05:48+07:00 — #479 trace-clean merged-main proof archived) +## 9. CURRENT STATUS (2026-06-01T15:35:00+07:00 — #504 handoff gate passed post-merge) - Repo truth stabilization merged in PR #465, UX-first doc sync merged in PR #468, first-minute click/title chrome proof merged in PR #470, local/Lexar/support-VM routing merged in PR #471, @@ -197,9 +205,10 @@ verifier; can revert the goal to "fix" anytime. `/app-status`, evidence export baseline, and docs; PR #494 added the dev-gated scripted provider; PR #495 added stable OpenWorlds accessibility / `data-worldos-testid` hooks; PR #496 added app playtest failure buckets plus RRI gate-split metadata. PR #498 synced takeover docs after those - merges. PR #499 then recorded the current-main built-app proof, and PR #500 fixed the Codex-DM - provider trace cancellations. The local app/private-art checkout `/Users/lume/ClawDnD-val` - was fast-forwarded to `f7ab6d7 == origin/main` after #500. + merges. PR #499 then recorded the current-main built-app proof, PR #500 fixed the Codex-DM + provider trace cancellations, PR #501 recorded that proof in the runbooks/scorecard, and PR #504 + added the hybrid 100/100 app handoff gate. The local app/private-art checkout + `/Users/lume/ClawDnD-val` was fast-forwarded to `4a0efe1 == origin/main` after #504. - The stale local pre-sync artifacts were preserved before the fast-forward at `/Volumes/LEXAR/Codex/worldos-local-checkout-snapshot-20260531T223923` and in `stash@{0}` (`pre-sync local takeover docs 2026-05-31`). Treat those as evidence, not current release truth. @@ -221,10 +230,14 @@ verifier; can revert the goal to "fix" anytime. macOS TCC attribution contamination: `responsible=dev.clawdnd.app`, but the actual accessor was `/usr/bin/find` launched by the test/diagnostic environment. Treat that screenshot prompt as harness contamination unless a clean run shows `WorldOSApp`/WebKit itself accessing a protected library path. -- The next gate evidence step is issue #466: a clean non-partial five-persona RRI from `f7ab6d7` or newer. +- The next gate evidence step is issue #466: a clean non-partial five-persona RRI from `4a0efe1` or newer. Heavy backend/persona sweeps belong on the owner-provided 32GB support VM (`support-vm-1`) once auth/config - are intentionally installed there; connection details are kept outside tracked docs. Mac-only built-app - launch/play proof stays on this Mac or macOS CI. + are intentionally installed there; connection details are kept outside tracked docs. In this Codex Desktop + session the local SSH alias for `support-vm-1` did not resolve; a read-only operator-endpoint scout reached + the VM and confirmed `evaos-support` has ~32 GB RAM, 16 CPUs, `git`, `python3`, `uv`, Node/npm, Codex CLI, + Playwright, and private art, but its WorldOS checkout is `4524b3e` (19 commits behind `4a0efe1`) and Codex + auth/config was not proven. Restore/verify VM routing, fast-forward the VM checkout, verify auth, and define + artifact return before the heavy sweep. Mac-only built-app launch/play proof stays on this Mac or macOS CI. - Built-app diagnostic evidence exists, but release truth is still absent. The PR #475 pre-merge app-code proof `8bd833f` (`codex-app-headproof-20260601T043909`) was trace-clean. The post-merge main proof `32ca561` (`post475-main-app-proof-20260601T051230`) was playable with private art, Alfira, five enabled @@ -245,6 +258,11 @@ verifier; can revert the goal to "fix" anytime. screenshot archived, `app-evidence/manifest.json` with no gaps, and `provider-errors.after-move.json` reporting zero parse errors plus zero failed/error tool calls. This is sufficient to close #479 as a merged-main diagnostic; it is still not an RRI release verdict. +- The post-#504 merged-main handoff gate `handoff-20260601T081016Z-4a0efe1` then proved the fast GUI + velocity loop on the current `main`: web scripted smoke 5 moves, built-app scripted smoke 5 moves, + built-app Codex playtest 1 move, private art present, Alfira active, five enabled actions, zero evidence + gaps across all three manifests, and Codex trace `failed_or_error_count=0`. This is the current best + app-wiring proof, but it remains diagnostic and cannot replace the full five-persona RRI. - The agent-grade testability layer now has real code merged: `GET /app-status` exposes the live run, campaign, provider, private-art presence, move sink, actor, enabled actions, readiness, and failure buckets without mutating state; the scripted provider can prove wiring behind a dev/test gate; and stable a11y/DOM diff --git a/WorldOS-RUNBOOK.md b/WorldOS-RUNBOOK.md index c6cb5608..14b0acde 100644 --- a/WorldOS-RUNBOOK.md +++ b/WorldOS-RUNBOOK.md @@ -13,8 +13,9 @@ > snapshots, and logs; do not make Lexar the default GUI runtime tree because external-drive > permissions can break local AI/browser tests. Heavy backend/persona sweeps belong on GitHub CI or > the owner-provided 32GB support VM (`support-vm-1`) after remote access and Codex config are -> intentionally installed and verified; connection details are kept outside tracked docs. Mac-only -> built-app proof remains local/macOS CI. +> intentionally installed and verified; connection details are kept outside tracked docs. A read-only +> scout reached the operator endpoint and found `evaos-support` suitable but stale (`4524b3e`, 19 commits +> behind `4a0efe1`) with Codex auth/config unproven. Mac-only built-app proof remains local/macOS CI. > **This is the compaction-resilience doc.** If you are an agent resuming this project > after a context reset, read this top-to-bottom before doing anything. It captures the @@ -25,7 +26,7 @@ > If an operator hands you local session notes or decision records, treat them as > private working artifacts unless they are intentionally promoted into tracked docs. > -> Last updated: 2026-06-01T08:05:48+07:00 (`main@f7ab6d7`; #475/#494/#495/#496/#498/#499/#500 merged; #479 trace-clean merged-main proof archived; release notes below are historical context). +> Last updated: 2026-06-01T15:35:00+07:00 (`main@4a0efe1`; #475/#494/#495/#496/#498/#499/#500/#501/#504 merged; #504 100/100 handoff gate passed post-merge; release notes below are historical context). > > **Graphics & game-types roadmap (canonical):** the long-term plan for the kinds of games > WorldOS can produce (GT0 narrative dashboard → GT1 SNES pixel → GT2 Pillars/BG isometric) @@ -310,16 +311,18 @@ Local routing sync merged as PR #471, native provider-selection sync merged as P state docs synced as PR #473, Codex-DM app observability merged as PR #475, scripted smoke provider merged as PR #494, stable agent UI hooks merged as PR #495, failure-bucket/RRI split metadata merged as PR #496, and takeover truth sync merged as PR #498, followed by PR #499 recording current-main built-app proof -and PR #500 fixing Codex-DM provider trace cancellations. +PR #500 fixing Codex-DM provider trace cancellations, PR #501 recording that proof in docs, and +PR #504 adding the 100/100 hybrid app handoff gate. The local app/private-art checkout is now synced at -`f7ab6d7 == origin/main`; the only current gate +`4a0efe1 == origin/main`; the only current gate truth lives in `WorldOS-OPERATING-GOAL.md` + `WorldOS-GUI-RUNBOOK.md` + `qa/SCORECARD.md`. Do not use this section to decide release state. The next sprint is UX-first (#467): -current-main built-app play is proven diagnostically on `f7ab6d7`, including private art, Codex DM, Alfira, -five enabled actions, one accepted/resolved `/move`, no app-evidence gaps, zero failed/error provider tool -calls, and no visible chronicle duplication in native accessibility review. With #479 proven on merged main, -run #466 only after Mac built-app proof plus support-VM preflight, -then prioritize clickability/chrome, launcher clarity, +current-main fast handoff play is proven diagnostically on `4a0efe1`, including private art, Codex DM, Alfira, +five enabled actions, one accepted/resolved `/move`, no evidence-manifest gaps, zero failed/error provider +trace events, and a post-merge `handoff_score=100`. With #479 proven and #504 merged, run #466 only after +support-VM routing/auth/config preflight is explicit; this session's read-only check found the local +`support-vm-1` SSH alias did not resolve and the operator-endpoint VM checkout was stale at `4524b3e`. +Then prioritize clickability/chrome, launcher clarity, live-response feel, and CRPG depth before more hardening/proxy/security work. **LATEST (2026-05-27 cont.3) — the Quest & Arc engine is COMPLETE + WIRED, all combat diff --git a/qa/SCORECARD.md b/qa/SCORECARD.md index 322fd082..a7eae993 100644 --- a/qa/SCORECARD.md +++ b/qa/SCORECARD.md @@ -16,6 +16,7 @@ | Run | Date | app/code SHA | Surface | Provider | Evidence | Result / notes | |---|---|---|---|---|---|---| +| handoff-20260601T081016Z-4a0efe1 | 2026-06-01 | 4a0efe1 | hybrid web + built `dist/WorldOS.app` local Mac | Scripted + Codex DM | `/Volumes/LEXAR/Codex/worldos-agent-grade-app-testability/handoff-20260601T081016Z-4a0efe1/` | **PASS 100/100 handoff-readiness gate after #504 merge, not a release verdict.** `handoff.json` reports `status=passed`, `handoff_score=100`, `dirty=false`, blockers empty, and all required gates on build `4a0efe1`: web scripted smoke 5 moves, built-app scripted smoke 5 moves, and built-app Codex-provider playtest 1 move. All three manifests report `verdict=passed`, `app_build_sha=4a0efe1`, `evidence_gaps=0`, `handoff_gate.ok=true`, and private art present. Final built-app app-status showed Alfira active, `can_act=true`, five enabled actions, console errors 0, network failures 0, image probe OK, and provider ready. Codex trace summary reported `trace_exists=true`, `line_count=175`, `failed_or_error_count=0`. Full #466 five-persona RRI remains required. | | codex-main-f7ab6d7-proof-20260601T010058Z | 2026-06-01 | f7ab6d7 | `dist/WorldOS.app` local Mac | Codex DM | `/Volumes/LEXAR/Codex/worldos-built-app-playtest/codex-main-f7ab6d7-proof-20260601T010058Z/` | **PASS trace-clean merged-main diagnostic for #479.** Private BG art present via `/Users/lume/ClawDnD-val`; built app minted Codex run `play-20260601010121` on port 8766; `/app-status` reported build `f7ab6d7`, live campaign `camp_a3e0a4b545a8`, writable move sink, Alfira active, five enabled actions, and no evidence gaps in `app-evidence/manifest.json`; one `/move` accepted and resolved; chat roles `dm, player, dm`; `/session-surface` remained live/actionable; native after-move screenshot archived. `provider-errors.after-move.json` reports zero parse errors, zero failed/error tool calls, and statuses `completed=10`, `in_progress=10`. Native accessibility review showed one opening narration row and one follow-up narration row, not duplicate chat/event prose. This closes #479 as a merged-main diagnostic, but it is not an RRI verdict. | | codex-479-traceclean-nodup-proof-20260601T003002Z | 2026-06-01 | b081092 | `dist/WorldOS.app` local Mac | Codex DM | `/Volumes/LEXAR/Codex/worldos-built-app-playtest/codex-479-traceclean-nodup-proof-20260601T003002Z/` | **PASS trace-clean branch diagnostic for #479.** Private BG art present via `/Users/lume/ClawDnD-val`; `/app-status` reported build `b081092`, live Codex run `play-20260601003008`, writable move sink, Alfira active, five enabled actions, and no evidence gaps in `app-evidence/manifest.json`; one `/move` accepted and resolved; chat roles `dm, player, dm`; `/session-surface` remained live/actionable; native after-move screenshot archived. `provider-errors.after-move.json` reports zero parse errors, zero failed/error tool calls, and statuses `completed=10`, `in_progress=10`. Native accessibility review showed one opening narration row and one follow-up narration row, not duplicate chat/event prose. Not an RRI verdict; superseded for #479 closure by the merged-main `f7ab6d7` proof above. | | codex-current-main-proof-20260531T234242Z | 2026-06-01 | 19c3fd0 | `dist/WorldOS.app` local Mac | Codex DM | `/Volumes/LEXAR/Codex/worldos-built-app-playtest/codex-current-main-proof-20260531T234242Z/` | **PASS playable diagnostic, but provider trace not clean.** Private BG art present; `/app-status` reported build `19c3fd0`, live Codex run, writable move sink, Alfira active, five enabled actions, and private-art root present; one `/move` accepted and resolved; chat roles `dm, player, dm`; `/session-surface` remained live/actionable; built-app screenshot and `app-evidence/manifest.json` archived. `provider-errors.after-move.json` recorded three failed/cancelled engine tool calls (`log_event`, `log_event`, `persist_beat`), so this is historical non-clean evidence superseded for #479 closure by `f7ab6d7`. | diff --git a/qa/release_readiness.py b/qa/release_readiness.py index 554789f7..2ac40d66 100644 --- a/qa/release_readiness.py +++ b/qa/release_readiness.py @@ -12,6 +12,7 @@ - --behavioral GREEN|RED (assert_behavioral.py exit) - --ui-audit PASS|FAIL (ui_audit_health.sh exit) - --palette-live true|false (a clean /session-surface read done by the CALLER, not here) + - --handoff-json handoff.json (optional Mac-built app proof from qa/app_handoff_gate.py) The two NEW signals the plan calls for — image-render-rate and palette-live — are computed here (image rate from score.json/network.ndjson) and passed in (palette-live), so this stays @@ -21,7 +22,7 @@ release_readiness.py --runs [,...] \ [--story story.json] [--mech mech.json] \ [--behavioral GREEN|RED] [--ui-audit PASS|FAIL] [--palette-live true|false] \ - [--build-sha SHA] [--expected-personas newbie,veteran,...] + [--build-sha SHA] [--expected-personas newbie,veteran,...] [--handoff-json handoff.json] [--out qa/RRI.json] [--scorecard-row] Targets for 10/10 (each dimension is a gate; all must hold on ONE build): @@ -39,6 +40,26 @@ REQUIRED_RELEASE_PERSONAS = ["newbie", "veteran", "adversarial", "narrative", "optimizer"] RELEASE_VERDICT_GATE = "full_five_persona_rri" +REQUIRED_HANDOFF_GATES = ["web_scripted_smoke", "built_app_scripted_smoke", "built_app_codex_playtest"] +REQUIRED_HANDOFF_EVIDENCE_KINDS = [ + "screenshots", + "app_status_snapshots", + "session_surface_snapshots", + "moves", + "provider_trace", + "console_logs", + "network_logs", + "action_logs", +] +REQUIRED_SCORE_FIELDS = { + "persona": "nonempty string", + "completed_intro_flow": "boolean", + "persona_satisfaction": "number", + "gave_up": "boolean", + "bug_reports_critical": "integer", + "console_errors": "integer", +} +MIN_BUILD_SHA_MATCH_CHARS = 7 GATE_SPLIT_CONTRACT = { "deterministic_built_app_smoke": { "scope": "fast built-app wiring proof with deterministic provider", @@ -68,6 +89,20 @@ def read_json(path: Path) -> dict: return {} +def read_json_with_error(path: Path) -> tuple[dict, str]: + if not path or not path.exists(): + return {}, "missing" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {}, f"invalid JSON: {exc.msg}" + except OSError as exc: + return {}, f"read failed: {exc}" + if not isinstance(payload, dict): + return {}, "JSON root is not an object" + return payload, "" + + def read_ndjson(path: Path) -> list[dict]: out: list[dict] = [] if not path or not path.exists(): @@ -107,6 +142,177 @@ def split_csv(value: str) -> list[str]: return [p.strip() for p in value.split(",") if p.strip()] +def build_sha_matches(reported: str, expected: str) -> bool: + reported = (reported or "").strip() + expected = (expected or "").strip() + if len(reported) < MIN_BUILD_SHA_MATCH_CHARS or len(expected) < MIN_BUILD_SHA_MATCH_CHARS: + return False + return reported == expected or reported.startswith(expected) or expected.startswith(reported) + + +def score_schema_errors(score: dict) -> list[str]: + errors: list[str] = [] + persona = score.get("persona") + if not isinstance(persona, str) or not persona.strip(): + errors.append("persona must be a nonempty string") + if not isinstance(score.get("completed_intro_flow"), bool): + errors.append("completed_intro_flow must be boolean") + sat = score.get("persona_satisfaction") + if not isinstance(sat, (int, float)) or isinstance(sat, bool): + errors.append("persona_satisfaction must be numeric") + if not isinstance(score.get("gave_up"), bool): + errors.append("gave_up must be boolean") + for field in ("bug_reports_critical", "console_errors"): + value = score.get(field) + if not isinstance(value, int) or isinstance(value, bool): + errors.append(f"{field} must be integer") + return errors + + +def resolve_manifest_path(handoff_path: Path, value: str) -> Path: + path = Path(value) + if path.is_absolute(): + return path + return handoff_path.parent / path + + +def resolve_evidence_file(manifest_path: Path, value: str) -> Path: + path = Path(value) + if path.is_absolute(): + return path + return manifest_path.parent / path + + +def handoff_gap(missing: str, detail: str) -> dict: + return {"gate": "native_gate", "missing": missing, "detail": detail} + + +def validate_handoff_json(handoff_json: str, expected_sha: str) -> tuple[dict, list[dict]]: + proof = { + "path": handoff_json or "", + "valid": False, + "status": "", + "handoff_score": 0, + "commit_sha": "", + "gates": {}, + } + if not handoff_json: + return proof, [] + + handoff_path = Path(handoff_json) + payload, error = read_json_with_error(handoff_path) + if error: + return proof, [handoff_gap(handoff_json, f"handoff JSON {error}")] + + gaps: list[dict] = [] + proof.update({ + "status": payload.get("status") or "", + "handoff_score": payload.get("handoff_score") or 0, + "commit_sha": payload.get("commit_sha") or "", + "release_verdict": bool(payload.get("release_verdict")), + }) + if payload.get("schema") != "worldos.app-handoff.v1": + gaps.append(handoff_gap(str(handoff_path), "handoff schema is missing or wrong")) + if payload.get("status") != "passed": + gaps.append(handoff_gap(str(handoff_path), f"handoff status is {payload.get('status') or 'missing'}")) + if payload.get("handoff_score") != 100: + gaps.append(handoff_gap(str(handoff_path), f"handoff_score is {payload.get('handoff_score')!r}, expected 100")) + if payload.get("dirty") is not False: + gaps.append(handoff_gap(str(handoff_path), "handoff evidence was recorded from a dirty worktree")) + if expected_sha and not build_sha_matches(str(payload.get("commit_sha") or ""), expected_sha): + gaps.append(handoff_gap(str(handoff_path), f"handoff commit_sha {payload.get('commit_sha') or 'missing'} does not match --build-sha {expected_sha}")) + + gates = payload.get("gates") + if not isinstance(gates, list): + gates = [] + gaps.append(handoff_gap(str(handoff_path), "handoff gates list is missing")) + by_name = {str(g.get("name") or ""): g for g in gates if isinstance(g, dict)} + proof["gates"] = {} + manifest_paths_seen: set[Path] = set() + for gate_name in REQUIRED_HANDOFF_GATES: + gate = by_name.get(gate_name) + if not gate: + gaps.append(handoff_gap(str(handoff_path), f"handoff missing required gate {gate_name}")) + continue + manifest_value = str(gate.get("evidence_manifest") or "") + manifest_path = resolve_manifest_path(handoff_path, manifest_value) if manifest_value else Path() + if manifest_path in manifest_paths_seen: + gaps.append(handoff_gap(gate_name, f"evidence_manifest reuses another gate's manifest: {manifest_path}")) + manifest_paths_seen.add(manifest_path) + gate_proof = { + "status": gate.get("status") or "", + "build_sha": gate.get("build_sha") or "", + "evidence_manifest": str(manifest_path) if manifest_value else "", + } + proof["gates"][gate_name] = gate_proof + if gate.get("status") != "passed": + gaps.append(handoff_gap(gate_name, f"handoff gate status is {gate.get('status') or 'missing'}")) + if gate.get("evidence_gaps"): + gaps.append(handoff_gap(gate_name, "handoff gate reported evidence_gaps")) + if expected_sha and not build_sha_matches(str(gate.get("build_sha") or ""), expected_sha): + gaps.append(handoff_gap(gate_name, f"gate build_sha {gate.get('build_sha') or 'missing'} does not match --build-sha {expected_sha}")) + if not manifest_value: + gaps.append(handoff_gap(gate_name, "evidence_manifest path is missing")) + continue + manifest, manifest_error = read_json_with_error(manifest_path) + if manifest_error: + gaps.append(handoff_gap(str(manifest_path), f"evidence manifest {manifest_error}")) + continue + gate_proof["manifest_verdict"] = manifest.get("verdict") or "" + if manifest.get("schema") != "worldos.app-evidence.v1": + gaps.append(handoff_gap(str(manifest_path), "manifest schema is missing or wrong")) + if manifest.get("gate_kind") != gate_name: + gaps.append(handoff_gap(str(manifest_path), f"manifest gate_kind {manifest.get('gate_kind') or 'missing'} does not match {gate_name}")) + if manifest.get("verdict") != "passed": + gaps.append(handoff_gap(str(manifest_path), f"manifest verdict is {manifest.get('verdict') or 'missing'}")) + if manifest.get("dirty") is not False: + gaps.append(handoff_gap(str(manifest_path), "manifest was recorded from a dirty worktree")) + if manifest.get("evidence_gaps"): + gaps.append(handoff_gap(str(manifest_path), "manifest evidence_gaps is not empty")) + manifest_failure = manifest.get("failure") if isinstance(manifest.get("failure"), dict) else {} + has_failure_fields = ( + ("failure_bucket" in manifest and "failure_detail" in manifest) + or ("failure_bucket" in manifest_failure and "failure_detail" in manifest_failure) + ) + if not has_failure_fields: + gaps.append(handoff_gap(str(manifest_path), "manifest missing failure bucket/detail fields")) + if expected_sha and not build_sha_matches(str(manifest.get("app_build_sha") or ""), expected_sha): + gaps.append(handoff_gap(str(manifest_path), f"manifest app_build_sha {manifest.get('app_build_sha') or 'missing'} does not match --build-sha {expected_sha}")) + + handoff_gate = manifest.get("handoff_gate") if isinstance(manifest.get("handoff_gate"), dict) else {} + art = manifest.get("art") if isinstance(manifest.get("art"), dict) else {} + live = manifest.get("live") if isinstance(manifest.get("live"), dict) else {} + evidence_files = manifest.get("evidence_files") if isinstance(manifest.get("evidence_files"), dict) else {} + checks = { + "handoff_gate.ok": handoff_gate.get("ok") is True, + "handoff_gate.app_status_ok": handoff_gate.get("app_status_ok") is True, + "handoff_gate.session_surface_ok": handoff_gate.get("session_surface_ok") is True, + "handoff_gate.move_sink_present": handoff_gate.get("move_sink_present") is True, + "handoff_gate.private_art_present": handoff_gate.get("private_art_present") is True, + "handoff_gate.can_act": handoff_gate.get("can_act") is True, + "art.private_root_present": art.get("private_root_present") is True, + "live.can_act": live.get("can_act") is True, + } + for check, ok in checks.items(): + if not ok: + gaps.append(handoff_gap(str(manifest_path), f"{check} was not true")) + if int(handoff_gate.get("enabled_action_count") or 0) <= 0 or int(live.get("enabled_action_count") or 0) <= 0: + gaps.append(handoff_gap(str(manifest_path), "manifest did not prove enabled player actions")) + if int(handoff_gate.get("evidence_gap_count") or 0) != 0: + gaps.append(handoff_gap(str(manifest_path), "handoff_gate evidence_gap_count was not zero")) + for evidence_kind in REQUIRED_HANDOFF_EVIDENCE_KINDS: + values = evidence_files.get(evidence_kind) + if not isinstance(values, list) or not values: + gaps.append(handoff_gap(str(manifest_path), f"manifest missing evidence_files.{evidence_kind}")) + continue + for value in values: + evidence_file = resolve_evidence_file(manifest_path, str(value)) + if not evidence_file.exists(): + gaps.append(handoff_gap(str(manifest_path), f"manifest evidence_files.{evidence_kind} entry missing on disk: {value}")) + proof["valid"] = not gaps + return proof, gaps + + def infer_persona(run_dir: Path) -> str: name = run_dir.name for persona in ("newbie", "veteran", "adversarial", "narrative", "optimizer"): @@ -127,6 +333,7 @@ def main() -> int: ap.add_argument("--behavioral-path", default="", help="path to the behavioral evidence source") ap.add_argument("--ui-audit-log", default="", help="path to the UI audit evidence source") ap.add_argument("--palette-source", default="", help="path or label for the palette-live evidence source") + ap.add_argument("--handoff-json", default="", help="Mac app handoff gate JSON proving built-app smoke/play evidence") ap.add_argument("--build-sha", dest="build_sha", default="") ap.add_argument("--out", default="qa/RRI.json") ap.add_argument("--scorecard-row", action="store_true") @@ -134,21 +341,23 @@ def main() -> int: run_dirs = [Path(p) for p in split_csv(args.runs)] expected_personas = split_csv(args.expected_personas) + handoff_proof, handoff_evidence_gaps = validate_handoff_json(args.handoff_json, args.build_sha) persona_scores = [] harness_failures = [] completed_personas: list[str] = [] for idx, rd in enumerate(run_dirs): expected_for_run = expected_personas[idx] if idx < len(expected_personas) else "" rj = read_json(rd / "run.json") - sc = read_json(rd / "score.json") - if not sc: + sc, score_error = read_json_with_error(rd / "score.json") + if score_error: part_a_obj = rj.get("part_a") or {} part_b_obj = rj.get("part_b") or {} part_b = part_b_obj.get("persona_loop") harness_failures.append({ "run": rd.name, "persona": expected_for_run or infer_persona(rd), - "missing": "score.json", + "missing": "score.json" if score_error == "missing" else "score.json invalid", + "detail": score_error, "part_a": part_a_obj.get("result") or "n/a", "part_b": part_b or "n/a", "part_a_failure_bucket": part_a_obj.get("failure_bucket") or "", @@ -157,6 +366,24 @@ def main() -> int: "part_b_failure_detail": part_b_obj.get("failure_detail") or "", }) continue + schema_errors = score_schema_errors(sc) + if schema_errors: + part_a_obj = rj.get("part_a") or {} + part_b_obj = rj.get("part_b") or {} + harness_failures.append({ + "run": rd.name, + "persona": expected_for_run or sc.get("persona") or infer_persona(rd), + "missing": "score.json required fields", + "detail": "; ".join(schema_errors), + "required_fields": REQUIRED_SCORE_FIELDS, + "part_a": part_a_obj.get("result") or "n/a", + "part_b": part_b_obj.get("persona_loop") or "n/a", + "part_a_failure_bucket": part_a_obj.get("failure_bucket") or "", + "part_a_failure_detail": part_a_obj.get("failure_detail") or "", + "part_b_failure_bucket": part_b_obj.get("failure_bucket") or "", + "part_b_failure_detail": part_b_obj.get("failure_detail") or "", + }) + continue rate, ok, total, image_source = image_render_rate(rd, sc) persona = sc.get("persona") or expected_for_run or infer_persona(rd) if persona: @@ -194,6 +421,8 @@ def main() -> int: avg_sat = sum(sats) / len(sats) if sats else 0.0 any_gave_up = any(p["gave_up"] for p in persona_scores) any_completed = any(p["completed_intro_flow"] for p in persona_scores) + score_pass_failed_personas = [str(p["persona"]) for p in persona_scores if not p.get("part_b_score_pass")] + score_pass_complete = bool(persona_scores) and not score_pass_failed_personas total_critical = sum(p["critical"] for p in persona_scores) total_console_errors = sum(p["console_errors"] for p in persona_scores) # weighted image rate across personas that recorded image traffic. A release @@ -209,19 +438,35 @@ def main() -> int: story_overall = float(story.get("overall", 0) or 0) mech_overall = float(mech.get("overall", 0) or 0) - # native gate: read part_a from any run.json present + # native gate: read part_a from persona run.json, or accept an explicit + # Mac-built app handoff proof when persona artifacts come from a VM/backend + # sweep and therefore cannot prove dist/WorldOS.app directly. native = "" native_detail = "" + native_source = "" + part_a_failures = [] for rd in run_dirs: rj = read_json(rd / "run.json") part_a_obj = rj.get("part_a") or {} pa = part_a_obj.get("result") if pa: - native = pa bucket = part_a_obj.get("failure_bucket") or "" detail = part_a_obj.get("failure_detail") or "" - native_detail = f" failure_bucket={bucket} failure_detail={detail}".strip() if bucket or detail else "" - break + if pa == "PASS" and not native: + native = pa + native_source = f"{rd}/run.json" + native_detail = f"part_a={pa}" + elif pa != "PASS": + part_a_failures.append({ + "run": rd.name, + "part_a": pa, + "failure_bucket": bucket, + "failure_detail": detail, + }) + if not native and handoff_proof.get("valid"): + native = "PASS" + native_source = args.handoff_json + native_detail = f"handoff_json={args.handoff_json} gates={','.join(REQUIRED_HANDOFF_GATES)}" evidence_gaps = [] build_shas = sorted({str(p["run_build_sha"]) for p in persona_scores if p.get("run_build_sha")}) @@ -239,7 +484,7 @@ def main() -> int: "detail": "missing run build_sha for: " + ", ".join(str(p.get("persona") or p.get("run")) for p in missing_build_sha), }) if args.build_sha: - mismatched = [p for p in persona_scores if p.get("run_build_sha") and p.get("run_build_sha") != args.build_sha] + mismatched = [p for p in persona_scores if p.get("run_build_sha") and not build_sha_matches(str(p.get("run_build_sha")), args.build_sha)] if mismatched: evidence_gaps.append({ "gate": "native_gate", @@ -271,8 +516,8 @@ def main() -> int: ) evidence_gaps.append({ "gate": "cross_persona_sat", - "missing": f"{h['run']}/score.json", - "detail": f"persona={h.get('persona') or 'unknown'} part_a={h.get('part_a') or 'n/a'} part_b={h.get('part_b') or 'n/a'} {buckets}".strip(), + "missing": f"{h['run']}/{h.get('missing') or 'score.json'}", + "detail": f"persona={h.get('persona') or 'unknown'} detail={h.get('detail') or ''} part_a={h.get('part_a') or 'n/a'} part_b={h.get('part_b') or 'n/a'} {buckets}".strip(), }) failed_part_b = [p for p in persona_scores if p.get("part_b_result") != "PASS"] for p in failed_part_b: @@ -283,11 +528,18 @@ def main() -> int: "missing": f"{p['run']}/run.json part_b PASS", "detail": f"persona={p.get('persona') or 'unknown'} part_b={p.get('part_b_result')} score_pass={p.get('part_b_score_pass')} failure_bucket={bucket} failure_detail={detail}".strip(), }) + evidence_gaps.extend(handoff_evidence_gaps) + for failure in part_a_failures: + evidence_gaps.append({ + "gate": "native_gate", + "missing": f"{failure['run']}/run.json part_a PASS", + "detail": f"part_a={failure.get('part_a')} failure_bucket={failure.get('failure_bucket') or ''} failure_detail={failure.get('failure_detail') or ''}".strip(), + }) if not native: evidence_gaps.append({ "gate": "native_gate", - "missing": "run.json part_a.result", - "detail": "no persona run recorded native built-app transition evidence", + "missing": "run.json part_a.result or --handoff-json", + "detail": "no persona run or Mac handoff bundle recorded native built-app transition evidence", }) elif native != "PASS": evidence_gaps.append({ @@ -328,15 +580,16 @@ def main() -> int: elif looks_like_path(args.palette_source) and not Path(args.palette_source).exists(): evidence_gaps.append({"gate": "palette_live", "missing": args.palette_source, "detail": "palette-live evidence source missing"}) evidence_gap_gates = {gap["gate"] for gap in evidence_gaps} + native_gate_detail = f"source={native_source or 'n/a'} {native_detail or 'part_a=' + (native or 'n/a')}".strip() # ---- the 11 gates (each contributes to RRI; all must hold for 10/10) ---- gates = { "native_gate": (native == "PASS" and "native_gate" not in evidence_gap_gates, - f"part_a={native or 'n/a'} {native_detail}".strip()), + native_gate_detail), "arc_completed": (any_completed and "arc_completed" not in evidence_gap_gates, f"completed_intro_flow on >=1 persona"), - "cross_persona_sat": (not missing_release_personas and expected_complete and avg_sat >= 7.0, - f"avg={avg_sat:.1f}/10 over {len(sats)}; missing={missing_personas or 'none'}; release_missing={missing_release_personas or 'none'}"), + "cross_persona_sat": (not missing_release_personas and expected_complete and avg_sat >= 7.0 and score_pass_complete, + f"avg={avg_sat:.1f}/10 over {len(sats)}; score_pass_failed={score_pass_failed_personas or 'none'}; missing={missing_personas or 'none'}; release_missing={missing_release_personas or 'none'}"), "no_give_up": (not any_gave_up and "no_give_up" not in evidence_gap_gates, f"any_gave_up={any_gave_up}"), "zero_critical": (total_critical == 0 and total_console_errors == 0 and "zero_critical" not in evidence_gap_gates, @@ -391,13 +644,17 @@ def main() -> int: "palette_live": args.palette_source or "argument", "story": args.story or "", "mechanical": args.mech or "", + "handoff_json": args.handoff_json or "", "runs": [str(p) for p in run_dirs], "images": sorted({p["image_source"] for p in persona_scores if p.get("image_source")}), }, "signals": { "native_gate": native, + "native_gate_source": native_source, + "handoff_proof": handoff_proof, "arc_completed": any_completed, "cross_persona_satisfaction": round(avg_sat, 1), + "score_pass_failed_personas": score_pass_failed_personas, "any_gave_up": any_gave_up, "total_critical_bugs": total_critical, "total_console_errors": total_console_errors, diff --git a/qa/test_release_readiness.py b/qa/test_release_readiness.py index 5ef94702..914e27f1 100644 --- a/qa/test_release_readiness.py +++ b/qa/test_release_readiness.py @@ -18,6 +18,143 @@ def run_rri(self, tmp: Path, *args: str) -> tuple[int, str, dict]: payload = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {} return proc.returncode, proc.stdout + proc.stderr, payload + def write_handoff_bundle( + self, + tmp: Path, + *, + sha: str = "deadbee", + manifest_sha: str | None = None, + reuse_manifest: bool = False, + ) -> Path: + handoff_root = tmp / "handoff" + handoff_root.mkdir() + gates = [] + first_manifest_path: Path | None = None + for gate_name in ("web_scripted_smoke", "built_app_scripted_smoke", "built_app_codex_playtest"): + evidence_dir = handoff_root / gate_name + manifest_path = first_manifest_path if reuse_manifest and first_manifest_path else evidence_dir / "app-evidence" / "manifest.json" + first_manifest_path = first_manifest_path or manifest_path + manifest_path.parent.mkdir(parents=True, exist_ok=True) + if not manifest_path.exists(): + manifest_path.write_text( + json.dumps( + { + "schema": "worldos.app-evidence.v1", + "verdict": "passed", + "dirty": False, + "app_build_sha": manifest_sha or sha, + "evidence_gaps": [], + "failure_bucket": "", + "failure_detail": "", + "failure": {"failure_bucket": "", "failure_detail": ""}, + "gate_kind": gate_name, + "art": {"private_root_present": True}, + "live": {"can_act": True, "enabled_action_count": 5}, + "handoff_gate": { + "ok": True, + "app_status_ok": True, + "session_surface_ok": True, + "move_sink_present": True, + "private_art_present": True, + "can_act": True, + "enabled_action_count": 5, + "evidence_gap_count": 0, + }, + "evidence_files": { + "screenshots": ["screenshots/final.png"], + "app_status_snapshots": ["app-status.final.json"], + "session_surface_snapshots": ["session-surface.final.json"], + "moves": ["moves.ndjson"], + "provider_trace": ["provider-trace-summary.json"], + "console_logs": ["console.ndjson"], + "network_logs": ["network.ndjson"], + "action_logs": ["actions.ndjson"], + }, + } + ), + encoding="utf-8", + ) + for evidence_file in ( + "screenshots/final.png", + "app-status.final.json", + "session-surface.final.json", + "moves.ndjson", + "provider-trace-summary.json", + "console.ndjson", + "network.ndjson", + "actions.ndjson", + ): + path = manifest_path.parent / evidence_file + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("ok\n", encoding="utf-8") + gates.append( + { + "name": gate_name, + "status": "passed", + "build_sha": sha, + "evidence_gaps": [], + "evidence_manifest": str(manifest_path), + } + ) + handoff = handoff_root / "handoff.json" + handoff.write_text( + json.dumps( + { + "schema": "worldos.app-handoff.v1", + "status": "passed", + "handoff_score": 100, + "dirty": False, + "commit_sha": sha, + "release_verdict": False, + "gates": gates, + } + ), + encoding="utf-8", + ) + return handoff + + def write_release_inputs(self, tmp: Path) -> tuple[Path, Path, Path, Path, Path]: + story = tmp / "story.json" + mech = tmp / "mech.json" + behavioral = tmp / "behavioral.txt" + audit = tmp / "audit.log" + palette = tmp / "session_surface.final.json" + story.write_text(json.dumps({"overall": 5}), encoding="utf-8") + mech.write_text(json.dumps({"overall": 5}), encoding="utf-8") + behavioral.write_text("GREEN\n", encoding="utf-8") + audit.write_text("PASS\n", encoding="utf-8") + palette.write_text(json.dumps({"can_act": True}), encoding="utf-8") + return story, mech, behavioral, audit, palette + + def write_persona_run(self, tmp: Path, persona: str, *, sha: str = "deadbee", include_part_a: bool = True) -> Path: + run = tmp / f"gate-{persona}" + player = run / "player" + player.mkdir(parents=True) + (run / "score.json").write_text( + json.dumps( + { + "run": f"gate-{persona}", + "persona": persona, + "completed_intro_flow": True, + "persona_satisfaction": 9, + "gave_up": False, + "bug_reports_critical": 0, + "console_errors": 0, + "image_404s": 0, + } + ), + encoding="utf-8", + ) + payload = {"build_sha": sha, "part_b": {"persona_loop": "PASS", "score_pass": True}} + if include_part_a: + payload["part_a"] = {"result": "PASS"} + (run / "run.json").write_text(json.dumps(payload), encoding="utf-8") + (player / "network.ndjson").write_text( + json.dumps({"url": f"http://127.0.0.1/image?scope={persona}", "status": 200}), + encoding="utf-8", + ) + return run + def test_missing_expected_persona_score_marks_partial_and_fails_release(self): with tempfile.TemporaryDirectory() as td: tmp = Path(td) @@ -34,6 +171,7 @@ def test_missing_expected_persona_score_marks_partial_and_fails_release(self): "persona_satisfaction": 9, "gave_up": False, "bug_reports_critical": 0, + "console_errors": 0, "image_404s": 0, } ), @@ -102,6 +240,7 @@ def test_image_rate_reads_player_network_ndjson(self): "persona_satisfaction": 9, "gave_up": False, "bug_reports_critical": 0, + "console_errors": 0, "image_404s": 0, } ), @@ -961,6 +1100,338 @@ def test_palette_source_label_does_not_require_filesystem_path(self): self.assertNotIn("palette_live", {gap["gate"] for gap in payload["evidence_gaps"]}) self.assertEqual(payload["artifact_sources"]["palette_live"], "session-surface-final") + def test_split_vm_persona_evidence_uses_handoff_json_for_native_gate(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona, include_part_a=False) for persona in personas] + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + handoff = self.write_handoff_bundle(tmp) + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--handoff-json", + str(handoff), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 0) + self.assertTrue(payload["release_ready"]) + self.assertEqual(payload["signals"]["native_gate"], "PASS") + self.assertEqual(payload["signals"]["native_gate_source"], str(handoff)) + self.assertEqual(payload["artifact_sources"]["handoff_json"], str(handoff)) + self.assertTrue(payload["signals"]["handoff_proof"]["valid"]) + self.assertIn("handoff_json=", payload["gate_detail"]["native_gate"]) + + def test_missing_handoff_json_blocks_native_gate_when_persona_runs_have_no_part_a(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona, include_part_a=False) for persona in personas] + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 1) + self.assertFalse(payload["release_ready"]) + self.assertIn("native_gate", payload["failed_gates"]) + self.assertIn("run.json part_a.result or --handoff-json", {gap["missing"] for gap in payload["evidence_gaps"]}) + + def test_handoff_sha_mismatch_blocks_native_gate(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona, include_part_a=False) for persona in personas] + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + handoff = self.write_handoff_bundle(tmp, sha="badcafe") + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--handoff-json", + str(handoff), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 1) + self.assertFalse(payload["release_ready"]) + self.assertIn("native_gate", payload["failed_gates"]) + self.assertIn("handoff commit_sha badcafe does not match --build-sha deadbee", " ".join(gap["detail"] for gap in payload["evidence_gaps"])) + self.assertFalse(payload["signals"]["handoff_proof"]["valid"]) + + def test_short_build_sha_prefix_cannot_mix_stale_vm_and_mac_handoff_evidence(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona, sha="4524b3e", include_part_a=False) for persona in personas] + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + handoff = self.write_handoff_bundle(tmp, sha="4a0efe1") + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--handoff-json", + str(handoff), + "--build-sha", + "4", + ) + + self.assertEqual(rc, 1) + self.assertFalse(payload["release_ready"]) + self.assertIn("native_gate", payload["failed_gates"]) + self.assertTrue(payload["evidence_gaps"]) + + def test_handoff_manifest_must_match_gate_and_not_be_reused(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona, include_part_a=False) for persona in personas] + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + handoff = self.write_handoff_bundle(tmp, reuse_manifest=True) + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--handoff-json", + str(handoff), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 1) + self.assertFalse(payload["release_ready"]) + details = " ".join(gap["detail"] for gap in payload["evidence_gaps"]) + self.assertIn("reuses another gate's manifest", details) + self.assertIn("manifest gate_kind web_scripted_smoke does not match built_app_scripted_smoke", details) + + def test_part_b_score_pass_false_blocks_release_even_with_high_scores(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona) for persona in personas] + run_json = json.loads((runs[2] / "run.json").read_text(encoding="utf-8")) + run_json["part_b"]["score_pass"] = False + (runs[2] / "run.json").write_text(json.dumps(run_json), encoding="utf-8") + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 1) + self.assertFalse(payload["release_ready"]) + self.assertFalse(payload["partial"]) + self.assertFalse(payload["harness_contaminated"]) + self.assertEqual(payload["failed_gates"], ["cross_persona_sat"]) + self.assertEqual(payload["signals"]["score_pass_failed_personas"], ["adversarial"]) + + def test_malformed_score_json_is_harness_contaminated_and_incomplete(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + personas = ("newbie", "veteran", "adversarial", "narrative", "optimizer") + runs = [self.write_persona_run(tmp, persona) for persona in personas] + (runs[1] / "score.json").write_text("{not json", encoding="utf-8") + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + ",".join(str(r) for r in runs), + "--expected-personas", + ",".join(personas), + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 1) + self.assertTrue(payload["partial"]) + self.assertTrue(payload["harness_contaminated"]) + self.assertIn("veteran", payload["missing_personas"]) + self.assertEqual(payload["harness_failures"][0]["missing"], "score.json invalid") + self.assertIn("invalid JSON", payload["harness_failures"][0]["detail"]) + + def test_score_missing_console_errors_is_harness_contaminated(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + run = self.write_persona_run(tmp, "newbie") + score = json.loads((run / "score.json").read_text(encoding="utf-8")) + score.pop("console_errors") + (run / "score.json").write_text(json.dumps(score), encoding="utf-8") + story, mech, behavioral, audit, palette = self.write_release_inputs(tmp) + + rc, _text, payload = self.run_rri( + tmp, + "--runs", + str(run), + "--expected-personas", + "newbie", + "--story", + str(story), + "--mech", + str(mech), + "--behavioral", + "GREEN", + "--behavioral-path", + str(behavioral), + "--ui-audit", + "PASS", + "--ui-audit-log", + str(audit), + "--palette-live", + "true", + "--palette-source", + str(palette), + "--build-sha", + "deadbee", + ) + + self.assertEqual(rc, 1) + self.assertTrue(payload["harness_contaminated"]) + self.assertEqual(payload["completed_personas"], []) + self.assertEqual(payload["harness_failures"][0]["missing"], "score.json required fields") + self.assertIn("console_errors must be integer", payload["harness_failures"][0]["detail"]) + if __name__ == "__main__": unittest.main()