qa(rri): detect 429 quota-aborts, run VM personas sequentially, never roll up a junk RRI (rc3 lesson)

qa/release_readiness.py

@@ -56,6 +56,7 @@
 
 import argparse
 import json
+import re
 import sys
 from pathlib import Path
 
@@ -559,6 +560,26 @@ def infer_persona(run_dir: Path) -> str:
     return name
 
 
+# A persona whose DM beats 429'd on the account session limit is INFRA-aborted, not a
+# product failure (the rc3 lesson — a 429-storm rolled up a misleading 1.8). Detect it so
+# the rollup attributes "quota" vs "broken build" correctly and never reads as a clean score.
+_QUOTA_RE = re.compile(r"session limit|HTTP 429|hit your (?:session|usage) limit", re.I)
+_RESET_RE = re.compile(r"resets [0-9: ]*[ap]m \(?(?:UTC|[A-Za-z/_]+)\)?", re.I)
+
+
+def infra_abort_hint(run_dir: Path) -> str:
+    """Return a non-empty reset hint (or the literal '429') if this run's backend 429'd, else ''."""
+    bl = run_dir / "backend.log"
+    try:
+        text = bl.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return ""
+    if not _QUOTA_RE.search(text):
+        return ""
+    m = _RESET_RE.search(text)
+    return m.group(0) if m else "429"
+
+
 def build_sha_evidence_gaps(persona_scores: list[dict], build_sha: str,
                             release_personas: list[str]) -> list[dict]:
     """native_gate's build-SHA contract, SCOPED to the canonical release personas.
@@ -609,6 +630,8 @@ def main() -> int:
     ap.add_argument("--handoff-json", default="", help="Mac app handoff gate JSON proving built-app smoke/play evidence")
     ap.add_argument("--support-preflight-json", default="", help="Support VM preflight JSON proving same-SHA heavy-lane readiness")
     ap.add_argument("--build-sha", dest="build_sha", default="")
+    ap.add_argument("--abort-marker", default="", help="path to a sweep QUOTA_ABORT marker; if it "
+                    "exists the rollup is forced to an ABORTED status (infra abort, not a product RRI)")
     ap.add_argument("--out", default="qa/RRI.json")
     ap.add_argument("--scorecard-row", action="store_true")
     args = ap.parse_args()
@@ -969,8 +992,37 @@ def main() -> int:
     # the gate logic into the helper and left this output reference dangling -> NameError at rollup time.
     build_shas = sorted({str(p["run_build_sha"]) for p in persona_scores if p.get("run_build_sha")})
 
+    # Infra-abort attribution (the rc3 lesson): a harness-failed persona whose DM beats 429'd on
+    # the account session limit is a QUOTA abort, not a broken build. An explicit --abort-marker
+    # (the sweep's QUOTA_ABORT file) also forces ABORTED. When infra-aborted, this rollup is NOT a
+    # product RRI — the status/verdict say so loudly so it can never be recorded as a clean score.
+    infra_aborted_personas = []
+    for rd in run_dirs:
+        hint = infra_abort_hint(rd)
+        if hint:
+            infra_aborted_personas.append(
+                {"persona": infer_persona(rd), "run": rd.name, "reset_hint": hint})
+    abort_marker_present = bool(args.abort_marker and Path(args.abort_marker).is_file())
+    abort_detail = ""
+    if abort_marker_present:
+        try:
+            abort_detail = Path(args.abort_marker).read_text(encoding="utf-8").strip()
+        except OSError:
+            abort_detail = "abort marker present"
+    elif infra_aborted_personas:
+        abort_detail = "; ".join(
+            f"{p['persona']}: {p['reset_hint']}" for p in infra_aborted_personas)
+    aborted = bool(infra_aborted_personas or abort_marker_present)
+    if aborted:
+        release_ready = False  # a quota-aborted sweep is never release-ready, regardless of gates
+
     result = {
         "rri": rri,
+        "status": "ABORTED" if aborted else ("READY" if release_ready else "NOT_READY"),
+        "aborted": aborted,
+        "abort_reason": "quota_session_limit" if aborted else "",
+        "abort_detail": abort_detail,
+        "infra_aborted_personas": infra_aborted_personas,
         "release_ready": release_ready,
         "release_verdict_gate": RELEASE_VERDICT_GATE,
         "gate_split_contract": GATE_SPLIT_CONTRACT,
@@ -1041,6 +1093,10 @@ def main() -> int:
     out.write_text(json.dumps(result, indent=2), encoding="utf-8")
 
     # human line
+    if aborted:
+        print(f"QUOTA-ABORTED — claude account session limit (HTTP 429): {abort_detail}")
+        print(f"  This is an INFRA abort, NOT a product RRI. The {rri}/10 below is NOT a measurement; "
+              f"re-run after the quota resets.")
     print(f"RRI {rri}/10  ({passed}/{total_gates} gates)  release_ready={release_ready}")
     if failed:
         details = []
@@ -1057,7 +1113,12 @@ def main() -> int:
 
     if args.scorecard_row:
         sha = (args.build_sha or "?")[:7]
-        verdict = "PARTIAL/HARNESS" if (missing_personas or evidence_gaps or harness_failures) else ("**GREEN**" if release_ready else "RED")
+        if aborted:
+            verdict = "QUOTA-ABORTED"
+        elif missing_personas or evidence_gaps or harness_failures:
+            verdict = "PARTIAL/HARNESS"
+        else:
+            verdict = "**GREEN**" if release_ready else "RED"
         row = (f"| RRI-{sha} | (date) | baldurs-gate | {len(expected_personas) or len(persona_scores)}-persona | sonnet | gate | "
                f"{verdict} | {story_overall or '—'} | "
                f"{mech_overall or '—'} | — | **{rri}** | "

qa/test_release_readiness.py

@@ -2437,6 +2437,111 @@ def test_score_missing_console_errors_is_harness_contaminated(self):
             self.assertEqual(payload["harness_failures"][0]["missing"], "score.json required fields")
             self.assertIn("console_errors must be integer", payload["harness_failures"][0]["detail"])
 
+    # --- Quota-abort attribution (the rc3 lesson): a 429-storm is an INFRA abort, not a
+    # product RRI. The rollup must say ABORTED/QUOTA-ABORTED, not roll up a misleading number. ---
+
+    def _seat_clean_canary(self, tmp: Path) -> Path:
+        newbie = tmp / "vm2-newbie"
+        newbie.mkdir()
+        (newbie / "score.json").write_text(json.dumps({
+            "run": "vm2-newbie", "persona": "newbie", "completed_intro_flow": True,
+            "persona_satisfaction": 7, "gave_up": False, "bug_reports_critical": 0,
+            "console_errors": 0, "image_404s": 0,
+        }), encoding="utf-8")
+        (newbie / "run.json").write_text(json.dumps(
+            {"build_sha": "deadbee", "part_a": {"result": "PASS"},
+             "part_b": {"persona_loop": "PASS", "score_pass": True}}), encoding="utf-8")
+        return newbie
+
+    def test_429_in_backend_log_marks_quota_aborted_not_a_product_rri(self):
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            self._seat_clean_canary(tmp)
+            opt = tmp / "vm2-optimizer"
+            opt.mkdir()
+            # No score.json (the backend never seated) + a 429 in backend.log = a QUOTA abort.
+            (opt / "run.json").write_text(json.dumps(
+                {"part_b": {"persona_loop": "backend_not_ready",
+                            "failure_bucket": "no_provider"}}), encoding="utf-8")
+            (opt / "backend.log").write_text(
+                "[dm-attempt] DM turn failed (rc=1): HTTP 429 is_error=true result=You've hit "
+                "your session limit · resets 3:50pm (UTC)\n", encoding="utf-8")
+            story = tmp / "story.json"; mech = tmp / "mech.json"
+            story.write_text(json.dumps({"overall": 5}), encoding="utf-8")
+            mech.write_text(json.dumps({"overall": 5}), encoding="utf-8")
+
+            rc, text, payload = self.run_rri(
+                tmp, "--runs", f"{tmp/'vm2-newbie'},{opt}",
+                "--expected-personas", "newbie,optimizer",
+                "--story", str(story), "--mech", str(mech),
+                "--behavioral", "GREEN", "--ui-audit", "PASS", "--palette-live", "true",
+                "--build-sha", "deadbee", "--scorecard-row",
+            )
+            self.assertEqual(rc, 1)
+            self.assertTrue(payload["aborted"])
+            self.assertEqual(payload["status"], "ABORTED")
+            self.assertEqual(payload["abort_reason"], "quota_session_limit")
+            self.assertFalse(payload["release_ready"])
+            personas = [p["persona"] for p in payload["infra_aborted_personas"]]
+            self.assertIn("optimizer", personas)
+            self.assertIn("resets 3:50pm (UTC)", payload["abort_detail"])
+            # The scorecard row + human line must NOT present this as a clean measurement.
+            self.assertIn("QUOTA-ABORTED", text)
+            self.assertIn("NOT a product RRI", text)
+
+    def test_backend_not_ready_without_429_stays_product_failure_not_quota_abort(self):
+        # The discriminator: a genuinely-broken seating (no 429 anywhere) must remain a
+        # product/harness failure, NOT get excused as an infra/quota abort.
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            self._seat_clean_canary(tmp)
+            opt = tmp / "vm2-optimizer"
+            opt.mkdir()
+            (opt / "run.json").write_text(json.dumps(
+                {"part_b": {"persona_loop": "backend_not_ready"}}), encoding="utf-8")
+            (opt / "backend.log").write_text(
+                "[B] backend never became player-ready (can_act=0 seatedPC=0)\n", encoding="utf-8")
+            story = tmp / "story.json"; mech = tmp / "mech.json"
+            story.write_text(json.dumps({"overall": 5}), encoding="utf-8")
+            mech.write_text(json.dumps({"overall": 5}), encoding="utf-8")
+
+            rc, text, payload = self.run_rri(
+                tmp, "--runs", f"{tmp/'vm2-newbie'},{opt}",
+                "--expected-personas", "newbie,optimizer",
+                "--story", str(story), "--mech", str(mech),
+                "--behavioral", "GREEN", "--ui-audit", "PASS", "--palette-live", "true",
+                "--build-sha", "deadbee", "--scorecard-row",
+            )
+            self.assertEqual(rc, 1)
+            self.assertFalse(payload["aborted"])
+            self.assertEqual(payload["status"], "NOT_READY")
+            self.assertEqual(payload["infra_aborted_personas"], [])
+            self.assertTrue(payload["harness_contaminated"])  # still a real product/harness failure
+            self.assertIn("PARTIAL/HARNESS", text)
+            self.assertNotIn("QUOTA-ABORTED", text)
+
+    def test_abort_marker_forces_aborted(self):
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            self._seat_clean_canary(tmp)
+            marker = tmp / "QUOTA_ABORT"
+            marker.write_text("optimizer resets 3:50pm (UTC)\n", encoding="utf-8")
+            story = tmp / "story.json"; mech = tmp / "mech.json"
+            story.write_text(json.dumps({"overall": 5}), encoding="utf-8")
+            mech.write_text(json.dumps({"overall": 5}), encoding="utf-8")
+            rc, _text, payload = self.run_rri(
+                tmp, "--runs", f"{tmp/'vm2-newbie'}",
+                "--expected-personas", "newbie",
+                "--story", str(story), "--mech", str(mech),
+                "--behavioral", "GREEN", "--ui-audit", "PASS", "--palette-live", "true",
+                "--build-sha", "deadbee", "--abort-marker", str(marker),
+            )
+            self.assertEqual(rc, 1)
+            self.assertTrue(payload["aborted"])
+            self.assertEqual(payload["status"], "ABORTED")
+            self.assertFalse(payload["release_ready"])
+            self.assertIn("resets 3:50pm (UTC)", payload["abort_detail"])
+
 
 if __name__ == "__main__":
     unittest.main()

qa/vm/sweep_v2.sh

@@ -39,7 +39,19 @@ cd /root/worldos-qa/WorldOS || { echo "NO REPO"; exit 1; }
 RES=/root/worldos-qa/results; mkdir -p "$RES"
 SHA="$(git rev-parse --short HEAD)"; LOG="$RES/sweep2.log"; : > "$LOG"
 note(){ echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; }
-rm -f "$RES/DONE" "$RES/CANARY_FAIL" 2>/dev/null
+rm -f "$RES/DONE" "$RES/CANARY_FAIL" "$RES/QUOTA_ABORT" 2>/dev/null
+
+# QUOTA-ABORT detection (the rc3 lesson). A `claude -p` DM beat that 429s on the account
+# session limit writes "session limit" / "HTTP 429" into the persona backend.log. A sweep
+# that 429s is INFRA-aborted, NOT a product measurement — detect it so we abort honestly
+# instead of rolling up quota-corpses into a misleading RRI (rc3 rolled a fake 1.8 from a
+# 429-storm). Sequential personas (below) mean the FIRST 429 aborts before the rest spend.
+quota_tripped(){  # $1 = a run dir or a log path; rc 0 if a session-limit/429 is present
+  grep -qriE "session limit|HTTP 429|hit your (session|usage) limit" "$1" 2>/dev/null
+}
+quota_reset_hint(){  # echoes e.g. "resets 3:50pm UTC" from the log(s), if present
+  grep -hroiE "resets [0-9: ]*[ap]m \(?(UTC|[A-Za-z/_]+)\)?" "$@" 2>/dev/null | head -1
+}
 
 # 0) kill the stuck v1 orchestrator + any stray vm- play procs; free ports
 note "killing v1 orchestrator + stray procs..."
@@ -112,22 +124,62 @@ run_persona(){  # $1=persona $2=port  -> writes results/score-$1.json
 # 1) CANARY: newbie alone. Verify scoring works before spending on the batch.
 note "CANARY: newbie (verifying part-B produces a score on the VM)..."
 run_persona newbie 8810
+CANARY_BL="qa/ui_playtest_runs/vm2-newbie/backend.log"
 if [ ! -f "$RES/score-newbie.json" ]; then
+  # Distinguish a QUOTA abort (account 429) from a genuine product/harness failure: a
+  # 429-killed canary means the account is already over its session limit, so the whole
+  # batch would 429 too — abort honestly with the reset hint rather than burning it.
+  if quota_tripped "$CANARY_BL"; then
+    note "QUOTA ABORT at the canary — claude account session limit ($(quota_reset_hint "$CANARY_BL")). The batch would 429 too; not spending it. INFRA abort, NOT a product measurement."
+    echo "newbie $(quota_reset_hint "$CANARY_BL")" > "$RES/QUOTA_ABORT"
+    touch "$RES/DONE"; exit 0
+  fi
   note "CANARY FAILED - no score-newbie.json. Aborting batch; see vm2-newbie.log for the cause."
   note "  --- vm2-newbie.log tail ---"; tail -25 "$RES/vm2-newbie.log" >> "$LOG" 2>/dev/null
   touch "$RES/CANARY_FAIL"; touch "$RES/DONE"; exit 0
 fi
-note "CANARY OK - scoring works. Launching the other 4 personas IN PARALLEL (staggered 30s)..."
+# Even a SCORED canary can be followed by a quota trip on its own retries; check before the batch.
+if quota_tripped "$CANARY_BL"; then
+  note "QUOTA ABORT — the canary scored but its backend 429'd ($(quota_reset_hint "$CANARY_BL")); the account is at its session limit. Not spending the batch."
+  echo "newbie $(quota_reset_hint "$CANARY_BL")" > "$RES/QUOTA_ABORT"
+  touch "$RES/DONE"; exit 0
+fi
+note "CANARY OK - scoring works. Running the other 4 personas SEQUENTIALLY (quota-safe)."
 
-# 2) Parallel batch: veteran/adversarial/narrative/optimizer, staggered starts
-i=0
+# 2) Sequential batch: veteran/adversarial/narrative/optimizer. WHY sequential, not parallel
+# (the rc3 lesson): cold-open is API-generation-bound, NOT VM-CPU-bound (worldos-latency-
+# forensics), so the old "parallel = use the 16 vCPUs" premise does not speed up generation —
+# it just bursts the account session-quota 4x and, when the limit trips mid-batch, wastes 3-4
+# cold-opens on 429 corpses AND rolls up a junk RRI. Sequential spends one cold-open at a time
+# and the FIRST 429 aborts the remainder. (Set SWEEP_PERSONA_CONCURRENCY>1 for a quota-rich window.)
 for pp in "veteran 8812" "adversarial 8814" "narrative 8816" "optimizer 8818"; do
   set -- $pp
-  run_persona "$1" "$2" &
-  i=$((i+1)); sleep 30
+  run_persona "$1" "$2"
+  bl="qa/ui_playtest_runs/vm2-$1/backend.log"
+  if quota_tripped "$bl"; then
+    note "QUOTA ABORT after '$1' — claude account session limit ($(quota_reset_hint "$bl")). Stopping; remaining personas NOT spent. INFRA abort, NOT a product result."
+    echo "$1 $(quota_reset_hint "$bl")" > "$RES/QUOTA_ABORT"
+    break
+  fi
 done
-wait
-note "all 5 personas done."
+note "persona batch done."
+
+# QUOTA-ABORT short-circuit: if the persona batch 429'd, do NOT spend the duo on a dead
+# account, and do NOT roll up an RRI from quota-corpses (rc3 emitted a misleading 1.8 this
+# way). Emit an explicit ABORTED status the ledger/scorecard can never read as a product score.
+if [ -f "$RES/QUOTA_ABORT" ]; then
+  note "=== RRI SKIPPED — QUOTA_ABORT ($(cat "$RES/QUOTA_ABORT")) — not a product measurement ==="
+  python3 - "$RES/RRI.json" "$SHA" "$(cat "$RES/QUOTA_ABORT")" <<'PY' 2>/dev/null
+import json, sys
+out, sha, detail = sys.argv[1], sys.argv[2], sys.argv[3]
+json.dump({"status": "ABORTED", "abort_reason": "quota_session_limit",
+           "detail": detail, "build_sha": sha, "release_ready": False,
+           "note": "claude account session limit (HTTP 429) tripped mid-sweep; "
+                   "this is an INFRA abort, NOT a product RRI. Re-run after the quota resets."},
+          open(out, "w"), indent=2)
+PY
+  note "=== SWEEP COMPLETE (QUOTA-ABORTED) -> $RES ==="; touch "$RES/DONE"; exit 0
+fi
 
 # 3) duo (story/mech) + behavioral + audit - run after personas (sequential, cheap-ish)
 note "3-lens duo..."