fix(qa): #842 quota circuit-breaker + stale-evidence hygiene (6 remaining gaps)

qa/run_duo.sh

@@ -372,6 +372,16 @@ $SETUP_DIRECTIVE OUTPUT DISCIPLINE — your final reply IS the opening scene: wr
 # line (empty final reply) — so a tool-final-but-narrated turn isn't mistaken for silence.
 worldos_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$WORLDOS_DM_REPLY"
 echo "[duo] DM opened: ${DMSG:0:120}…"
+# #842 Fix E (quota circuit-breaker): if the DM cold-open hit the account session limit (HTTP 429),
+# the per-attempt stream-json ($COMBINED) and/or the DM stderr ($T/$RUN.dm.err) carry the
+# "session limit" / "HTTP 429" marker. An empty/recovered reply on top of a 429 is NOT a product
+# failure — it is an INFRA abort. Detect it BEFORE scoring so we never burn the 3-lens scorer on a
+# quota corpse (or, worse, cap-RED a quota'd run as a 2.5 product score). Log a marker the VM sweep
+# greps for and exit rc=2 (distinct from the rc=1 genuine-no-opening abort below).
+if grep -qiE "session limit|HTTP 429|hit your (session|usage) limit" "$COMBINED" "$T/$RUN.dm.err" 2>/dev/null; then
+  echo "[duo] QUOTA ABORT — DM cold-open hit the account session limit (HTTP 429). Skipping scoring; this is an INFRA abort, NOT a product measurement." >&2
+  exit 2
+fi
 # SYN-01: an empty resolved reply is a FAILED beat (error-class result, recycled-only prose, or
 # nothing recovered). Record the wrapper-authored VISIBLE failure row — never the error text,
 # never a blank/hidden row — then abort loudly as before.
@@ -495,6 +505,17 @@ if [ -n "$SNAP" ]; then cp "$SNAP" "$T/$RUN.state.json"; else echo '{"warning":"
 [ -s "$PLAY" ] && "$SCORE_SCRIPT" "$PLAY" "$T/$RUN.state.json" qa/rubric_tolkien.md qa/score_schema_tolkien.json "$T/$RUN.tolkien.json" 1.50 &
 [ -f "$T/$RUN.md" ] && "$SCORE_SCRIPT" "$T/$RUN.md" "$T/$RUN.state.json" qa/rubric_angry_dm.md qa/score_schema_angry_dm.json "$T/$RUN.angrydm.json" 1.50 &
 wait
+# #842 Fix F (caller half): score.sh now FAILS FAST on a 429, writing a {"quota_exhausted":true,…}
+# sentinel into its OUT and exiting rc=2 — so the scorer can quota-trip even when the DM cold-open
+# itself didn't (e.g. the account hits the limit AFTER the play, during scoring). Any lens carrying
+# that sentinel is NOT a valid scorecard — short-circuit to the same QUOTA ABORT path Fix E uses
+# (log the marker the VM sweep greps + exit rc=2) instead of scoring/gating on a quota corpse.
+for _scf in "$T/$RUN.tolkien.json" "$T/$RUN.score.json" "$T/$RUN.angrydm.json"; do
+  if [ -f "$_scf" ] && jq -e '.quota_exhausted == true' "$_scf" >/dev/null 2>&1; then
+    echo "[duo] QUOTA ABORT — the scorer hit the account session limit (HTTP 429) on $(basename "$_scf"). Skipping the gate + scorecards; INFRA abort, NOT a product measurement." >&2
+    exit 2
+  fi
+done
 # Behavioral gate — flip RED on a structurally broken run (treat it like software).
 python3 qa/assert_behavioral.py "$COMBINED" "$T/$RUN.state.json" "$T/$RUN.chat.jsonl" "$MOVES" | tee "$T/$RUN.gate.txt"; GATE=${PIPESTATUS[0]}
 # Honest scoring: a gate-RED (non-progressing/structurally broken) run must NOT display as 4.1.

qa/score.sh

@@ -116,6 +116,18 @@ while [ "$attempt" -lt 3 ]; do
   # GUARD: distinguish a genuine API error / E2BIG / dead process from a transient blip,
   # and FAIL LOUDLY in the non-transient cases instead of silently calling it "auth/rate".
   api_err="$(jq -r 'select(.is_error == true) | .api_error_status // .subtype // "error"' "$RAW" 2>/dev/null)"
+  # #842 Fix F (quota circuit-breaker): a 429 account-session-limit must FAIL FAST — do NOT burn the
+  # 3 retries (re-hitting a quota'd account just wastes the window AND, on the LAST retry, would fall
+  # through to the generic "FAILED after N attempts" path that callers can't tell from a real product
+  # failure). Detect a 429 (api_error_status==429, or a "session limit"/"429" body) and short-circuit:
+  # write an explicit quota sentinel to $OUT (the caller checks .quota_exhausted) and exit rc=2 so a
+  # quota corpse can never be mistaken for a valid scorecard.
+  if [ "$api_err" = "429" ] || jq -e 'select(.is_error == true) | (.result // "") | test("session limit|HTTP 429|hit your (session|usage) limit"; "i")' "$RAW" >/dev/null 2>&1; then
+    echo "[score] QUOTA EXHAUSTED (HTTP 429 account session limit) for $(basename "$OUT") — failing fast (no retries). Writing a quota sentinel + exiting rc=2." >&2
+    printf '{"quota_exhausted":true,"api_error_status":429}\n' > "$OUT"
+    rm -f "$RAW"
+    exit 2
+  fi
   if [ ! -s "$RAW" ]; then
     # No envelope at all → claude itself never produced output (E2BIG, killed, exec fail).
     echo "[score] attempt $attempt: EMPTY output for $(basename "$OUT") — claude wrote NOTHING to stdout (E2BIG / killed / TIMED OUT at ${WORLDOS_SCORE_TIMEOUT:-300}s). Retrying. stderr tail:" >&2

qa/test_release_gate_static.py

@@ -182,5 +182,117 @@ def test_solo_play_contract_does_not_silently_recruit_companion(self):
         self.assertIn('exec "$ROOT/scripts/play.sh" "${ARGS[@]}"', party)
 
 
+class QuotaCircuitBreakerStaticContractTests(unittest.TestCase):
+    """#842: a 429 (account session limit) must yield a QUOTA abort, never a junk RRI/score,
+    and a quota-aborted sweep must never republish stale evidence. Grep-the-shell-source contracts
+    (mirrors the release-gate static style — no live runs)."""
+
+    def test_sweep_cleanup_wipes_stale_rri_json(self):
+        # Fix A: the sweep-start cleanup rm must include RRI.json so a quota abort before a fresh
+        # rollup can never leave the PREVIOUS run's RRI.json in place (the rc3 stale-RRI bug).
+        source = (ROOT / "qa" / "vm" / "sweep_v2.sh").read_text(encoding="utf-8")
+        self.assertIn('rm -f "$RES/DONE" "$RES/CANARY_FAIL" "$RES/QUOTA_ABORT" "$RES/RRI.json"', source)
+
+    def test_sweep_canary_abort_writes_aborted_rri(self):
+        # Fix B: the canary-abort path must ALSO write the {"status":"ABORTED",…} RRI.json (it
+        # previously touched DONE + exited leaving any stale RRI.json behind). The shared
+        # write_aborted_rri helper carries the ABORTED status; the canary-abort path must call it.
+        source = (ROOT / "qa" / "vm" / "sweep_v2.sh").read_text(encoding="utf-8")
+        self.assertIn("write_aborted_rri()", source)
+        self.assertIn('"status": "ABORTED"', source)
+        self.assertIn('"abort_reason": "quota_session_limit"', source)
+        # #842 review (load-bearing): evidence_audit.py keys on `aborted:true` + `abort_detail`
+        # (NOT `detail`). Without them the ABORTED RRI reads as RELEASE_READY — the exact masking
+        # #842 prevents. Lock the contract statically + functionally (below).
+        self.assertIn('"aborted": True', source)
+        self.assertIn('"abort_detail"', source)
+        self.assertNotIn('"detail": detail', source)  # the old wrong key must be gone
+        # the canary-abort branch (QUOTA ABORT at the canary) must call the writer before exiting.
+        canary_idx = source.index("QUOTA ABORT at the canary")
+        # the next write_aborted_rri call after the canary-abort message proves the path stamps it.
+        self.assertIn("write_aborted_rri", source[canary_idx:canary_idx + 600])
+
+    def test_aborted_rri_shape_reads_as_aborted_in_evidence_audit(self):
+        # #842 review (the end-to-end contract the static greps back): the ABORTED RRI the sweep
+        # writes MUST be classified as aborted (NOT release-ready) by qa/evidence_audit.py.
+        # Reproduce the helper's exact shape and assert evidence_audit does not call it ready.
+        import json, subprocess, tempfile, os
+        rri = {"status": "ABORTED", "aborted": True, "abort_reason": "quota_session_limit",
+               "abort_detail": "newbie — quota resets ~3h", "build_sha": "deadbeef",
+               "release_ready": False, "note": "infra abort, not a product RRI"}
+        fd, path = tempfile.mkstemp(suffix=".json")
+        try:
+            with os.fdopen(fd, "w") as f:
+                json.dump(rri, f)
+            out = subprocess.run(
+                ["python3", str(ROOT / "qa" / "evidence_audit.py"), "--rri", path],
+                capture_output=True, text=True, timeout=30)
+            combined = (out.stdout + out.stderr).upper()
+            self.assertNotIn("RELEASE_READY", combined,
+                             f"ABORTED RRI mis-classified as release-ready: {combined}")
+            self.assertIn("ABORT", combined, f"evidence_audit did not flag the abort: {combined}")
+        finally:
+            os.unlink(path)
+
+    def test_sweep_wipes_stale_duo_artifacts_before_duo_call(self):
+        # Fix C: the duo-artifact rm must PRECEDE the run_duo.sh call so the `[ -f ] && cp` below
+        # can only copy CURRENT-run output (rc3 republished rc2's byte-identical lens scores).
+        source = (ROOT / "qa" / "vm" / "sweep_v2.sh").read_text(encoding="utf-8")
+        self.assertIn('rm -f "$RES/duo-tolkien.json" "$RES/duo-angrydm.json" "$RES/duo-latency.json"', source)
+        self.assertIn('"qa/transcripts/vm2-duo.tolkien.json" "qa/transcripts/vm2-duo.angrydm.json"', source)
+        # the wipe must come before the run_duo invocation.
+        self.assertLess(
+            source.index('rm -f "$RES/duo-tolkien.json"'),
+            source.index("bash qa/run_duo.sh vm2-duo baldurs-gate veteran"),
+        )
+
+    def test_ui_playtest_app_has_quota_exhausted_bucket(self):
+        # Fix D: quota_exhausted must be a known failure bucket AND the poll loop must detect a
+        # 429 in backend.log, drop the QUOTA_EXHAUSTED sentinel, and bucket it as quota_exhausted
+        # (not the generic backend_not_ready / no_actor mis-bucketing).
+        source = (ROOT / "qa" / "ui_playtest_app.sh").read_text(encoding="utf-8")
+        self.assertIn('"quota_exhausted"', source)
+        self.assertIn('APP_FAILURE_BUCKETS_JSON=', source)
+        # the buckets JSON literal carries quota_exhausted.
+        buckets_line = next(
+            l for l in source.splitlines() if l.startswith("APP_FAILURE_BUCKETS_JSON=")
+        )
+        self.assertIn("quota_exhausted", buckets_line)
+        # the poll loop drops the sentinel and the readiness-failure path buckets it.
+        self.assertIn('touch "$RUNDIR/QUOTA_EXHAUSTED"', source)
+        self.assertIn('[ -f "$RUNDIR/QUOTA_EXHAUSTED" ]', source)
+        self.assertIn('PART_B_RESULT="quota_exhausted"', source)
+
+    def test_score_sh_has_429_fast_fail_arm(self):
+        # Fix F: score.sh must have a 429 fast-fail arm (NO 3 retries) that writes the quota
+        # sentinel and exits rc=2.
+        source = (ROOT / "qa" / "score.sh").read_text(encoding="utf-8")
+        self.assertIn('[ "$api_err" = "429" ]', source)
+        self.assertIn('printf \'{"quota_exhausted":true,"api_error_status":429}\\n\' > "$OUT"', source)
+        self.assertIn("exit 2", source)
+        # the fast-fail arm must sit BEFORE the generic retry-loop tail (the empty/api_err branches)
+        # so a 429 short-circuits instead of burning the 3 attempts.
+        self.assertLess(source.index('[ "$api_err" = "429" ]'), source.index('if [ ! -s "$RAW" ]; then'))
+
+    def test_run_duo_checks_for_quota_abort_before_scoring(self):
+        # Fix E + Fix F (caller half): run_duo.sh must (1) detect a DM cold-open 429 and emit the
+        # "[duo] QUOTA ABORT" marker + exit rc=2 BEFORE the empty-reply abort, and (2) treat the
+        # score.sh quota sentinel as a quota abort (not a valid scorecard) before the behavioral gate.
+        source = (ROOT / "qa" / "run_duo.sh").read_text(encoding="utf-8")
+        self.assertIn("[duo] QUOTA ABORT", source)
+        self.assertIn("session limit|HTTP 429|hit your (session|usage) limit", source)
+        self.assertIn(".quota_exhausted == true", source)
+        # the cold-open quota check must precede the empty-reply abort (DM produced no opening).
+        self.assertLess(
+            source.index("[duo] QUOTA ABORT"),
+            source.index("DM produced no opening"),
+        )
+        # the scorer-sentinel quota check must precede the behavioral gate (no gating a quota corpse).
+        self.assertLess(
+            source.index(".quota_exhausted == true"),
+            source.index("python3 qa/assert_behavioral.py"),
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

qa/ui_playtest_app.sh

@@ -229,7 +229,7 @@ esac
 
 # Agent-readable failure buckets for built-app smoke. Keep these crisp and stable; the
 # detailed shell/native result still travels separately as original_result.
-APP_FAILURE_BUCKETS_JSON='["no_app","no_launcher","no_provider","no_art","no_actor","no_actions","move_rejected","no_narration","console_error","permission_prompt"]'
+APP_FAILURE_BUCKETS_JSON='["no_app","no_launcher","no_provider","no_art","no_actor","no_actions","move_rejected","no_narration","console_error","permission_prompt","quota_exhausted"]'  # #842 Fix D: quota_exhausted — a backend DM cold-open that 429s on the account session limit is an INFRA abort, NEVER a no_actor/no_provider product miss
 
 bucket_pair() { printf '%s|%s\n' "$1" "$2"; }
 
@@ -966,9 +966,26 @@ run_part_b() {
       fi
     fi
     kill -0 "$B_BACKEND" 2>/dev/null || { log "[B] backend exited early — see $RUNDIR/backend.log"; break; }
+    # #842 Fix D (quota circuit-breaker): a DM cold-open that 429s on the account session limit writes
+    # "session limit" / "HTTP 429" into backend.log. WITHOUT this the poll just runs out its ~10-min cap
+    # and mis-buckets the corpse as no_actor/no_provider (the rc3 misattribution). Detect it INSIDE the
+    # loop, drop a QUOTA_EXHAUSTED sentinel, and break early so we abort honestly instead of waiting +
+    # mis-bucketing. This is an INFRA abort, NOT a product-readiness miss.
+    if grep -qiE "session limit|HTTP 429|hit your (session|usage) limit" "$RUNDIR/backend.log" 2>/dev/null; then
+      log "[B] QUOTA EXHAUSTED — backend DM cold-open hit the account session limit (HTTP 429); see $RUNDIR/backend.log. Aborting the ready-wait (INFRA abort, not a product miss)."
+      touch "$RUNDIR/QUOTA_EXHAUSTED"
+      break
+    fi
     sleep 3
   done
   if [ "$ready" != "1" ]; then
+    # #842 Fix D: a quota 429 short-circuit takes precedence over the generic backend_not_ready
+    # classification — bucket it as quota_exhausted so the rollup attributes an INFRA abort, never a
+    # no_actor/no_provider product failure (the rc3 mis-bucketing the quota circuit-breaker exists to kill).
+    if [ -f "$RUNDIR/QUOTA_EXHAUSTED" ]; then
+      log "[B] backend never became player-ready — QUOTA_EXHAUSTED (account session limit) — see $RUNDIR/backend.log"
+      PART_B_RESULT="quota_exhausted"; set_bucket_pair B "$(bucket_pair quota_exhausted 'DM cold-open hit the account session limit (HTTP 429) — INFRA abort, not a product readiness miss')"; return 1
+    fi
     log "[B] backend never became player-ready (can_act=$saw_canact seatedPC=$saw_pc) — see $RUNDIR/backend.log"
     PART_B_RESULT="backend_not_ready"; set_bucket_pair B "$(classify_part_b_readiness_failure "$saw_canact" "$saw_pc" "${chat_lines:-0}")"; return 1
   fi