fix(qa): de-flake behavioral A8 — recovered transient rejection is WARN, not RED (#897)

qa/BEHAVIORAL_GATE_TAXONOMY.json

@@ -198,7 +198,7 @@
       "category": "HARNESS_WIRING",
       "likely_code_locations": ["servers/engine/server.py", "servers/engine/models.py", "qa/run_duo.sh"],
       "retest": "bash qa/run_duo.sh duo-retest",
-      "hint": "A tool call was REJECTED with a schema/validation error (extra_forbidden => version skew or a wrong field). The DM's intent silently did not take effect — classic version-skew between the DM's expected tool schema and the engine's models. Check the tool param schema in server.py / models.py vs what the DM sent. FATAL."
+      "hint": "A tool was REJECTED with a schema/validation error (extra_forbidden => version skew or a wrong field) in a SYSTEMATIC pattern — classic version-skew between the DM's expected tool schema and the engine's models. De-flaked (#897): FATAL only when the rejection is UNRECOVERED (the same tool was NEVER successfully called) OR REPEATED (the same tool rejected >=2x = real skew). A SINGLE rejection the DM immediately retried correctly is a recovered transient => WARN, not RED (it was false-RED-capping whole runs on one invisible flub). Check the tool param schema in server.py / models.py vs what the DM sent for the named tool(s). FATAL (systematic) / WARN (recovered transient)."
     },
     "engine_guards_hit": {
       "category": "DM_ADHERENCE",

qa/assert_behavioral.py

@@ -854,20 +854,68 @@ def _has_spells(c: dict) -> bool:
     def _quest_reward_already_awarded(q: dict) -> bool:
         return any(bool(q.get(k)) for k in ("milestone_awarded", "awarded", "rewarded", "xp_awarded"))
 
-    # A8 (FATAL) — any tool call REJECTED with a schema/validation error (extra_forbidden ⇒
-    # version-skew or a wrong field). The DM's intent silently did not take effect; this is the
-    # class of failure that has produced 2 RED-capped runs historically. Benign engine guards
-    # the DM is EXPECTED to hit and recover from (travel-graph rejections etc.) are split off to
-    # a WARN so healthy recovery never false-REDs.
+    # A8 — a tool call REJECTED with a schema/validation error (extra_forbidden ⇒ version-skew
+    # or a wrong field). The DM's intent for that call silently did not take effect.
+    #
+    # DE-FLAKE (#897, mirrors #1030's discriminator-aware severity). Behavioral is computed from
+    # ONE stochastic duo; the bare "ANY schema rejection ⇒ FATAL" rule made a SINGLE recovered
+    # transient (the DM emits one malformed call, immediately retries the SAME tool correctly, the
+    # session completes cleanly — invisible to the player) RED-cap EVERY lens to 2.5 and swing the
+    # headline RRI by ~1.0 (observed twice). That is a precision bug, not a real integrity signal.
+    #
+    # A rejection now counts toward the FATAL set only when it is a PATTERN, not a recovered blip:
+    #   • UNRECOVERED — the offending tool was NEVER successfully called (is_error=False) anywhere
+    #     in the run, so the DM's intent for that tool silently never took effect (the genuine
+    #     version-skew defect: a stale signature the DM could not get right). [corpus fixture]
+    #   • REPEATED — the SAME tool was rejected with a schema/validation error >=2x across the run
+    #     = a systematic skew (the DM keeps re-using a stale/wrong signature). Repetition is the
+    #     real-skew signal, so this stays FATAL EVEN IF a later call eventually succeeds.
+    # A SINGLE rejection of a tool the DM then successfully retried (recovered transient) ⇒ WARN,
+    # never RED. This is a PRECISION improvement (distinguish player-felt skew from invisible
+    # recovered transients), NOT a leniency hack — the unrecovered + repeated classes the gate was
+    # built for still flip RED; the corpus fixture (an unrecovered update_character) still REDs.
     errors = [(n, text) for (n, inp, r, err, text) in evs if err]
     if errors:
-        fatal_errs = [(n, t) for (n, t) in errors
-                      if "extra_forbidden" in t or "validation error" in t.lower()]
-        benign = [(n, t) for (n, t) in errors if (n, t) not in fatal_errs]
-        chk("no_rejected_tool_calls", not fatal_errs,
-            f"{len(fatal_errs)} tool call(s) rejected with a schema/validation error "
-            f"(extra_forbidden ⇒ version skew or wrong field): {[n for n, _ in fatal_errs]}; "
-            f"first: {fatal_errs[0][1][:160] if fatal_errs else ''}", fatal=True)
+        schema_errs = [(n, t) for (n, t) in errors
+                       if "extra_forbidden" in t or "validation error" in t.lower()]
+        benign = [(n, t) for (n, t) in errors if (n, t) not in schema_errs]
+        # Which tools were EVER called successfully (is_error=False) anywhere in the run? A schema
+        # rejection of tool X is "recovered" iff X also appears with a clean result somewhere —
+        # the DM got the call right (order-independent: a clean call before or after a flub both
+        # prove the DM CAN issue that tool; the rejected intent itself is what we score).
+        succeeded_tools = {n for (n, inp, r, err, _) in evs if not err}
+        # How many times was each tool rejected with a schema/validation error?
+        schema_reject_counts: Counter = Counter(n for (n, _t) in schema_errs)
+        # FATAL set: a rejection is fatal if its tool was NEVER successfully called (unrecovered)
+        # OR its tool was rejected >=2x (repeated = systematic skew). De-dup by tool for the
+        # message (the per-tool classification is what matters, not the raw rejection count).
+        fatal_tools = sorted({
+            n for (n, _t) in schema_errs
+            if n not in succeeded_tools or schema_reject_counts[n] >= 2
+        })
+        # Recovered transients: a tool rejected exactly once that was later (or earlier) called
+        # cleanly — surfaced as a WARN so the flub is never silently dropped.
+        recovered_tools = sorted({
+            n for (n, _t) in schema_errs
+            if n in succeeded_tools and schema_reject_counts[n] < 2
+        })
+        if fatal_tools:
+            # Detail names which fatal class each tool fell into, so a RED is diagnosable.
+            why = ", ".join(
+                f"{n}(" + ("repeated x%d" % schema_reject_counts[n]
+                           if schema_reject_counts[n] >= 2 else "unrecovered") + ")"
+                for n in fatal_tools)
+            first = next((t for (n, t) in schema_errs if n in set(fatal_tools)), "")
+            chk("no_rejected_tool_calls", False,
+                f"{len(fatal_tools)} tool(s) with a SYSTEMATIC schema/validation rejection "
+                f"(extra_forbidden ⇒ version skew / wrong field): {why}; first: {first[:160]}",
+                fatal=True)
+        elif recovered_tools:
+            # All schema rejections were single + recovered ⇒ GREEN, but WARN so it's visible.
+            chk("no_rejected_tool_calls", False,
+                f"{len(recovered_tools)} RECOVERED transient schema rejection(s) "
+                f"(flubbed once, retried the same tool successfully ⇒ invisible to the player): "
+                f"{recovered_tools} — surfaced, not RED-capped (#897)", fatal=False)
         if benign:
             chk("engine_guards_hit", False,
                 f"{len(benign)} engine guard rejection(s) (recoverable, DM expected to retry): "

qa/gate_corpus/builder.py

@@ -325,6 +325,11 @@ def case_no_duplicate_companion():
 def case_no_rejected_tool_calls():
     # chk A8: a tool call REJECTED with an extra_forbidden schema/validation error (version skew /
     # wrong field). Modeled on ow-swB-123842's update_character rejection that RED-capped the run.
+    # NOTE (#897 de-flake): this fixture is deliberately UNRECOVERED — update_character is rejected
+    # and NEVER successfully retried, so it is the genuine version-skew defect that must STAY FATAL.
+    # (A recovered-transient variant lives in the GREEN corpus as
+    # case_no_rejected_tool_calls_recovered_warn.) Do NOT add a clean update_character call here or
+    # this case would correctly demote to a WARN and stop RED-ing.
     err = ("Error executing tool update_character: 1 validation error for Character\n"
            "skills\n  Extra inputs are not permitted "
            "[type=extra_forbidden, input_value=['Arcana'], input_type=list]")
@@ -335,6 +340,27 @@ def case_no_rejected_tool_calls():
     return events, _clean_player_state(), None, None
 
 
+def case_no_rejected_tool_calls_recovered_warn():
+    # GREEN fixture (#897) — the A8 de-flake scope guard. A SINGLE malformed update_character
+    # (extra_forbidden) that the DM immediately RETRIES correctly (the same tool, is_error=False)
+    # is a recovered transient — invisible to the player. Under the #897 de-flake A8 is demoted to
+    # a WARN (recovered transient), so the gate must exit GREEN with no_rejected_tool_calls as a
+    # [WARN] line — NOT a RED cap. A future edit that re-promotes recovered transients to FATAL
+    # (re-introducing the high-variance false-RED) flips this case RED and the green test fails.
+    err = ("Error executing tool update_character: 1 validation error for Character\n"
+           "skills\n  Extra inputs are not permitted "
+           "[type=extra_forbidden, input_value=['Arcana'], input_type=list]")
+    events = _roll() + [
+        _assistant_tool_use("t_uc_bad", "mcp__engine__update_character", {"skills": ["Arcana"]}),
+        _user_tool_result("t_uc_bad", err, is_error=True),
+        # the DM retries the SAME tool correctly -> recovered.
+        _assistant_tool_use("t_uc_ok", "mcp__engine__update_character",
+                            {"skill_proficiencies": ["Arcana"]}),
+        _user_tool_result("t_uc_ok", json.dumps({"ok": True})),
+    ]
+    return events, _clean_player_state(), None, None
+
+
 def case_end_combat_no_living_hostiles():
     # chk A3: end_combat called, combat NOT active, a LIVING hostile (monster, hp>0, not dead)
     # remains, and NO flee/surrender/retreat resolution declared. Keep moves < MIN_BEATS so the
@@ -472,6 +498,8 @@ def case_xp_awarded_on_progression():
 _GREEN_CASES_SPEC: list[tuple] = [
     ("structural_completeness_authored_warn", case_structural_completeness_authored_warn,
      "structural_completeness"),
+    ("no_rejected_tool_calls_recovered_warn", case_no_rejected_tool_calls_recovered_warn,
+     "no_rejected_tool_calls"),
 ]
 
 

qa/gate_corpus/cases/no_rejected_tool_calls_recovered_warn/run.jsonl

@@ -0,0 +1,6 @@
+{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "t_roll", "name": "mcp__engine__roll", "input": {"sides": 20}}]}}
+{"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "t_roll", "content": [{"type": "text", "text": "{\"total\": 14}"}], "is_error": false}]}}
+{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "t_uc_bad", "name": "mcp__engine__update_character", "input": {"skills": ["Arcana"]}}]}}
+{"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "t_uc_bad", "content": [{"type": "text", "text": "Error executing tool update_character: 1 validation error for Character\nskills\n  Extra inputs are not permitted [type=extra_forbidden, input_value=['Arcana'], input_type=list]"}], "is_error": true}]}}
+{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "t_uc_ok", "name": "mcp__engine__update_character", "input": {"skill_proficiencies": ["Arcana"]}}]}}
+{"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "t_uc_ok", "content": [{"type": "text", "text": "{\"ok\": true}"}], "is_error": false}]}}

qa/gate_corpus/cases/no_rejected_tool_calls_recovered_warn/state.json

@@ -0,0 +1,27 @@
+{
+  "party": [
+    "pc1"
+  ],
+  "leveling_mode": "xp",
+  "day": 2,
+  "time_of_day": "evening",
+  "current_location_id": "loc_camp",
+  "characters": {
+    "pc1": {
+      "name": "Dal",
+      "kind": "player",
+      "xp": 300,
+      "location_id": "loc_camp"
+    }
+  },
+  "locations": {
+    "loc_start": {
+      "name": "Tavern",
+      "visited": true
+    },
+    "loc_camp": {
+      "name": "Camp",
+      "visited": true
+    }
+  }
+}

qa/gate_corpus/manifest.json

@@ -198,6 +198,15 @@
         "state.json"
       ],
       "note": "must exit GREEN (rc 0) with warn_check as a [WARN] \u2014 locks a deliberate FATAL->WARN scope guard so a re-promotion to FATAL is caught (#1036)."
+    },
+    {
+      "case_dir": "no_rejected_tool_calls_recovered_warn",
+      "warn_check": "no_rejected_tool_calls",
+      "artifacts": [
+        "run.jsonl",
+        "state.json"
+      ],
+      "note": "must exit GREEN (rc 0) with warn_check as a [WARN] \u2014 locks a deliberate FATAL->WARN scope guard so a re-promotion to FATAL is caught (#1036)."
     }
   ]
 }