diff --git a/qa/assert_behavioral.py b/qa/assert_behavioral.py index 44c14a8c..52ae2b39 100644 --- a/qa/assert_behavioral.py +++ b/qa/assert_behavioral.py @@ -67,6 +67,44 @@ def _tally(events: list[dict]) -> tuple[Counter, int]: return tools, dm_text_turns +def _dm_narration_texts(events: list[dict]) -> list[str]: + """The DM's player-facing prose turns — each assistant top-level text block (what the player + actually reads). The OOC-leak gate scans these. Mirrors _tally's text extraction.""" + out: list[str] = [] + for ev in events: + if ev.get("type") != "assistant": + continue + for b in (ev.get("message", {}) or {}).get("content") or []: + if isinstance(b, dict) and b.get("type") == "text" and (b.get("text") or "").strip(): + out.append(b["text"]) + return out + + +# OOC craft-scaffolding leaking into PLAYER-FACING prose — the defect class the LLM story scorer +# reliably "blends away" (a 2026-06-17 adversarial audit found 5+ first-person authoring preambles +# in a 4.6-prose run, entirely un-gated, inflating the score to 4.8). The DM SKILL.md FICTION-ONLY +# rule is the source fix; this is the deterministic backstop the scorer can't be trusted for. +# EVERY pattern is OOC-ONLY phrasing that does NOT occur in in-character D&D fiction or dialogue — +# so a clean beat (even one full of combat prose, or a character saying "let me introduce you") +# scores 0 and can never false-RED. (Kept deliberately high-confidence over exhaustive.) An +# adversarial machine-checked sweep (2026-06-17) tightened three over-broad patterns: a bare +# `through the engine` matched literal machinery (Gond/artificer/Steel-Watch fiction) — now +# verb/noun-anchored to the game-engine sense; `as the pc` collided with any in-world "PC" +# initialism — now full-word only; and `your nat...` collided with fictional stammering / `spine +# hook` with a literal flensing tool — both dropped from the gate (the SKILL.md prose rule still +# bans the raw die-vocab + craft jargon at the source). +_NARRATION_LEAK = [ + r"\bas the player character\b", # "seat as the player character" + r"\b(?:advancement|leveling|progression|run it|route it|resolve it|process it" + r"|the (?:move|action|roll|turn)) through the engine\b", # the game-ENGINE sense, not machinery + r"\bcontinuity check\b", # first-person authoring self-correction + r"\bhere'?s how round \w+ (?:actually )?went\b", # OOC combat-replay framing (round-anchored) + r"\blet me set the order of it\b", # OOC initiative / turn-ordering preamble + r"\binciting incident\b", # plot-craft jargon (never in player fiction) +] +_NARRATION_LEAK_RE = [re.compile(p, re.I) for p in _NARRATION_LEAK] + + def _tool_events(events: list[dict]) -> list[tuple[str, dict, object, bool, str]]: """Ordered (short_name, input, result_obj_or_None, is_error, raw_text). @@ -153,6 +191,22 @@ def chk(name: str, ok: bool, detail: str = "", fatal: bool = True) -> None: chk("dm_produced_output", dm_text > 0 or sum(tools.values()) > 0, f"dm_text_turns={dm_text} tool_calls={sum(tools.values())}") + # 1b) OOC narration-leak floor (2026-06-17 craft audit): craft scaffolding / first-person + # authoring preambles ("Now let me seat as the player character", "Continuity check — let me + # correct that", "Here's how round one actually went") / raw system vocab leaking into the + # player-facing prose is a real felt-quality defect the LLM story scorer blends away (it scored + # a 5-leak / 4.6-prose run 4.8). Proportionate: 0 leaks pass; 1-2 incidental leaks WARN; >=3 in + # a substantial run is a pervasively-broken player surface -> RED (caps the lenses like any + # structural break). High-confidence OOC-only patterns ⇒ a clean in-fiction beat is always 0. + leak_hits = [t for t in _dm_narration_texts(events) if any(rx.search(t) for rx in _NARRATION_LEAK_RE)] + n_leak = len(leak_hits) + _leak_red = n_leak >= 3 and dm_text >= MIN_BEATS + chk("narration_no_ooc_leak", n_leak == 0, + f"{n_leak} player-facing beat(s) leaked OOC craft-scaffolding/bookkeeping/system-vocab" + + (f" — e.g. {' '.join(leak_hits[0].split())[:90]!r}" if leak_hits else "") + + (" [pervasive ⇒ RED]" if _leak_red else (" [WARN]" if n_leak else "")), + fatal=_leak_red) + # 2) two-sided duo runs: BOTH the player and the DM took turns (catches the # "1 player turn, 0 DM turns" botch). Only when a chat log exists. if chat: diff --git a/qa/test_assert_behavioral.py b/qa/test_assert_behavioral.py index 9ca4cb06..483fc3b4 100644 --- a/qa/test_assert_behavioral.py +++ b/qa/test_assert_behavioral.py @@ -748,3 +748,81 @@ def test_party_traveled_still_red_when_arc_resolved_but_too_few_beats(tmp_path): rc, out = _run_gate(tmp_path, events, state) assert "[FAIL] party_traveled" in out, out assert rc == 1, out + + +# ── narration_no_ooc_leak (2026-06-17 craft audit) — OOC scaffolding in player-facing prose ────── +# The 5 VERBATIM leak lines the audit found in the 4.8-scored gs-ember-deep run. +_LEAK_SEAT = "Now let me seat Kield Vant as the player character — an ex-Flaming Fist soldier fits a Fighter build in this world." +_LEAK_CONTINUITY = "Continuity check — authored Garrick Donn is the *miller* (the dazed man up at the mill), so the cart in town is run by his hand. Let me correct that and bring a new face on-screen." +_LEAK_ORDER = "The blade comes out and the room tips into violence. Let me set the order of it." +_LEAK_REPLAY = "The grey miller never gets his answer, and neither do you — not yet. Here's how round one actually went: the dead mill-hand came at you first." +_LEAK_ADVANCE = "The party hardens on the long way down — let me set their advancement through the engine before the climax." +_ALL_LEAKS = [_LEAK_SEAT, _LEAK_CONTINUITY, _LEAK_ORDER, _LEAK_REPLAY, _LEAK_ADVANCE] + +# Clean in-fiction prose engineered to brush AGAINST the patterns without being a leak — the +# false-positive guard that keeps the gate from ever RED-ing a clean beat. +_CLEAN_NEARMISS = [ + '"Let me introduce you to the captain," she says; "he\'ll want to hear this."', # dialogue "let me introduce" + "The siege engine groaned as the crew hauled it through the gate.", # engine, but not "through the engine" + "A natural spring fed the pool where the deer drank at dusk.", # "natural", not "natural 1" + '"Let me set the table," the innkeep mutters, clearing the mugs.', # "let me set", not "the order of it" + "The bridge cracked at its midpoint and pitched you toward the black water.", # "midpoint" in fiction + "He recounted how the night went, every ugly detail of it.", # "how the night went", not "round" + "You read the room: three farmers, a nervous barkeep, the door at your back.", # clean tension prose + # the 2026-06-17 adversarial sweep's machine-checked killers — now clean after hardening: + "Steam screamed through the engine block as the Steel Watcher lurched upright.", # literal machinery, not the game-engine sense + "She would serve as the PC — the Principal Courier — until the writ cleared.", # in-world "PC" initialism, not "player character" + '"Your nat... your natural gift for trouble," he sighed, trailing off.', # fictional stammer on a "nat-" word + "The torturer's spine hook glinted on the rack, wet and patient.", # literal flensing implement, not craft jargon +] + + +def _dm_leak(text: str) -> dict: + return {"type": "assistant", "message": {"content": [{"type": "text", "text": text}]}} + + +def test_narration_leak_patterns_match_all_five_verbatim_leaks(): + for line in _ALL_LEAKS: + assert any(rx.search(line) for rx in ab._NARRATION_LEAK_RE), f"missed leak: {line!r}" + + +def test_narration_leak_patterns_do_not_match_clean_fiction(): + for line in _CLEAN_NEARMISS: + hit = [rx.pattern for rx in ab._NARRATION_LEAK_RE if rx.search(line)] + assert not hit, f"false positive on clean line {line!r}: {hit}" + + +def test_dm_narration_texts_extracts_assistant_prose(): + events = [_dm_leak("First beat of fiction."), _assistant_tool_use("t1", "roll", {}), + _dm_leak("Second beat of fiction.")] + assert ab._dm_narration_texts(events) == ["First beat of fiction.", "Second beat of fiction."] + + +def test_narration_leak_three_plus_in_substantial_run_is_red(tmp_path): + # 3 distinct OOC leaks across a substantial run (>= MIN_BEATS dm_text) -> the leak check is FATAL. + events = [_dm_leak(_LEAK_SEAT), _dm_leak(_LEAK_ORDER), _dm_leak(_LEAK_ADVANCE)] + _dm_text_turns(6) + rc, out = _run_gate(tmp_path, events, _with_party({"leveling_mode": "milestone"})) + assert "[FAIL] narration_no_ooc_leak" in out, out + + +def test_narration_leak_one_or_two_is_warn_not_red(tmp_path): + # A single incidental leak in a substantial run is a WARN (surfaced), never a RED. + events = [_dm_leak(_LEAK_CONTINUITY)] + _dm_text_turns(6) + rc, out = _run_gate(tmp_path, events, _with_party({"leveling_mode": "milestone"})) + assert "[WARN] narration_no_ooc_leak" in out, out + assert "[FAIL] narration_no_ooc_leak" not in out, out + + +def test_narration_leak_three_in_short_run_is_only_warn(tmp_path): + # 3 leaks but only 3 dm_text turns (< MIN_BEATS) -> WARN, not RED: a tiny smoke isn't RED-capped. + events = [_dm_leak(_LEAK_SEAT), _dm_leak(_LEAK_ORDER), _dm_leak(_LEAK_REPLAY)] + rc, out = _run_gate(tmp_path, events, _with_party({"leveling_mode": "milestone"})) + assert "[WARN] narration_no_ooc_leak" in out, out + assert "[FAIL] narration_no_ooc_leak" not in out, out + + +def test_narration_leak_clean_run_passes(tmp_path): + # A clean session (even with near-miss fiction) PASSES the leak gate. + events = _dm_text_turns(4) + [_dm_leak(t) for t in _CLEAN_NEARMISS] + rc, out = _run_gate(tmp_path, events, _with_party({"leveling_mode": "milestone"})) + assert "[PASS] narration_no_ooc_leak" in out, out diff --git a/skills/dungeon-master/SKILL.md b/skills/dungeon-master/SKILL.md index ff2735db..0993c3da 100644 --- a/skills/dungeon-master/SKILL.md +++ b/skills/dungeon-master/SKILL.md @@ -76,12 +76,16 @@ The mechanics above are the floor; these make it a *scene* and not a log. They a This is **not** "every tense scene is a brawl." A parley, a threat the player talks down, a foe who has **not** attacked and isn't being attacked, a stealth/rescue approach the player *chooses over* a fight — those stay social/skill/exploration beats (the arc above was partly that: the player de-escalated). The trigger fires on a **declared or narrated attack**, or **present-and-imminent violence the player engages** — not on mere danger in the air. But the instant the line is crossed, the dice roll in the engine. - **Resolve only what the player declared — never play their part.** Voice the *world* and adjudicate with the dice; **never put words in the player's mouth, take an action they didn't declare, or decide their next move.** Don't ask "what do you do?" and then answer it yourself. Make every choice you offer **real, not illusory** — it must visibly *bend* the scene, or it isn't a choice. - **Dice live inside the tools.** The player reads the *felt outcome* ("the near one's boots are too clean for this room"), never a bare "Perception 16 / nat 1" label dropped into the prose. -- **The player-facing narration is FICTION ONLY — your craft scaffolding is PRIVATE and NEVER leaks into it.** Everything the player reads is in-world prose + quoted dialogue. The act/beat structure, the dice math, and the "what the scene is doing" bookkeeping are *your* private reasoning — keep them entirely out of the narration. Three things that must NEVER appear in a player-facing beat (this is a system-prompt-style leak, the same class as echoing a Director nudge or a tool name): +- **The player-facing narration is FICTION ONLY — your craft scaffolding is PRIVATE and NEVER leaks into it.** Everything the player reads is in-world prose + quoted dialogue. The act/beat structure, the dice math, and the "what the scene is doing" bookkeeping are *your* private reasoning — keep them entirely out of the narration. Five things that must NEVER appear in a player-facing beat (this is a system-prompt-style leak, the same class as echoing a Director nudge or a tool name): - **Dice / check tallies** — never "three failed social checks", "after two missed Perception rolls", "she's failed every Insight check so far". The player feels the *outcome* in the fiction ("Zevlor's jaw stays set; whatever you've tried, the door in his face hasn't opened"), never the count. - **Plot-structure / craft jargon** — never name the machinery: *spine hook*, *rib*, *cold open*, *act* / *act one/two/three*, *beat*, *midpoint*, *reversal*, *setup*, *payoff*, *inciting incident*, *scene N*, *the Threshold/the Call* (as labels). The structure is **felt, never labeled** (see `reference/storycraft.md`) — narrate the moment, not the moment's role in the arc. - **Stage-directions / status summaries** — never a meta line announcing the scene's state: "Meeting beat of the cold open complete", "beat complete", "Act 1 wraps", "this connects to the spine hook", "X of the Y complete", "scene resolved — advancing". When a beat lands, you simply move into the next beat of fiction; you do not stamp it done. + - **First-person authoring / bookkeeping preambles** — never narrate the GM move you are about to make (or are making): *"Let me set the order of it"*, *"Now let me seat Kield as the player character"*, *"Continuity check — authored Garrick is the miller… let me correct that"*, *"Here's how round one actually went:"*, *"let me set their advancement through the engine."* Turn-ordering, character-seating, continuity fixes, leveling, and combat-sequencing are mechanics you perform **silently** around the tool calls — the player sees only the fiction the mechanics produce. This family is the most insidious because it rides on the **back** of good prose: *"The blade comes out and the room tips into violence. Let me set the order of it."* — the first sentence is the scene; the trailing clause is the leak. Write the fiction, then go quiet and call the tool; never say what you're doing in the engine. + - **Raw system vocabulary** — never *"natural 1"*, *"nat 20"*, *"a d20"*, *"DC 15"*, *"AC 13"*, *"rolled a 7"*, an HP total, or a "day 98" in the prose. The player feels the result as fiction ("the blow glances off his gorget" — not "you rolled an 8, under his AC 13"). If you begin a mechanic word, rewrite the WHOLE sentence rather than leave a truncated token — a stray *"your nat…"* is the worst of both worlds. Verbatim example of the leak to NEVER produce: *"Zevlor held silence after three failed social checks; … connecting directly to the spine hook. Meeting beat of the cold open complete."* That is your scratchpad bleeding onto the page. Hold the scaffolding in your head; give the player only the world. + + **This holds at SESSION CLOSE too.** After `end_session`, your final player-facing text is the in-fiction denouement ONLY — a last quiet image, and at most a single bare close marker. NEVER an author's-note / "a good stopping point" / arc-grading coda, NEVER craft jargon (*reversal*, *midpoint*, *threads tied off*, *the arc closed*), NEVER raw mechanics (HP totals, "the 82-HP herald", "day 98", scheduled-consequence days). The arc reflection and the live-thread bookkeeping belong in `end_session`'s `summary` argument (player-invisible) and the scheduled `add_consequence` note — not on the page the player reads. - **End each beat on a live, open moment dramatized IN the scene** — the situation in front of them and the choice it forces ("his hand drifts toward his coat; the back door is six feet behind you"). **Never** end on a bare sign-off tag on its own line — "*Your move.*" / "*What do you do?*" repeated every beat is dead air masquerading as agency. Let the open question live in the concrete detail, then stop and let the player act. **This holds DOUBLY at a fork or a session-closing beat: don't formalize the choice into an alignment-tagged, DC-stamped option menu in the narration** ("*Grease his palm (corrupt) / Call the shakedown (lawful) / Walk away (cautious)*") — that HUD dropped into a lived scene is the single most-flagged scene-craft miss, and it pre-digests the gut-punch the player should feel *before* they choose. (The GUI's own options panel still surfaces the structured skills/DCs from the engine scaffold — that's *its* job, separate from your prose.) Still call `generate_parley_options` for the scaffold and route the pick through the engine the same as any action (`skill_check` / `social_check` / `record_decision`); the player can always act off-menu. In the *telling*, voice the branches as embodied moves inside the prose ("you could lean on the debt his house owes yours, or read the fear under the bluster and let the silence do the work") — kept *distinct* enough that the player feels the real fork, never blurred into one vague sentence — with the tags/DCs held in `record_decision`, and let the moment stay open. - **Your turn's FINAL output is ALWAYS 2nd-person player-facing narration — never a tool call and never a 3rd-person status line.** Resolve the move through the engine (roll/cast/attack), then **write the player-facing prose, and run `persist_beat` (state only) LAST** (step 7) — so your turn CLOSES on the in-world prose the player reads, addressed to *you* ("You step into the warren…"), present and concrete, never on a tool call. Do **not** let the last thing you emit be a tool invocation, an engine result, or a meta status summary like "The party has moved to Heapside; the Insight check failed." (a 3rd-person log line is the *scratchpad*, not the scene — and when it's the last thing you say, the player's chat shows **nothing playable**). **The same prose lives in two places by design — and that is fine: you stream it live via `log_event` (steps 2 + 6a) AND speak it as your reply text. The dashboard de-duplicates them by text, so each paragraph shows EXACTLY ONCE (the live copy wins; the reply copy is recognized as the turn's resolution and dropped). For that de-dup to work the reply text must be the SAME prose you streamed — narrate the beat once and let both the live log and the reply carry that one telling; do NOT write a *different* second version for the reply (a reworded reply defeats the de-dup and the player sees the beat twice).** Every resolved beat ends in prose the player can read and act on. - **The WORLD MOVES — don't run a whole session in one room at one hour** (session-scope, not per-beat, but non-negotiable across the arc). A living world *progresses*: (1) **the clock advances** — see step 7; a session still at *morning* in the *opening location* is frozen; (2) **the party travels to ≥2 locations** — `travel_to` along connections (or `add_location(make_current=True)` for somewhere new), with `advance_time=True` when it's a real journey; (3) **new named faces enter** — `create_character` an NPC (give them a name + a line; mark `met=True` when the party meets them on-screen). The seeded roster is a *starting cast*, not the whole world — keep peopling it. A session frozen in the opening scene, in one place, with no one new is a FAILED session no matter how good the prose, and the QA gate now flips it RED.