diff --git a/qa/dm_narration_fallback.py b/qa/dm_narration_fallback.py index b29bf4f4..d9cd9e0d 100755 --- a/qa/dm_narration_fallback.py +++ b/qa/dm_narration_fallback.py @@ -38,6 +38,18 @@ import re import sys +# #749: the wrapper progress heartbeat ("Your move lands; attention gathers…") is a canned +# liveness row the play/QA wrappers log BEFORE the DM turn — never recoverable prose. The +# shared constants live in servers/engine/wrapper_progress.py (re-exported by the sibling +# qa/wrapper_progress_lines.py). This script is best-effort plumbing in the beat path, so a +# missing shim degrades to "no wrapper filtering" rather than a crash; the engine test suite +# (tests/test_dm_narration_fallback.py) proves the import works in-repo. +try: + from wrapper_progress_lines import is_wrapper_progress_line +except Exception: # pragma: no cover - only on a broken checkout + def is_wrapper_progress_line(_text): + return False + # Engine session-log kinds (SessionLogEntry.kind): narration | dialogue | roll | system | combat. # Only narration + dialogue are player-facing prose; the rest are bookkeeping the player never reads. PROSE_KINDS = {"narration", "dialogue"} @@ -123,7 +135,14 @@ def _recover(snap_path): # scratchpad, not a scene — treat it like bookkeeping: it breaks the trailing # block and is never recovered (showing the player notation is worse than blank). # Dialogue rows are always a quoted character line, so they're never system-notation. - if kind == "narration" and text and _is_system_notation(text): + # #749: the wrapper progress heartbeat gets the SAME treatment — it is canned + # liveness filler logged BEFORE the DM turn. Crucially it BREAKS the block (it is + # not transparently skipped): a heartbeat-only (dead) beat must recover NOTHING, + # because stitching the PRIOR beat's stale prose under a fresh heartbeat would + # mask the dead beat as 'resolved'. + if kind == "narration" and text and ( + _is_system_notation(text) or is_wrapper_progress_line(text) + ): block = [] continue if kind in PROSE_KINDS and text: diff --git a/qa/lib_beat_driver.sh b/qa/lib_beat_driver.sh index 21c5d74d..d25a10e7 100644 --- a/qa/lib_beat_driver.sh +++ b/qa/lib_beat_driver.sh @@ -126,6 +126,24 @@ clawdnd_dm_narration_or_fallback() { fi } +# clawdnd_resolve_dm_reply REPLY STATE_DIR — the DIRECT-call front door over +# clawdnd_dm_narration_or_fallback (#749c fallback honesty). Sets: +# CLAWDND_DM_REPLY — the resolved reply text (the DM's own, or the recovered prose) +# CLAWDND_FALLBACK_RECOVERED — 1 IFF the #357 fallback recovered the prose (the DM's reply was +# blank and the engine log supplied it), else 0 +# Call it DIRECTLY (never in a command substitution — a subshell would drop both globals), then +# read CLAWDND_DM_REPLY. The flag is consumed (and reset) by the next record_dm_reply / +# clawdnd_chatlog_dm, which stamps {"fallback_recovered":true} on the dm chat row so behavioral +# tallies can later discount a masked-dead beat that was "resolved" with recovered prose. +clawdnd_resolve_dm_reply() { + local original="$1" + CLAWDND_DM_REPLY="$(clawdnd_dm_narration_or_fallback "$1" "$2")" + CLAWDND_FALLBACK_RECOVERED=0 + if [ -z "${original//[[:space:]]/}" ] && [ -n "${CLAWDND_DM_REPLY//[[:space:]]/}" ]; then + CLAWDND_FALLBACK_RECOVERED=1 + fi +} + # CHRONICLE WRITE + ENGINE-LOG TRUTHFULNESS GUARD (issue #720 — the ONE shared impl). # # The cold-open opening prose lands in TWO viewer-read sources: the engine per-session log @@ -199,6 +217,7 @@ import os import sys import server +import wrapper_progress campaign_id, text = sys.argv[1], sys.argv[2] norm = " ".join(text.split()) @@ -229,6 +248,16 @@ try: except Exception: already = False # any read failure → fall through to a normal append (no regression) +# #749(d): the wrapper progress heartbeat legitimately REPEATS — the 4-line rotation means +# beat N+4 re-emits beat N's exact text, and a run of DEAD beats logs ONLY heartbeats, so the +# repeat always sits inside the last-8 tail scanned above. It is a liveness signal, never +# duplicated prose: dedup-dropping it silently stops the player's spinner from flipping on +# cadence-aligned beats (exactly when the heartbeat is the only life sign). The extra rows are +# inert — they never render (app.jsx flips progress and returns null) and every engine memory +# consumer exact-match filters them (recap / FTS ledger / lean tail / narration fallback). +if wrapper_progress.is_wrapper_progress_line(text): + already = False + if not already: server.log_event(campaign_id, "narration", text) PY @@ -239,14 +268,35 @@ PY # de-dup the /chat blob against /events). On failure → an UNFLAGGED row (byte-identical to the # pre-#720 behavior; the client's eventsStreamedThisTurnRef backstop still applies). NEVER stamp # the flag unconditionally — that would suppress a legitimately /chat-only beat to zero rows. +# #749(c): when the preceding clawdnd_resolve_dm_reply recovered this prose from the engine log +# (CLAWDND_FALLBACK_RECOVERED=1), BOTH branches additionally stamp fallback_recovered:true so +# behavioral tallies can discount masked-dead beats. Consume-once: the flag resets here. record_dm_reply() { - local campaign_id="$1" text="$2" phase="$3" + local campaign_id="$1" text="$2" phase="$3" extra='{"engine_logged":true}' plain_extra='' + if [ "${CLAWDND_FALLBACK_RECOVERED:-0}" = "1" ]; then + extra='{"engine_logged":true,"fallback_recovered":true}' + plain_extra='{"fallback_recovered":true}' + fi if log_engine_narration "$campaign_id" "$text"; then - chatlog dm "$text" '{"engine_logged":true}' + chatlog dm "$text" "$extra" else echo "[worldos] warning: could not record ${phase} narration through engine — chat row written without engine_logged" >&2 - chatlog dm "$text" + chatlog dm "$text" "$plain_extra" + fi + CLAWDND_FALLBACK_RECOVERED=0 +} + +# clawdnd_chatlog_dm TEXT — `chatlog dm TEXT` for the runners that write the dm row directly +# (run_duo / run_party / ui_playtest), stamping {"fallback_recovered":true} when the preceding +# clawdnd_resolve_dm_reply recovered the prose (#749c). Consume-once, mirroring record_dm_reply. +# With the flag unset the row is byte-identical to the plain `chatlog dm` it replaces. +clawdnd_chatlog_dm() { + if [ "${CLAWDND_FALLBACK_RECOVERED:-0}" = "1" ]; then + chatlog dm "$1" '{"fallback_recovered":true}' + else + chatlog dm "$1" fi + CLAWDND_FALLBACK_RECOVERED=0 } # LIVE-PROGRESS + WRAPPER HEARTBEAT (#623 — the ONE shared implementation of the perceived-latency fix). @@ -304,11 +354,15 @@ clawdnd_progress_beat_text() { } # clawdnd_emit_progress_heartbeat CAMPAIGN_ID FIRST BEAT_INDEX — write the wrapper-authored progress beat -# to the engine session log (via log_engine_narration) so /events has a row to render BEFORE the model's -# long think. Best-effort + idempotent: a blank campaign id no-ops; log_engine_narration's substring guard -# (it already de-dups against the recent narration tail) means a heartbeat the DM happens to echo isn't -# double-logged. ALWAYS returns 0 — a heartbeat failure must never fail a beat. Reads ambient $STATE_DIR / -# $ROOT exactly as log_engine_narration / record_dm_reply do. $1=campaign_id $2=first?(1/0) $3=beat index. +# to the engine session log (via log_engine_narration) so /events has a row for the viewer to flip its +# live-progress state on BEFORE the model's long think (the row itself never renders — app.jsx returns +# null on the exact wrapper lines). Best-effort: a blank campaign id no-ops. #749(d): heartbeats are +# EXEMPT from log_engine_narration's #727 substring dedup (the 4-line rotation legitimately repeats on +# cadence-aligned beats, and a run of dead beats logs ONLY heartbeats — dropping the repeat would kill +# the only liveness signal); the repeated rows are inert (never rendered, filtered from recap/FTS/lean +# tail/fallback). ALWAYS returns 0 — a heartbeat failure must never fail a beat. Reads ambient +# $STATE_DIR / $ROOT exactly as log_engine_narration / record_dm_reply do. $1=campaign_id $2=first?(1/0) +# $3=beat index. clawdnd_emit_progress_heartbeat() { local campaign_id="$1" first="${2:-0}" idx="${3:-0}" text [ -n "${campaign_id//[[:space:]]/}" ] || return 0 diff --git a/qa/run_duo.sh b/qa/run_duo.sh index aa57dcf1..e27ef533 100755 --- a/qa/run_duo.sh +++ b/qa/run_duo.sh @@ -244,10 +244,10 @@ $PMSG Do the setup now: start_world(\"$WORLD\"), start_session, then seat THEIR character as the PLAYER CHARACTER (the PC). The player ALWAYS plays a REAL, LIVING CANON NPC — their persona names one (e.g. Aubree, a Flaming Fist ranger). Seat that exact figure via load_canon_character(their canon name, kind=\"player\", add_to_party=true) so they get a real backstory + ingested portrait — NEVER create_character / invent a custom PC, NEVER seat the player's own character as a companion or NPC, and NEVER a canon-DEAD figure (a corpse like Dal Lightspark is rejected as a PC; if the seat returns an error, pick a living canon NPC instead). A companion is a DIFFERENT character the player MEETS. Then OPEN the scene — human-scale and personal — grounded in the world's canon, responding to their stated intent. A companion should ENTER as part of that opening scene: someone the player MEETS on-screen (voiced, with a real wound and a reason they fall in together) — recruit_companion / load_canon_character(kind=\"companion\") as that meeting lands, NOT a silent name dropped into the party before the player has met anyone. End by handing the moment to the player. OUTPUT DISCIPLINE — your final reply IS the opening scene: write it as 2nd-person in-fiction PROSE + quoted dialogue ONLY. NEVER narrate your own setup/process — no \"State is grounded\", no \"the cold open is on the dashboard\", no \"Closing my turn on the scene\", no 3rd-person status line. The very first words the player reads must be INSIDE the fiction.")" # #357: recover the engine's logged narration if the DM turn ended on a tool call / status # line (empty final reply) — so a tool-final-but-narrated turn isn't mistaken for silence. -DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" +clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" echo "[duo] DM opened: ${DMSG:0:120}…" [ -z "$DMSG" ] && { echo "[duo] DM produced no opening — aborting (see $COMBINED)" >&2; exit 1; } -chatlog dm "$DMSG" +clawdnd_chatlog_dm "$DMSG" # Resolve the campaign id the cold open just minted (for the lean re-ground; harmless when # CLAWDND_LEAN_BEATS=0). D1's start_world wrote the snapshot to @@ -324,10 +324,10 @@ $DIRECTOR $EVENT_ADV")" # #357: recover engine-logged narration before the silence check, so a turn that ended on a # tool call but logged real prose isn't mis-flagged as a silent DM (and isn't blank in chat). - DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" + clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" echo "[duo] beat $b DM: ${DMSG:0:100}…" [ -z "$DMSG" ] && { echo "[duo] DM went silent at beat $b; stopping early"; break; } - chatlog dm "$DMSG" + clawdnd_chatlog_dm "$DMSG" # C — soft clock-tick backstop: if the DM didn't move the clock this beat, advance one # phase via the engine (sole writer). Defers to the DM when it advanced time in-fiction. diff --git a/qa/run_party.sh b/qa/run_party.sh index f1b47473..332e75f4 100755 --- a/qa/run_party.sh +++ b/qa/run_party.sh @@ -304,9 +304,9 @@ $beat0_block Resolve each declared move through the engine; voice the world and any NPC; let the companions be PRESENT (the player and companions are separate people with their own agency — you narrate the RESULT of their declared moves, never invent a companion's internal choice). End by handing the open moment to the PLAYER.")" # #357: recover engine-logged narration if the DM turn ended on a tool call (empty reply). -DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" +clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" [ -z "$DMSG" ] && { echo "[party] DM produced no opening — aborting (see $COMBINED)" >&2; exit 1; } -chatlog dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1)) +clawdnd_chatlog_dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1)) echo "[party] DM opened: ${DMSG:0:120}…" # --- main loop: player + each living companion act, then the DM resolves the beat --- @@ -327,10 +327,10 @@ $PARTY_BLOCK Then PLAY the next beat as a full lived scene — NOT a fragment: any NPC (or companion) present SPEAKS at least one quoted line in their own voice; let them push back when it's real. Narrate the RESULT of each declared move (never invent a companion's choice). Weave the open moment back to the PLAYER inside the scene — never a bare 'Your move.'")" # #357: recover engine-logged narration before the silence check (tool-final-but-narrated # turn ≠ silence; keeps the chat non-blank on a resolved beat). - DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" + clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" echo "[party] beat $b DM: ${DMSG:0:120}…" [ -z "$DMSG" ] && { echo "[party] DM went silent at beat $b; stopping early"; break; } - chatlog dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1)) + clawdnd_chatlog_dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1)) done # --- wrap + score (same artifacts as run_duo) --------------------------------------- diff --git a/qa/ui_playtest.sh b/qa/ui_playtest.sh index 1d9206a1..3e68c92c 100755 --- a/qa/ui_playtest.sh +++ b/qa/ui_playtest.sh @@ -143,9 +143,10 @@ chatlog() { python3 -c 'import json,sys;open(sys.argv[1],"a").write(json.dumps({ # never RESOLVED on the client, leaving the player on the (now-fixed-but-slower) client stall path with # no backend recovery at all. Here we (1) wall-clock the beat with `timeout` (tiered off the cold-open # `first` signal via the shared clawdnd_dm_timeout; a frozen process is KILLED at the deadline so the -# turn returns), and (2) if the killed/failed beat left empty result text, stitch the engine-logged -# narration tail as a fallback reply (clawdnd_dm_narration_or_fallback) so `chatlog dm` always writes a -# real turn-END line → the client's pending clears + the bar re-enables. CLAWDND_DM_MODEL lets the +# turn returns), and (2) if the killed/failed beat left empty result text, the CALLER stitches the +# engine-logged narration tail as a fallback reply (clawdnd_resolve_dm_reply, which also flags the +# recovery — #749c) so the dm chat row always carries a real turn-END line → the client's pending +# clears + the bar re-enables. CLAWDND_DM_MODEL lets the # timeout helper pick the opus cold-open tier. Bash 3.2-safe (timeout(1) from coreutils; ${arr[@]+…}). CLAWDND_DM_MODEL="$DM_MODEL" dm_turn() { @@ -160,9 +161,11 @@ dm_turn() { rc=$? [ "$rc" -ne 0 ] && echo "[uipt] DM turn rc=$rc (timeout=${beat_timeout}s) — relying on engine-logged narration fallback" >&2 cat "$out" >> "$COMBINED" - # If the beat was killed/failed with no final result text, recover the engine-logged narration tail so - # the turn STILL resolves on /chat (never an indefinite hang). A healthy beat returns its result verbatim. - clawdnd_dm_narration_or_fallback "$(jq -rs 'map(select(.type=="result"))[-1].result // ""' "$out" 2>/dev/null)" "$STATE_DIR" + # Echo the beat's RAW final result text. The #357 fallback (recover the engine-logged narration + # tail when a killed/failed beat left this empty, so the turn STILL resolves on /chat) is applied + # by the CALLER via clawdnd_resolve_dm_reply — a direct call, because dm_turn runs in a command + # substitution where the #749c fallback_recovered flag (a global) could never escape the subshell. + jq -rs 'map(select(.type=="result"))[-1].result // ""' "$out" 2>/dev/null } # --- DM opens the scene so a LIVE, playable game exists (the launcher's Chronicles @@ -174,8 +177,11 @@ echo "[uipt] DM opening the scene…" DMSG="$(dm_turn 1 "$DM_BRIEF Begin a SOLO session for a brand-new human player in this world: start_world(\"$WORLD\"), start_session, seat a fitting level-3 PLAYER CHARACTER (a LIVING canon figure via load_canon_character(kind=\"player\", add_to_party=true) — NEVER a dead/fallen character; apply sensible skills/spells), and bring in ONE roster companion the player meets in the scene. Then open a human-scale, personal scene with real quoted dialogue and hand the player an open moment + a clear choice. Their actions will arrive next as tagged moves.")" +# #357/#749c: recover the engine-logged narration tail when the turn died with no result text; +# a recovered reply stamps fallback_recovered:true on the dm chat row (clawdnd_chatlog_dm). +clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" [ -z "$DMSG" ] && echo "[uipt] WARN: DM produced no opening (see $RUNDIR/dm/dm.err) — the player may land in a thin scene." >&2 -chatlog dm "${DMSG:-The scene is set. What do you do?}" +clawdnd_chatlog_dm "${DMSG:-The scene is set. What do you do?}" # --- background DM-resolver loop: tail $MOVES, resolve each new move, append narration # to $CHAT (the UI shows it via /chat). Identical shape to play_human.sh's loop. Runs @@ -194,7 +200,9 @@ chatlog dm "${DMSG:-The scene is set. What do you do?}" $PMSG Resolve it through the engine (roll checks, apply casts/attacks, voice NPCs) and narrate the next beat as a played scene. Hand the moment back to the player.")" - chatlog dm "${DMSG:-...}" + # #357/#749c: same recovery + honesty stamp as the opening turn (direct call, see dm_turn). + clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" + clawdnd_chatlog_dm "${DMSG:-...}" else sleep 2 fi diff --git a/qa/wrapper_progress_lines.py b/qa/wrapper_progress_lines.py new file mode 100644 index 00000000..9f3487d8 --- /dev/null +++ b/qa/wrapper_progress_lines.py @@ -0,0 +1,32 @@ +"""qa-side accessor for the wrapper progress-heartbeat lines (#749). + +The ONE python source of truth is ``servers/engine/wrapper_progress.py`` (the engine's +memory filters import it directly; the engine must keep filtering even if it were ever +deployed without qa/). This shim loads that module by file path — qa scripts run under +the system ``python3`` with no engine on ``sys.path`` — and re-exports its names so +stdlib-only qa tools (``qa/dm_narration_fallback.py``) share the exact same constants. + +``tests/test_wrapper_progress_sync.py`` pins this re-export (and the jsx/sh copies) +byte-identical to the canonical module. +""" + +from __future__ import annotations + +import importlib.util +import os + +_CANONICAL = os.path.normpath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", "servers", "engine", "wrapper_progress.py", + ) +) + +_spec = importlib.util.spec_from_file_location("_worldos_wrapper_progress", _CANONICAL) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +WRAPPER_OPENING_PROGRESS_LINE = _mod.WRAPPER_OPENING_PROGRESS_LINE +WRAPPER_MOVE_PROGRESS_LINES = _mod.WRAPPER_MOVE_PROGRESS_LINES +WRAPPER_PROGRESS_LINES = _mod.WRAPPER_PROGRESS_LINES +is_wrapper_progress_line = _mod.is_wrapper_progress_line diff --git a/scripts/play.sh b/scripts/play.sh index cfba4073..6606b54c 100755 --- a/scripts/play.sh +++ b/scripts/play.sh @@ -412,7 +412,7 @@ Their actions will arrive as tagged moves — [say] (their dialogue), [do] (an a fi # #357: same empty-reply fallback as the beat loop — recover the engine's logged opening # narration if the DM's first turn ended on a tool call rather than prose. -DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" +clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" # Resolve the campaign id the DM just minted, BEFORE writing the opening to the chronicle — # record_dm_reply (#720) needs it to log the opening narration to the engine session log so the @@ -496,7 +496,7 @@ $RUNBOOK" "$CAMPAIGN_ID")" # #357: if the DM turn ended on a tool call / 3rd-person status line, its final reply text is # empty — fall back to the player-facing narration the engine logged this beat so the chat is # never blank on a resolved move (engine stays the sole writer; this only READS its log). - DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" + clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" # #720: route the per-beat DM reply through record_dm_reply (engine_logged stamp on success). record_dm_reply "$CAMPAIGN_ID" "$DMSG" beat; DM_TURNS=$((DM_TURNS + 1)) # C — soft clock-tick backstop: advance one phase via the engine only if the DM left the diff --git a/scripts/play_party.sh b/scripts/play_party.sh index 454e73bf..a25d1cd4 100755 --- a/scripts/play_party.sh +++ b/scripts/play_party.sh @@ -471,7 +471,7 @@ CRITICAL — your FINAL output THIS turn MUST BE the opening SCENE itself, writt Each beat, declarations arrive as tagged moves — [say] (dialogue), [do] (an attempt), [check] (roll that skill), [cast]/[use]/[attack] (resolve via the engine) — from the HUMAN (their PC) and from each companion (banner-tagged with the companion's name). Resolve EACH actor's moves through the engine.")" # #357: recover the engine's logged opening narration if the DM's first turn ended on a tool # call rather than prose — BEFORE the abort check, so a tool-final-but-narrated opener stands. -DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" +clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" [ -z "$DMSG" ] && { echo "[play-party] DM produced no opening — aborting (see $COMBINED)" >&2; exit 1; } # #720: route the opening through record_dm_reply — engine_logged stamp on success so the # OpenWorlds client renders the cold-open opening ONCE (this is the dup source on the VM sweep). @@ -520,7 +520,7 @@ if ! pc_seated; then - use campaign_id=$CAMPAIGN_ID for EVERY engine call. DO NOT call start_world (it would mint a NEW campaign id and ORPHAN the pre-seeded companions). The companions already present are: $COMP_NAME_LIST. - SEAT THE PLAYER CHARACTER: generate_ability_scores + create_character with kind=\"player\" and add_to_party=true, apply_srd_defaults, sensible skills/spells. Pick a fitting concept and tell the player who they are. This is the ONLY character you create. The party MUST contain the human's kind=\"player\" PC when this turn ends. - Then CLOSE the turn by writing the opening SCENE as 2nd-person player-facing prose addressed to \"you\" (where the player IS, what they see/hear/smell, who is present + a real quoted line), ending on a clear open moment + choice. NEVER end on a tool call or a 3rd-person setup brief.")" - RESEAT_DMSG="$(clawdnd_dm_narration_or_fallback "$RESEAT_DMSG" "$STATE_DIR")" + clawdnd_resolve_dm_reply "$RESEAT_DMSG" "$STATE_DIR"; RESEAT_DMSG="$CLAWDND_DM_REPLY" AGENT_TURNS=$((AGENT_TURNS + 1)) # #720: the reseat turn re-writes the opening scene — route it through record_dm_reply too. if [ -n "$RESEAT_DMSG" ]; then DMSG="$RESEAT_DMSG"; record_dm_reply "$CAMPAIGN_ID" "$DMSG" reseat; echo "[play-party] reseat turn opened: ${DMSG:0:120}…"; fi @@ -567,7 +567,7 @@ $INTRO_BLOCK Narrate the RESULT of each declared move (never invent a companion's internal choice), then weave the open moment back to the human PLAYER inside the scene — never a bare 'Your move.'")" # #357: recover engine-logged narration if this DM turn ended on a tool call. - DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" + clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" # #720: route the after-intros DM beat through record_dm_reply (engine_logged stamp on success). [ -n "$DMSG" ] && { record_dm_reply "$CAMPAIGN_ID" "$DMSG" after_intros; AGENT_TURNS=$((AGENT_TURNS + 1)); echo "[play-party] DM after intros: ${DMSG:0:120}…"; } fi @@ -648,7 +648,7 @@ For EACH companion this beat, call check_companion_arc(companion_id) — the eng Then PLAY the next beat as a full lived scene — NOT a fragment: any NPC (or companion) present SPEAKS at least one quoted line in their own voice; let them push back when it's real. Narrate the RESULT of each declared move (never invent a companion's choice). Weave the open moment back to the human PLAYER inside the scene — never a bare 'Your move.' ALWAYS end your turn on 2nd-person player-facing narration (addressed to \"you\"), never on a tool call or a 3rd-person status line — the player reads your final reply text as the scene, so the beat's prose MUST be in it. Your reply IS the scene: write FLOWING 2nd-person PROSE, NEVER your planning notes or terse scaffolding. (Wrong — internal shorthand the player must never see: \"Devella presses Renn on the seal. Renn: the rangers made that call — no log filed.\" Right — render it lived: her jaw tightening, the quoted line in her own voice, the weight of the answer in the room.)")" # #357: if the DM turn ended on a tool call / 3rd-person status line, recover the # player-facing narration the engine logged this beat so the chat is never blank. - DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")" + clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY" # #720: route the per-beat DM reply through record_dm_reply (engine_logged stamp on success). record_dm_reply "$CAMPAIGN_ID" "$DMSG" beat; AGENT_TURNS=$((AGENT_TURNS + 1)) # Remember this beat's location so the next beat's runbook can detect a stuck party (travel cue). diff --git a/servers/engine/ledger.py b/servers/engine/ledger.py index 1e48e7b0..8511be25 100644 --- a/servers/engine/ledger.py +++ b/servers/engine/ledger.py @@ -25,6 +25,7 @@ from typing import Optional import store +from wrapper_progress import is_wrapper_progress_line KINDS = ("events", "dialogue", "decision", "npc_fact", "quest_milestone", "consequence", "lore") @@ -188,6 +189,10 @@ def _ins(kind, text, who="", ref="", day=0): for sid in campaign.session_ids: for e in store.read_log(campaign_id, sid): if e.kind in ("narration", "dialogue", "combat", "system"): + # #749: never index the wrapper progress heartbeat — it is mid-turn + # liveness filler, not campaign memory; recall must never surface it. + if is_wrapper_progress_line(e.text): + continue _ins("dialogue" if e.kind == "dialogue" else "events", e.text, who=e.speaker or "") for ch in campaign.characters.values(): for fact in ch.memory: diff --git a/servers/engine/recap.py b/servers/engine/recap.py index 174fff41..9c09e71d 100644 --- a/servers/engine/recap.py +++ b/servers/engine/recap.py @@ -11,6 +11,7 @@ from __future__ import annotations from models import SessionLogEntry +from wrapper_progress import is_wrapper_progress_line # Kinds that carry the story. Rolls and system messages are bookkeeping noise we # leave out of a "Previously on..." recap. @@ -52,7 +53,13 @@ def format_recap(entries: list[SessionLogEntry], max_entries: int = 12) -> str: max_entries = 1 # Keep only story beats, then take the most recent `max_entries` of them. - story = [e for e in entries if e.kind in _STORY_KINDS] + # #749: the wrapper progress heartbeat ("Your move lands; attention gathers…") is a + # liveness signal the QA/play wrappers log mid-turn, not story — reciting it in a + # "Previously on…" recap reads as canned filler. Exact-match excluded. + story = [ + e for e in entries + if e.kind in _STORY_KINDS and not is_wrapper_progress_line(e.text) + ] recent = story[-max_entries:] lines = [b for b in (_beat(e) for e in recent) if b] diff --git a/servers/engine/server.py b/servers/engine/server.py index 6f52e9f9..5fc3b297 100644 --- a/servers/engine/server.py +++ b/servers/engine/server.py @@ -45,6 +45,7 @@ import travel import wander import worldsim +import wrapper_progress as _wrapper_progress_mod from models import ( SKILL_ABILITIES, Ability, @@ -9040,7 +9041,14 @@ def _scene_recent_narration(c: Campaign, limit: int) -> list[dict]: if limit <= 0: return [] entries = read_log_all(c.id, getattr(c, "session_ids", None)) - facing = [e for e in entries if e.kind in ("narration", "dialogue")] + # #749: drop the wrapper progress heartbeat (exact-match) — it is the QA/play wrappers' + # mid-turn liveness filler, not the DM's prose. Feeding it back here told a lean + # (transcript-free) DM that canned filler was its own canon. + facing = [ + e for e in entries + if e.kind in ("narration", "dialogue") + and not _wrapper_progress_mod.is_wrapper_progress_line(e.text) + ] return [ {"text": e.text, **({"speaker": e.speaker} if e.speaker else {})} for e in facing[-limit:] diff --git a/servers/engine/tests/test_dm_narration_fallback.py b/servers/engine/tests/test_dm_narration_fallback.py index 62fc68ba..c9268f1f 100644 --- a/servers/engine/tests/test_dm_narration_fallback.py +++ b/servers/engine/tests/test_dm_narration_fallback.py @@ -204,3 +204,51 @@ def test_real_2nd_person_prose_with_innocent_parens_survives(tmp_path, monkeypat out = _run(snap) assert out.startswith("You duck beneath the awning") assert "hooded figure" in out + + +# --- #749: the wrapper progress heartbeat is a liveness signal, never recoverable prose ------- + + +def _wrapper_lines(): + import wrapper_progress + return wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE, list( + wrapper_progress.WRAPPER_MOVE_PROGRESS_LINES + ) + + +def test_wrapper_heartbeat_breaks_block_so_a_dead_beat_recovers_nothing(tmp_path, monkeypatch): + """A fully-DEAD beat logs ONLY the wrapper heartbeat after the prior beat's prose. The + heartbeat must break the trailing block like bookkeeping — recovering the PRIOR beat's + stale prose (+ filler) would mask the dead beat as 'resolved' (#749 root 2).""" + opening, moves = _wrapper_lines() + entries = [ + SessionLogEntry(t=1.0, kind="narration", text="You slip through the postern gate."), + SessionLogEntry(t=2.0, kind="narration", text=moves[0]), # the dead beat's only row + ] + snap = _seed(tmp_path, monkeypatch, campaign_id="camp_k", session_id="sess_k", entries=entries) + out = _run(snap) + assert out == "", ( + "a heartbeat-only (dead) beat must recover NOTHING — stale prose + filler masks the " + f"failure. Recovered: {out!r}" + ) + + +def test_prose_after_wrapper_heartbeat_survives(tmp_path, monkeypatch): + """The normal healthy shape: heartbeat first (pre-turn), then the DM's real prose. Only + the prose is recovered; the filler never reaches the chat.""" + opening, moves = _wrapper_lines() + entries = [ + SessionLogEntry(t=1.0, kind="narration", text=opening), + SessionLogEntry( + t=2.0, kind="narration", + text="You step off the gangplank into the riot of the Gray Harbor fishmarket.", + ), + SessionLogEntry(t=3.0, kind="dialogue", text="Mind the eels.", speaker="Dockhand"), + ] + snap = _seed(tmp_path, monkeypatch, campaign_id="camp_l", session_id="sess_l", entries=entries) + out = _run(snap) + assert out == ( + "You step off the gangplank into the riot of the Gray Harbor fishmarket.\n\n" + "Dockhand: Mind the eels." + ) + assert opening not in out diff --git a/servers/engine/tests/test_dm_session_remint.py b/servers/engine/tests/test_dm_session_remint.py index b9be4f07..d1662193 100644 --- a/servers/engine/tests/test_dm_session_remint.py +++ b/servers/engine/tests/test_dm_session_remint.py @@ -387,11 +387,18 @@ def test_ui_playtest_dm_turn_wraps_claude_in_a_bounded_timeout(): def test_ui_playtest_dm_turn_falls_back_to_engine_narration_on_a_stalled_beat(): - """A killed/empty beat must still RESOLVE on /chat: dm_turn routes its result through the shared - clawdnd_dm_narration_or_fallback so the engine-logged narration tail becomes the turn-END line.""" + """A killed/empty beat must still RESOLVE on /chat: each dm_turn result routes through the shared + fallback front door (#749c: clawdnd_resolve_dm_reply — a DIRECT call wrapping + clawdnd_dm_narration_or_fallback, so the recovery flag survives) and the engine-logged narration + tail becomes the turn-END line, with a recovered reply stamped fallback_recovered on the chat row.""" src = _src("qa/ui_playtest.sh") - assert "clawdnd_dm_narration_or_fallback" in src, ( - "ui_playtest.sh dm_turn must recover via the shared narration fallback so a stalled beat still resolves" + assert src.count('clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"') >= 2, ( + "ui_playtest.sh must recover BOTH the opening and per-beat turns via the shared fallback " + "front door so a stalled beat still resolves" + ) + assert "clawdnd_chatlog_dm" in src, ( + "ui_playtest.sh must write dm rows via clawdnd_chatlog_dm so a recovered reply carries " + "the fallback_recovered honesty stamp (#749c)" ) diff --git a/servers/engine/tests/test_heartbeat_repair.py b/servers/engine/tests/test_heartbeat_repair.py new file mode 100644 index 00000000..b38548b5 --- /dev/null +++ b/servers/engine/tests/test_heartbeat_repair.py @@ -0,0 +1,272 @@ +"""#749 heartbeat repair — engine-memory decontamination, dedup-bypass, fallback honesty. + +The #743 wrapper heartbeat writes real ``kind=narration`` rows ("Your move lands; attention +gathers…") into the engine session log so /events has a row mid-turn. Those rows are a +LIVENESS SIGNAL, not story — but before #749 every engine memory consumer treated them as +canon: + + - recap recited the filler in "Previously on your adventure…", + - the FTS ledger indexed it for recall, + - ``scene_context``'s lean re-ground tail fed it back to the DM as its own prose, + - and ``qa/dm_narration_fallback.py`` recovered it as a beat's "narration". + +These tests seed REAL campaigns through the engine's own writers and assert each consumer +now excludes the exact wrapper lines while REAL prose still flows. They also pin the (d) +dedup interaction: ``qa/lib_beat_driver.sh``'s ``log_engine_narration`` #727 substring +guard must NOT silently swallow a cadence-aligned heartbeat (the 4-line rotation repeats +every 4 beats — a run of dead beats logs ONLY heartbeats, so beat N+4's text is always in +the last-8 tail), while still de-duping the DM's own echoed prose. And the (c) fallback +honesty contract: a chat row whose prose was RECOVERED from the engine log (not the DM's +own reply) carries ``{"fallback_recovered": true}`` so behavioral tallies can later +discount masked-dead beats. + +Bash helpers run under ``/bin/bash`` exactly as the play/QA wrappers invoke them (macOS +system bash 3.2-clean), mirroring tests/test_dm_reply_engine_logged.py. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +import recap +import server +import wrapper_progress +from models import SessionLogEntry + +ENGINE_DIR = Path(__file__).resolve().parents[1] +REPO_ROOT = ENGINE_DIR.parents[1] +LIB = REPO_ROOT / "qa" / "lib_beat_driver.sh" + +OPENING = wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE +MOVES = list(wrapper_progress.WRAPPER_MOVE_PROGRESS_LINES) +PROSE_1 = "You step into the Heapside warren as lamplight gutters along the brick." +PROSE_2 = "Mirelda lowers her voice; the ledger between you suddenly feels heavier." + + +@pytest.fixture +def state(tmp_path, monkeypatch): + monkeypatch.setenv("CLAWDND_STATE_DIR", str(tmp_path)) + return tmp_path + + +@pytest.fixture +def cid(state): + return server.start_adventure("cellar-rats")["campaign_id"] + + +def _bash(script: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["/bin/bash", "-c", script], capture_output=True, text=True, cwd=str(REPO_ROOT) + ) + + +def _session_rows(state: Path, cid: str) -> list[dict]: + rows: list[dict] = [] + for p in sorted((state / "campaigns" / cid / "sessions").glob("*.jsonl")): + for line in p.read_text(encoding="utf-8").splitlines(): + if line.strip(): + rows.append(json.loads(line)) + return rows + + +def _chat_rows(chat: Path) -> list[dict]: + return [ + json.loads(line) + for line in chat.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +# --- (b) recap: "Previously on…" must not recite the heartbeat filler ----------------------- + + +def test_recap_excludes_wrapper_lines_store_backed(cid): + server.log_event(cid, "narration", OPENING) + server.log_event(cid, "narration", PROSE_1) + server.log_event(cid, "narration", MOVES[0]) + server.log_event(cid, "narration", PROSE_2) + out = server.session_recap(cid)["recap"] + assert "Heapside" in out # real prose surfaces + assert "Mirelda" in out + assert OPENING not in out + for m in MOVES: + assert m not in out, f"recap recited wrapper filler: {m!r}" + + +def test_format_recap_unit_excludes_wrapper_lines(): + entries = [ + SessionLogEntry(t=1.0, kind="narration", text=OPENING), + SessionLogEntry(t=2.0, kind="narration", text=PROSE_1), + SessionLogEntry(t=3.0, kind="narration", text=MOVES[1]), + SessionLogEntry(t=4.0, kind="dialogue", text="Stay close.", speaker="Lyra"), + ] + out = recap.format_recap(entries) + assert "Stay close." in out + assert OPENING not in out and MOVES[1] not in out + + +def test_format_recap_only_wrapper_lines_is_new_adventure(): + entries = [ + SessionLogEntry(t=float(i), kind="narration", text=line) + for i, line in enumerate([OPENING, *MOVES]) + ] + out = recap.format_recap(entries) + assert "start of a new adventure" in out.lower() + + +# --- (b) FTS ledger: recall must not index the heartbeat filler ----------------------------- + + +def test_fts_recall_excludes_wrapper_lines(cid): + server.log_event(cid, "narration", OPENING) + server.log_event(cid, "narration", MOVES[2]) + server.log_event(cid, "narration", "The lich raised a barrow-wight from the ashen mound.") + # A query built from the wrapper line's own words must not surface it… + hits = server.recall(cid, "scene gathers voices risks choices focus")["hits"] + wrapper_texts = set(wrapper_progress.WRAPPER_PROGRESS_LINES) + assert not any(h["text"].strip() in wrapper_texts for h in hits), hits + # …while real prose stays recallable (the index itself is intact). + hits = server.recall(cid, "lich barrow wight")["hits"] + assert any("lich" in h["text"].lower() for h in hits) + + +# --- (b) lean re-ground tail: scene_context must not feed filler back as canon -------------- + + +def test_scene_context_recent_narration_excludes_wrapper_lines(cid): + server.log_event(cid, "narration", OPENING) + server.log_event(cid, "narration", PROSE_1) + for m in MOVES: + server.log_event(cid, "narration", m) + server.log_event(cid, "narration", PROSE_2) + tail = server.scene_context(cid, recent_narration=10)["recent_narration"] + texts = [t["text"] for t in tail] + assert texts == [PROSE_1, PROSE_2], ( + "the lean re-ground tail must carry ONLY real prose — wrapper heartbeat filler is " + f"not the DM's canon. Got: {texts}" + ) + + +# --- (d) dedup interaction: the #727 guard must not swallow a cadence-aligned heartbeat ----- + + +def _log_engine_narration_script(state_dir: Path, text: str, cid: str, times: int) -> str: + calls = "\n".join( + f'log_engine_narration {cid!r} {text!r} || echo "CALL_FAILED" >&2' for _ in range(times) + ) + return ( + f'set -u; ROOT="{REPO_ROOT}"; STATE_DIR="{state_dir}"; CHAT="{state_dir}/chat.jsonl"; . "{LIB}"\n' + f"{calls}\n" + ) + + +def test_dedup_guard_never_drops_a_repeated_heartbeat(state, cid): + """A run of dead beats logs ONLY heartbeats: rotation of 4 ⇒ beat N+4 repeats beat N's + exact text inside the #727 last-8 tail. The guard must let the repeat through (it is a + liveness signal, not duplicated prose) — otherwise the player's spinner never flips on + cadence-aligned beats.""" + r = _bash(_log_engine_narration_script(state, MOVES[0], cid, times=2)) + assert r.returncode == 0, r.stderr + assert "CALL_FAILED" not in r.stderr + rows = [e for e in _session_rows(state, cid) if e.get("text") == MOVES[0]] + assert len(rows) == 2, ( + f"the #727 dedup guard swallowed a repeated heartbeat (got {len(rows)} rows) — " + "cadence-aligned beats lose their liveness signal" + ) + + +def test_dedup_guard_still_dedups_real_prose(state, cid): + """The #727 guard's actual job is untouched: the DM's own echoed prose logs ONCE.""" + r = _bash(_log_engine_narration_script(state, PROSE_1, cid, times=2)) + assert r.returncode == 0, r.stderr + rows = [e for e in _session_rows(state, cid) if e.get("text") == PROSE_1] + assert len(rows) == 1, f"real prose must still dedup (got {len(rows)} rows)" + + +# --- (c) fallback honesty: recovered prose is flagged on the chat row ----------------------- + + +def _seed_fallback_campaign(state: Path, prose: str) -> None: + """A minimal snapshot + session log shaped exactly like the engine's on-disk format, + so clawdnd_dm_narration_or_fallback (via qa/dm_narration_fallback.py) recovers prose.""" + camp = state / "campaigns" / "c1" + (camp / "sessions").mkdir(parents=True) + (camp / "snapshot.json").write_text( + json.dumps({"id": "c1", "active_session_id": "s1", "day": 1}), encoding="utf-8" + ) + (camp / "sessions" / "s1.jsonl").write_text( + json.dumps({"t": 1.0, "kind": "narration", "text": prose}) + "\n", encoding="utf-8" + ) + + +def test_resolve_dm_reply_flags_recovery_and_stamps_chat_row(tmp_path): + """Empty DM reply + engine-logged prose ⇒ the resolved reply is the recovered prose AND + the dm chat row carries fallback_recovered:true (so tallies can discount masked beats).""" + _seed_fallback_campaign(tmp_path, PROSE_1) + chat = tmp_path / "chat.jsonl" + script = ( + f'set -u; ROOT="{REPO_ROOT}"; STATE_DIR="{tmp_path}"; CHAT="{chat}"; . "{LIB}"\n' + f'clawdnd_resolve_dm_reply "" "$STATE_DIR"\n' + f'echo "recovered=$CLAWDND_FALLBACK_RECOVERED"\n' + f'clawdnd_chatlog_dm "$CLAWDND_DM_REPLY"\n' + ) + r = _bash(script) + assert r.returncode == 0, r.stderr + assert "recovered=1" in r.stdout + rows = _chat_rows(chat) + assert rows == [{"role": "dm", "text": PROSE_1, "fallback_recovered": True}], rows + + +def test_resolve_dm_reply_no_flag_when_dm_replied(tmp_path): + """A DM that ended on its own prose is NOT flagged — the row stays byte-identical to + the legacy {role,text} shape.""" + _seed_fallback_campaign(tmp_path, PROSE_1) + chat = tmp_path / "chat.jsonl" + script = ( + f'set -u; ROOT="{REPO_ROOT}"; STATE_DIR="{tmp_path}"; CHAT="{chat}"; . "{LIB}"\n' + f'clawdnd_resolve_dm_reply {PROSE_2!r} "$STATE_DIR"\n' + f'echo "recovered=$CLAWDND_FALLBACK_RECOVERED"\n' + f'clawdnd_chatlog_dm "$CLAWDND_DM_REPLY"\n' + ) + r = _bash(script) + assert r.returncode == 0, r.stderr + assert "recovered=0" in r.stdout + assert _chat_rows(chat) == [{"role": "dm", "text": PROSE_2}] + + +def test_record_dm_reply_failure_path_carries_flag_and_consumes_it(tmp_path): + """record_dm_reply merges fallback_recovered into BOTH its branches; here the engine-log + failure path (blank campaign id). The flag is consume-once: the next row is unflagged.""" + chat = tmp_path / "chat.jsonl" + script = ( + f'set -u; ROOT="{REPO_ROOT}"; STATE_DIR="{tmp_path}"; CHAT="{chat}"; . "{LIB}"\n' + f"CLAWDND_FALLBACK_RECOVERED=1\n" + f'record_dm_reply "" {PROSE_1!r} beat\n' + f'record_dm_reply "" {PROSE_2!r} beat\n' + ) + r = _bash(script) + assert r.returncode == 0, r.stderr + rows = _chat_rows(chat) + assert rows[0] == {"role": "dm", "text": PROSE_1, "fallback_recovered": True}, rows + assert rows[1] == {"role": "dm", "text": PROSE_2}, rows + + +def test_record_dm_reply_success_path_carries_both_flags(state, cid): + """SUCCESS path: recovered prose that also engine-logs carries engine_logged AND + fallback_recovered (the client de-dups it; the tally can still discount it).""" + chat = state / "chat.jsonl" + script = ( + f'set -u; ROOT="{REPO_ROOT}"; STATE_DIR="{state}"; CHAT="{chat}"; . "{LIB}"\n' + f"CLAWDND_FALLBACK_RECOVERED=1\n" + f"record_dm_reply {cid!r} {PROSE_1!r} beat\n" + ) + r = _bash(script) + assert r.returncode == 0, r.stderr + rows = _chat_rows(chat) + assert rows == [ + {"role": "dm", "text": PROSE_1, "engine_logged": True, "fallback_recovered": True} + ], rows diff --git a/servers/engine/tests/test_wrapper_progress_sync.py b/servers/engine/tests/test_wrapper_progress_sync.py new file mode 100644 index 00000000..138697af --- /dev/null +++ b/servers/engine/tests/test_wrapper_progress_sync.py @@ -0,0 +1,129 @@ +"""#749 SYNC TEST — the wrapper progress-heartbeat lines must be IDENTICAL everywhere. + +The wrapper-authored heartbeat (#743) emits a small rotation of "the scene is arriving" +lines from the play/QA bash wrappers, and FOUR other surfaces must agree on those exact +strings or the repair silently regresses: + + - ``servers/engine/wrapper_progress.py`` — the ONE python source of truth; the engine's + memory filters (recap / FTS ledger / lean re-ground tail) exact-match against it. + - ``qa/wrapper_progress_lines.py`` — the qa-side accessor (re-export) used by + ``qa/dm_narration_fallback.py`` (a stdlib-only standalone script). + - ``viewer/openworlds/screen-table.jsx`` — ``_WRAPPER_PROGRESS_LINES`` (sanitize drops + these from rendering; app.jsx flips the live-progress state on them at /events ingest). + - ``qa/lib_beat_driver.sh`` — ``CLAWDND_OPENING_PROGRESS_TEXT`` + + ``CLAWDND_MOVE_PROGRESS_TEXTS`` (the claude-DM wrappers' emit rotation). + - ``scripts/play_codex_dm.sh`` — ``OPENING_PROGRESS_TEXT`` + + ``MOVE_PROGRESS_TEXTS`` (the codex-DM wrapper's emit rotation). + +If ANY side drifts (a reworded teaser, an added rotation line), the heartbeat would either +render as canned prose to the player or leak into engine memory — so this test parses the +jsx and both shell files with regexes and FAILS on the first divergent byte. +""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path + +import wrapper_progress + +ENGINE_DIR = Path(__file__).resolve().parents[1] # servers/engine +REPO_ROOT = ENGINE_DIR.parents[1] # repo root +QA_SHIM = REPO_ROOT / "qa" / "wrapper_progress_lines.py" +SCREEN_TABLE = REPO_ROOT / "viewer" / "openworlds" / "screen-table.jsx" +LIB_BEAT_DRIVER = REPO_ROOT / "qa" / "lib_beat_driver.sh" +PLAY_CODEX_DM = REPO_ROOT / "scripts" / "play_codex_dm.sh" + + +def _quoted_strings(block: str) -> list[str]: + """All double-quoted string literals in a source block, in order.""" + return [m.group(1) for m in re.finditer(r'"((?:[^"\\]|\\.)*)"', block)] + + +def _jsx_lines() -> list[str]: + src = SCREEN_TABLE.read_text(encoding="utf-8") + m = re.search(r"_WRAPPER_PROGRESS_LINES\s*=\s*new Set\(\[(.*?)\]\)", src, re.S) + assert m, "screen-table.jsx no longer defines _WRAPPER_PROGRESS_LINES = new Set([...])" + return _quoted_strings(m.group(1)) + + +def _sh_rotation(path: Path, opening_var: str, moves_var: str) -> tuple[str, list[str]]: + src = path.read_text(encoding="utf-8") + mo = re.search(rf'^{opening_var}="([^"]*)"', src, re.M) + assert mo, f"{path.name} no longer defines {opening_var}" + mm = re.search(rf"^{moves_var}=\((.*?)^\)", src, re.M | re.S) + assert mm, f"{path.name} no longer defines {moves_var}=( ... )" + moves = _quoted_strings(mm.group(1)) + assert moves, f"{path.name}: empty {moves_var} rotation" + return mo.group(1), moves + + +# --- the python canonical is sane --------------------------------------------------------- + + +def test_python_canonical_shape(): + assert wrapper_progress.WRAPPER_PROGRESS_LINES[0] == wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE + assert ( + tuple(wrapper_progress.WRAPPER_PROGRESS_LINES[1:]) + == tuple(wrapper_progress.WRAPPER_MOVE_PROGRESS_LINES) + ) + # No duplicates — a duplicate would silently shrink the jsx Set comparison. + assert len(set(wrapper_progress.WRAPPER_PROGRESS_LINES)) == len(wrapper_progress.WRAPPER_PROGRESS_LINES) + + +def test_is_wrapper_progress_line_exact_trim_match(): + line = wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE + assert wrapper_progress.is_wrapper_progress_line(line) + assert wrapper_progress.is_wrapper_progress_line(f" {line}\n") # trim, like the jsx .trim() + assert not wrapper_progress.is_wrapper_progress_line(line + " More prose.") # exact, not substring + assert not wrapper_progress.is_wrapper_progress_line("You step into the warren.") + assert not wrapper_progress.is_wrapper_progress_line("") + assert not wrapper_progress.is_wrapper_progress_line(None) + + +# --- cross-surface sync ------------------------------------------------------------------- + + +def test_jsx_set_matches_python(): + assert set(_jsx_lines()) == set(wrapper_progress.WRAPPER_PROGRESS_LINES), ( + "screen-table.jsx _WRAPPER_PROGRESS_LINES drifted from servers/engine/wrapper_progress.py" + ) + + +def test_jsx_exports_shared_window_constant(): + src = SCREEN_TABLE.read_text(encoding="utf-8") + assert "window.isWrapperProgressLine" in src, ( + "screen-table.jsx must export window.isWrapperProgressLine (app.jsx's /events ingest uses it)" + ) + assert "window.WRAPPER_PROGRESS_LINES" in src, ( + "screen-table.jsx must export window.WRAPPER_PROGRESS_LINES (the shared constant)" + ) + + +def test_lib_beat_driver_rotation_matches_python(): + opening, moves = _sh_rotation( + LIB_BEAT_DRIVER, "CLAWDND_OPENING_PROGRESS_TEXT", "CLAWDND_MOVE_PROGRESS_TEXTS" + ) + assert opening == wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE + assert moves == list(wrapper_progress.WRAPPER_MOVE_PROGRESS_LINES), ( + "qa/lib_beat_driver.sh CLAWDND_MOVE_PROGRESS_TEXTS drifted from wrapper_progress.py " + "(order matters: the emit rotation indexes by beat)" + ) + + +def test_play_codex_dm_rotation_matches_python(): + opening, moves = _sh_rotation(PLAY_CODEX_DM, "OPENING_PROGRESS_TEXT", "MOVE_PROGRESS_TEXTS") + assert opening == wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE + assert moves == list(wrapper_progress.WRAPPER_MOVE_PROGRESS_LINES), ( + "scripts/play_codex_dm.sh MOVE_PROGRESS_TEXTS drifted from wrapper_progress.py" + ) + + +def test_qa_shim_reexports_canonical(): + spec = importlib.util.spec_from_file_location("qa_wrapper_progress_lines", QA_SHIM) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + assert tuple(mod.WRAPPER_PROGRESS_LINES) == tuple(wrapper_progress.WRAPPER_PROGRESS_LINES) + assert mod.is_wrapper_progress_line(wrapper_progress.WRAPPER_OPENING_PROGRESS_LINE) + assert not mod.is_wrapper_progress_line("You step into the warren.") diff --git a/servers/engine/wrapper_progress.py b/servers/engine/wrapper_progress.py new file mode 100644 index 00000000..a46b27a1 --- /dev/null +++ b/servers/engine/wrapper_progress.py @@ -0,0 +1,52 @@ +"""Wrapper-authored progress-heartbeat lines — the ONE python source of truth (#749). + +The play/QA wrappers (qa/lib_beat_driver.sh, scripts/play_codex_dm.sh) write a short +canned "the scene is arriving" ``kind=narration`` row to the engine session log BEFORE +the DM model starts (#743), so the viewer's /events poll has a row to flip the player's +spinner within ~1s of a move. Those rows are a LIVENESS SIGNAL, not story: + + - the viewer must flip its live-progress state on them and NEVER render them + (viewer/openworlds/app.jsx /events ingest + screen-table.jsx sanitize), and + - the engine's memory consumers must EXCLUDE them — recap would otherwise recite the + filler in "Previously on…", the FTS ledger would index it for recall, the lean + re-ground tail (``scene_context``'s ``recent_narration``) would tell the DM the + filler is its own canon, and qa/dm_narration_fallback.py would recover it as a + dead beat's "prose". + +Every surface exact-matches against THIS module (qa/wrapper_progress_lines.py re-exports +it for the stdlib-only qa scripts; the jsx + sh copies are pinned byte-identical by +tests/test_wrapper_progress_sync.py — edit the rotation HERE first, then mirror it in +screen-table.jsx, lib_beat_driver.sh and play_codex_dm.sh or that sync test fails). + +Matching is EXACT on the trimmed line (mirroring screen-table.jsx's ``.trim()`` set +lookup), never substring — real DM prose that merely *mentions* a teaser survives. +Pure constants: no engine imports, safe for any consumer. +""" + +from __future__ import annotations + +WRAPPER_OPENING_PROGRESS_LINE = ( + "The first scene gathers around you; voices, risks, and choices come into focus." +) + +# Continuing-beat rotation — ORDER MATTERS (the wrappers index it by beat number). +WRAPPER_MOVE_PROGRESS_LINES: tuple[str, ...] = ( + "Your choice takes hold; nearby voices, risks, and consequences begin to answer.", + "The world turns with your action; the scene shifts toward its answer.", + "Your move lands; attention gathers around what changes next.", + "Momentum carries through the scene; consequences are beginning to surface.", +) + +WRAPPER_PROGRESS_LINES: tuple[str, ...] = ( + WRAPPER_OPENING_PROGRESS_LINE, + *WRAPPER_MOVE_PROGRESS_LINES, +) + +_WRAPPER_PROGRESS_SET = frozenset(WRAPPER_PROGRESS_LINES) + + +def is_wrapper_progress_line(text: object) -> bool: + """True iff ``text`` is exactly (after trimming) a wrapper-authored heartbeat line.""" + if not isinstance(text, str): + return False + return text.strip() in _WRAPPER_PROGRESS_SET diff --git a/viewer/openworlds/app.jsx b/viewer/openworlds/app.jsx index dae8266f..ea1d329e 100644 --- a/viewer/openworlds/app.jsx +++ b/viewer/openworlds/app.jsx @@ -693,7 +693,22 @@ function useLiveSession(state) { .map((e) => { const kind = (e && (e.kind || e.type)) || "narration"; if (kind !== "narration" && kind !== "dialogue") return null; - const clean = sanitize(e && (e.text || e.detail)); + const raw = e && (e.text || e.detail); + // #749: the wrapper progress heartbeat (#743) — a canned "the scene is arriving" + // row the play/QA wrappers log BEFORE the DM model starts — is a LIVENESS signal, + // not prose. It must flip the pending turn's streaming/progress state and NEVER + // render. This check runs BEFORE the sanitize-drop below (sanitize excises the + // exact wrapper lines, so the old order silently swallowed the row and the + // heartbeat was a no-op for the player). Deliberately NOT setting + // eventsStreamedThisTurnRef / dmBeatCountRef here: no prose streamed, so a + // dead-DM beat's recovered /chat text must still render (it would otherwise be + // suppressed to zero rows). Shared predicate from screen-table.jsx (loaded + // first); if absent we fall through to sanitize — today's drop, no regression. + if (typeof window.isWrapperProgressLine === "function" && window.isWrapperProgressLine(raw)) { + notePendingProgress(); + return null; + } + const clean = sanitize(raw); if (!clean) return null; // #405/BUG2: dedup by the STABLE, SESSION-SCOPED composite key `${sid}:${seq}` — NOT by // prose. So a paragraph re-ingested by a windowing re-mount or a session-rotation cursor diff --git a/viewer/openworlds/screen-table.jsx b/viewer/openworlds/screen-table.jsx index 31720115..f73bb5b3 100644 --- a/viewer/openworlds/screen-table.jsx +++ b/viewer/openworlds/screen-table.jsx @@ -175,6 +175,16 @@ const _WRAPPER_PROGRESS_LINES = new Set([ function _isWrapperProgressLine(line) { return _WRAPPER_PROGRESS_LINES.has((line || "").trim()); } +// #749: share the wrapper-line predicate + constant (window-guarded globals, like +// sanitizeNarration). app.jsx's /events ingest needs the predicate BEFORE its sanitize-drop: +// a heartbeat row must flip the live-progress state (notePendingProgress) and never render — +// sanitize alone silently swallowed the row, making the #743 heartbeat a no-op for the player. +// The strings themselves are pinned byte-identical to servers/engine/wrapper_progress.py and +// the two sh emit rotations by servers/engine/tests/test_wrapper_progress_sync.py. +if (typeof window !== "undefined") { + window.WRAPPER_PROGRESS_LINES = Array.from(_WRAPPER_PROGRESS_LINES); + window.isWrapperProgressLine = _isWrapperProgressLine; +} function sanitizeNarration(text) { if (typeof text !== "string" || !text) return ""; const kept = text diff --git a/viewer/tests/test_live_narration_stream.py b/viewer/tests/test_live_narration_stream.py index 823d01c5..1a9602ef 100644 --- a/viewer/tests/test_live_narration_stream.py +++ b/viewer/tests/test_live_narration_stream.py @@ -352,6 +352,52 @@ def test_streamed_beat_marks_pending_streaming(self): self.assertFalse(out["stuck"], "fresh streamed prose proves the turn is alive — it must not be stuck") + # --- #749 FIX 1: a wrapper heartbeat row flips the progress state WITHOUT rendering -------- + # The #743 wrapper heartbeat writes a canned "the scene is arriving" narration row to the + # session log BEFORE the DM model starts, precisely so the player's spinner flips within ~1s. + # But sanitize() drops those exact lines (screen-table's _isWrapperProgressLine), and the old + # ingest sanitized BEFORE notePendingProgress — so the heartbeat was a complete no-op for the + # player. Now the ingest special-cases the wrapper lines first: flip streaming, never render. + def test_wrapper_heartbeat_flips_progress_without_rendering(self): + out = self._run( + "h.arm('I push open the door');" + # The wrapper heartbeat row, exactly as lib_beat_driver emits it (read from the SHARED + # window constant so this test can never drift from the real rotation). + "var hb = sandbox.window.WRAPPER_PROGRESS_LINES[1];" + "h.enqueue('/events', { entries: [{ kind: 'narration', text: hb, seq: 0 }], next: 1 });" + "await h.tick();" + "var p = h.pending();" + "return ({ texts: h.narrationTexts(), pending_present: !!p, streaming: !!(p && p.streaming), stuck: !!(p && p.stuck) });" + ) + self.assertEqual(out["texts"], [], + "the wrapper heartbeat is a liveness signal — it must NEVER render as prose") + self.assertTrue(out["pending_present"], + "the heartbeat must not resolve the turn — the bar stays gated") + self.assertTrue(out["streaming"], + "the heartbeat's whole purpose: flip the pending turn to `streaming` (the " + "spinner reads 'the scene is arriving') — it must call notePendingProgress") + self.assertFalse(out["stuck"], "a heartbeat proves the turn is alive") + + # --- #749 GUARD: a heartbeat must NOT count as 'this turn streamed prose' ------------------- + # eventsStreamedThisTurnRef suppresses the turn-END /chat copy of ALREADY-STREAMED prose. A + # heartbeat streams no prose — if it set that flag, a dead-DM beat whose only /events row was + # the heartbeat would suppress the recovered /chat text to ZERO rendered rows. + def test_wrapper_heartbeat_does_not_suppress_chat_only_prose(self): + out = self._run( + "h.arm('I push open the door');" + "var hb = sandbox.window.WRAPPER_PROGRESS_LINES[2];" + "h.enqueue('/events', { entries: [{ kind: 'narration', text: hb, seq: 0 }], next: 1 });" + "await h.tick();" + # turn-END: the beat's prose arrives ONLY on /chat (nothing real streamed). + "h.enqueue('/chat', { items: [{ role: 'dm', text: 'You awaken to birdsong.' }], next: 1 });" + "await h.tick();" + "return ({ texts: h.narrationTexts(), pending: h.pending() });" + ) + self.assertEqual(out["texts"], ["You awaken to birdsong."], + "a heartbeat-only stream must not mark the turn as 'streamed' — the " + "/chat-only prose would be suppressed to zero rows") + self.assertIsNone(out["pending"], "the /chat DM line still resolves the turn") + # A fresh turn must NOT inherit the prior turn's `streaming` flag — armPending starts a clean # pending object, so the affordance re-derives "the scene is arriving" from THIS turn's own # /events arrivals (otherwise every later turn would falsely claim it's already streaming).