fix(qa+engine): make the beat heartbeat real — flip progress at ingest, decontaminate recap/FTS/lean-tail/fallback (#749)

qa/dm_narration_fallback.py

@@ -38,6 +38,18 @@
 import re
 import sys
 
+# #749: the wrapper progress heartbeat ("Your move lands; attention gathers…") is a canned
+# liveness row the play/QA wrappers log BEFORE the DM turn — never recoverable prose. The
+# shared constants live in servers/engine/wrapper_progress.py (re-exported by the sibling
+# qa/wrapper_progress_lines.py). This script is best-effort plumbing in the beat path, so a
+# missing shim degrades to "no wrapper filtering" rather than a crash; the engine test suite
+# (tests/test_dm_narration_fallback.py) proves the import works in-repo.
+try:
+    from wrapper_progress_lines import is_wrapper_progress_line
+except Exception:  # pragma: no cover - only on a broken checkout
+    def is_wrapper_progress_line(_text):
+        return False
+
 # Engine session-log kinds (SessionLogEntry.kind): narration | dialogue | roll | system | combat.
 # Only narration + dialogue are player-facing prose; the rest are bookkeeping the player never reads.
 PROSE_KINDS = {"narration", "dialogue"}
@@ -123,7 +135,14 @@ def _recover(snap_path):
                 # scratchpad, not a scene — treat it like bookkeeping: it breaks the trailing
                 # block and is never recovered (showing the player notation is worse than blank).
                 # Dialogue rows are always a quoted character line, so they're never system-notation.
-                if kind == "narration" and text and _is_system_notation(text):
+                # #749: the wrapper progress heartbeat gets the SAME treatment — it is canned
+                # liveness filler logged BEFORE the DM turn. Crucially it BREAKS the block (it is
+                # not transparently skipped): a heartbeat-only (dead) beat must recover NOTHING,
+                # because stitching the PRIOR beat's stale prose under a fresh heartbeat would
+                # mask the dead beat as 'resolved'.
+                if kind == "narration" and text and (
+                    _is_system_notation(text) or is_wrapper_progress_line(text)
+                ):
                     block = []
                     continue
                 if kind in PROSE_KINDS and text:

qa/lib_beat_driver.sh

@@ -126,6 +126,24 @@ clawdnd_dm_narration_or_fallback() {
   fi
 }
 
+# clawdnd_resolve_dm_reply REPLY STATE_DIR — the DIRECT-call front door over
+# clawdnd_dm_narration_or_fallback (#749c fallback honesty). Sets:
+#   CLAWDND_DM_REPLY            — the resolved reply text (the DM's own, or the recovered prose)
+#   CLAWDND_FALLBACK_RECOVERED  — 1 IFF the #357 fallback recovered the prose (the DM's reply was
+#                                 blank and the engine log supplied it), else 0
+# Call it DIRECTLY (never in a command substitution — a subshell would drop both globals), then
+# read CLAWDND_DM_REPLY. The flag is consumed (and reset) by the next record_dm_reply /
+# clawdnd_chatlog_dm, which stamps {"fallback_recovered":true} on the dm chat row so behavioral
+# tallies can later discount a masked-dead beat that was "resolved" with recovered prose.
+clawdnd_resolve_dm_reply() {
+  local original="$1"
+  CLAWDND_DM_REPLY="$(clawdnd_dm_narration_or_fallback "$1" "$2")"
+  CLAWDND_FALLBACK_RECOVERED=0
+  if [ -z "${original//[[:space:]]/}" ] && [ -n "${CLAWDND_DM_REPLY//[[:space:]]/}" ]; then
+    CLAWDND_FALLBACK_RECOVERED=1
+  fi
+}
+
 # CHRONICLE WRITE + ENGINE-LOG TRUTHFULNESS GUARD (issue #720 — the ONE shared impl).
 #
 # The cold-open opening prose lands in TWO viewer-read sources: the engine per-session log
@@ -199,6 +217,7 @@ import os
 import sys
 
 import server
+import wrapper_progress
 
 campaign_id, text = sys.argv[1], sys.argv[2]
 norm = " ".join(text.split())
@@ -229,6 +248,16 @@ try:
 except Exception:
     already = False  # any read failure → fall through to a normal append (no regression)
 
+# #749(d): the wrapper progress heartbeat legitimately REPEATS — the 4-line rotation means
+# beat N+4 re-emits beat N's exact text, and a run of DEAD beats logs ONLY heartbeats, so the
+# repeat always sits inside the last-8 tail scanned above. It is a liveness signal, never
+# duplicated prose: dedup-dropping it silently stops the player's spinner from flipping on
+# cadence-aligned beats (exactly when the heartbeat is the only life sign). The extra rows are
+# inert — they never render (app.jsx flips progress and returns null) and every engine memory
+# consumer exact-match filters them (recap / FTS ledger / lean tail / narration fallback).
+if wrapper_progress.is_wrapper_progress_line(text):
+    already = False
+
 if not already:
     server.log_event(campaign_id, "narration", text)
 PY
@@ -239,14 +268,35 @@ PY
 # de-dup the /chat blob against /events). On failure → an UNFLAGGED row (byte-identical to the
 # pre-#720 behavior; the client's eventsStreamedThisTurnRef backstop still applies). NEVER stamp
 # the flag unconditionally — that would suppress a legitimately /chat-only beat to zero rows.
+# #749(c): when the preceding clawdnd_resolve_dm_reply recovered this prose from the engine log
+# (CLAWDND_FALLBACK_RECOVERED=1), BOTH branches additionally stamp fallback_recovered:true so
+# behavioral tallies can discount masked-dead beats. Consume-once: the flag resets here.
 record_dm_reply() {
-  local campaign_id="$1" text="$2" phase="$3"
+  local campaign_id="$1" text="$2" phase="$3" extra='{"engine_logged":true}' plain_extra=''
+  if [ "${CLAWDND_FALLBACK_RECOVERED:-0}" = "1" ]; then
+    extra='{"engine_logged":true,"fallback_recovered":true}'
+    plain_extra='{"fallback_recovered":true}'
+  fi
   if log_engine_narration "$campaign_id" "$text"; then
-    chatlog dm "$text" '{"engine_logged":true}'
+    chatlog dm "$text" "$extra"
   else
     echo "[worldos] warning: could not record ${phase} narration through engine — chat row written without engine_logged" >&2
-    chatlog dm "$text"
+    chatlog dm "$text" "$plain_extra"
+  fi
+  CLAWDND_FALLBACK_RECOVERED=0
+}
+
+# clawdnd_chatlog_dm TEXT — `chatlog dm TEXT` for the runners that write the dm row directly
+# (run_duo / run_party / ui_playtest), stamping {"fallback_recovered":true} when the preceding
+# clawdnd_resolve_dm_reply recovered the prose (#749c). Consume-once, mirroring record_dm_reply.
+# With the flag unset the row is byte-identical to the plain `chatlog dm` it replaces.
+clawdnd_chatlog_dm() {
+  if [ "${CLAWDND_FALLBACK_RECOVERED:-0}" = "1" ]; then
+    chatlog dm "$1" '{"fallback_recovered":true}'
+  else
+    chatlog dm "$1"
   fi
+  CLAWDND_FALLBACK_RECOVERED=0
 }
 
 # LIVE-PROGRESS + WRAPPER HEARTBEAT (#623 — the ONE shared implementation of the perceived-latency fix).
@@ -304,11 +354,15 @@ clawdnd_progress_beat_text() {
 }
 
 # clawdnd_emit_progress_heartbeat CAMPAIGN_ID FIRST BEAT_INDEX — write the wrapper-authored progress beat
-# to the engine session log (via log_engine_narration) so /events has a row to render BEFORE the model's
-# long think. Best-effort + idempotent: a blank campaign id no-ops; log_engine_narration's substring guard
-# (it already de-dups against the recent narration tail) means a heartbeat the DM happens to echo isn't
-# double-logged. ALWAYS returns 0 — a heartbeat failure must never fail a beat. Reads ambient $STATE_DIR /
-# $ROOT exactly as log_engine_narration / record_dm_reply do. $1=campaign_id $2=first?(1/0) $3=beat index.
+# to the engine session log (via log_engine_narration) so /events has a row for the viewer to flip its
+# live-progress state on BEFORE the model's long think (the row itself never renders — app.jsx returns
+# null on the exact wrapper lines). Best-effort: a blank campaign id no-ops. #749(d): heartbeats are
+# EXEMPT from log_engine_narration's #727 substring dedup (the 4-line rotation legitimately repeats on
+# cadence-aligned beats, and a run of dead beats logs ONLY heartbeats — dropping the repeat would kill
+# the only liveness signal); the repeated rows are inert (never rendered, filtered from recap/FTS/lean
+# tail/fallback). ALWAYS returns 0 — a heartbeat failure must never fail a beat. Reads ambient
+# $STATE_DIR / $ROOT exactly as log_engine_narration / record_dm_reply do. $1=campaign_id $2=first?(1/0)
+# $3=beat index.
 clawdnd_emit_progress_heartbeat() {
   local campaign_id="$1" first="${2:-0}" idx="${3:-0}" text
   [ -n "${campaign_id//[[:space:]]/}" ] || return 0

qa/run_duo.sh

@@ -244,10 +244,10 @@ $PMSG
 Do the setup now: start_world(\"$WORLD\"), start_session, then seat THEIR character as the PLAYER CHARACTER (the PC). The player ALWAYS plays a REAL, LIVING CANON NPC — their persona names one (e.g. Aubree, a Flaming Fist ranger). Seat that exact figure via load_canon_character(their canon name, kind=\"player\", add_to_party=true) so they get a real backstory + ingested portrait — NEVER create_character / invent a custom PC, NEVER seat the player's own character as a companion or NPC, and NEVER a canon-DEAD figure (a corpse like Dal Lightspark is rejected as a PC; if the seat returns an error, pick a living canon NPC instead). A companion is a DIFFERENT character the player MEETS. Then OPEN the scene — human-scale and personal — grounded in the world's canon, responding to their stated intent. A companion should ENTER as part of that opening scene: someone the player MEETS on-screen (voiced, with a real wound and a reason they fall in together) — recruit_companion / load_canon_character(kind=\"companion\") as that meeting lands, NOT a silent name dropped into the party before the player has met anyone. End by handing the moment to the player. OUTPUT DISCIPLINE — your final reply IS the opening scene: write it as 2nd-person in-fiction PROSE + quoted dialogue ONLY. NEVER narrate your own setup/process — no \"State is grounded\", no \"the cold open is on the dashboard\", no \"Closing my turn on the scene\", no 3rd-person status line. The very first words the player reads must be INSIDE the fiction.")"
 # #357: recover the engine's logged narration if the DM turn ended on a tool call / status
 # line (empty final reply) — so a tool-final-but-narrated turn isn't mistaken for silence.
-DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")"
+clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
 echo "[duo] DM opened: ${DMSG:0:120}…"
 [ -z "$DMSG" ] && { echo "[duo] DM produced no opening — aborting (see $COMBINED)" >&2; exit 1; }
-chatlog dm "$DMSG"
+clawdnd_chatlog_dm "$DMSG"
 
 # Resolve the campaign id the cold open just minted (for the lean re-ground; harmless when
 # CLAWDND_LEAN_BEATS=0). D1's start_world wrote the snapshot to
@@ -324,10 +324,10 @@ $DIRECTOR
 $EVENT_ADV")"
   # #357: recover engine-logged narration before the silence check, so a turn that ended on a
   # tool call but logged real prose isn't mis-flagged as a silent DM (and isn't blank in chat).
-  DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")"
+  clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
   echo "[duo] beat $b DM: ${DMSG:0:100}…"
   [ -z "$DMSG" ] && { echo "[duo] DM went silent at beat $b; stopping early"; break; }
-  chatlog dm "$DMSG"
+  clawdnd_chatlog_dm "$DMSG"
 
   # C — soft clock-tick backstop: if the DM didn't move the clock this beat, advance one
   # phase via the engine (sole writer). Defers to the DM when it advanced time in-fiction.

qa/run_party.sh

@@ -304,9 +304,9 @@ $beat0_block
 
 Resolve each declared move through the engine; voice the world and any NPC; let the companions be PRESENT (the player and companions are separate people with their own agency — you narrate the RESULT of their declared moves, never invent a companion's internal choice). End by handing the open moment to the PLAYER.")"
 # #357: recover engine-logged narration if the DM turn ended on a tool call (empty reply).
-DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")"
+clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
 [ -z "$DMSG" ] && { echo "[party] DM produced no opening — aborting (see $COMBINED)" >&2; exit 1; }
-chatlog dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1))
+clawdnd_chatlog_dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1))
 echo "[party] DM opened: ${DMSG:0:120}…"
 
 # --- main loop: player + each living companion act, then the DM resolves the beat ---
@@ -327,10 +327,10 @@ $PARTY_BLOCK
 Then PLAY the next beat as a full lived scene — NOT a fragment: any NPC (or companion) present SPEAKS at least one quoted line in their own voice; let them push back when it's real. Narrate the RESULT of each declared move (never invent a companion's choice). Weave the open moment back to the PLAYER inside the scene — never a bare 'Your move.'")"
   # #357: recover engine-logged narration before the silence check (tool-final-but-narrated
   # turn ≠ silence; keeps the chat non-blank on a resolved beat).
-  DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")"
+  clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
   echo "[party] beat $b DM: ${DMSG:0:120}…"
   [ -z "$DMSG" ] && { echo "[party] DM went silent at beat $b; stopping early"; break; }
-  chatlog dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1))
+  clawdnd_chatlog_dm "$DMSG"; AGENT_TURNS=$((AGENT_TURNS + 1))
 done
 
 # --- wrap + score (same artifacts as run_duo) ---------------------------------------

qa/ui_playtest.sh

@@ -143,9 +143,10 @@ chatlog() { python3 -c 'import json,sys;open(sys.argv[1],"a").write(json.dumps({
 # never RESOLVED on the client, leaving the player on the (now-fixed-but-slower) client stall path with
 # no backend recovery at all. Here we (1) wall-clock the beat with `timeout` (tiered off the cold-open
 # `first` signal via the shared clawdnd_dm_timeout; a frozen process is KILLED at the deadline so the
-# turn returns), and (2) if the killed/failed beat left empty result text, stitch the engine-logged
-# narration tail as a fallback reply (clawdnd_dm_narration_or_fallback) so `chatlog dm` always writes a
-# real turn-END line → the client's pending clears + the bar re-enables. CLAWDND_DM_MODEL lets the
+# turn returns), and (2) if the killed/failed beat left empty result text, the CALLER stitches the
+# engine-logged narration tail as a fallback reply (clawdnd_resolve_dm_reply, which also flags the
+# recovery — #749c) so the dm chat row always carries a real turn-END line → the client's pending
+# clears + the bar re-enables. CLAWDND_DM_MODEL lets the
 # timeout helper pick the opus cold-open tier. Bash 3.2-safe (timeout(1) from coreutils; ${arr[@]+…}).
 CLAWDND_DM_MODEL="$DM_MODEL"
 dm_turn() {
@@ -160,9 +161,11 @@ dm_turn() {
   rc=$?
   [ "$rc" -ne 0 ] && echo "[uipt] DM turn rc=$rc (timeout=${beat_timeout}s) — relying on engine-logged narration fallback" >&2
   cat "$out" >> "$COMBINED"
-  # If the beat was killed/failed with no final result text, recover the engine-logged narration tail so
-  # the turn STILL resolves on /chat (never an indefinite hang). A healthy beat returns its result verbatim.
-  clawdnd_dm_narration_or_fallback "$(jq -rs 'map(select(.type=="result"))[-1].result // ""' "$out" 2>/dev/null)" "$STATE_DIR"
+  # Echo the beat's RAW final result text. The #357 fallback (recover the engine-logged narration
+  # tail when a killed/failed beat left this empty, so the turn STILL resolves on /chat) is applied
+  # by the CALLER via clawdnd_resolve_dm_reply — a direct call, because dm_turn runs in a command
+  # substitution where the #749c fallback_recovered flag (a global) could never escape the subshell.
+  jq -rs 'map(select(.type=="result"))[-1].result // ""' "$out" 2>/dev/null
 }
 
 # --- DM opens the scene so a LIVE, playable game exists (the launcher's Chronicles
@@ -174,8 +177,11 @@ echo "[uipt] DM opening the scene…"
 DMSG="$(dm_turn 1 "$DM_BRIEF
 
 Begin a SOLO session for a brand-new human player in this world: start_world(\"$WORLD\"), start_session, seat a fitting level-3 PLAYER CHARACTER (a LIVING canon figure via load_canon_character(kind=\"player\", add_to_party=true) — NEVER a dead/fallen character; apply sensible skills/spells), and bring in ONE roster companion the player meets in the scene. Then open a human-scale, personal scene with real quoted dialogue and hand the player an open moment + a clear choice. Their actions will arrive next as tagged moves.")"
+# #357/#749c: recover the engine-logged narration tail when the turn died with no result text;
+# a recovered reply stamps fallback_recovered:true on the dm chat row (clawdnd_chatlog_dm).
+clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
 [ -z "$DMSG" ] && echo "[uipt] WARN: DM produced no opening (see $RUNDIR/dm/dm.err) — the player may land in a thin scene." >&2
-chatlog dm "${DMSG:-The scene is set. What do you do?}"
+clawdnd_chatlog_dm "${DMSG:-The scene is set. What do you do?}"
 
 # --- background DM-resolver loop: tail $MOVES, resolve each new move, append narration
 # to $CHAT (the UI shows it via /chat). Identical shape to play_human.sh's loop. Runs
@@ -194,7 +200,9 @@ chatlog dm "${DMSG:-The scene is set. What do you do?}"
 $PMSG
 
 Resolve it through the engine (roll checks, apply casts/attacks, voice NPCs) and narrate the next beat as a played scene. Hand the moment back to the player.")"
-      chatlog dm "${DMSG:-...}"
+      # #357/#749c: same recovery + honesty stamp as the opening turn (direct call, see dm_turn).
+      clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
+      clawdnd_chatlog_dm "${DMSG:-...}"
     else
       sleep 2
     fi

qa/wrapper_progress_lines.py

@@ -0,0 +1,32 @@
+"""qa-side accessor for the wrapper progress-heartbeat lines (#749).
+
+The ONE python source of truth is ``servers/engine/wrapper_progress.py`` (the engine's
+memory filters import it directly; the engine must keep filtering even if it were ever
+deployed without qa/). This shim loads that module by file path — qa scripts run under
+the system ``python3`` with no engine on ``sys.path`` — and re-exports its names so
+stdlib-only qa tools (``qa/dm_narration_fallback.py``) share the exact same constants.
+
+``tests/test_wrapper_progress_sync.py`` pins this re-export (and the jsx/sh copies)
+byte-identical to the canonical module.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+
+_CANONICAL = os.path.normpath(
+    os.path.join(
+        os.path.dirname(os.path.abspath(__file__)),
+        "..", "servers", "engine", "wrapper_progress.py",
+    )
+)
+
+_spec = importlib.util.spec_from_file_location("_worldos_wrapper_progress", _CANONICAL)
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+
+WRAPPER_OPENING_PROGRESS_LINE = _mod.WRAPPER_OPENING_PROGRESS_LINE
+WRAPPER_MOVE_PROGRESS_LINES = _mod.WRAPPER_MOVE_PROGRESS_LINES
+WRAPPER_PROGRESS_LINES = _mod.WRAPPER_PROGRESS_LINES
+is_wrapper_progress_line = _mod.is_wrapper_progress_line

scripts/play.sh

@@ -412,7 +412,7 @@ Their actions will arrive as tagged moves — [say] (their dialogue), [do] (an a
 fi
 # #357: same empty-reply fallback as the beat loop — recover the engine's logged opening
 # narration if the DM's first turn ended on a tool call rather than prose.
-DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")"
+clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
 
 # Resolve the campaign id the DM just minted, BEFORE writing the opening to the chronicle —
 # record_dm_reply (#720) needs it to log the opening narration to the engine session log so the
@@ -496,7 +496,7 @@ $RUNBOOK" "$CAMPAIGN_ID")"
     # #357: if the DM turn ended on a tool call / 3rd-person status line, its final reply text is
     # empty — fall back to the player-facing narration the engine logged this beat so the chat is
     # never blank on a resolved move (engine stays the sole writer; this only READS its log).
-    DMSG="$(clawdnd_dm_narration_or_fallback "$DMSG" "$STATE_DIR")"
+    clawdnd_resolve_dm_reply "$DMSG" "$STATE_DIR"; DMSG="$CLAWDND_DM_REPLY"
     # #720: route the per-beat DM reply through record_dm_reply (engine_logged stamp on success).
     record_dm_reply "$CAMPAIGN_ID" "$DMSG" beat; DM_TURNS=$((DM_TURNS + 1))
     # C — soft clock-tick backstop: advance one phase via the engine only if the DM left the