fix(reliability): recover a stalled-mid-stream DM beat — never lock the player out (#623 deeper layer)

qa/ui_playtest.sh

@@ -136,15 +136,33 @@ echo "[uipt] viewer ready."
 DSID="$(python3 -c 'import uuid;print(uuid.uuid4())')"
 DM_BRIEF="$(cat "$ROOT/qa/play_dm_duo.txt")"
 chatlog() { python3 -c 'import json,sys;open(sys.argv[1],"a").write(json.dumps({"role":sys.argv[2],"text":sys.argv[3]})+"\n")' "$CHAT" "$1" "$2"; }
+# #745 (the newbie mid-stream-stall give-up): the GUI-sweep DM driver MUST bound every beat exactly
+# like scripts/play.sh's dm_turn — previously this helper ran `claude -p` with NO `timeout`, NO retry,
+# and NO fallback, so a DM turn that streamed partial prose via /events and then FROZE mid-generation
+# hung FOREVER: dm_turn never returned → `chatlog dm` (the turn-END /chat line) never fired → the turn
+# never RESOLVED on the client, leaving the player on the (now-fixed-but-slower) client stall path with
+# no backend recovery at all. Here we (1) wall-clock the beat with `timeout` (tiered off the cold-open
+# `first` signal via the shared clawdnd_dm_timeout; a frozen process is KILLED at the deadline so the
+# turn returns), and (2) if the killed/failed beat left empty result text, stitch the engine-logged
+# narration tail as a fallback reply (clawdnd_dm_narration_or_fallback) so `chatlog dm` always writes a
+# real turn-END line → the client's pending clears + the bar re-enables. CLAWDND_DM_MODEL lets the
+# timeout helper pick the opus cold-open tier. Bash 3.2-safe (timeout(1) from coreutils; ${arr[@]+…}).
+CLAWDND_DM_MODEL="$DM_MODEL"
 dm_turn() {
-  local first="$1" msg="$2" out resume=()
+  local first="$1" msg="$2" out resume=() beat_timeout rc
   [ "$first" = "0" ] && resume=(--resume "$DSID") || resume=(--session-id "$DSID")
+  beat_timeout="$(clawdnd_dm_timeout "$first")"
   out="$RUNDIR/dm/turn.$(date +%s%N).jsonl"
-  claude -p "$msg" "${resume[@]}" --plugin-dir "$ROOT" --mcp-config "$DM_CFG" --strict-mcp-config \
+  timeout "$beat_timeout" \
+    claude -p "$msg" "${resume[@]}" --plugin-dir "$ROOT" --mcp-config "$DM_CFG" --strict-mcp-config \
     --model "$DM_MODEL" --permission-mode bypassPermissions --max-budget-usd "$DM_BUDGET" \
     --output-format stream-json --verbose > "$out" 2>> "$RUNDIR/dm/dm.err"
+  rc=$?
+  [ "$rc" -ne 0 ] && echo "[uipt] DM turn rc=$rc (timeout=${beat_timeout}s) — relying on engine-logged narration fallback" >&2
   cat "$out" >> "$COMBINED"
-  jq -rs 'map(select(.type=="result"))[-1].result // ""' "$out" 2>/dev/null
+  # If the beat was killed/failed with no final result text, recover the engine-logged narration tail so
+  # the turn STILL resolves on /chat (never an indefinite hang). A healthy beat returns its result verbatim.
+  clawdnd_dm_narration_or_fallback "$(jq -rs 'map(select(.type=="result"))[-1].result // ""' "$out" 2>/dev/null)" "$STATE_DIR"
 }
 
 # --- DM opens the scene so a LIVE, playable game exists (the launcher's Chronicles

servers/engine/tests/test_dm_session_remint.py

@@ -362,3 +362,52 @@ def test_play_sh_emits_the_heartbeat_before_each_dm_turn():
     assert play.count("clawdnd_emit_progress_heartbeat") >= 2, (
         "play.sh must emit the heartbeat on the cold-open AND per-beat paths"
     )
+
+
+# --- #745: the GUI-sweep DM driver must BOUND every beat (the newbie mid-stream-stall give-up) ----
+# The lone v1.0.4-rc2 RRI holdout @c92a393 (newbie) hit a DM beat that STREAMED partial prose and then
+# FROZE mid-generation. qa/ui_playtest.sh's dm_turn ran `claude -p` with NO `timeout`, NO retry, and NO
+# fallback — unlike scripts/play.sh's dm_turn — so the frozen process hung forever: dm_turn never
+# returned -> `chatlog dm` (the turn-END /chat line) never fired -> the turn never RESOLVED on the
+# client, so the backend offered NO recovery (the client-side stall ceiling, #745 app.jsx, then had to
+# carry the whole burden). These guard that the GUI driver now wall-clocks the beat and falls back to
+# the engine-logged narration tail so a stalled beat always resolves on /chat.
+
+def test_ui_playtest_dm_turn_wraps_claude_in_a_bounded_timeout():
+    """The GUI-sweep dm_turn must wrap `claude -p` in `timeout` (tiered via the shared clawdnd_dm_timeout)
+    so a frozen beat is KILLED at a deadline and dm_turn returns — never an indefinite hang."""
+    src = _src("qa/ui_playtest.sh")
+    # the shared, tiered deadline helper is resolved…
+    assert 'clawdnd_dm_timeout "$first"' in src, "ui_playtest.sh must resolve the per-beat deadline via the shared helper"
+    # …and the claude invocation is wrapped in `timeout "$beat_timeout"` (the line continuation puts the
+    # `claude -p` on the following line, so assert both tokens are present near each other).
+    assert 'timeout "$beat_timeout"' in src, "ui_playtest.sh dm_turn must wall-clock `claude -p` with `timeout`"
+    # a bare, unbounded `claude -p \\` (no timeout on the same logical line) must NOT remain.
+    assert "\n  claude -p " not in src, "ui_playtest.sh must not invoke `claude -p` unbounded (no timeout wrapper)"
+
+
+def test_ui_playtest_dm_turn_falls_back_to_engine_narration_on_a_stalled_beat():
+    """A killed/empty beat must still RESOLVE on /chat: dm_turn routes its result through the shared
+    clawdnd_dm_narration_or_fallback so the engine-logged narration tail becomes the turn-END line."""
+    src = _src("qa/ui_playtest.sh")
+    assert "clawdnd_dm_narration_or_fallback" in src, (
+        "ui_playtest.sh dm_turn must recover via the shared narration fallback so a stalled beat still resolves"
+    )
+
+
+def test_ui_playtest_dm_timeout_is_bash32_clean_and_sourced():
+    """The driver sources the shared lib and clawdnd_dm_timeout resolves a positive integer under the
+    macOS system bash 3.2 — proving the new wiring is 3.2-clean and actually reachable from this script."""
+    lib = ROOT / "qa" / "lib_beat_driver.sh"
+    script = (
+        f'set -u; . "{lib}"\n'
+        # cold-open tier (first=1) and continuing tier (first=0) must both yield a positive integer.
+        'CLAWDND_DM_MODEL=opus; co="$(clawdnd_dm_timeout 1)"; bt="$(clawdnd_dm_timeout 0)"\n'
+        'echo "co=$co bt=$bt"\n'
+        'case "$co" in (*[!0-9]*|"") echo BAD_CO; exit 1;; esac\n'
+        'case "$bt" in (*[!0-9]*|"") echo BAD_BT; exit 1;; esac\n'
+        '[ "$co" -gt 0 ] && [ "$bt" -gt 0 ] && echo OK\n'
+    )
+    r = _bash(script)
+    assert r.returncode == 0, (r.stdout, r.stderr)
+    assert "OK" in r.stdout, (r.stdout, r.stderr)

viewer/openworlds/app.jsx

@@ -228,6 +228,22 @@ window.neutralizeMarkup = window.neutralizeMarkup || function neutralizeMarkup(r
 const PENDING_RECOVERY_MS = 180 * 1000;           // #399: later-beat stall window (worst-case DM turns run ~90–120s; was 90s/#342).
 const PENDING_RECOVERY_FIRST_MS = 4 * 60 * 1000;  // #348: first-beat (Act-opening) window — fits the multi-minute cold open.
 const PENDING_BACKSTOP_MS = 12 * 60 * 1000;       // …with the original hard backstop as a final net.
+// #745 (the newbie mid-stream-stall give-up): a HARD stuck ceiling from submit that flags `stuck` (the
+// recoverable "Try again" affordance) and — unlike the per-progress recovery timer — is NOT reset by
+// streamed progress. Root cause it fixes: notePendingProgress re-arms the FULL recovery window (180s/240s)
+// AND clears `stuck` on EVERY streamed /events paragraph. So a beat that streams several partial paragraphs
+// ("You give the sergeant your own name… The charcoal touches the paper… That's the arithmetic o—") and
+// then FREEZES mid-generation keeps pushing the stuck deadline forward with each partial; with partials
+// <window apart, `stuck` never fires and recovery is deferred to the 12-min PENDING_BACKSTOP_MS — which
+// CLEARS pending to null (a plain re-enabled bar, NO "Try again", the partial narration stranded). That is
+// the ~12–15-min lockout the newbie gave up on. This ceiling bounds TOTAL stall from submit regardless of
+// how many partials trickle in (progress does not reset it), and resolves to the SAME recoverable `stuck`
+// state (the bar re-opens as "Try again"). It is generous enough to clear a worst-case HEALTHY beat (which
+// RESOLVES on /chat → clearPending cancels every timer long before this fires), so it never false-positives
+// on a slow-but-alive turn; it only ever fires on a genuine freeze. Strictly between the position windows
+// and the 12-min null-backstop, so the ordering is: position recovery (resettable) < stuck-backstop (hard,
+// recoverable) < null-backstop (hard, last-resort clear).
+const PENDING_STUCK_BACKSTOP_MS = 5 * 60 * 1000;  // #745: hard stuck ceiling from submit (progress does NOT reset it).
 // #648: a JUST-armed narrating turn is protected from a SPURIOUS same-tick clear (the immediate
 // post-armPending surface poll, a /chat cursor-reset re-reading the prior resolved turn's line as a
 // fresh resolution, or a transient campaignId flip tripping the per-run reset) for this long — so the
@@ -336,6 +352,7 @@ function useLiveSession(state) {
   const eventsStreamedThisTurnRef = React.useRef(false);
   const recoveryTimer = React.useRef(null);
   const backstopTimer = React.useRef(null);
+  const stuckBackstopTimer = React.useRef(null);  // #745: hard stuck ceiling from submit (not reset by progress)
 
   // sanitizeNarration lives in screen-table.jsx (loaded first); fall back to identity if absent.
   const sanitize = (txt) => (typeof window.sanitizeNarration === "function" ? window.sanitizeNarration(txt) : (txt || ""));
@@ -378,6 +395,10 @@ function useLiveSession(state) {
   const clearTimers = React.useCallback(() => {
     clearRecoveryTimer();
     if (backstopTimer.current) { window.clearTimeout(backstopTimer.current); backstopTimer.current = null; }
+    // #745: the hard stuck-backstop is disarmed alongside the others — a real resolution (clearPending →
+    // clearTimers) or a retry re-arm (armPending → clearTimers) must cancel it so it can't fire on a turn
+    // that already resolved/re-armed.
+    if (stuckBackstopTimer.current) { window.clearTimeout(stuckBackstopTimer.current); stuckBackstopTimer.current = null; }
   }, [clearRecoveryTimer]);
 
   // #393: a ref mirror of `pending` so a poll callback (whose effect deps deliberately EXCLUDE
@@ -428,6 +449,16 @@ function useLiveSession(state) {
     recoveryTimer.current = window.setTimeout(() => {
       setPendingState((p) => (p ? { ...p, stuck: true } : p));
     }, recoveryMs);
+    // #745: a HARD stuck ceiling from submit. Unlike recoveryTimer (re-armed by every streamed paragraph
+    // in notePendingProgress), this is armed ONCE here and progress does NOT reset it — so a beat that
+    // streams a partial trickle and then FREEZES mid-stream still surfaces the recoverable `stuck` "Try
+    // again" affordance within a bounded time, instead of the trickle deferring recovery to the 12-min
+    // null-backstop (which strands the partial narration behind a plain re-enabled bar). It is generous
+    // enough that a healthy turn always RESOLVES (clearPending → clearTimers) first, so it never trips a
+    // slow-but-alive beat. Fires only when nothing has resolved by its deadline.
+    stuckBackstopTimer.current = window.setTimeout(() => {
+      setPendingState((p) => (p ? { ...p, stuck: true } : p));
+    }, PENDING_STUCK_BACKSTOP_MS);
     backstopTimer.current = window.setTimeout(() => setPendingState(null), PENDING_BACKSTOP_MS);
   }, [clearTimers, setPendingState]);
 
@@ -450,6 +481,11 @@ function useLiveSession(state) {
     // long-but-healthy streaming turn keeps resetting 'stuck' (it's plainly alive) while the
     // absolute cap still fires at its original deadline.
     clearRecoveryTimer();
+    // The per-progress recovery timer re-arms to the FULL position window on each streamed paragraph —
+    // a long-but-healthy streaming turn (prose landing every few seconds) is plainly alive, so it must
+    // NOT be falsely flagged stuck. The mid-stream-FREEZE case (a trickle that pushes this window forward
+    // forever) is caught instead by the #745 stuck-backstop armed ONCE in armPending — a hard ceiling
+    // from submit that progress does NOT reset (so it can't be deferred by a trickle). See armPending.
     const recoveryMs = recoveryWindowMs(Boolean(p.firstBeat));
     recoveryTimer.current = window.setTimeout(() => {
       setPendingState((q) => (q ? { ...q, stuck: true } : q));
@@ -681,7 +717,11 @@ function useLiveSession(state) {
     return () => { cancelled = true; stop(); document.removeEventListener("visibilitychange", onVisibility); };
   }, [campaignId, source, runId, notePendingProgress, claimNarration]);
 
-  return { chatBeats, log, pending, armPending, clearPending, recordPlayerEcho };
+  // #745: expose notePendingProgress so the live-progress signal is part of the hook's public surface
+  // (consistent with armPending/clearPending; the /events poll calls the same ref). Purely additive —
+  // existing consumers destructure named fields, so nothing breaks; it also makes the mid-stream stall
+  // ceiling unit-testable without reaching into the hook's internals.
+  return { chatBeats, log, pending, armPending, clearPending, recordPlayerEcho, notePendingProgress };
 }
 window.useLiveSession = useLiveSession;
 // #348: expose the recovery-timing contract for tests (and devtools introspection). Purely
@@ -691,7 +731,8 @@ window.__PENDING_TIMING__ = {
   recoveryMs: PENDING_RECOVERY_MS,
   recoveryFirstMs: PENDING_RECOVERY_FIRST_MS,
   backstopMs: PENDING_BACKSTOP_MS,
-  armGraceMs: PENDING_ARM_GRACE_MS,  // #648: the just-armed-turn protection window
+  armGraceMs: PENDING_ARM_GRACE_MS,            // #648: the just-armed-turn protection window
+  stuckBackstopMs: PENDING_STUCK_BACKSTOP_MS,  // #745: hard stuck ceiling from submit (progress does NOT reset it)
 };
 // #402: expose the live-tail bound for tests/devtools introspection (purely additive — the hook
 // closes over the consts directly; nothing in the running app reads these off window).