Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 32 additions & 16 deletions qa/dryrun_coldopen_timeout_proof.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,16 @@
# `timeout` wrapper VERBATIM — scripts/play.sh dm_turn AND scripts/play_party.sh turn() — plus the
# player / companion facade turn (which is NEVER wrapped in a per-beat timeout), with a stub
# `timeout`+`claude` that just prints the argv they would have run. We assert:
# (1) the COLD-OPEN DM argv is `timeout 400 claude …` (default cold-open deadline);
# (2) a ROUTINE/continuing DM argv is `timeout 200 claude …` (routine deadline, unchanged);
# (1) the COLD-OPEN DM argv is `timeout <co> claude …` (model-aware cold-open deadline: opus 500,
# non-opus 550 — F12-2);
# (2) a ROUTINE/continuing DM argv is `timeout 360 claude …` (routine deadline — F12-1 raised the
# flat 200s to 360s; this proof is updated to match);
# (3) the env override works: WORLDOS_COLDOPEN_TIMEOUT bumps the cold open, CLAWDND_BEAT_TIMEOUT
# bumps the routine tier, independently;
# (4) the PLAYER / COMPANION turn argv has NO `timeout` wrapper at all (player turn unaffected);
# (5) the SAME resolved deadline (not the stale global) is echoed in the retry log line.
# (5) the SAME resolved deadline (not the stale global) is echoed in the retry log line;
# (6) F12-2 model-aware cold-open margin: opus cold open = 500, sonnet (non-opus) cold open = 550
# (the sonnet default cleared its documented 400s band top, the thin-margin bug).
set -uo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
. "$ROOT/qa/lib_beat_driver.sh"
Expand Down Expand Up @@ -76,44 +80,56 @@ party_turn_argv() {

hr() { printf '\n========== %s ==========\n' "$1"; }

hr "S1 play.sh COLD OPEN (first=1), no env -> timeout 400"
# Default CLAWDND_DM_MODEL here is "sonnet" (set above) -> the cold-open default is the non-opus 550s.
hr "S1 play.sh COLD OPEN (first=1, sonnet), no env -> timeout 550"
out1="$(play_dm_turn_argv 1 'Begin the session.')"; printf '%s\n' "$out1"
hr "S2 play.sh ROUTINE beat (first=0), no env -> timeout 200"
hr "S2 play.sh ROUTINE beat (first=0), no env -> timeout 360"
out2="$(play_dm_turn_argv 0 'The player does: opens the door.')"; printf '%s\n' "$out2"
hr "S3 play.sh COLD OPEN + WORLDOS_COLDOPEN_TIMEOUT=600 -> timeout 600"
out3="$(WORLDOS_COLDOPEN_TIMEOUT=600 play_dm_turn_argv 1 'Begin the session.')"; printf '%s\n' "$out3"
hr "S4 play.sh ROUTINE + CLAWDND_BEAT_TIMEOUT=150 -> timeout 150 (cold open unaffected)"
out4="$(CLAWDND_BEAT_TIMEOUT=150 play_dm_turn_argv 0 'The player does: opens the door.')"; printf '%s\n' "$out4"
hr "S5 play_party.sh COLD OPEN DM (first=1) -> timeout 400"
hr "S5 play_party.sh COLD OPEN DM (first=1, sonnet) -> timeout 550"
out5="$(party_turn_argv dm 1 'You are the Dungeon Master. Begin.' "$DSID")"; printf '%s\n' "$out5"
hr "S6 play_party.sh ROUTINE DM beat (first=0) -> timeout 200"
hr "S6 play_party.sh ROUTINE DM beat (first=0) -> timeout 360"
out6="$(party_turn_argv dm 0 'This beat, the party acts.' "$DSID")"; printf '%s\n' "$out6"
hr "S7 play_party.sh COMPANION facade turn -> NO timeout wrapper (player turn unaffected)"
out7="$(party_turn_argv actor 0 'Take your action through your tools.' "$CSID" "$COMP_CFG")"; printf '%s\n' "$out7"
# F12-2: the cold-open deadline is model-aware. Opus (the DEFAULT DM model in every lane) -> 500;
# sonnet / any non-opus (the explicit A/B opt-in) -> 550 (cleared the 400s band-top thin-margin bug).
hr "S8 play.sh COLD OPEN with CLAWDND_DM_MODEL=opus -> timeout 500"
out8="$(CLAWDND_DM_MODEL=opus play_dm_turn_argv 1 'Begin the session.')"; printf '%s\n' "$out8"
hr "S9 play.sh COLD OPEN with CLAWDND_DM_MODEL=sonnet -> timeout 550 (NOT the old 400 band-top)"
out9="$(CLAWDND_DM_MODEL=sonnet play_dm_turn_argv 1 'Begin the session.')"; printf '%s\n' "$out9"

# ---- Assertions -------------------------------------------------------------------
hr "ASSERTIONS"
fail=0
chk() { if eval "$2"; then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi; }

# (1) cold open -> 400; (2) routine -> 200; and the cold open is NOT 200 (the bug it fixes).
chk "S1 play.sh cold open wraps timeout 400" 'printf "%s" "$out1" | grep -q -- "TIMEOUT-WRAP «400»"'
chk "S1 play.sh cold open is NOT timeout 200" '! printf "%s" "$out1" | grep -q -- "TIMEOUT-WRAP «200»"'
# (1) cold open (sonnet) -> 550; (2) routine -> 360; and the cold open is NOT the routine 360 (tiered).
chk "S1 play.sh cold open wraps timeout 550" 'printf "%s" "$out1" | grep -q -- "TIMEOUT-WRAP «550»"'
chk "S1 play.sh cold open is NOT routine 360" '! printf "%s" "$out1" | grep -q -- "TIMEOUT-WRAP «360»"'
chk "S1 play.sh cold open still has --effort max" 'printf "%s" "$out1" | grep -A1 -- "«--effort»" | grep -q -- "«max»"'
chk "S2 play.sh routine wraps timeout 200" 'printf "%s" "$out2" | grep -q -- "TIMEOUT-WRAP «200»"'
chk "S2 play.sh routine wraps timeout 360" 'printf "%s" "$out2" | grep -q -- "TIMEOUT-WRAP «360»"'
chk "S2 play.sh routine still --effort medium" 'printf "%s" "$out2" | grep -A1 -- "«--effort»" | grep -q -- "«medium»"'
# (3) env overrides, independent per tier.
chk "S3 WORLDOS_COLDOPEN_TIMEOUT=600 -> timeout 600" 'printf "%s" "$out3" | grep -q -- "TIMEOUT-WRAP «600»"'
chk "S4 CLAWDND_BEAT_TIMEOUT=150 -> routine timeout 150" 'printf "%s" "$out4" | grep -q -- "TIMEOUT-WRAP «150»"'
# (5) retry log line uses the RESOLVED deadline (400 on the cold open, 200 on routine) — not stale.
chk "S1 retry log line says timeout=400s" 'printf "%s" "$out1" | grep -q -- "timeout=400s"'
chk "S2 retry log line says timeout=200s" 'printf "%s" "$out2" | grep -q -- "timeout=200s"'
# (5) retry log line uses the RESOLVED deadline (the cold open's, the routine's) — not stale.
chk "S1 retry log line says timeout=550s" 'printf "%s" "$out1" | grep -q -- "timeout=550s"'
chk "S2 retry log line says timeout=360s" 'printf "%s" "$out2" | grep -q -- "timeout=360s"'
# play_party parity.
chk "S5 play_party cold open wraps timeout 400" 'printf "%s" "$out5" | grep -q -- "TIMEOUT-WRAP «400»"'
chk "S6 play_party routine wraps timeout 200" 'printf "%s" "$out6" | grep -q -- "TIMEOUT-WRAP «200»"'
chk "S5 play_party cold open wraps timeout 550" 'printf "%s" "$out5" | grep -q -- "TIMEOUT-WRAP «550»"'
chk "S6 play_party routine wraps timeout 360" 'printf "%s" "$out6" | grep -q -- "TIMEOUT-WRAP «360»"'
# (4) the player / companion facade turn is NEVER timeout-wrapped (player turn unaffected).
chk "S7 companion turn has NO timeout wrapper" '! printf "%s" "$out7" | grep -q -- "TIMEOUT-WRAP"'
chk "S7 companion turn still runs claude" 'printf "%s" "$out7" | grep -q -- "CLAUDE-ARGV-BEGIN"'
# (6) F12-2 model-aware cold-open margin: opus -> 500 (unchanged, the shipped default); sonnet -> 550
# (the bump). The sonnet cold open must NOT be the old 400s band-top — the thin-margin bug.
chk "S8 opus cold open wraps timeout 500" 'printf "%s" "$out8" | grep -q -- "TIMEOUT-WRAP «500»"'
chk "S9 sonnet cold open wraps timeout 550" 'printf "%s" "$out9" | grep -q -- "TIMEOUT-WRAP «550»"'
chk "S9 sonnet cold open is NOT the old 400 band-top" '! printf "%s" "$out9" | grep -q -- "TIMEOUT-WRAP «400»"'

hr "RESULT"
[ "$fail" = 0 ] && echo "ALL ASSERTIONS PASSED" || echo "SOME ASSERTIONS FAILED"
Expand Down
94 changes: 87 additions & 7 deletions qa/lib_beat_driver.sh
Original file line number Diff line number Diff line change
Expand Up @@ -622,16 +622,17 @@ clawdnd_dm_effort_arg() {
# • the COLD OPEN (first != 0) is the one-time, --effort max, full world-build — generate the
# world, scene, PC, opening NPCs + portraits. That MAX-EFFORT cold open routinely runs ~280–400s;
# the routine deadline KILLS it mid-build (the masked "cold-open reproducibly broken" mode),
# so the cold open gets a generous deadline: WORLDOS_COLDOPEN_TIMEOUT (default 400s).
# so the cold open gets a generous, model-aware deadline: WORLDOS_COLDOPEN_TIMEOUT
# (default 500s opus / 550s non-opus — see the F12-2 note in clawdnd_dm_timeout below).
# • CONTINUING / routine beats (first = 0) are --effort medium and resolve one move against
# established canon — faster — so they get the per-beat deadline CLAWDND_BEAT_TIMEOUT
# (default 360s — see the F12-1 note below).
# Keyed off the SAME `first` signal as the effort + lean levers, so cold-open=full+max+400s and the
# long tail=lean+medium+360s stay in lock-step. Applies ONLY to the DM turn (player/companion turns
# Keyed off the SAME `first` signal as the effort + lean levers, so cold-open=full+max+(500/550)s and
# the long tail=lean+medium+360s stay in lock-step. Applies ONLY to the DM turn (player/companion turns
# are never wrapped in a per-beat timeout at all).
#
# Both knobs resolve through the same WORLDOS_/CLAWDND_ fallback as everything else (worldos_env):
# WORLDOS_COLDOPEN_TIMEOUT (default 400) — the cold open's deadline, in seconds.
# WORLDOS_COLDOPEN_TIMEOUT (default 500 opus / 550 non-opus) — the cold open's deadline, in seconds.
# CLAWDND_BEAT_TIMEOUT (default 360) — every continuing beat's deadline (today's knob, kept).
# Note: CLAWDND_BEAT_TIMEOUT keeps its CLAWDND_ name (it predates the WorldOS rename and is the
# documented routine knob); only the NEW cold-open knob takes the WORLDOS_ name. worldos_env still
Expand All @@ -648,9 +649,16 @@ clawdnd_dm_effort_arg() {
clawdnd_dm_timeout() {
local first="$1"
if [ "$first" != "0" ]; then
# Cold-open deadline is model-aware. Opus-high cold-open measured ~300s; give it margin (500s) for
# per-world/per-run variance so it is never killed mid-build. Sonnet keeps 400s (max runs ~280–400s).
local _co_timeout=400
# Cold-open deadline is model-aware. Opus-high cold-open measured ~300s here; give it margin (500s)
# for per-world/per-run variance so it is never killed mid-build.
# F12-2 (audit 2026-06-11): the NON-opus default was 400s — but a sonnet max-effort cold open's
# OWN documented band is "~280–400s" (the prior comment), so 400 == the band TOP (zero margin vs
# the band, ~8% vs the 370s measured max) and a slow sonnet cold open was killed at the same mark
# the band predicted. Bump the non-opus default to 550s (~38% over the 400 band-top), so the sonnet
# A/B arm gets proportional margin to opus's (opus: 500 vs ~300 measured ≈ +67%; sonnet: 550 vs 400
# band-top ≈ +38%). Opus is unchanged (still the default DM model in every lane), so the shipped
# path is byte-identical; only the explicit sonnet opt-in widens. Env override still wins unchanged.
local _co_timeout=550
case "${CLAWDND_DM_MODEL:-}" in *opus*) _co_timeout=500 ;; esac
worldos_env COLDOPEN_TIMEOUT "$_co_timeout"
else
Expand Down Expand Up @@ -717,6 +725,78 @@ except KeyboardInterrupt:
' "$_secs" "$@"
}

# PROVIDER-STATUS SIDECAR (F12-10): write the provider lifecycle sidecar the OpenWorlds viewer reads
# (provider_status.json, schema worldos.provider-status.v1). Before this, ONLY the codex DM wrapper
# wrote it; the CLAUDE lanes (scripts/play.sh + scripts/play_party.sh) never did, so on a turn-cap /
# budget stop or a crash the viewer fell back to status "unknown" — which is NOT in the
# {stopped,failed,exhausted} set the viewer buckets as `no_provider` (viewer/server.py), so the app
# showed a live-looking-but-dead dashboard instead of the "DM provider is no longer running" surface.
# This is the ONE shared writer for the claude lanes (the codex wrapper keeps its own richer copy with
# the seed fixture). The sidecar is DERIVED / atomic state, NOT campaign state — it is the viewer's
# read-only health view, never read back as truth — so writing it does not violate engine-sole-writer.
#
# Atomic: write a sibling tmp under the SAME dir, fsync, replace, fsync the dir — so a concurrent
# viewer read never sees a torn file. Reads ambient globals with safe defaults (every field the
# viewer's _provider_status_summary expects), so a caller that hasn't set a field still writes a valid
# row. Best-effort: a write failure never fails a beat (returns 0). Bash 3.2-clean: standalone python
# (no heredoc-in-$()), no arrays. $1=PROVIDER_STATUS path $2=status $3=reason $4=detail $5=turns
# $6=wrapper-name (optional; defaults to the calling script's basename — display-only field).
clawdnd_write_provider_status() {
local path="$1" status="$2" reason="$3" detail="$4" turns="${5:-0}" wrapper="${6:-${BASH_SOURCE[1]:-}}"
local provider model max_turns sha actor_model scorer_model
provider="$(worldos_env PROVIDER claude)"
model="$(worldos_env DM_MODEL '')"
actor_model="$(worldos_env ACTOR_MODEL '')"
scorer_model="$(worldos_env SCORER_MODEL '')"
max_turns="${MAX_TURNS:-}"
sha="${WORLDOS_BUILD_SHA:-${CLAWDND_BUILD_SHA:-$(git rev-parse --short HEAD 2>/dev/null || printf 'unknown')}}"
python3 - "$path" "$status" "$reason" "$detail" "$provider" "$model" "$actor_model" "$scorer_model" "$max_turns" "$turns" "$sha" "$wrapper" 2>/dev/null <<'PY' || true
import json, os, sys, time
from pathlib import Path
(path, status, reason, detail, provider, model, actor_model, scorer_model,
max_turns, turns, sha, wrapper) = sys.argv[1:]
path = Path(path)
wrapper_name = os.path.basename(wrapper) or wrapper or "play.sh"
payload = {
"schema": "worldos.provider-status.v1",
"provider": provider,
"provider_family": "anthropic-claude" if "codex" not in provider.lower() else "codex-openai",
"auth_surface": "claude-cli",
"model": model,
"player_model": actor_model,
"scorer_model": scorer_model,
"wrapper": ("scripts/%s" % wrapper_name) if not wrapper_name.startswith("scripts/") else wrapper_name,
"fixture": {},
"status": status,
"reason": reason,
"detail": detail,
"max_turns": int(max_turns) if str(max_turns).isdigit() else (max_turns or None),
"dm_turns": int(turns) if str(turns).isdigit() else turns,
"updated_at": time.time(),
}
try:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(".%s.%s.tmp" % (path.name, os.getpid()))
with tmp.open("w", encoding="utf-8") as fh:
fh.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
fh.flush()
os.fsync(fh.fileno())
tmp.replace(path)
try:
dfd = os.open(str(path.parent), os.O_RDONLY)
except OSError:
dfd = None
if dfd is not None:
try:
os.fsync(dfd)
finally:
os.close(dfd)
except OSError:
sys.exit(0)
PY
return 0
}

# RE-MINT SESSION ON RETRY (the ONE shared implementation of "never reuse a CONSUMED session id").
# A `claude -p` attempt that fails AFTER startup STILL registered its --session-id on disk
# (~/.claude/projects/<proj>/<uuid>.jsonl), so a retry that re-passes that SAME --session-id dies
Expand Down
41 changes: 34 additions & 7 deletions qa/run_duo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,30 @@ turn() {
# signal as lean. DM turn ONLY — the player branch below never gets --effort.
clawdnd_dm_effort_arg "$first"
out="$T/$RUN.dm.$(date +%s%N).jsonl"
claude -p "$msg" ${resume[@]+"${resume[@]}"} ${extra[@]+"${extra[@]}"} --plugin-dir "$ROOT" --mcp-config "$DM_CFG" --strict-mcp-config \
--model "$CLAWDND_DM_MODEL" ${CLAWDND_DM_EFFORT[@]+"${CLAWDND_DM_EFFORT[@]}"} --permission-mode bypassPermissions --max-budget-usd "$BUDGET" \
--output-format stream-json --verbose > "$out" 2>> "$T/$RUN.dm.err"
# F12-11 (audit 2026-06-11): the DM turn was UNBOUNDED here — run_duo had no per-beat deadline at
# all, so a wedged DM beat (hung MCP startup, a stuck model call) hung the whole sweep, and
# turn_retry's empty-output retry never fired because a hang never RETURNS empty (it never
# returns). Bound the DM turn through the SAME worldos_timeout shim + clawdnd_dm_timeout tier the
# product lanes use (cold-open vs routine, model-aware), so a hang is killed at the deadline (rc=124)
# → no result event → clawdnd_dm_final_text echoes empty → turn_retry's empty-output retry fires.
# Player turn stays unbounded (it is a fast facade turn and was never the hang source).
local beat_timeout; beat_timeout="$(clawdnd_dm_timeout "$first")"
worldos_timeout "$beat_timeout" \
claude -p "$msg" ${resume[@]+"${resume[@]}"} ${extra[@]+"${extra[@]}"} --plugin-dir "$ROOT" --mcp-config "$DM_CFG" --strict-mcp-config \
--model "$CLAWDND_DM_MODEL" ${CLAWDND_DM_EFFORT[@]+"${CLAWDND_DM_EFFORT[@]}"} --permission-mode bypassPermissions --max-budget-usd "$BUDGET" \
--output-format stream-json --verbose > "$out" 2>> "$T/$RUN.dm.err"
rc=$?
cat "$out" >> "$COMBINED"
# F12-11: surface the REAL failure cause on a nonzero rc with NO error-class result (a timeout
# rc=124 writes no result event; a CLI crash; a rate-limit exit) — these were MASKED because
# clawdnd_dm_final_text below reports only an error-class RESULT event, so a hang/timeout left the
# structured cause buried in $out and the duo loop showed only "empty turn". The
# is-error guard avoids a DOUBLE report when the result IS error-class (final_text handles that
# one). Read-only; echoes a "[dm-attempt] …" reason (+ the 401/403 re-auth hint) to stderr. The
# rc==0 path is byte-identical to before.
if [ "$rc" -ne 0 ] && ! clawdnd_dm_result_is_error "$out"; then
clawdnd_report_attempt_failure "$out" "$rc"
fi
# SYN-01: the shared classification front door (qa/lib_beat_driver.sh) — notes $out for the
# caller's clawdnd_resolve_dm_reply and echoes NOTHING on an error-class result (a 401's
# "result" text is the API's error string, never a reply), so turn_retry's empty-only retry
Expand All @@ -192,11 +211,19 @@ turn_retry() {
if [ -z "$r" ]; then
echo "[duo] empty turn ($1) — retrying once…" >&2
# A cold-open ($3=1) retry must NOT reuse $2's already-registered --session-id (a failed but
# registered attempt → "Session ID … is already in use." → empty output again). Re-mint a fresh
# id for the retry. Continuing beats ($3=0) use --resume (safe to repeat); lean continuing beats
# already mint their own fresh id inside turn(), so only the cold open needs a swap here.
# registered attempt → "Session ID … is already in use." → empty output again). F12-11: re-mint
# via the SHARED clawdnd_dm_remint_session_on_retry (qa/lib_beat_driver.sh) — the SAME re-mint
# implementation scripts/play.sh + play_party.sh use, so the three harnesses can't drift. The
# helper inspects the prior turn's resume MODE (which `turn` built from $3): on a cold open the
# mode is `--session-id $2`, so it populates CLAWDND_DM_RETRY_SESSION with a FRESH `--session-id
# <uuid>` we hand back to turn as the new sid. Continuing beats ($3=0) use --resume (safe to
# repeat) — the helper leaves the array empty and we retry verbatim; lean continuing beats already
# mint their own fresh id inside turn(). The empty-output trigger above is preserved (it now ALSO
# fires on a timeout, which clawdnd_dm_final_text turns into an empty reply).
if [ "${3:-}" = "1" ]; then
local _fresh; _fresh="$(uuidgen 2>/dev/null || python3 -c 'import uuid;print(uuid.uuid4())')"
clawdnd_dm_remint_session_on_retry --session-id "$2"
local _fresh="$2"
[ "${#CLAWDND_DM_RETRY_SESSION[@]}" -ge 2 ] && _fresh="${CLAWDND_DM_RETRY_SESSION[1]}"
r="$(turn "$1" "$_fresh" "$3" "${@:4}")"
else
r="$(turn "$@")"
Expand Down
Loading
Loading