From 95795c5fd431e0149b534ceabe307345ac627a4b Mon Sep 17 00:00:00 2001 From: Luke Schneider Date: Wed, 15 Jul 2026 17:24:17 +0800 Subject: [PATCH 1/6] feat(spawn): add memory headroom tripwire before dispatch Refuse a crewmate/scout/secondmate launch before any side effect when used memory is at or above FM_SPAWN_MEM_MAX_PCT (default 80), printing a DEFERRED: message; FM_SPAWN_MEM_FORCE=1 bypasses for emergencies. Prevents OOM-driven fleet kills on this machine's tight WSL2 memory cap. --- bin/fm-mem-guard.sh | 58 +++++++++++++++++ bin/fm-spawn.sh | 12 ++++ tests/fm-mem-guard.test.sh | 126 +++++++++++++++++++++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100755 bin/fm-mem-guard.sh create mode 100644 tests/fm-mem-guard.test.sh diff --git a/bin/fm-mem-guard.sh b/bin/fm-mem-guard.sh new file mode 100755 index 000000000..8bbe63388 --- /dev/null +++ b/bin/fm-mem-guard.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# fm-mem-guard.sh - dispatch-time memory headroom tripwire. Source this file +# and call fm_mem_guard_check before any fm-spawn.sh side effect (window, +# worktree lease, meta, hook install). This is a TRIPWIRE, not a queue: it +# only refuses a single launch attempt so the caller can exit non-zero; +# firstmate's backlog is already the queue and retries the dispatch later. +# +# Background: this machine runs WSL2 under a deliberately tight memory cap +# set in /mnt/c/Users/snydi/.wslconfig (read live if you need the cap value; +# it changes, never hard-code it here). Two 2026-07-10 OOM incidents killed +# the whole fleet (tmux server, no-mistakes daemon, dbus) from concurrent +# dispatch with no headroom check. +# +# Measurement: used% = (MemTotal - MemAvailable) / MemTotal * 100, read +# straight from /proc/meminfo with bash builtins only (no forks, no jq/awk). +# +# FM_SPAWN_MEM_MAX_PCT - defer threshold, percent of MemTotal used. Default 80. +# FM_SPAWN_MEM_FORCE=1 - skip the check entirely, for a captain-ordered +# emergency dispatch. +# FM_MEMINFO_PATH_OVERRIDE - read this path instead of /proc/meminfo; the +# hermetic test seam, never set outside tests. +# +# Fails OPEN: a missing or unparseable meminfo file lets the spawn proceed. +# The tripwire must never brick dispatch on a weird environment. + +fm_mem_guard_check() { + [ "${FM_SPAWN_MEM_FORCE:-0}" = 1 ] && return 0 + + local meminfo total avail key val pct threshold used + meminfo="${FM_MEMINFO_PATH_OVERRIDE:-/proc/meminfo}" + [ -r "$meminfo" ] || return 0 + + total= + avail= + while IFS=: read -r key val; do + case "$key" in + MemTotal) total=${val%kB}; total=${total// /} ;; + MemAvailable) avail=${val%kB}; avail=${avail// /} ;; + esac + { [ -z "$total" ] || [ -z "$avail" ]; } || break + done < "$meminfo" + + case "$total" in ''|*[!0-9]*) return 0 ;; esac + case "$avail" in ''|*[!0-9]*) return 0 ;; esac + [ "$total" -gt 0 ] || return 0 + + used=$((total - avail)) + pct=$((used * 100 / total)) + + threshold="${FM_SPAWN_MEM_MAX_PCT:-80}" + case "$threshold" in ''|*[!0-9]*) threshold=80 ;; esac + + if [ "$pct" -ge "$threshold" ]; then + echo "DEFERRED: memory headroom tripwire tripped - used ${pct}% >= threshold ${threshold}%; firstmate should retry this dispatch once headroom recovers (bypass with FM_SPAWN_MEM_FORCE=1 for a captain-ordered emergency dispatch)" >&2 + return 1 + fi + return 0 +} diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index b8b1fd673..587bb2668 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -87,6 +87,10 @@ # On success prints: spawned harness= kind= mode= yolo= window= worktree= # mode/yolo are resolved per-project from data/projects.md for ship/scout tasks; # secondmate spawns record mode=secondmate, yolo=off, home=, and projects=. +# Before any side effect, a memory headroom tripwire (bin/fm-mem-guard.sh) +# refuses the launch and exits non-zero with a DEFERRED: message when used +# memory is at or above FM_SPAWN_MEM_MAX_PCT (default 80); bypass with +# FM_SPAWN_MEM_FORCE=1. See that script's header for the full contract. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -116,6 +120,8 @@ SUB_HOME_MARKER=".fm-secondmate-home" . "$SCRIPT_DIR/fm-gate-refuse-lib.sh" # shellcheck source=bin/fm-pr-lib.sh . "$SCRIPT_DIR/fm-pr-lib.sh" +# shellcheck source=bin/fm-mem-guard.sh +. "$SCRIPT_DIR/fm-mem-guard.sh" # Fail closed before any fleet mutation: a no-mistakes gate agent must never spawn # a direct report (see bin/fm-gate-refuse-lib.sh). fm_refuse_if_gate_agent @@ -287,6 +293,12 @@ if [ "${#POS[@]}" -gt 0 ] && [ "${POS[0]}" != "$idpart" ] && case "$idpart" in * done exit "$rc" fi +# Memory headroom tripwire (bin/fm-mem-guard.sh): refuse before any side +# effect (window, worktree lease, meta, hook install). Batch pairs each +# re-exec this script in single-task mode above, so this check runs once per +# task launch, letting a batch land its first task and defer the rest as +# pressure rises. +fm_mem_guard_check || exit 1 ID=${POS[0]} fm_task_id_creation_valid "$ID" || { echo "error: invalid task id" >&2; exit 2; } PROJ= diff --git a/tests/fm-mem-guard.test.sh b/tests/fm-mem-guard.test.sh new file mode 100644 index 000000000..4bae9e4ae --- /dev/null +++ b/tests/fm-mem-guard.test.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# tests/fm-mem-guard.test.sh - unit tests for bin/fm-mem-guard.sh's +# fm_mem_guard_check, the dispatch-time memory headroom tripwire fm-spawn.sh +# calls before any spawn side effect (AGENTS.md task lifecycle, "Spawn"). +# +# Hermetic like the JQ_DIR-guard pattern in tests/fm-x-mode.test.sh: every case +# points FM_MEMINFO_PATH_OVERRIDE at a fixture file this suite writes, so the +# real /proc/meminfo (and this machine's actual memory pressure) is never read. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# shellcheck source=bin/fm-mem-guard.sh +. "$ROOT/bin/fm-mem-guard.sh" + +TMP_ROOT=$(fm_test_tmproot fm-mem-guard) +mkdir -p "$TMP_ROOT" + +# write_meminfo writes a minimal fixture with just +# the two fields fm_mem_guard_check reads, in the real file's key/space/unit form. +write_meminfo() { + local path=$1 total=$2 avail=$3 + { + printf 'MemTotal: %s kB\n' "$total" + printf 'MemFree: 1234 kB\n' + printf 'MemAvailable: %s kB\n' "$avail" + } > "$path" +} + +# Isolate every case from the ambient environment: unset the bypass/threshold +# knobs so a leftover export from an earlier case (or the caller's shell) +# never leaks in. +run_check() { + local meminfo=$1; shift + ( unset FM_SPAWN_MEM_FORCE FM_SPAWN_MEM_MAX_PCT + FM_MEMINFO_PATH_OVERRIDE=$meminfo + export FM_MEMINFO_PATH_OVERRIDE + for kv in "$@"; do + export "$kv" + done + fm_mem_guard_check + ) +} + +test_under_threshold_proceeds() { + local mi out status + mi="$TMP_ROOT/under.meminfo" + write_meminfo "$mi" 100000 21000 # used 79% + out=$(run_check "$mi") ; status=$? + [ "$status" -eq 0 ] || fail "under-threshold usage should proceed, got exit $status" + [ -z "$out" ] || fail "under-threshold usage should print nothing, got: $out" + pass "fm_mem_guard_check: usage below the default 80% threshold proceeds silently" +} + +test_at_threshold_defers() { + local mi out status + mi="$TMP_ROOT/at.meminfo" + write_meminfo "$mi" 100000 20000 # used exactly 80% + out=$(run_check "$mi" 2>&1); status=$? + [ "$status" -ne 0 ] || fail "at-threshold usage should defer (non-zero exit)" + printf '%s\n' "$out" | grep -qF 'DEFERRED:' || fail "missing DEFERRED: message, got: $out" + printf '%s\n' "$out" | grep -qF 'used 80%' || fail "message should state measured used%, got: $out" + printf '%s\n' "$out" | grep -qF 'threshold 80%' || fail "message should state the threshold, got: $out" + printf '%s\n' "$out" | grep -qiF 'retry' || fail "message should say firstmate should retry, got: $out" + pass "fm_mem_guard_check: usage at the threshold defers with a DEFERRED: message and non-zero exit" +} + +test_over_threshold_defers() { + local mi out status + mi="$TMP_ROOT/over.meminfo" + write_meminfo "$mi" 100000 5000 # used 95% + out=$(run_check "$mi" 2>&1); status=$? + [ "$status" -ne 0 ] || fail "over-threshold usage should defer" + printf '%s\n' "$out" | grep -qF 'DEFERRED:' || fail "missing DEFERRED: message, got: $out" + pass "fm_mem_guard_check: usage over the threshold defers" +} + +test_force_bypass_proceeds() { + local mi out status + mi="$TMP_ROOT/force.meminfo" + write_meminfo "$mi" 100000 1000 # used 99%, would defer without the bypass + out=$(run_check "$mi" FM_SPAWN_MEM_FORCE=1 2>&1); status=$? + [ "$status" -eq 0 ] || fail "FM_SPAWN_MEM_FORCE=1 should proceed even under heavy pressure, got exit $status" + [ -z "$out" ] || fail "forced bypass should print nothing, got: $out" + pass "fm_mem_guard_check: FM_SPAWN_MEM_FORCE=1 bypasses the check" +} + +test_unreadable_meminfo_proceeds() { + local out status + out=$(run_check "$TMP_ROOT/does-not-exist.meminfo" 2>&1); status=$? + [ "$status" -eq 0 ] || fail "unreadable meminfo must fail open (proceed), got exit $status" + [ -z "$out" ] || fail "fail-open path should print nothing, got: $out" + pass "fm_mem_guard_check: unreadable /proc/meminfo fails open and proceeds" +} + +test_unparseable_meminfo_proceeds() { + local mi out status + mi="$TMP_ROOT/garbage.meminfo" + printf 'not meminfo at all\n' > "$mi" + out=$(run_check "$mi" 2>&1); status=$? + [ "$status" -eq 0 ] || fail "unparseable meminfo must fail open (proceed), got exit $status" + [ -z "$out" ] || fail "fail-open path should print nothing, got: $out" + pass "fm_mem_guard_check: unparseable meminfo fails open and proceeds" +} + +test_custom_threshold_honored() { + local mi out status + mi="$TMP_ROOT/custom.meminfo" + write_meminfo "$mi" 100000 40000 # used 60% + out=$(run_check "$mi" FM_SPAWN_MEM_MAX_PCT=50 2>&1); status=$? + [ "$status" -ne 0 ] || fail "60% used should defer against a custom 50% threshold" + printf '%s\n' "$out" | grep -qF 'threshold 50%' || fail "message should honor the custom threshold, got: $out" + + out=$(run_check "$mi" FM_SPAWN_MEM_MAX_PCT=90 2>&1); status=$? + [ "$status" -eq 0 ] || fail "60% used should proceed against a custom 90% threshold, got exit $status" + pass "fm_mem_guard_check: FM_SPAWN_MEM_MAX_PCT overrides the default threshold" +} + +test_under_threshold_proceeds +test_at_threshold_defers +test_over_threshold_defers +test_force_bypass_proceeds +test_unreadable_meminfo_proceeds +test_unparseable_meminfo_proceeds +test_custom_threshold_honored From 401307c961357cce020629a7897917cd37a797ba Mon Sep 17 00:00:00 2001 From: Luke Schneider Date: Wed, 15 Jul 2026 17:34:53 +0800 Subject: [PATCH 2/6] no-mistakes(review): fix mem-guard test exec bit and warn on bad knob values --- bin/fm-mem-guard.sh | 23 ++++++++++++++++++----- tests/fm-mem-guard.test.sh | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) mode change 100644 => 100755 tests/fm-mem-guard.test.sh diff --git a/bin/fm-mem-guard.sh b/bin/fm-mem-guard.sh index 8bbe63388..c92498a35 100755 --- a/bin/fm-mem-guard.sh +++ b/bin/fm-mem-guard.sh @@ -15,8 +15,10 @@ # straight from /proc/meminfo with bash builtins only (no forks, no jq/awk). # # FM_SPAWN_MEM_MAX_PCT - defer threshold, percent of MemTotal used. Default 80. +# A set-but-unparseable value warns on stderr and falls back to the default. # FM_SPAWN_MEM_FORCE=1 - skip the check entirely, for a captain-ordered -# emergency dispatch. +# emergency dispatch. Only the literal 1 bypasses; any other set value (a +# typo'd true/yes) warns on stderr and is ignored, so the check still runs. # FM_MEMINFO_PATH_OVERRIDE - read this path instead of /proc/meminfo; the # hermetic test seam, never set outside tests. # @@ -24,9 +26,15 @@ # The tripwire must never brick dispatch on a weird environment. fm_mem_guard_check() { - [ "${FM_SPAWN_MEM_FORCE:-0}" = 1 ] && return 0 + local meminfo total avail key val pct threshold used force + + force="${FM_SPAWN_MEM_FORCE-0}" + case "$force" in + 1) return 0 ;; + ''|0) ;; + *) echo "WARNING: FM_SPAWN_MEM_FORCE='${force}' ignored - only FM_SPAWN_MEM_FORCE=1 bypasses the memory headroom check" >&2 ;; + esac - local meminfo total avail key val pct threshold used meminfo="${FM_MEMINFO_PATH_OVERRIDE:-/proc/meminfo}" [ -r "$meminfo" ] || return 0 @@ -47,8 +55,13 @@ fm_mem_guard_check() { used=$((total - avail)) pct=$((used * 100 / total)) - threshold="${FM_SPAWN_MEM_MAX_PCT:-80}" - case "$threshold" in ''|*[!0-9]*) threshold=80 ;; esac + threshold="${FM_SPAWN_MEM_MAX_PCT-80}" + case "$threshold" in + ''|*[!0-9]*) + echo "WARNING: FM_SPAWN_MEM_MAX_PCT='${threshold}' is not a non-negative integer - falling back to the default 80% memory headroom threshold" >&2 + threshold=80 + ;; + esac if [ "$pct" -ge "$threshold" ]; then echo "DEFERRED: memory headroom tripwire tripped - used ${pct}% >= threshold ${threshold}%; firstmate should retry this dispatch once headroom recovers (bypass with FM_SPAWN_MEM_FORCE=1 for a captain-ordered emergency dispatch)" >&2 diff --git a/tests/fm-mem-guard.test.sh b/tests/fm-mem-guard.test.sh old mode 100644 new mode 100755 index 4bae9e4ae..0fb7b9d0a --- a/tests/fm-mem-guard.test.sh +++ b/tests/fm-mem-guard.test.sh @@ -37,7 +37,7 @@ run_check() { FM_MEMINFO_PATH_OVERRIDE=$meminfo export FM_MEMINFO_PATH_OVERRIDE for kv in "$@"; do - export "$kv" + export "${kv?}" done fm_mem_guard_check ) @@ -117,6 +117,37 @@ test_custom_threshold_honored() { pass "fm_mem_guard_check: FM_SPAWN_MEM_MAX_PCT overrides the default threshold" } +test_bad_force_warns_and_still_checks() { + local mi out status + mi="$TMP_ROOT/badforce.meminfo" + write_meminfo "$mi" 100000 1000 # used 99% + out=$(run_check "$mi" FM_SPAWN_MEM_FORCE=true 2>&1); status=$? + [ "$status" -ne 0 ] || fail "a non-1 FM_SPAWN_MEM_FORCE must not bypass the check" + printf '%s\n' "$out" | grep -qF "FM_SPAWN_MEM_FORCE='true' ignored" || fail "should warn the bad bypass value was ignored, got: $out" + printf '%s\n' "$out" | grep -qF 'DEFERRED:' || fail "check should still defer after the warning, got: $out" + + write_meminfo "$mi" 100000 90000 # used 10%, well under threshold + out=$(run_check "$mi" FM_SPAWN_MEM_FORCE=0 2>&1); status=$? + [ "$status" -eq 0 ] || fail "FM_SPAWN_MEM_FORCE=0 is the documented off value, got exit $status" + [ -z "$out" ] || fail "FM_SPAWN_MEM_FORCE=0 is honored as off and should not warn, got: $out" + pass "fm_mem_guard_check: only FM_SPAWN_MEM_FORCE=1 bypasses; another set value warns and is ignored" +} + +test_bad_threshold_warns_and_defaults() { + local mi out status + mi="$TMP_ROOT/badthreshold.meminfo" + write_meminfo "$mi" 100000 21000 # used 79%, under the default 80 + out=$(run_check "$mi" FM_SPAWN_MEM_MAX_PCT=95% 2>&1); status=$? + [ "$status" -eq 0 ] || fail "a bad threshold should fall back to the default 80 and proceed at 79%, got exit $status" + printf '%s\n' "$out" | grep -qF "FM_SPAWN_MEM_MAX_PCT='95%'" || fail "warning should name the bad value, got: $out" + printf '%s\n' "$out" | grep -qF 'default 80%' || fail "warning should name the default fallback, got: $out" + + out=$(run_check "$mi" FM_SPAWN_MEM_MAX_PCT= 2>&1); status=$? + [ "$status" -eq 0 ] || fail "an empty threshold should fall back to the default 80, got exit $status" + printf '%s\n' "$out" | grep -qF "FM_SPAWN_MEM_MAX_PCT=''" || fail "an explicitly empty threshold should warn, got: $out" + pass "fm_mem_guard_check: an unparseable FM_SPAWN_MEM_MAX_PCT warns and falls back to 80" +} + test_under_threshold_proceeds test_at_threshold_defers test_over_threshold_defers @@ -124,3 +155,5 @@ test_force_bypass_proceeds test_unreadable_meminfo_proceeds test_unparseable_meminfo_proceeds test_custom_threshold_honored +test_bad_force_warns_and_still_checks +test_bad_threshold_warns_and_defaults From 1b0cb019969e8c88e768a38bff30a799820b535b Mon Sep 17 00:00:00 2001 From: Luke Schneider Date: Wed, 15 Jul 2026 17:59:50 +0800 Subject: [PATCH 3/6] no-mistakes(test): fix host-dependent node and jq assumptions in tests --- tests/fm-dispatch-select.test.sh | 8 ++++++++ tests/fm-session-start.test.sh | 12 ++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index a9ae5b7bf..3f34ba214 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -6,6 +6,14 @@ set -u . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} +# jq is an unconditional dependency of the script under test, never the subject +# of a fixture: the fakebin only ever shadows quota-axi. Keep jq reachable on +# hosts that install it outside the sandbox PATH's system dirs, so a quota +# fallback case cannot fail as an unrelated "jq is required" exit 2. +if ! (PATH="$BASE_PATH"; command -v jq >/dev/null 2>&1); then + JQ_BIN=$(command -v jq 2>/dev/null || true) + [ -n "$JQ_BIN" ] && BASE_PATH="$BASE_PATH:$(dirname "$JQ_BIN")" +fi TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests) mkdir -p "$TMP_ROOT" diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 6ae0d124b..063d35aab 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -401,8 +401,6 @@ $rec EOF make_fake_toolchain "$fakebin" make_fake_ps_claude "$fakebin" - # Force a MISSING diagnostic line so the bootstrap section is non-trivial. - rm -f "$fakebin/node" printf 'window=fm-sess:w1\nkind=ship\n' > "$home/state/task-a.meta" @@ -425,7 +423,9 @@ EOF [ "$context_line" -lt "$fleet_line" ] || fail "CONTEXT did not precede FLEET STATE" [ "$fleet_line" -lt "$next_line" ] || fail "FLEET STATE did not precede NEXT STEP" - missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: node' | head -1 | cut -d: -f1) + # The fakebin's PATH excludes tasks-axi, so this MISSING line fires + # deterministically regardless of what is installed on the test host. + missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: tasks-axi (install:' | head -1 | cut -d: -f1) [ -n "$missing_line" ] || fail "MISSING diagnostic did not appear at all" [ "$missing_line" -lt "$fleet_line" ] || fail "actionable MISSING diagnostic was buried after the bulk fleet-state digest" @@ -585,7 +585,6 @@ $rec EOF make_fake_toolchain "$fakebin" make_fake_ps_claude "$fakebin" - rm -f "$fakebin/node" append_wake "$home/state" signal task-z "needs-decision: pick a library" @@ -593,8 +592,9 @@ EOF # fm-lock.sh's own exact success text. assert_contains "$out" "lock acquired: harness pid" "fm-lock.sh's real output did not appear (composition, not reimplementation)" - # fm-bootstrap.sh's own exact MISSING-tool line format. - assert_contains "$out" "MISSING: node (install:" "fm-bootstrap.sh's real detect line did not appear verbatim" + # fm-bootstrap.sh's own exact MISSING-tool line format. tasks-axi is absent + # from the fakebin's PATH, so the line fires on any test host. + assert_contains "$out" "MISSING: tasks-axi (install:" "fm-bootstrap.sh's real detect line did not appear verbatim" # fm-wake-drain.sh's real drained record (raw tab-separated queue line). assert_contains "$out" "$(printf 'signal\ttask-z\tneeds-decision: pick a library')" "fm-wake-drain.sh's real drained record did not appear" From c411c455d85bbd139fce660dc6ea03bc5024754d Mon Sep 17 00:00:00 2001 From: Luke Schneider Date: Wed, 15 Jul 2026 18:30:35 +0800 Subject: [PATCH 4/6] no-mistakes(test): skip herdr lab tests without a running default session --- tests/fm-afk-inject-herdr-e2e.test.sh | 1 + tests/fm-afk-launch.test.sh | 4 ++++ tests/fm-backend-autodetect-smoke.test.sh | 1 + tests/fm-backend-herdr-eventwait-smoke.test.sh | 1 + tests/fm-backend-herdr-prune-safety-e2e.test.sh | 1 + tests/fm-backend-herdr-respawn-idem-e2e.test.sh | 1 + tests/fm-backend-herdr-smoke.test.sh | 1 + .../fm-backend-herdr-workspace-per-home-e2e.test.sh | 1 + tests/herdr-test-safety.sh | 12 ++++++++++++ 9 files changed, 23 insertions(+) diff --git a/tests/fm-afk-inject-herdr-e2e.test.sh b/tests/fm-afk-inject-herdr-e2e.test.sh index 5e48ab43e..dc90f681e 100755 --- a/tests/fm-afk-inject-herdr-e2e.test.sh +++ b/tests/fm-afk-inject-herdr-e2e.test.sh @@ -40,6 +40,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } pass() { printf 'ok - %s\n' "$1"; } diff --git a/tests/fm-afk-launch.test.sh b/tests/fm-afk-launch.test.sh index 3baaa5cff..acd3a750b 100755 --- a/tests/fm-afk-launch.test.sh +++ b/tests/fm-afk-launch.test.sh @@ -771,6 +771,10 @@ e2e_herdr() { command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (herdr e2e)"; return 0; } # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" + herdr_lab_precondition_ok || { + echo "skip: no running default Herdr session for the fleet-state tripwire (herdr e2e)" + return 0 + } # shellcheck source=bin/fm-backend.sh . "$ROOT/bin/fm-backend.sh" diff --git a/tests/fm-backend-autodetect-smoke.test.sh b/tests/fm-backend-autodetect-smoke.test.sh index e3413b9c1..987ff6e21 100755 --- a/tests/fm-backend-autodetect-smoke.test.sh +++ b/tests/fm-backend-autodetect-smoke.test.sh @@ -44,6 +44,7 @@ command -v treehouse >/dev/null 2>&1 || { echo "skip: treehouse not found (requi # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } # TMP_ROOT is physically resolved (mktemp -d "$(pwd -P)"-relative) to keep this # real-herdr smoke fixture free of unrelated OS symlink noise. diff --git a/tests/fm-backend-herdr-eventwait-smoke.test.sh b/tests/fm-backend-herdr-eventwait-smoke.test.sh index 60f784aec..1771df07e 100755 --- a/tests/fm-backend-herdr-eventwait-smoke.test.sh +++ b/tests/fm-backend-herdr-eventwait-smoke.test.sh @@ -24,6 +24,7 @@ command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found (required # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } SESSION="fm-lab-eventwait-smoke-$$" export HERDR_SESSION="$SESSION" diff --git a/tests/fm-backend-herdr-prune-safety-e2e.test.sh b/tests/fm-backend-herdr-prune-safety-e2e.test.sh index 1498e9869..6a61b419f 100755 --- a/tests/fm-backend-herdr-prune-safety-e2e.test.sh +++ b/tests/fm-backend-herdr-prune-safety-e2e.test.sh @@ -32,6 +32,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } SESSION="fm-lab-prune-safety-e2e-$$" export HERDR_SESSION="$SESSION" diff --git a/tests/fm-backend-herdr-respawn-idem-e2e.test.sh b/tests/fm-backend-herdr-respawn-idem-e2e.test.sh index 0f316d641..d4f789d28 100755 --- a/tests/fm-backend-herdr-respawn-idem-e2e.test.sh +++ b/tests/fm-backend-herdr-respawn-idem-e2e.test.sh @@ -44,6 +44,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } SESSION="fm-lab-respawn-idem-e2e-$$" export HERDR_SESSION="$SESSION" diff --git a/tests/fm-backend-herdr-smoke.test.sh b/tests/fm-backend-herdr-smoke.test.sh index db1f5202f..022de57ad 100755 --- a/tests/fm-backend-herdr-smoke.test.sh +++ b/tests/fm-backend-herdr-smoke.test.sh @@ -26,6 +26,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } SESSION="fm-lab-backend-smoke-$$" export HERDR_SESSION="$SESSION" diff --git a/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh b/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh index de26ea33b..dff53068d 100755 --- a/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +++ b/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh @@ -53,6 +53,7 @@ command -v treehouse >/dev/null 2>&1 || { echo "skip: treehouse not found (requi # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; } # TMP_ROOT is physically resolved (mktemp -d "$(pwd -P)"-relative) for the same # low-noise scratch fixture shape used by diff --git a/tests/herdr-test-safety.sh b/tests/herdr-test-safety.sh index eabc5f730..f8660c67f 100644 --- a/tests/herdr-test-safety.sh +++ b/tests/herdr-test-safety.sh @@ -18,6 +18,18 @@ herdr_refuse_if_default() { # fm_herdr_lab_refuse_if_default "$1" } +# The fleet-state tripwire snapshots the running default session, so +# fm_herdr_lab_prepare cannot provision a lab session on a host where herdr is +# installed but no default server is running. That is host state these tests +# must not change - starting or stopping the captain's default server is the +# very thing the tripwire guards - so report it as a skip precondition, +# alongside the herdr/jq absence gates each real-Herdr test already carries. +herdr_lab_precondition_ok() { + command -v herdr >/dev/null 2>&1 || return 1 + command -v jq >/dev/null 2>&1 || return 1 + fm_herdr_lab_fleet_state "fm-lab-precondition-probe-$$" >/dev/null 2>&1 +} + herdr_safe_stop_and_delete() { # fm_herdr_lab_teardown "$1" } From 397c45e916a8272621982880845012989e844d27 Mon Sep 17 00:00:00 2001 From: Luke Schneider Date: Wed, 15 Jul 2026 18:51:15 +0800 Subject: [PATCH 5/6] no-mistakes(test): trap TERM in watcher so its lock is always released --- bin/fm-watch.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 4ea6da70a..51de550b6 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -692,7 +692,12 @@ watcher_cleanup() { fm_lock_release "$WATCH_LOCK" } trap watcher_cleanup EXIT -trap 'exit 1' HUP INT TERM +# Bash runs the EXIT trap on an UNtrapped terminating signal only when it wins a +# race with its own dying foreground child, so a bounded checkpoint or an +# fm-watch-arm.sh --restart would intermittently leak this singleton lock and +# lock out the next watcher. Trapping the signal explicitly makes the shell +# handle it at a safe point and always fall through to the cleanup above. +trap 'exit 143' TERM INT HUP # This watcher's own pid, as recorded in the lock by fm_lock_claim (which writes # ${BASHPID:-$$} from this same main shell). Read directly, never via a command # substitution, so it matches the stored holder pid for the self-eviction check. From 107e526f55f37c342d59dfcbb480e4e43ff11883 Mon Sep 17 00:00:00 2001 From: Luke Schneider Date: Wed, 15 Jul 2026 19:19:53 +0800 Subject: [PATCH 6/6] no-mistakes(document): document memory headroom tripwire and watcher signal trap --- docs/architecture.md | 11 +++++++++++ docs/configuration.md | 9 +++++++++ docs/scripts.md | 1 + 3 files changed, 21 insertions(+) diff --git a/docs/architecture.md b/docs/architecture.md index 6e8ad3f0f..dc2f8346a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,6 +50,7 @@ That block owns the live wait shape for the running primary harness: Claude and `bin/fm-watch-arm.sh` remains the verified arm wrapper for protocols that call it; it forks the watcher as a tracked child, verifies it is genuinely alive with a fresh liveness beacon, and prints exactly one honest status line (`started` / `attached` / restart-only `healthy` / `FAILED`, the last exiting non-zero). On `attached` it stays live until that existing cycle ends so background-notify harnesses do not get an empty false wake from a healthy no-op exit. Its `--restart` mode signals only the watcher recorded in the current home's `state/.watch.lock`, so restarting one home cannot kill sibling secondmate watchers. +`fm-watch.sh` traps `TERM`, `INT`, and `HUP` explicitly and exits 143 through them, so a signalled watcher always falls through to its own singleton-lock release instead of intermittently leaking the lock and locking out the next watcher. A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, or if tasks are in flight and that watcher stops running or queued wakes are waiting to be drained. The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on stale watcher liveness. It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the stale-watcher banner/reminder policy so repeated guarded commands stay noisy without reprinting the full watcher-down banner in the same episode. @@ -129,6 +130,16 @@ Secondmate launches are exempt because they resolve the secondmate harness and a Unsupported effort values are still recorded in task meta when passed to `fm-spawn.sh`, but the launch template omits any effort flag that the selected harness does not accept. That keeps spawn launch compatible across claude, codex, grok, pi, and opencode while preserving the requested profile for later audit. +## Dispatch memory headroom + +Every agent launch is a real process, so concurrent dispatch on a memory-capped host can exhaust the box and take the whole fleet down with it. +`fm-spawn.sh` sources `bin/fm-mem-guard.sh` and checks used memory against `FM_SPAWN_MEM_MAX_PCT` (default 80) once per task launch, before any side effect: no window, no worktree lease, no meta file, and no hook install. +On a trip it prints a `DEFERRED:` line naming the measured used percent and the threshold, then exits 1. +This is a tripwire, not a queue: it refuses one launch attempt, and firstmate retries the dispatch later from the backlog it already keeps. +The check runs per task launch rather than per invocation, so a batch `id=repo` dispatch keeps its one-failure-does-not-stop-the-rest contract and can land its first tasks while deferring the rest as pressure rises. +It applies identically to `--secondmate` launches, and `FM_SPAWN_MEM_FORCE=1` bypasses it for a captain-ordered emergency dispatch. +The tripwire fails open on an unreadable or unparseable meminfo file; [configuration.md](configuration.md#environment-variables) owns the tuning knobs and fallback details. + ## Optional secondmates `data/secondmates.md` records persistent domain supervisors with natural-language scopes, project clone lists, and home paths. diff --git a/docs/configuration.md b/docs/configuration.md index 6920d919c..5105ca0cc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -358,6 +358,9 @@ FM_ZELLIJ_SESSION=firstmate # zellij-only: named session for normal backend ops FM_BACKEND_CMUX_COMPOSER_LINES=20 # cmux-only: tail lines scanned to locate the composer row for submit verification FM_BACKEND_CMUX_IDLE_RE='^Type a message\.\.\.$' # cmux-only: empty-composer placeholder regex after border/prompt stripping CMUX_SOCKET_PASSWORD= # cmux-only: socket password fallback when config/cmux-socket-password is absent (docs/cmux-backend.md) +FM_SPAWN_MEM_MAX_PCT=80 # used-memory percent at or above which fm-spawn.sh defers a launch before any side effect; a blank or non-integer value warns and falls back to 80 +FM_SPAWN_MEM_FORCE= # set to the literal 1 to skip the memory headroom check for a captain-ordered emergency dispatch; any value other than 1, 0, or blank warns and is ignored +FM_MEMINFO_PATH_OVERRIDE= # meminfo path read instead of /proc/meminfo; the hermetic test seam, never set outside tests FM_SESSION_START_STATUS_TAIL=5 # state/*.status lines printed per task in the session-start digest FM_BOOTSTRAP_DETECT_ONLY=0 # internal/read-only session-start mode: skip bootstrap's mutating sweeps and print advisory TANGLE wording FM_GUARD_READ_ONLY=0 # internal/read-only guard mode: keep alarms but suppress drain, supervision repair, and checkout repair commands @@ -431,6 +434,12 @@ FM_LOG_MAX_BYTES=1048576 # daemon log size that triggers trimming FM_LOG_KEEP_LINES=2000 # daemon log lines kept when trimming ``` +`bin/fm-mem-guard.sh` measures `used% = (MemTotal - MemAvailable) / MemTotal` from `/proc/meminfo` with bash builtins only, and `fm-spawn.sh` calls it once per task launch before any side effect. +`FM_SPAWN_MEM_MAX_PCT` accepts a nonnegative integer percent; unset uses the default of 80 silently, while a blank or non-integer value warns on stderr and falls back to that same default. +Only the literal `FM_SPAWN_MEM_FORCE=1` bypasses the check; unset, blank, and `0` leave it running silently, and any other value warns on stderr and is ignored, so the check still runs. +The tripwire fails open: a missing, unreadable, or unparseable meminfo file lets the spawn proceed, so it can never brick dispatch on an environment without `/proc/meminfo`. +The script's header owns the full contract. + `fm-teardown.sh` retries only Git's `Unable to create '...index.lock': File exists` return failure up to `FM_TREEHOUSE_RETURN_LOCK_RETRIES` times. `FM_TREEHOUSE_RETURN_LOCK_RETRIES` accepts a nonnegative integer, and an unset, blank, or invalid value uses the default of 3. `FM_TREEHOUSE_RETURN_LOCK_RETRY_WAIT_SECS` accepts nonnegative whole or fractional seconds between attempts. diff --git a/docs/scripts.md b/docs/scripts.md index d0815745c..bce6a3699 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -42,6 +42,7 @@ The shared no-mistakes gate refusal used by `fm-spawn.sh`, `fm-send.sh`, and `fm | `fm-review-diff.sh` | Review a crewmate branch or recorded PR head against the authoritative base | | `fm-marker-lib.sh` | Shared from-firstmate request marker, detector, and idempotent transformation | | `fm-gate-refuse-lib.sh` | Shared no-mistakes gate-context refusal for fleet lifecycle entrypoints | +| `fm-mem-guard.sh` | Shared dispatch-time memory headroom tripwire sourced by `fm-spawn.sh` | | `fm-watch-arm.sh` | Verified home-scoped watcher arm wrapper with honest status reporting | | `fm-watch-checkpoint.sh` | Run one bounded foreground watcher checkpoint for Codex-style supervision | | `fm-watch.sh` | Singleton-safe always-on watcher: absorb benign wakes, queue and exit on actionable ones |