feat(spawn): add memory headroom tripwire before dispatch

bin/fm-mem-guard.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# fm-mem-guard.sh - dispatch-time memory headroom tripwire. Source this file
+# and call fm_mem_guard_check before any fm-spawn.sh side effect (window,
+# worktree lease, meta, hook install). This is a TRIPWIRE, not a queue: it
+# only refuses a single launch attempt so the caller can exit non-zero;
+# firstmate's backlog is already the queue and retries the dispatch later.
+#
+# Background: this machine runs WSL2 under a deliberately tight memory cap
+# set in /mnt/c/Users/snydi/.wslconfig (read live if you need the cap value;
+# it changes, never hard-code it here). Two 2026-07-10 OOM incidents killed
+# the whole fleet (tmux server, no-mistakes daemon, dbus) from concurrent
+# dispatch with no headroom check.
+#
+# Measurement: used% = (MemTotal - MemAvailable) / MemTotal * 100, read
+# straight from /proc/meminfo with bash builtins only (no forks, no jq/awk).
+#
+# FM_SPAWN_MEM_MAX_PCT - defer threshold, percent of MemTotal used. Default 80.
+#   A set-but-unparseable value warns on stderr and falls back to the default.
+# FM_SPAWN_MEM_FORCE=1 - skip the check entirely, for a captain-ordered
+#   emergency dispatch. Only the literal 1 bypasses; any other set value (a
+#   typo'd true/yes) warns on stderr and is ignored, so the check still runs.
+# FM_MEMINFO_PATH_OVERRIDE - read this path instead of /proc/meminfo; the
+#   hermetic test seam, never set outside tests.
+#
+# Fails OPEN: a missing or unparseable meminfo file lets the spawn proceed.
+# The tripwire must never brick dispatch on a weird environment.
+
+fm_mem_guard_check() {
+  local meminfo total avail key val pct threshold used force
+
+  force="${FM_SPAWN_MEM_FORCE-0}"
+  case "$force" in
+    1) return 0 ;;
+    ''|0) ;;
+    *) echo "WARNING: FM_SPAWN_MEM_FORCE='${force}' ignored - only FM_SPAWN_MEM_FORCE=1 bypasses the memory headroom check" >&2 ;;
+  esac
+
+  meminfo="${FM_MEMINFO_PATH_OVERRIDE:-/proc/meminfo}"
+  [ -r "$meminfo" ] || return 0
+
+  total=
+  avail=
+  while IFS=: read -r key val; do
+    case "$key" in
+      MemTotal) total=${val%kB}; total=${total// /} ;;
+      MemAvailable) avail=${val%kB}; avail=${avail// /} ;;
+    esac
+    { [ -z "$total" ] || [ -z "$avail" ]; } || break
+  done < "$meminfo"
+
+  case "$total" in ''|*[!0-9]*) return 0 ;; esac
+  case "$avail" in ''|*[!0-9]*) return 0 ;; esac
+  [ "$total" -gt 0 ] || return 0
+
+  used=$((total - avail))
+  pct=$((used * 100 / total))
+
+  threshold="${FM_SPAWN_MEM_MAX_PCT-80}"
+  case "$threshold" in
+    ''|*[!0-9]*)
+      echo "WARNING: FM_SPAWN_MEM_MAX_PCT='${threshold}' is not a non-negative integer - falling back to the default 80% memory headroom threshold" >&2
+      threshold=80
+      ;;
+  esac
+
+  if [ "$pct" -ge "$threshold" ]; then
+    echo "DEFERRED: memory headroom tripwire tripped - used ${pct}% >= threshold ${threshold}%; firstmate should retry this dispatch once headroom recovers (bypass with FM_SPAWN_MEM_FORCE=1 for a captain-ordered emergency dispatch)" >&2
+    return 1
+  fi
+  return 0
+}

bin/fm-spawn.sh

@@ -87,6 +87,10 @@
 # On success prints: spawned <id> harness=<name> kind=<ship|scout|secondmate> mode=<mode> yolo=<on|off> window=<backend-target> worktree=<path>
 # mode/yolo are resolved per-project from data/projects.md for ship/scout tasks;
 # secondmate spawns record mode=secondmate, yolo=off, home=, and projects=.
+# Before any side effect, a memory headroom tripwire (bin/fm-mem-guard.sh)
+# refuses the launch and exits non-zero with a DEFERRED: message when used
+# memory is at or above FM_SPAWN_MEM_MAX_PCT (default 80); bypass with
+# FM_SPAWN_MEM_FORCE=1. See that script's header for the full contract.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -116,6 +120,8 @@ SUB_HOME_MARKER=".fm-secondmate-home"
 . "$SCRIPT_DIR/fm-gate-refuse-lib.sh"
 # shellcheck source=bin/fm-pr-lib.sh
 . "$SCRIPT_DIR/fm-pr-lib.sh"
+# shellcheck source=bin/fm-mem-guard.sh
+. "$SCRIPT_DIR/fm-mem-guard.sh"
 # Fail closed before any fleet mutation: a no-mistakes gate agent must never spawn
 # a direct report (see bin/fm-gate-refuse-lib.sh).
 fm_refuse_if_gate_agent
@@ -287,6 +293,12 @@ if [ "${#POS[@]}" -gt 0 ] && [ "${POS[0]}" != "$idpart" ] && case "$idpart" in *
   done
   exit "$rc"
 fi
+# Memory headroom tripwire (bin/fm-mem-guard.sh): refuse before any side
+# effect (window, worktree lease, meta, hook install). Batch pairs each
+# re-exec this script in single-task mode above, so this check runs once per
+# task launch, letting a batch land its first task and defer the rest as
+# pressure rises.
+fm_mem_guard_check || exit 1
 ID=${POS[0]}
 fm_task_id_creation_valid "$ID" || { echo "error: invalid task id" >&2; exit 2; }
 PROJ=

bin/fm-watch.sh

@@ -692,7 +692,12 @@ watcher_cleanup() {
   fm_lock_release "$WATCH_LOCK"
 }
 trap watcher_cleanup EXIT
-trap 'exit 1' HUP INT TERM
+# Bash runs the EXIT trap on an UNtrapped terminating signal only when it wins a
+# race with its own dying foreground child, so a bounded checkpoint or an
+# fm-watch-arm.sh --restart would intermittently leak this singleton lock and
+# lock out the next watcher. Trapping the signal explicitly makes the shell
+# handle it at a safe point and always fall through to the cleanup above.
+trap 'exit 143' TERM INT HUP
 # This watcher's own pid, as recorded in the lock by fm_lock_claim (which writes
 # ${BASHPID:-$$} from this same main shell). Read directly, never via a command
 # substitution, so it matches the stored holder pid for the self-eviction check.

docs/architecture.md

@@ -50,6 +50,7 @@ That block owns the live wait shape for the running primary harness: Claude and
 `bin/fm-watch-arm.sh` remains the verified arm wrapper for protocols that call it; it forks the watcher as a tracked child, verifies it is genuinely alive with a fresh liveness beacon, and prints exactly one honest status line (`started` / `attached` / restart-only `healthy` / `FAILED`, the last exiting non-zero).
 On `attached` it stays live until that existing cycle ends so background-notify harnesses do not get an empty false wake from a healthy no-op exit.
 Its `--restart` mode signals only the watcher recorded in the current home's `state/.watch.lock`, so restarting one home cannot kill sibling secondmate watchers.
+`fm-watch.sh` traps `TERM`, `INT`, and `HUP` explicitly and exits 143 through them, so a signalled watcher always falls through to its own singleton-lock release instead of intermittently leaking the lock and locking out the next watcher.
 A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, or if tasks are in flight and that watcher stops running or queued wakes are waiting to be drained.
 The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on stale watcher liveness.
 It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the stale-watcher banner/reminder policy so repeated guarded commands stay noisy without reprinting the full watcher-down banner in the same episode.
@@ -129,6 +130,16 @@ Secondmate launches are exempt because they resolve the secondmate harness and a
 Unsupported effort values are still recorded in task meta when passed to `fm-spawn.sh`, but the launch template omits any effort flag that the selected harness does not accept.
 That keeps spawn launch compatible across claude, codex, grok, pi, and opencode while preserving the requested profile for later audit.
 
+## Dispatch memory headroom
+
+Every agent launch is a real process, so concurrent dispatch on a memory-capped host can exhaust the box and take the whole fleet down with it.
+`fm-spawn.sh` sources `bin/fm-mem-guard.sh` and checks used memory against `FM_SPAWN_MEM_MAX_PCT` (default 80) once per task launch, before any side effect: no window, no worktree lease, no meta file, and no hook install.
+On a trip it prints a `DEFERRED:` line naming the measured used percent and the threshold, then exits 1.
+This is a tripwire, not a queue: it refuses one launch attempt, and firstmate retries the dispatch later from the backlog it already keeps.
+The check runs per task launch rather than per invocation, so a batch `id=repo` dispatch keeps its one-failure-does-not-stop-the-rest contract and can land its first tasks while deferring the rest as pressure rises.
+It applies identically to `--secondmate` launches, and `FM_SPAWN_MEM_FORCE=1` bypasses it for a captain-ordered emergency dispatch.
+The tripwire fails open on an unreadable or unparseable meminfo file; [configuration.md](configuration.md#environment-variables) owns the tuning knobs and fallback details.
+
 ## Optional secondmates
 
 `data/secondmates.md` records persistent domain supervisors with natural-language scopes, project clone lists, and home paths.

docs/configuration.md

@@ -358,6 +358,9 @@ FM_ZELLIJ_SESSION=firstmate  # zellij-only: named session for normal backend ops
 FM_BACKEND_CMUX_COMPOSER_LINES=20  # cmux-only: tail lines scanned to locate the composer row for submit verification
 FM_BACKEND_CMUX_IDLE_RE='^Type a message\.\.\.$'  # cmux-only: empty-composer placeholder regex after border/prompt stripping
 CMUX_SOCKET_PASSWORD=   # cmux-only: socket password fallback when config/cmux-socket-password is absent (docs/cmux-backend.md)
+FM_SPAWN_MEM_MAX_PCT=80  # used-memory percent at or above which fm-spawn.sh defers a launch before any side effect; a blank or non-integer value warns and falls back to 80
+FM_SPAWN_MEM_FORCE=      # set to the literal 1 to skip the memory headroom check for a captain-ordered emergency dispatch; any value other than 1, 0, or blank warns and is ignored
+FM_MEMINFO_PATH_OVERRIDE=   # meminfo path read instead of /proc/meminfo; the hermetic test seam, never set outside tests
 FM_SESSION_START_STATUS_TAIL=5   # state/*.status lines printed per task in the session-start digest
 FM_BOOTSTRAP_DETECT_ONLY=0   # internal/read-only session-start mode: skip bootstrap's mutating sweeps and print advisory TANGLE wording
 FM_GUARD_READ_ONLY=0    # internal/read-only guard mode: keep alarms but suppress drain, supervision repair, and checkout repair commands
@@ -431,6 +434,12 @@ FM_LOG_MAX_BYTES=1048576           # daemon log size that triggers trimming
 FM_LOG_KEEP_LINES=2000             # daemon log lines kept when trimming
 ```
 
+`bin/fm-mem-guard.sh` measures `used% = (MemTotal - MemAvailable) / MemTotal` from `/proc/meminfo` with bash builtins only, and `fm-spawn.sh` calls it once per task launch before any side effect.
+`FM_SPAWN_MEM_MAX_PCT` accepts a nonnegative integer percent; unset uses the default of 80 silently, while a blank or non-integer value warns on stderr and falls back to that same default.
+Only the literal `FM_SPAWN_MEM_FORCE=1` bypasses the check; unset, blank, and `0` leave it running silently, and any other value warns on stderr and is ignored, so the check still runs.
+The tripwire fails open: a missing, unreadable, or unparseable meminfo file lets the spawn proceed, so it can never brick dispatch on an environment without `/proc/meminfo`.
+The script's header owns the full contract.
+
 `fm-teardown.sh` retries only Git's `Unable to create '...index.lock': File exists` return failure up to `FM_TREEHOUSE_RETURN_LOCK_RETRIES` times.
 `FM_TREEHOUSE_RETURN_LOCK_RETRIES` accepts a nonnegative integer, and an unset, blank, or invalid value uses the default of 3.
 `FM_TREEHOUSE_RETURN_LOCK_RETRY_WAIT_SECS` accepts nonnegative whole or fractional seconds between attempts.

docs/scripts.md

@@ -42,6 +42,7 @@ The shared no-mistakes gate refusal used by `fm-spawn.sh`, `fm-send.sh`, and `fm
 | `fm-review-diff.sh`      | Review a crewmate branch or recorded PR head against the authoritative base          |
 | `fm-marker-lib.sh`       | Shared from-firstmate request marker, detector, and idempotent transformation         |
 | `fm-gate-refuse-lib.sh`  | Shared no-mistakes gate-context refusal for fleet lifecycle entrypoints               |
+| `fm-mem-guard.sh`        | Shared dispatch-time memory headroom tripwire sourced by `fm-spawn.sh`                |
 | `fm-watch-arm.sh`        | Verified home-scoped watcher arm wrapper with honest status reporting                |
 | `fm-watch-checkpoint.sh` | Run one bounded foreground watcher checkpoint for Codex-style supervision            |
 | `fm-watch.sh`            | Singleton-safe always-on watcher: absorb benign wakes, queue and exit on actionable ones |

tests/fm-afk-inject-herdr-e2e.test.sh

@@ -40,6 +40,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; }
 pass() { printf 'ok - %s\n' "$1"; }

tests/fm-afk-launch.test.sh

@@ -771,6 +771,10 @@ e2e_herdr() {
   command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (herdr e2e)"; return 0; }
   # shellcheck source=tests/herdr-test-safety.sh
   . "$ROOT/tests/herdr-test-safety.sh"
+  herdr_lab_precondition_ok || {
+    echo "skip: no running default Herdr session for the fleet-state tripwire (herdr e2e)"
+    return 0
+  }
   # shellcheck source=bin/fm-backend.sh
   . "$ROOT/bin/fm-backend.sh"
 

tests/fm-backend-autodetect-smoke.test.sh

@@ -44,6 +44,7 @@ command -v treehouse >/dev/null 2>&1 || { echo "skip: treehouse not found (requi
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 # TMP_ROOT is physically resolved (mktemp -d "$(pwd -P)"-relative) to keep this
 # real-herdr smoke fixture free of unrelated OS symlink noise.

tests/fm-backend-herdr-eventwait-smoke.test.sh

@@ -24,6 +24,7 @@ command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found (required
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 SESSION="fm-lab-eventwait-smoke-$$"
 export HERDR_SESSION="$SESSION"

tests/fm-backend-herdr-prune-safety-e2e.test.sh

@@ -32,6 +32,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 SESSION="fm-lab-prune-safety-e2e-$$"
 export HERDR_SESSION="$SESSION"

tests/fm-backend-herdr-respawn-idem-e2e.test.sh

@@ -44,6 +44,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 SESSION="fm-lab-respawn-idem-e2e-$$"
 export HERDR_SESSION="$SESSION"

tests/fm-backend-herdr-smoke.test.sh

@@ -26,6 +26,7 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 SESSION="fm-lab-backend-smoke-$$"
 export HERDR_SESSION="$SESSION"

tests/fm-backend-herdr-workspace-per-home-e2e.test.sh

@@ -53,6 +53,7 @@ command -v treehouse >/dev/null 2>&1 || { echo "skip: treehouse not found (requi
 
 # shellcheck source=tests/herdr-test-safety.sh
 . "$ROOT/tests/herdr-test-safety.sh"
+herdr_lab_precondition_ok || { echo "skip: no running default Herdr session for the fleet-state tripwire"; exit 0; }
 
 # TMP_ROOT is physically resolved (mktemp -d "$(pwd -P)"-relative) for the same
 # low-noise scratch fixture shape used by

tests/fm-dispatch-select.test.sh

@@ -6,6 +6,14 @@ set -u
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
 BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin}
+# jq is an unconditional dependency of the script under test, never the subject
+# of a fixture: the fakebin only ever shadows quota-axi. Keep jq reachable on
+# hosts that install it outside the sandbox PATH's system dirs, so a quota
+# fallback case cannot fail as an unrelated "jq is required" exit 2.
+if ! (PATH="$BASE_PATH"; command -v jq >/dev/null 2>&1); then
+  JQ_BIN=$(command -v jq 2>/dev/null || true)
+  [ -n "$JQ_BIN" ] && BASE_PATH="$BASE_PATH:$(dirname "$JQ_BIN")"
+fi
 TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests)
 mkdir -p "$TMP_ROOT"