fix(herdr): record a stopped default session in the lab fleet-state tripwire

bin/fm-brief.sh

@@ -206,8 +206,9 @@ HERDR_SECTION=$(printf '%s\n' \
 '   It re-checks refuse-default immediately before stop and again immediately before delete, and fails closed on ambiguity.' \
 '4. If an experiment requires a deliberate mid-run session stop, use only `"$HERDR_LAB_HELPER" stop "$HERDR_LAB_SESSION"`; it performs the same immediate refuse-default check.' \
 '5. Forbidden commands: direct `herdr server stop`, every other server-global operation such as `herdr server live-handoff` or reload/update operations, direct `herdr session stop`, direct `herdr session delete`, and any Herdr call scoped only by ambient or inline `HERDR_SESSION`.' \
-'6. The helper records the live default session before provisioning and verifies the identical fleet state after teardown.' \
-'   A missing, stopped, or changed default session is a hard tripwire failure, never a cleanup warning to ignore.' \
+'6. The helper records the default session, running or not, before provisioning and verifies the identical fleet state after teardown.' \
+'   A missing or changed default session is a hard tripwire failure, never a cleanup warning to ignore.' \
+'   A default session that is already stopped is recorded as-is, so it starting or stopping during lab work still trips teardown.' \
 '' \
 'Never bypass the helper, even for a read-only lifecycle probe or cleanup after failure.' \
 'The captain fleet uses the running `default` session.')

bin/fm-herdr-lab.sh

@@ -19,8 +19,8 @@
 # delete is available only through teardown.
 # Both paths perform a fresh refuse-default check immediately before each
 # destructive call.
-# Provision records the running default session as a fleet-state tripwire and
-# teardown requires that record to be identical afterward.
+# Provision records the default session as a fleet-state tripwire, running or
+# not, and teardown requires that record to be identical afterward.
 set -u
 
 fm_herdr_lab_error() {
@@ -62,15 +62,19 @@ fm_herdr_lab_fleet_state() { # <session>
     fm_herdr_lab_error "cannot read Herdr sessions for the fleet-state tripwire"
     return 1
   }
+  # The snapshot keeps .running so a lab session that starts or stops the
+  # default server trips the before/after comparison, but a stopped default is
+  # a legitimate state to record: requiring it to be running here would refuse
+  # every isolated lab session on a host that simply has no live default server.
   snapshot=$(printf '%s' "$sessions" | jq -c '
     [.sessions[]? | select(.default == true)]
-    | if length == 1 and .[0].name == "default" and .[0].running == true
+    | if length == 1 and .[0].name == "default"
       then .[0] | {name, default, running, socket_path}
       else empty
       end
   ' 2>/dev/null)
   [ -n "$snapshot" ] || {
-    fm_herdr_lab_error "fleet-state tripwire requires exactly one running default session"
+    fm_herdr_lab_error "fleet-state tripwire requires exactly one default session named 'default'"
     return 1
   }
   printf '%s\n' "$snapshot"

docs/herdr-backend.md

@@ -427,7 +427,8 @@ The fix, verified against the real binary in an isolated session (both a genuine
 The same guard is now a first-class production helper, `bin/fm-herdr-lab.sh`, not just test scaffolding.
 It provisions an isolated never-`default` lab session (names must start with `fm-lab-`), runs every task command through `run <session> ...` with a mandatory trailing `--session` appended, and refuses caller-supplied `--session`, any leading option before the subcommand, and every server or session-lifecycle subcommand.
 Destructive teardown goes only through `teardown <session>` (or a deliberate mid-run `stop <session>`), each re-running the refuse-default check immediately before every stop and delete.
-It also adds a before/after fleet-state tripwire: `provision` records the live `default` session before creating the lab session, and `teardown` verifies that recorded state is byte-identical afterward before clearing it, treating any missing, stopped, or changed default session as a hard failure rather than a warning.
+It also adds a before/after fleet-state tripwire: `provision` records the `default` session before creating the lab session, and `teardown` verifies that recorded state is byte-identical afterward before clearing it, treating any missing or changed default session as a hard failure rather than a warning.
+The recorded snapshot keeps the default session's `running` flag, so a lab session that starts or stops the default server still trips teardown; a default session that is merely stopped at provision time is a legitimate state to record, because requiring a running one would refuse every isolated lab session on a host with no live default server.
 Crewmate briefs for tasks that drive Herdr lifecycle get this exact contract embedded by scaffolding with `bin/fm-brief.sh --herdr-lab`; every crewmate brief scaffolded without the flag instead carries a loud not-enabled gate, because the scaffold cannot detect from the caller-supplied repo string whether the task will touch Herdr lifecycle.
 
 ## ID stability across a server restart

tests/fm-brief.test.sh

@@ -145,7 +145,7 @@ test_herdr_lab_contract_is_explicit_and_complete() {
     "Herdr lab brief missing the per-call trailing session contract"
   assert_grep "direct \`herdr server stop\`" "$brief" \
     "Herdr lab brief missing the forbidden server-global command list"
-  assert_grep "records the live default session before provisioning" "$brief" \
+  assert_grep "records the default session, running or not, before provisioning" "$brief" \
     "Herdr lab brief missing the before tripwire"
   assert_grep "verifies the identical fleet state after teardown" "$brief" \
     "Herdr lab brief missing the after tripwire"

tests/fm-dispatch-select.test.sh

@@ -5,9 +5,13 @@ set -u
 # shellcheck source=tests/lib.sh
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin}
 TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests)
 mkdir -p "$TMP_ROOT"
+# fm-dispatch-select.sh requires the real jq; make it resolvable regardless of
+# where it is installed, which the bare BASE_PATH may not include. Only jq is
+# imported, so test_quota_missing_falls_back_to_first still sees no quota-axi
+# even on a host that installs both under one prefix.
+BASE_PATH=$(fm_test_base_path "$TMP_ROOT" jq)
 
 write_quota() {
   local file=$1 claude_status=$2 claude_five=$3 claude_week=$4 codex_status=$5 codex_five=$6 codex_week=$7

tests/fm-herdr-lab.test.sh

@@ -13,6 +13,7 @@ TRIPWIRES="$TMP_ROOT/tripwires"
 REAL_SLEEP=$(command -v sleep)
 mkdir -p "$FAKE_STATE"
 printf '%s\n' '/Users/test/.config/herdr/herdr.sock' > "$FAKE_STATE/default-socket"
+printf '%s\n' true > "$FAKE_STATE/default-running"
 : > "$FAKE_LOG"
 
 cat > "$FAKEBIN/herdr" <<'SH'
@@ -28,18 +29,21 @@ done
 [ "${previous:-}" = --session ] || { echo "fake herdr: missing trailing --session" >&2; exit 90; }
 session=$last
 default_socket=$(cat "$state/default-socket")
+default_running=$(cat "$state/default-running")
 lab_state=absent
 [ ! -f "$state/$session" ] || lab_state=$(cat "$state/$session")
 
 case "$1 ${2:-}" in
   "session list")
     if [ "$lab_state" = absent ] || [ "$lab_state" = deleted ]; then
-      jq -nc --arg socket "$default_socket" '{sessions:[{default:true,name:"default",running:true,socket_path:$socket}]}'
+      jq -nc --arg socket "$default_socket" --argjson default_running "$default_running" \
+        '{sessions:[{default:true,name:"default",running:$default_running,socket_path:$socket}]}'
     else
       running=false
       [ "$lab_state" = running ] && running=true
       jq -nc --arg socket "$default_socket" --arg name "$session" --argjson running "$running" \
-        '{sessions:[{default:true,name:"default",running:true,socket_path:$socket},{default:false,name:$name,running:$running,socket_path:("/tmp/" + $name + ".sock")}]}'
+        --argjson default_running "$default_running" \
+        '{sessions:[{default:true,name:"default",running:$default_running,socket_path:$socket},{default:false,name:$name,running:$running,socket_path:("/tmp/" + $name + ".sock")}]}'
     fi
     ;;
   "server --session")
@@ -178,6 +182,26 @@ test_changed_default_trips_after_teardown() {
   pass "fm-herdr-lab: changed default fleet state is a hard failure"
 }
 
+test_stopped_default_still_provisions_and_trips() {
+  local name="fm-lab-default-stopped-$$" status=0
+  : > "$FAKE_LOG"
+  printf '%s\n' false > "$FAKE_STATE/default-running"
+  run_with_fake fm_herdr_lab_provision "$name" || fail "a stopped default session must not block an isolated lab"
+  assert_present "$TRIPWIRES/$name.fleet-state.json" "provision against a stopped default recorded no tripwire"
+  grep -q '"running":false' "$TRIPWIRES/$name.fleet-state.json" ||
+    fail "tripwire did not record the default session's stopped state"
+  # The relaxed precondition must not weaken the tripwire: the default session
+  # coming up during lab work is still a fleet-state change.
+  printf '%s\n' true > "$FAKE_STATE/default-running"
+  run_with_fake fm_herdr_lab_teardown "$name" >/dev/null 2>&1 || status=$?
+  expect_code 1 "$status" "a default session that started during lab work must fail teardown"
+  printf '%s\n' false > "$FAKE_STATE/default-running"
+  run_with_fake fm_herdr_lab_teardown "$name" || fail "teardown against an unchanged stopped default failed"
+  assert_absent "$TRIPWIRES/$name.fleet-state.json" "successful teardown left its tripwire behind"
+  printf '%s\n' true > "$FAKE_STATE/default-running"
+  pass "fm-herdr-lab: a stopped default session provisions and still trips on change"
+}
+
 test_stopped_owned_lab_can_reprovision() {
   local name="fm-lab-reprovision-$$"
   : > "$FAKE_LOG"
@@ -235,6 +259,7 @@ test_refuses_unsafe_names
 test_provision_run_and_guarded_teardown
 test_missing_tripwire_blocks_destruction
 test_changed_default_trips_after_teardown
+test_stopped_default_still_provisions_and_trips
 test_stopped_owned_lab_can_reprovision
 test_failed_delete_retains_tripwire
 test_timed_out_provision_cancels_late_launch

tests/fm-session-start.test.sh

@@ -394,7 +394,7 @@ EOF
 # --- output ordering ----------------------------------------------------------
 
 test_output_ordering_diagnostics_lead() {
-  local rec root home fakebin out lock_line boot_line wake_line context_line fleet_line next_line
+  local rec root home fakebin mask out lock_line boot_line wake_line context_line fleet_line next_line
   rec=$(new_world ordering)
   IFS='|' read -r root home fakebin <<EOF
 $rec
@@ -403,10 +403,14 @@ EOF
   make_fake_ps_claude "$fakebin"
   # Force a MISSING diagnostic line so the bootstrap section is non-trivial.
   rm -f "$fakebin/node"
+  # A real node may live on a BASE_PATH dir (e.g. apt's /usr/bin/node), which
+  # would satisfy detection and silently defeat this forced-missing case, so the
+  # diagnostic must fire independent of the host's real tool layout.
+  mask=$(fm_test_command_mask "$home/mask-node.bash" node)
 
   printf 'window=fm-sess:w1\nkind=ship\n' > "$home/state/task-a.meta"
 
-  out=$(run_session_start "$home" "$root" "$fakebin:$BASE_PATH")
+  out=$(BASH_ENV="$mask" run_session_start "$home" "$root" "$fakebin:$BASE_PATH")
 
   lock_line=$(printf '%s\n' "$out" | grep -n '^LOCK$' | head -1 | cut -d: -f1)
   boot_line=$(printf '%s\n' "$out" | grep -n '^BOOTSTRAP$' | head -1 | cut -d: -f1)
@@ -444,15 +448,7 @@ EOF
     rm -f "$fakebin/tmux"
     fm_fake_exit0 "$fakebin" herdr jq
     printf '%s\n' manual > "$home/config/backlog-backend"
-    mask="$home/mask-tmux.bash"
-    cat > "$mask" <<'SH'
-command() {
-  if [ "${1:-}" = -v ] && [ "${2:-}" = tmux ]; then
-    return 1
-  fi
-  builtin command "$@"
-}
-SH
+    mask=$(fm_test_command_mask "$home/mask-tmux.bash" tmux)
     if [ "$mode" = configured ]; then
       printf '%s\n' herdr > "$home/config/backend"
       out=$(TMUX='' HERDR_ENV='' BASH_ENV="$mask" run_session_start "$home" "$root" "$fakebin:$BASE_PATH")
@@ -578,18 +574,21 @@ EOF
 # --- composition: real scripts run, not reimplemented ------------------------
 
 test_composition_invokes_real_scripts() {
-  local rec root home fakebin out
+  local rec root home fakebin mask out
   rec=$(new_world composition)
   IFS='|' read -r root home fakebin <<EOF
 $rec
 EOF
   make_fake_toolchain "$fakebin"
   make_fake_ps_claude "$fakebin"
   rm -f "$fakebin/node"
+  # See test_output_ordering_diagnostics_lead: mask `command -v node` so a real
+  # node on a BASE_PATH dir (e.g. apt's /usr/bin/node) cannot satisfy detection.
+  mask=$(fm_test_command_mask "$home/mask-node.bash" node)
 
   append_wake "$home/state" signal task-z "needs-decision: pick a library"
 
-  out=$(run_session_start "$home" "$root" "$fakebin:$BASE_PATH")
+  out=$(BASH_ENV="$mask" run_session_start "$home" "$root" "$fakebin:$BASE_PATH")
 
   # fm-lock.sh's own exact success text.
   assert_contains "$out" "lock acquired: harness pid" "fm-lock.sh's real output did not appear (composition, not reimplementation)"

tests/fm-x-mode.test.sh

@@ -13,13 +13,11 @@ set -u
 # shellcheck source=tests/lib.sh
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin}
-# The client under test uses the real jq; make it resolvable regardless of where
-# it is installed (Homebrew, Nix profile bins, etc.), which the bare BASE_PATH may
-# not include. Prepended after the fakebin so the fake curl still wins.
-JQ_DIR=$(command -v jq 2>/dev/null) && JQ_DIR=$(dirname "$JQ_DIR") || JQ_DIR=
-[ -n "$JQ_DIR" ] && BASE_PATH="$JQ_DIR:$BASE_PATH"
 TMP_ROOT=$(fm_test_tmproot fm-x-mode-tests)
+# The client under test uses the real jq; fm_test_base_path makes it resolvable
+# regardless of where it is installed, which the bare BASE_PATH may not include.
+# Prepended after the fakebin so the fake curl still wins.
+BASE_PATH=$(fm_test_base_path "$TMP_ROOT" jq)
 
 # A fakebin `curl` that mimics the relay: it reads its behavior from env
 # (FAKE_POLL_CODE/FAKE_POLL_BODY/FAKE_ANSWER_CODE, and

tests/lib.sh

@@ -76,6 +76,60 @@ fm_test_tmproot() {
   printf '%s\n' "$root"
 }
 
+# --- BASE_PATH / real-tool resolution ---------------------------------------
+
+# fm_test_base_path <dir> [tool...]: echo a BASE_PATH-style PATH for tests that
+# shadow PATH with a fakebin dir. Starts from FM_TEST_BASE_PATH (or the standard
+# system dirs) and makes each named tool that resolves on the host reachable, so
+# a test stays hermetic to fakebin overrides while still finding a real tool
+# installed somewhere the bare BASE_PATH may not cover (Homebrew, Nix profile
+# bins, ~/.local/bin, etc.). Generalizes the JQ_DIR pattern from
+# fm-x-mode.test.sh; callers that need the real tool should pass it here rather
+# than re-deriving its directory.
+#
+# Only the named tools are imported: each is symlinked into <dir>/realbin and
+# that one dir is prepended, never the tool's own prefix dir. Prepending the
+# prefix dir would drag every unrelated binary beside it (a shared ~/.local/bin
+# or Nix profile holds dozens) ahead of the system dirs, so a tool a test needs
+# ABSENT could resolve purely because of the host's layout.
+fm_test_base_path() {
+  local dir=$1 base=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} tool resolved realbin
+  shift
+  realbin="$dir/realbin"
+  for tool in "$@"; do
+    resolved=$(command -v "$tool" 2>/dev/null) || continue
+    # `command -v` echoes a bare name for a builtin, and the whole definition for
+    # a function; only an absolute path names a real file to link.
+    case $resolved in
+      /*) ;;
+      *) continue ;;
+    esac
+    mkdir -p "$realbin"
+    ln -sf "$resolved" "$realbin/$tool"
+  done
+  [ -d "$realbin" ] && base="$realbin:$base"
+  printf '%s\n' "$base"
+}
+
+# --- command -v masking -----------------------------------------------------
+
+# fm_test_command_mask <file> <tool>: write a BASH_ENV snippet at <file> that
+# makes `command -v <tool>` fail, and echo <file>. Use it when a test forces a
+# tool's absence from a fakebin PATH but a real one may still sit on a BASE_PATH
+# dir (e.g. apt's /usr/bin/node) and silently satisfy detection.
+fm_test_command_mask() {
+  local file=$1 tool=$2
+  cat > "$file" <<SH
+command() {
+  if [ "\${1:-}" = -v ] && [ "\${2:-}" = $tool ]; then
+    return 1
+  fi
+  builtin command "\$@"
+}
+SH
+  printf '%s\n' "$file"
+}
+
 # --- fakebin / PATH shims ---------------------------------------------------
 #
 # fm_fakebin <dir> creates <dir>/fakebin and echoes it; prepend it to PATH to