feat(bootstrap): surface fleet daemon health alerts at cold start

AGENTS.md

@@ -179,6 +179,7 @@ Otherwise it prints one line per problem or capability fact; handle each:
   This mirrors `/updatefirstmate`'s `nudge-secondmates:` report: it is a gentle steer, never an interruption, and the fast-forward already landed safely.
   A secondmate that was skipped, already current, or whose advance changed no instructions is not listed and must not be disturbed.
 - `FMX: X mode on ...` / `FMX: X mode off ...` - bootstrap confirmed or removed the local X-mode poll artifacts; follow section 14 for watcher cadence restart only when a running watcher needs the transition applied immediately.
+- `ALERT: <problem> - <suggested fix>` - a live fleet-health problem surfaced at cold start (the counterpart of `fm-guard.sh`'s alarms): afk is on but the away-mode daemon is not running (its pid in `state/.supervise-daemon.pid` is missing or dead, so walk-away escalations are dark), the watcher beacon (`.last-watcher-beat`) is stale past `FM_GUARD_GRACE` while tasks are in flight, a `no-mistakes-daemon-*` systemd unit is crash-looping (NRestarts at or above `FM_NM_CRASH_THRESHOLD`, default 50), or multiple no-mistakes daemons serve one `--root` (stale-cache push risk). Like `MISSING`/`STUCK` it is surfaced to the captain and never a hard block - bootstrap still exits 0. The gate runs even in detect-only and read-only sessions because it mutates nothing, and each systemd- or pgrep-based check degrades silently where its detection mechanism is unavailable (non-Linux or stripped host).
 
 Bootstrap's fleet refresh is bounded by `FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT` seconds, default 20; a timeout is reported as a `FLEET_SYNC` skip and does not block startup.
 

bin/fm-bootstrap.sh

@@ -12,6 +12,14 @@
 #                 "SECONDMATE_SYNC: secondmate <id>: skipped: <reason>",
 #                 "NUDGE_SECONDMATES: <window-targets...>",
 #                 "FMX: X mode on ..." or "FMX: X mode off ...".
+#                 "ALERT: ..." = a live fleet-health problem (afk on but the
+#                 away-mode daemon is dark, the watcher beacon is stale with
+#                 tasks in flight, a crash-looping no-mistakes daemon unit, or
+#                 duplicate no-mistakes daemons on one root) with a suggested fix.
+#                 Like MISSING/STUCK it is surfaced to the captain, not a hard
+#                 block; bootstrap still exits 0. The gate runs even in
+#                 detect-only mode since it mutates nothing and the cold-start
+#                 counterpart of fm-guard.sh's alarms belong in every digest.
 #          A NUDGE_SECONDMATES line lists the RUNNING secondmate windows whose
 #          worktree was fast-forwarded to firstmate's own current default-branch
 #          commit (a purely LOCAL fast-forward, never an origin fetch) AND whose
@@ -60,6 +68,15 @@ FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}"
 PROJECTS="${FM_PROJECTS_OVERRIDE:-$FM_HOME/projects}"
 CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}"
 STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}"
+
+# Portable mtime in epoch seconds; mirrors fm-guard.sh (the `stat -f || stat -c`
+# fallback breaks on Linux where only one form exists).
+if [ "$(uname)" = Darwin ]; then
+  _bootstrap_stat_mtime() { stat -f %m "$1" 2>/dev/null; }
+else
+  _bootstrap_stat_mtime() { stat -c %Y "$1" 2>/dev/null; }
+fi
+
 # shellcheck source=bin/fm-tasks-axi-lib.sh
 . "$SCRIPT_DIR/fm-tasks-axi-lib.sh"
 # shellcheck source=bin/fm-tangle-lib.sh
@@ -171,6 +188,86 @@ secondmate_sync() {
   return 0
 }
 
+# Fleet daemon + watcher health gate. Runs LAST so it never blocks tool detection
+# or fleet sync, and surfaces one ALERT line per LIVE problem. Like MISSING and
+# STUCK lines it is surfaced to the captain; bootstrap still exits 0 (it warns,
+# never blocks - mirroring bin/fm-guard.sh's contract). Every check degrades
+# silently when its detection mechanism is unavailable (no systemd --user, no
+# pgrep), so a non-Linux or stripped host simply skips the systemd-based checks.
+daemon_health_check() {
+  [ -d "$STATE" ] || return 0
+  local grace=${FM_GUARD_GRACE:-300} now m age meta in_flight=0 pid
+
+  # (1) afk daemon alive when afk is active. If state/.afk is present the
+  #     away-mode engine MUST be running or its escalation path is silently dark
+  #     (a routine wake during a walk-away would never reach the captain). The
+  #     afk skill starts the daemon with `nohup bin/fm-supervise-daemon.sh &` and
+  #     records its pid in state/.supervise-daemon.pid, so liveness is read from
+  #     that pidfile: a missing/empty pidfile or a dead pid is the dark state.
+  if [ -e "$STATE/.afk" ]; then
+    pid=$(cat "$STATE/.supervise-daemon.pid" 2>/dev/null || true)
+    if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
+      echo "ALERT: afk is on but the away-mode daemon is not running - escalations are dark; re-enter /afk or run: nohup bin/fm-supervise-daemon.sh >/dev/null 2>&1 &"
+    fi
+  fi
+
+  # (2) watcher beacon fresh when work is in flight. .last-watcher-beat is
+  #     touched every watcher poll; stale or missing with tasks in flight means
+  #     supervision is off (the same condition bin/fm-guard.sh banners, surfaced
+  #     here at cold start so recovery does not silently proceed unsupervised).
+  for meta in "$STATE"/*.meta; do
+    [ -e "$meta" ] || continue
+    in_flight=$((in_flight + 1))
+  done
+  if [ "$in_flight" -gt 0 ] && [ -e "$STATE/.last-watcher-beat" ]; then
+    m=$(_bootstrap_stat_mtime "$STATE/.last-watcher-beat")
+    if [ -n "$m" ]; then
+      now=$(date +%s); age=$(( now - m ))
+      if [ "$age" -ge "$grace" ]; then
+        echo "ALERT: watcher beacon is ${age}s stale (grace ${grace}s) with ${in_flight} task(s) in flight - supervision is off; re-arm: bin/fm-watch-arm.sh as a harness-tracked background task"
+      fi
+    fi
+  fi
+
+  _bootstrap_nm_health
+}
+
+# no-mistakes daemon health (crash-loops + duplicate-root daemons). systemd
+# --user only; degrades silently otherwise. The two failure modes are the live
+# resilience gaps: a leaked unit flapping against a deleted binary burns CPU +
+# journals, and more than one daemon serving one --root is the stale-cache
+# push-failure mechanism.
+_bootstrap_nm_health() {
+  command -v systemctl >/dev/null 2>&1 || return 0
+  systemctl --user show-environment >/dev/null 2>&1 || return 0
+
+  # (3) crash-looping nm units: a high NRestarts means a unit is flapping.
+  #     Threshold defaults to 50; FM_NM_CRASH_THRESHOLD overrides.
+  local threshold=${FM_NM_CRASH_THRESHOLD:-50} unit nrestarts crashing=""
+  while IFS= read -r unit; do
+    [ -n "$unit" ] || continue
+    nrestarts=$(systemctl --user show -p NRestarts --value "$unit" 2>/dev/null || echo 0)
+    case "$nrestarts" in ''|*[!0-9]*) nrestarts=0 ;; esac
+    [ "$nrestarts" -ge "$threshold" ] && crashing="$crashing $unit(${nrestarts})"
+  done < <(systemctl --user list-units --type=service --all --no-legend --no-pager 2>/dev/null | awk '$1 ~ /^no-mistakes-daemon-/ {print $1}')
+  if [ -n "$crashing" ]; then
+    echo "ALERT: no-mistakes daemon unit(s) crash-looping (high NRestarts):$crashing - investigate/reset: systemctl --user stop '<unit>' && systemctl --user reset-failed '<unit>'"
+  fi
+
+  # (4) duplicate nm daemons on one root. More than one process serving the same
+  #     --root lets an old daemon with stale in-memory gate config serve pushes
+  #     after the on-disk gate was re-initialized.
+  command -v pgrep >/dev/null 2>&1 || return 0
+  local dup line
+  dup=$(pgrep -af 'no-mistakes daemon run --root' 2>/dev/null \
+        | sed -nE 's/.*--root ([^ ]+).*/\1/p' | sort | uniq -d)
+  [ -z "$dup" ] && return 0
+  for line in $dup; do
+    [ -n "$line" ] || continue
+    echo "ALERT: multiple no-mistakes daemons serve root '$line' (stale-cache push risk) - reconcile: keep the systemd-managed one, kill the stray nohup processes"
+  done
+}
+
 install_cmd() {
   case "$1" in
     tmux|node|gh|curl|jq) echo "brew install $1  # or the platform's package manager" ;;
@@ -427,4 +524,5 @@ if [ "${FM_BOOTSTRAP_DETECT_ONLY:-0}" != 1 ]; then
   x_mode_setup
   fleet_sync
 fi
+daemon_health_check
 exit 0

docs/configuration.md

@@ -202,6 +202,7 @@ FMX_X_THREAD_MAX=25     # maximum tweets in one auto-split X reply thread
 FMX_FOLLOWUP_MAX_AGE_SECS=86400   # local window for posting one X completion follow-up
 FM_LOCK_STALE_AFTER=2   # seconds before dead-pid lock records can be reclaimed; mid-acquire locks keep at least 2s grace
 FM_GUARD_GRACE=300      # seconds before guard warnings and arm health checks treat a watcher beacon as stale
+FM_NM_CRASH_THRESHOLD=50   # NRestarts at/above this marks a no-mistakes-daemon-* unit as crash-looping in bootstrap's ALERT gate
 FM_ARM_CONFIRM_TIMEOUT=10   # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED
 FM_WATCHER_STALE_GRACE=300   # defaults to FM_GUARD_GRACE; seconds a live watcher lock may have a stale beacon before re-arm errors
 FM_SIGNAL_GRACE=30      # seconds to coalesce nearby status and turn-end signals into one wake

docs/scripts.md

@@ -6,7 +6,7 @@ Each file also starts with a short header comment.
 | Script                   | Description                                                                                                         |
 | ------------------------ | ------------------------------------------------------------------------------------------------------------------- |
 | `fm-session-start.sh`    | The one command AGENTS.md sections 3 and 5 run at every session start: composes `fm-lock.sh`, `fm-bootstrap.sh` (its three mutating sweeps gated on holding the lock via `FM_BOOTSTRAP_DETECT_ONLY`), and `fm-wake-drain.sh`, then prints a full context digest (`data/projects.md`, `data/secondmates.md`, `data/captain.md`, `data/learnings.md`, each `ABSENT`-marked when missing) and fleet-state digest (`data/backlog.md`, every `state/*.meta`, a bounded `state/*.status` tail, `state/.afk`, and a cheap per-task endpoint-liveness read); prints a loud read-only banner and skips every mutating step when the lock is held elsewhere; never arms the watcher itself |
-| `fm-bootstrap.sh`        | Detect required toolchain and version problems, dispatch profile JSON errors or active-rule blocks, default backlog-backend status, primary-checkout `TANGLE:` problems, and actionable clone refresh outcomes; refresh project clones best-effort; locally sync live secondmate homes and propagate declared inheritable config; set up opt-in X mode; install tools only after consent; `FM_BOOTSTRAP_DETECT_ONLY=1` skips the three mutating sweeps and prints advisory-only `TANGLE:` wording without a checkout command |
+| `fm-bootstrap.sh`        | Detect required toolchain and version problems, dispatch profile JSON errors or active-rule blocks, default backlog-backend status, primary-checkout `TANGLE:` problems, and actionable clone refresh outcomes; refresh project clones best-effort; locally sync live secondmate homes and propagate declared inheritable config; set up opt-in X mode; surface live fleet-health `ALERT:` lines from the daemon + watcher health gate (afk-on-but-daemon-dark, stale watcher beacon with tasks in flight, crash-looping no-mistakes daemon units, or duplicate daemons on one root), which runs last and warns even in detect-only mode; install tools only after consent; `FM_BOOTSTRAP_DETECT_ONLY=1` skips the three mutating sweeps and prints advisory-only `TANGLE:` wording without a checkout command |
 | `fm-fleet-sync.sh`       | Fetch clones, fast-forward safe default-branch states, self-heal clean detached ancestor drift, report unsafe drift as `STUCK:`, and safely prune branches whose remote is gone |
 | `fm-update.sh`           | Self-update the running firstmate repo and registered secondmate homes with fast-forward-only pulls from origin     |
 | `fm-backlog-handoff.sh`  | Move already-judged in-scope queued backlog items from the main home into a seeded secondmate home                 |

tests/fm-bootstrap.test.sh

@@ -207,3 +207,82 @@ test_bootstrap_reporting
 test_no_mistakes_min_version
 test_crew_dispatch_active_rules_are_surfaced
 test_crew_dispatch_validation
+
+# --- daemon/watcher health gate (ALERT lines) ---------------------------------
+#
+# The health gate is the cold-start counterpart of bin/fm-guard.sh: it surfaces
+# one ALERT line per live fleet-health problem and stays silent when healthy.
+# A FAILING fake systemctl makes the systemd-based no-mistakes checks degrade
+# silently; the afk-dark ALERT is driven by the pidfile (a dead pid), so each
+# case below exercises both a real ALERT and the silent-degradation path at once.
+add_failing_systemctl() {
+  local fakebin=$1
+  printf '#!/usr/bin/env bash\nexit 1\n' > "$fakebin/systemctl"
+  chmod +x "$fakebin/systemctl"
+}
+
+# make_state_home <dir>: lay down an empty FM_HOME shell with state/ + projects/.
+make_state_home() {
+  local dir=$1
+  mkdir -p "$dir/home/state" "$dir/home/projects"
+}
+
+test_health_gate_silent_when_healthy() {
+  local case_dir fakebin out
+  case_dir="$TMP_ROOT/hg-healthy"
+  make_state_home "$case_dir"
+  fakebin=$(make_fake_toolchain "$case_dir")
+  add_failing_systemctl "$fakebin"
+  # No .afk and no in-flight task: the health gate must print nothing.
+  out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \
+    FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh")
+  case "$out" in
+    *ALERT*) fail "health gate emitted an ALERT on a healthy fleet: $out" ;;
+  esac
+  pass "health gate is silent on a healthy fleet"
+}
+
+test_health_gate_afk_daemon_dark() {
+  local case_dir fakebin out
+  case_dir="$TMP_ROOT/hg-afk-dark"
+  make_state_home "$case_dir"
+  fakebin=$(make_fake_toolchain "$case_dir")
+  add_failing_systemctl "$fakebin"
+  : > "$case_dir/home/state/.afk"           # afk on ...
+  # pidfile points at a PID that is not alive (and not us): daemon dark.
+  echo 999999 > "$case_dir/home/state/.supervise-daemon.pid"
+  out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \
+    FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh")
+  case "$out" in
+    *"ALERT: afk is on but the away-mode daemon is not running"*) ;;
+    *) fail "afk-dark ALERT missing; got: $out" ;;
+  esac
+  # The systemd-based nm checks must NOT emit a spurious ALERT when systemd is
+  # unavailable (silent degradation).
+  case "$out" in
+    *"no-mistakes daemon unit"*) fail "nm crash-loop ALERT leaked without systemd: $out" ;;
+  esac
+  pass "health gate alerts when afk is on but the daemon is dark"
+}
+
+test_health_gate_watcher_stale() {
+  local case_dir fakebin out
+  case_dir="$TMP_ROOT/hg-watcher-stale"
+  make_state_home "$case_dir"
+  fakebin=$(make_fake_toolchain "$case_dir")
+  add_failing_systemctl "$fakebin"
+  echo "window=firstmate:fm-demo" > "$case_dir/home/state/demo.meta"  # one task in flight
+  touch "$case_dir/home/state/.last-watcher-beat"
+  touch -d @1 "$case_dir/home/state/.last-watcher-beat"   # epoch -> far past grace
+  out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \
+    FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh")
+  case "$out" in
+    *"ALERT: watcher beacon is"*"stale"*"supervision is off"*) ;;
+    *) fail "watcher-stale ALERT missing; got: $out" ;;
+  esac
+  pass "health gate alerts when the watcher beacon is stale with work in flight"
+}
+
+test_health_gate_silent_when_healthy
+test_health_gate_afk_daemon_dark
+test_health_gate_watcher_stale

tests/fm-session-start.test.sh

@@ -256,8 +256,11 @@ $rec
 EOF
   make_fake_toolchain "$fakebin"
   make_fake_ps_claude "$fakebin"
-  # Force a MISSING diagnostic line so the bootstrap section is non-trivial.
-  rm -f "$fakebin/node"
+  # The fake toolchain omits tasks-axi (a firstmate-specific npm tool, never in
+  # the system BASE_PATH), so bootstrap deterministically emits a
+  # MISSING: tasks-axi line regardless of host tooling - a non-trivial
+  # diagnostic to assert ordering against (host-installed node would otherwise
+  # make removing fakebin/node unreliable).
 
   printf 'window=fm-sess:w1\nkind=ship\n' > "$home/state/task-a.meta"
 
@@ -280,7 +283,7 @@ EOF
   [ "$context_line" -lt "$fleet_line" ] || fail "CONTEXT did not precede FLEET STATE"
   [ "$fleet_line" -lt "$next_line" ] || fail "FLEET STATE did not precede NEXT STEP"
 
-  missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: node' | head -1 | cut -d: -f1)
+  missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: tasks-axi' | head -1 | cut -d: -f1)
   [ -n "$missing_line" ] || fail "MISSING diagnostic did not appear at all"
   [ "$missing_line" -lt "$fleet_line" ] || fail "actionable MISSING diagnostic was buried after the bulk fleet-state digest"
 
@@ -400,16 +403,19 @@ $rec
 EOF
   make_fake_toolchain "$fakebin"
   make_fake_ps_claude "$fakebin"
-  rm -f "$fakebin/node"
 
   append_wake "$home/state" signal task-z "needs-decision: pick a library"
 
   out=$(run_session_start "$home" "$root" "$fakebin:$BASE_PATH")
 
   # fm-lock.sh's own exact success text.
   assert_contains "$out" "lock acquired: harness pid" "fm-lock.sh's real output did not appear (composition, not reimplementation)"
-  # fm-bootstrap.sh's own exact MISSING-tool line format.
-  assert_contains "$out" "MISSING: node (install:" "fm-bootstrap.sh's real detect line did not appear verbatim"
+  # fm-bootstrap.sh's own exact MISSING-tool line format. tasks-axi is omitted
+  # from the fake toolchain and is a firstmate-specific npm tool never present
+  # in the system BASE_PATH, so this line appears deterministically regardless
+  # of host tooling (host-installed node would make removing fakebin/node
+  # unreliable).
+  assert_contains "$out" "MISSING: tasks-axi (install:" "fm-bootstrap.sh's real detect line did not appear verbatim"
   # fm-wake-drain.sh's real drained record (raw tab-separated queue line).
   assert_contains "$out" "$(printf 'signal\ttask-z\tneeds-decision: pick a library')" "fm-wake-drain.sh's real drained record did not appear"