diff --git a/AGENTS.md b/AGENTS.md index d657a01b..578ee1e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -179,6 +179,7 @@ Otherwise it prints one line per problem or capability fact; handle each: This mirrors `/updatefirstmate`'s `nudge-secondmates:` report: it is a gentle steer, never an interruption, and the fast-forward already landed safely. A secondmate that was skipped, already current, or whose advance changed no instructions is not listed and must not be disturbed. - `FMX: X mode on ...` / `FMX: X mode off ...` - bootstrap confirmed or removed the local X-mode poll artifacts; follow section 14 for watcher cadence restart only when a running watcher needs the transition applied immediately. +- `ALERT: - ` - a live fleet-health problem surfaced at cold start (the counterpart of `fm-guard.sh`'s alarms): afk is on but the away-mode daemon is not running (its pid in `state/.supervise-daemon.pid` is missing or dead, so walk-away escalations are dark), the watcher beacon (`.last-watcher-beat`) is stale past `FM_GUARD_GRACE` while tasks are in flight, a `no-mistakes-daemon-*` systemd unit is crash-looping (NRestarts at or above `FM_NM_CRASH_THRESHOLD`, default 50), or multiple no-mistakes daemons serve one `--root` (stale-cache push risk). Like `MISSING`/`STUCK` it is surfaced to the captain and never a hard block - bootstrap still exits 0. The gate runs even in detect-only and read-only sessions because it mutates nothing, and each systemd- or pgrep-based check degrades silently where its detection mechanism is unavailable (non-Linux or stripped host). Bootstrap's fleet refresh is bounded by `FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT` seconds, default 20; a timeout is reported as a `FLEET_SYNC` skip and does not block startup. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 2ef62d87..b1141a7e 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -12,6 +12,14 @@ # "SECONDMATE_SYNC: secondmate : skipped: ", # "NUDGE_SECONDMATES: ", # "FMX: X mode on ..." or "FMX: X mode off ...". +# "ALERT: ..." = a live fleet-health problem (afk on but the +# away-mode daemon is dark, the watcher beacon is stale with +# tasks in flight, a crash-looping no-mistakes daemon unit, or +# duplicate no-mistakes daemons on one root) with a suggested fix. +# Like MISSING/STUCK it is surfaced to the captain, not a hard +# block; bootstrap still exits 0. The gate runs even in +# detect-only mode since it mutates nothing and the cold-start +# counterpart of fm-guard.sh's alarms belong in every digest. # A NUDGE_SECONDMATES line lists the RUNNING secondmate windows whose # worktree was fast-forwarded to firstmate's own current default-branch # commit (a purely LOCAL fast-forward, never an origin fetch) AND whose @@ -60,6 +68,15 @@ FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" PROJECTS="${FM_PROJECTS_OVERRIDE:-$FM_HOME/projects}" CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" + +# Portable mtime in epoch seconds; mirrors fm-guard.sh (the `stat -f || stat -c` +# fallback breaks on Linux where only one form exists). +if [ "$(uname)" = Darwin ]; then + _bootstrap_stat_mtime() { stat -f %m "$1" 2>/dev/null; } +else + _bootstrap_stat_mtime() { stat -c %Y "$1" 2>/dev/null; } +fi + # shellcheck source=bin/fm-tasks-axi-lib.sh . "$SCRIPT_DIR/fm-tasks-axi-lib.sh" # shellcheck source=bin/fm-tangle-lib.sh @@ -171,6 +188,86 @@ secondmate_sync() { return 0 } +# Fleet daemon + watcher health gate. Runs LAST so it never blocks tool detection +# or fleet sync, and surfaces one ALERT line per LIVE problem. Like MISSING and +# STUCK lines it is surfaced to the captain; bootstrap still exits 0 (it warns, +# never blocks - mirroring bin/fm-guard.sh's contract). Every check degrades +# silently when its detection mechanism is unavailable (no systemd --user, no +# pgrep), so a non-Linux or stripped host simply skips the systemd-based checks. +daemon_health_check() { + [ -d "$STATE" ] || return 0 + local grace=${FM_GUARD_GRACE:-300} now m age meta in_flight=0 pid + + # (1) afk daemon alive when afk is active. If state/.afk is present the + # away-mode engine MUST be running or its escalation path is silently dark + # (a routine wake during a walk-away would never reach the captain). The + # afk skill starts the daemon with `nohup bin/fm-supervise-daemon.sh &` and + # records its pid in state/.supervise-daemon.pid, so liveness is read from + # that pidfile: a missing/empty pidfile or a dead pid is the dark state. + if [ -e "$STATE/.afk" ]; then + pid=$(cat "$STATE/.supervise-daemon.pid" 2>/dev/null || true) + if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then + echo "ALERT: afk is on but the away-mode daemon is not running - escalations are dark; re-enter /afk or run: nohup bin/fm-supervise-daemon.sh >/dev/null 2>&1 &" + fi + fi + + # (2) watcher beacon fresh when work is in flight. .last-watcher-beat is + # touched every watcher poll; stale or missing with tasks in flight means + # supervision is off (the same condition bin/fm-guard.sh banners, surfaced + # here at cold start so recovery does not silently proceed unsupervised). + for meta in "$STATE"/*.meta; do + [ -e "$meta" ] || continue + in_flight=$((in_flight + 1)) + done + if [ "$in_flight" -gt 0 ] && [ -e "$STATE/.last-watcher-beat" ]; then + m=$(_bootstrap_stat_mtime "$STATE/.last-watcher-beat") + if [ -n "$m" ]; then + now=$(date +%s); age=$(( now - m )) + if [ "$age" -ge "$grace" ]; then + echo "ALERT: watcher beacon is ${age}s stale (grace ${grace}s) with ${in_flight} task(s) in flight - supervision is off; re-arm: bin/fm-watch-arm.sh as a harness-tracked background task" + fi + fi + fi + + _bootstrap_nm_health +} + +# no-mistakes daemon health (crash-loops + duplicate-root daemons). systemd +# --user only; degrades silently otherwise. The two failure modes are the live +# resilience gaps: a leaked unit flapping against a deleted binary burns CPU + +# journals, and more than one daemon serving one --root is the stale-cache +# push-failure mechanism. +_bootstrap_nm_health() { + command -v systemctl >/dev/null 2>&1 || return 0 + systemctl --user show-environment >/dev/null 2>&1 || return 0 + + # (3) crash-looping nm units: a high NRestarts means a unit is flapping. + # Threshold defaults to 50; FM_NM_CRASH_THRESHOLD overrides. + local threshold=${FM_NM_CRASH_THRESHOLD:-50} unit nrestarts crashing="" + while IFS= read -r unit; do + [ -n "$unit" ] || continue + nrestarts=$(systemctl --user show -p NRestarts --value "$unit" 2>/dev/null || echo 0) + case "$nrestarts" in ''|*[!0-9]*) nrestarts=0 ;; esac + [ "$nrestarts" -ge "$threshold" ] && crashing="$crashing $unit(${nrestarts})" + done < <(systemctl --user list-units --type=service --all --no-legend --no-pager 2>/dev/null | awk '$1 ~ /^no-mistakes-daemon-/ {print $1}') + if [ -n "$crashing" ]; then + echo "ALERT: no-mistakes daemon unit(s) crash-looping (high NRestarts):$crashing - investigate/reset: systemctl --user stop '' && systemctl --user reset-failed ''" + fi + + # (4) duplicate nm daemons on one root. More than one process serving the same + # --root lets an old daemon with stale in-memory gate config serve pushes + # after the on-disk gate was re-initialized. + command -v pgrep >/dev/null 2>&1 || return 0 + local dup line + dup=$(pgrep -af 'no-mistakes daemon run --root' 2>/dev/null \ + | sed -nE 's/.*--root ([^ ]+).*/\1/p' | sort | uniq -d) + [ -z "$dup" ] && return 0 + for line in $dup; do + [ -n "$line" ] || continue + echo "ALERT: multiple no-mistakes daemons serve root '$line' (stale-cache push risk) - reconcile: keep the systemd-managed one, kill the stray nohup processes" + done +} + install_cmd() { case "$1" in tmux|node|gh|curl|jq) echo "brew install $1 # or the platform's package manager" ;; @@ -427,4 +524,5 @@ if [ "${FM_BOOTSTRAP_DETECT_ONLY:-0}" != 1 ]; then x_mode_setup fleet_sync fi +daemon_health_check exit 0 diff --git a/docs/configuration.md b/docs/configuration.md index 5649c7a6..89d40430 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -202,6 +202,7 @@ FMX_X_THREAD_MAX=25 # maximum tweets in one auto-split X reply thread FMX_FOLLOWUP_MAX_AGE_SECS=86400 # local window for posting one X completion follow-up FM_LOCK_STALE_AFTER=2 # seconds before dead-pid lock records can be reclaimed; mid-acquire locks keep at least 2s grace FM_GUARD_GRACE=300 # seconds before guard warnings and arm health checks treat a watcher beacon as stale +FM_NM_CRASH_THRESHOLD=50 # NRestarts at/above this marks a no-mistakes-daemon-* unit as crash-looping in bootstrap's ALERT gate FM_ARM_CONFIRM_TIMEOUT=10 # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watcher lock may have a stale beacon before re-arm errors FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake diff --git a/docs/scripts.md b/docs/scripts.md index b46e6a61..14b8e48d 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -6,7 +6,7 @@ Each file also starts with a short header comment. | Script | Description | | ------------------------ | ------------------------------------------------------------------------------------------------------------------- | | `fm-session-start.sh` | The one command AGENTS.md sections 3 and 5 run at every session start: composes `fm-lock.sh`, `fm-bootstrap.sh` (its three mutating sweeps gated on holding the lock via `FM_BOOTSTRAP_DETECT_ONLY`), and `fm-wake-drain.sh`, then prints a full context digest (`data/projects.md`, `data/secondmates.md`, `data/captain.md`, `data/learnings.md`, each `ABSENT`-marked when missing) and fleet-state digest (`data/backlog.md`, every `state/*.meta`, a bounded `state/*.status` tail, `state/.afk`, and a cheap per-task endpoint-liveness read); prints a loud read-only banner and skips every mutating step when the lock is held elsewhere; never arms the watcher itself | -| `fm-bootstrap.sh` | Detect required toolchain and version problems, dispatch profile JSON errors or active-rule blocks, default backlog-backend status, primary-checkout `TANGLE:` problems, and actionable clone refresh outcomes; refresh project clones best-effort; locally sync live secondmate homes and propagate declared inheritable config; set up opt-in X mode; install tools only after consent; `FM_BOOTSTRAP_DETECT_ONLY=1` skips the three mutating sweeps and prints advisory-only `TANGLE:` wording without a checkout command | +| `fm-bootstrap.sh` | Detect required toolchain and version problems, dispatch profile JSON errors or active-rule blocks, default backlog-backend status, primary-checkout `TANGLE:` problems, and actionable clone refresh outcomes; refresh project clones best-effort; locally sync live secondmate homes and propagate declared inheritable config; set up opt-in X mode; surface live fleet-health `ALERT:` lines from the daemon + watcher health gate (afk-on-but-daemon-dark, stale watcher beacon with tasks in flight, crash-looping no-mistakes daemon units, or duplicate daemons on one root), which runs last and warns even in detect-only mode; install tools only after consent; `FM_BOOTSTRAP_DETECT_ONLY=1` skips the three mutating sweeps and prints advisory-only `TANGLE:` wording without a checkout command | | `fm-fleet-sync.sh` | Fetch clones, fast-forward safe default-branch states, self-heal clean detached ancestor drift, report unsafe drift as `STUCK:`, and safely prune branches whose remote is gone | | `fm-update.sh` | Self-update the running firstmate repo and registered secondmate homes with fast-forward-only pulls from origin | | `fm-backlog-handoff.sh` | Move already-judged in-scope queued backlog items from the main home into a seeded secondmate home | diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index d1d96ef1..91d85bc4 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -207,3 +207,82 @@ test_bootstrap_reporting test_no_mistakes_min_version test_crew_dispatch_active_rules_are_surfaced test_crew_dispatch_validation + +# --- daemon/watcher health gate (ALERT lines) --------------------------------- +# +# The health gate is the cold-start counterpart of bin/fm-guard.sh: it surfaces +# one ALERT line per live fleet-health problem and stays silent when healthy. +# A FAILING fake systemctl makes the systemd-based no-mistakes checks degrade +# silently; the afk-dark ALERT is driven by the pidfile (a dead pid), so each +# case below exercises both a real ALERT and the silent-degradation path at once. +add_failing_systemctl() { + local fakebin=$1 + printf '#!/usr/bin/env bash\nexit 1\n' > "$fakebin/systemctl" + chmod +x "$fakebin/systemctl" +} + +# make_state_home : lay down an empty FM_HOME shell with state/ + projects/. +make_state_home() { + local dir=$1 + mkdir -p "$dir/home/state" "$dir/home/projects" +} + +test_health_gate_silent_when_healthy() { + local case_dir fakebin out + case_dir="$TMP_ROOT/hg-healthy" + make_state_home "$case_dir" + fakebin=$(make_fake_toolchain "$case_dir") + add_failing_systemctl "$fakebin" + # No .afk and no in-flight task: the health gate must print nothing. + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") + case "$out" in + *ALERT*) fail "health gate emitted an ALERT on a healthy fleet: $out" ;; + esac + pass "health gate is silent on a healthy fleet" +} + +test_health_gate_afk_daemon_dark() { + local case_dir fakebin out + case_dir="$TMP_ROOT/hg-afk-dark" + make_state_home "$case_dir" + fakebin=$(make_fake_toolchain "$case_dir") + add_failing_systemctl "$fakebin" + : > "$case_dir/home/state/.afk" # afk on ... + # pidfile points at a PID that is not alive (and not us): daemon dark. + echo 999999 > "$case_dir/home/state/.supervise-daemon.pid" + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") + case "$out" in + *"ALERT: afk is on but the away-mode daemon is not running"*) ;; + *) fail "afk-dark ALERT missing; got: $out" ;; + esac + # The systemd-based nm checks must NOT emit a spurious ALERT when systemd is + # unavailable (silent degradation). + case "$out" in + *"no-mistakes daemon unit"*) fail "nm crash-loop ALERT leaked without systemd: $out" ;; + esac + pass "health gate alerts when afk is on but the daemon is dark" +} + +test_health_gate_watcher_stale() { + local case_dir fakebin out + case_dir="$TMP_ROOT/hg-watcher-stale" + make_state_home "$case_dir" + fakebin=$(make_fake_toolchain "$case_dir") + add_failing_systemctl "$fakebin" + echo "window=firstmate:fm-demo" > "$case_dir/home/state/demo.meta" # one task in flight + touch "$case_dir/home/state/.last-watcher-beat" + touch -d @1 "$case_dir/home/state/.last-watcher-beat" # epoch -> far past grace + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") + case "$out" in + *"ALERT: watcher beacon is"*"stale"*"supervision is off"*) ;; + *) fail "watcher-stale ALERT missing; got: $out" ;; + esac + pass "health gate alerts when the watcher beacon is stale with work in flight" +} + +test_health_gate_silent_when_healthy +test_health_gate_afk_daemon_dark +test_health_gate_watcher_stale diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index a5acede4..12c25e38 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -256,8 +256,11 @@ $rec EOF make_fake_toolchain "$fakebin" make_fake_ps_claude "$fakebin" - # Force a MISSING diagnostic line so the bootstrap section is non-trivial. - rm -f "$fakebin/node" + # The fake toolchain omits tasks-axi (a firstmate-specific npm tool, never in + # the system BASE_PATH), so bootstrap deterministically emits a + # MISSING: tasks-axi line regardless of host tooling - a non-trivial + # diagnostic to assert ordering against (host-installed node would otherwise + # make removing fakebin/node unreliable). printf 'window=fm-sess:w1\nkind=ship\n' > "$home/state/task-a.meta" @@ -280,7 +283,7 @@ EOF [ "$context_line" -lt "$fleet_line" ] || fail "CONTEXT did not precede FLEET STATE" [ "$fleet_line" -lt "$next_line" ] || fail "FLEET STATE did not precede NEXT STEP" - missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: node' | head -1 | cut -d: -f1) + missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: tasks-axi' | head -1 | cut -d: -f1) [ -n "$missing_line" ] || fail "MISSING diagnostic did not appear at all" [ "$missing_line" -lt "$fleet_line" ] || fail "actionable MISSING diagnostic was buried after the bulk fleet-state digest" @@ -400,7 +403,6 @@ $rec EOF make_fake_toolchain "$fakebin" make_fake_ps_claude "$fakebin" - rm -f "$fakebin/node" append_wake "$home/state" signal task-z "needs-decision: pick a library" @@ -408,8 +410,12 @@ EOF # fm-lock.sh's own exact success text. assert_contains "$out" "lock acquired: harness pid" "fm-lock.sh's real output did not appear (composition, not reimplementation)" - # fm-bootstrap.sh's own exact MISSING-tool line format. - assert_contains "$out" "MISSING: node (install:" "fm-bootstrap.sh's real detect line did not appear verbatim" + # fm-bootstrap.sh's own exact MISSING-tool line format. tasks-axi is omitted + # from the fake toolchain and is a firstmate-specific npm tool never present + # in the system BASE_PATH, so this line appears deterministically regardless + # of host tooling (host-installed node would make removing fakebin/node + # unreliable). + assert_contains "$out" "MISSING: tasks-axi (install:" "fm-bootstrap.sh's real detect line did not appear verbatim" # fm-wake-drain.sh's real drained record (raw tab-separated queue line). assert_contains "$out" "$(printf 'signal\ttask-z\tneeds-decision: pick a library')" "fm-wake-drain.sh's real drained record did not appear"