diff --git a/AGENTS.md b/AGENTS.md index cff0dc95f3..8c611ed4ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,7 +73,7 @@ config/cmux-socket-password optional cmux control-socket password; LOCAL, gitig config/present-daemon optional presence flag for the present-mode supervision daemon; LOCAL, gitignored; absent means the session arms the watcher itself per turn, as before. Not inherited by secondmate homes; see docs/configuration.md and bin/fm-present-daemon.sh config/liveness-watchdog optional presence flag for the external liveness watchdog (bin/fm-liveness-watchdog.sh), which runs OUTSIDE the agent tree and, when the primary dies with work in flight, re-wakes the primary's own supervisor pane (recorded at session start in state/.supervisor-target) and writes a durable local escalation (state/.liveness-escalation) surfaced at next session start - NO phone push; LOCAL, gitignored; absent means the feature is inert. Optional config/liveness-resume gives a relaunch command for a dead-shell supervisor pane. Not inherited by secondmate homes; see docs/liveness-watchdog.md config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md -config/heavy-run-slots how many heavy runs (suites, lint, builds) may execute at once in this home; LOCAL, gitignored; absent or malformed = 1; not inherited into secondmate homes, whose queues are their own (docs/configuration.md "Heavy-run serialization") +config/heavy-run-slots how many heavy runs (suites, lint, builds) may execute at once on the host; LOCAL, gitignored; absent or malformed = 1; the ledger is host-global and every home resolves one shared ceiling from the primary home's copy of this file (docs/configuration.md "Heavy-run serialization") config/watcher-cadence optional supervision-watcher cadence knobs (signal_grace, poll, heartbeat) as key=value seconds; LOCAL, gitignored; present overrides, absent uses built-in defaults (signal_grace 240, poll 300, heartbeat 600), malformed value or unknown key falls back to the default and is reported loudly; read by bin/fm-watch.sh so no env prefix is needed at arm time (docs/configuration.md "Watcher cadence") config/x-mode.env generated X-mode watcher cadence; LOCAL, gitignored; source before arming watcher when present data/ personal fleet records; LOCAL, gitignored as a whole @@ -104,7 +104,7 @@ state/ volatile runtime signals; gitignored .pr-check-migration-scan-v1 private marker proving the non-executing scan disabled every unsafe legacy check; .pr-check-migration-v1 separately records completed private repairs x-watch.check.sh generated X-mode relay poll shim; present only when opted in (section 14) pending-replies/ parent-owned secondmate pending-reply records (correlation id, delivery vs reply, recovery, escalation); fm-pending-reply-lib.sh - heavy-runs/ heavy-run lease queue: one record per running or queued suite plus its admission lock; bin/fm-heavy-run.sh owns it, never edit by hand + heavy-runs/ heavy-run lease queue when FM_HEAVY_RUN_DIR points here; by default the ledger is host-global outside any home ($TMPDIR/fm-heavy-runs-), one record per running or queued suite plus its admission lock; bin/fm-heavy-run.sh owns it, never edit by hand (docs/configuration.md "Heavy-run serialization") x-inbox/ generated X-mode pending mention payloads; fmx-respond drains it (section 14) x-context/ generated X-mode durable per-request reply context and one-wake offer markers, keyed by request_id; survives inbox cleanup and expires within seven days (section 14; bin/fm-x-lib.sh) x-outbox/ generated X-mode dry-run reply and dismiss previews; inspect it when FMX_DRY_RUN is set (section 14) diff --git a/bin/fm-heavy-run.sh b/bin/fm-heavy-run.sh index 7e9966c9a0..492f66cb46 100755 --- a/bin/fm-heavy-run.sh +++ b/bin/fm-heavy-run.sh @@ -36,12 +36,28 @@ # crewmates being unblocked at once and all starting a suite before the next # reading is ever taken. # -# LEASE RECORDS. state/heavy-runs/q/.entry, one per participant, each with -# seq, pid, a PID identity (start time plus command line), state=waiting or -# state=running, started, task, label, and a truncated command string. The -# admission lock is held only while records are read and rewritten, never while -# a command runs. Records are written to a temp file and renamed, so an -# unlocked --status never reads a half-written record. +# WHY THE LEDGER IS HOST-GLOBAL, NOT PER-HOME. The machine is the resource being +# protected, and a fleet runs several operational homes (a primary plus one or +# more secondmate homes) on one host. A per-home queue would give each home its +# own N slots, so the host-wide count would be N times the number of homes - +# exactly the multiplication this cap exists to prevent. The default queue +# therefore lives at a fixed host-global path outside any home +# (/tmp/fm-heavy-runs-, one per operating user so a shared box cannot cross- +# contaminate or be hijacked), and every home's runs share one running count. +# FM_HEAVY_RUN_DIR still overrides it (a test seam, and the way to scope a queue +# to something other than the whole host). The ceiling VALUE is read from the +# primary home's config so the homes sharing the ledger also share one cap: +# FM_HEAVY_SLOTS_FILE points at that authoritative config, and when it is unset +# or unreadable the resolver falls back to this home's own config, then to 1. +# +# LEASE RECORDS. /.entry, one per participant, each with seq, pid, a +# PID identity (start time plus command line), state=waiting or state=running, +# started, the operational home that owns the run, task, label, and a truncated +# command string. The home field is attribution only, so --status and firstmate +# can see which home each shared-ledger run belongs to. The admission lock is +# held only while records are read and rewritten, never while a command runs. +# Records are written to a temp file and renamed, so an unlocked --status never +# reads a half-written record. # # FAIRNESS AND VISIBILITY. Sequence numbers are monotonic and the lowest waiting # sequence is admitted first, so a queued run cannot be starved by later @@ -57,6 +73,22 @@ # The one process it does signal is its own child, which it forwards TERM, INT, # and HUP to, so a killed requester does not orphan a running suite. # +# HOST-PRESSURE GUARD. Admission also reads the watcher's CACHED sustained +# pressure verdict (state/.resource-status, the word the resource probe already +# published on its own cadence). A free slot is NOT granted while that cached +# verdict is a fresh `critical`, because starting new heavy work into a host that +# is already thrashing is the second failure mode this control exists to prevent. +# It reads the cache only and never samples afresh at acquire time, so it honours +# the sustained-sampling rule rather than reacting to a momentary spike. It fails +# OPEN: an absent, stale, unreadable, or non-critical verdict lets admission +# proceed normally, so a home with no resource monitor is never wedged by it. +# +# RELEASE NUDGE. When a run that HELD a slot releases it and a waiter is still +# queued, the releasing process enqueues one `check heavy-run-slot-free` wake, so +# firstmate nudges a waiter parked on `paused: awaiting test slot` to retry. +# Firstmate is the nudger, never the granter: the waiter still re-acquires +# through ordinary admission, and there is no separate FIFO ticket queue. +# # REFUSALS. An unusable queue directory, an unobtainable admission lock, a # vanished own record, or an exceeded --max-wait all refuse WITHOUT running the # command: proceeding unserialized is exactly the failure this exists to @@ -67,7 +99,11 @@ # 64 usage error # 69 refused: the queue could not be brought to a safe state; nothing ran # 75 refused: --max-wait elapsed while still queued; nothing ran +# 76 refused: the host is under sustained critical pressure; nothing ran # +# ADRs docs/adr/0001-heavy-run-refuse-by-default-admission.md and +# docs/adr/0002-heavy-run-host-global-ledger.md record why admission refuses by +# default and why the ledger deliberately lives outside home isolation. # docs/configuration.md owns the config/heavy-run-slots knob and the FM_HEAVY_* # environment variables; this header owns the mechanism. set -u @@ -81,11 +117,23 @@ CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" -HEAVY_DIR="${FM_HEAVY_RUN_DIR:-$STATE/heavy-runs}" +# The default ledger is host-global (see the header): one queue for the whole +# machine, scoped per operating user so a shared host cannot cross-contaminate. +# FM_HEAVY_RUN_DIR overrides it, which is also how the tests isolate. +HEAVY_GLOBAL_DEFAULT="${TMPDIR:-/tmp}/fm-heavy-runs-$(id -u 2>/dev/null || printf '%s' 0)" +HEAVY_DIR="${FM_HEAVY_RUN_DIR:-$HEAVY_GLOBAL_DEFAULT}" QUEUE="$HEAVY_DIR/q" LOCK="$HEAVY_DIR/admit.lock" SEQ_FILE="$HEAVY_DIR/seq" +# This home's own ceiling file, and the authoritative pointer that overrides it +# so every home sharing the host-global ledger resolves one cap (see the header). SLOTS_FILE="$CONFIG/heavy-run-slots" +SLOTS_POINTER="${FM_HEAVY_SLOTS_FILE:-}" + +# Cached sustained host-pressure verdict the resource probe publishes, and the +# freshness bound the heartbeat annotation already uses (2 * probe interval). +RESOURCE_STATUS_FILE="${FM_HEAVY_RESOURCE_STATUS:-$STATE/.resource-status}" +RESOURCE_INTERVAL_OVERRIDE="${FM_HEAVY_RESOURCE_INTERVAL:-}" POLL=${FM_HEAVY_POLL:-2} NOTICE_EVERY=${FM_HEAVY_NOTICE:-30} @@ -113,6 +161,11 @@ refuse() { exit 69 } +refuse_pressure() { + log "refusing without running the command: $*" + exit 76 +} + # One line, no tabs or newlines, bounded length: lease records are line-oriented # key=value and are read by --status without a lock. one_line() { @@ -121,16 +174,22 @@ one_line() { # --- ceiling ---------------------------------------------------------------- # -# FM_HEAVY_SLOTS (operator override and test seam), then the first non-empty -# line of config/heavy-run-slots, then 1. A malformed value falls back to 1 -# rather than to a permissive number: the failure mode of guessing high is the -# host thrash this exists to prevent. +# FM_HEAVY_SLOTS (operator override and test seam), then the authoritative +# ceiling file, then 1. The authoritative file is FM_HEAVY_SLOTS_FILE (the +# primary home's config/heavy-run-slots) when it is set and readable, so every +# home sharing the host-global ledger resolves ONE cap; otherwise this home's own +# config/heavy-run-slots. A malformed value falls back to 1 rather than to a +# permissive number: the failure mode of guessing high is the host thrash this +# exists to prevent. resolve_slots() { - local raw= + local raw='' source_file=$SLOTS_FILE + if [ -n "$SLOTS_POINTER" ] && [ -f "$SLOTS_POINTER" ] && [ -r "$SLOTS_POINTER" ]; then + source_file=$SLOTS_POINTER + fi if [ -n "${FM_HEAVY_SLOTS:-}" ]; then raw=$FM_HEAVY_SLOTS - elif [ -f "$SLOTS_FILE" ]; then - raw=$(grep -v '^[[:space:]]*$' "$SLOTS_FILE" 2>/dev/null | head -n 1 | tr -d '[:space:]') + elif [ -f "$source_file" ]; then + raw=$(grep -v '^[[:space:]]*$' "$source_file" 2>/dev/null | head -n 1 | tr -d '[:space:]') fi [ -n "$raw" ] || { printf '1\n'; return 0; } case "$raw" in @@ -160,10 +219,10 @@ identity_of() { # one_line "$(fm_pid_identity "$1" 2>/dev/null || true)" } -E_SEQ=; E_PID=; E_IDENTITY=; E_STATE=; E_STARTED=; E_TASK=; E_LABEL=; E_CMD= +E_SEQ=; E_PID=; E_IDENTITY=; E_STATE=; E_STARTED=; E_HOME=; E_TASK=; E_LABEL=; E_CMD= entry_read() { # : populate E_*; non-zero when the record is unusable local file=$1 line - E_SEQ=; E_PID=; E_IDENTITY=; E_STATE=; E_STARTED=; E_TASK=; E_LABEL=; E_CMD= + E_SEQ=; E_PID=; E_IDENTITY=; E_STATE=; E_STARTED=; E_HOME=; E_TASK=; E_LABEL=; E_CMD= [ -f "$file" ] || return 1 while IFS= read -r line; do case "$line" in @@ -172,6 +231,7 @@ entry_read() { # : populate E_*; non-zero when the record is unusable identity=*) E_IDENTITY=${line#identity=} ;; state=*) E_STATE=${line#state=} ;; started=*) E_STARTED=${line#started=} ;; + home=*) E_HOME=${line#home=} ;; task=*) E_TASK=${line#task=} ;; label=*) E_LABEL=${line#label=} ;; cmd=*) E_CMD=${line#cmd=} ;; @@ -191,6 +251,7 @@ entry_write() { # : atomic rewrite preserving this run's fields printf 'identity=%s\n' "$MY_IDENTITY" printf 'state=%s\n' "$state" printf 'started=%s\n' "$MY_STARTED" + printf 'home=%s\n' "$MY_HOME" printf 'task=%s\n' "$TASK" printf 'label=%s\n' "$LABEL" printf 'cmd=%s\n' "$CMD_TEXT" @@ -256,8 +317,11 @@ ensure_queue() { umask 077 mkdir -p "$QUEUE" 2>/dev/null umask "$prior" - [ -d "$QUEUE" ] && [ ! -L "$QUEUE" ] && [ -w "$QUEUE" ] \ - || refuse "queue directory $QUEUE is unusable" + local d + for d in "$HEAVY_DIR" "$QUEUE"; do + [ -d "$d" ] && [ ! -L "$d" ] && [ -O "$d" ] && [ -w "$d" ] \ + || refuse "ledger directory $d is unusable or not owned by this user" + done } # --- status ----------------------------------------------------------------- @@ -278,11 +342,11 @@ cmd_status() { entry_read "$f" || continue if [ "$E_STATE" = running ]; then running=$(( running + 1 )) - run_lines+=("run seq=$E_SEQ pid=$E_PID task=$E_TASK started=$E_STARTED label=$E_LABEL cmd=$E_CMD") + run_lines+=("run seq=$E_SEQ pid=$E_PID home=$E_HOME task=$E_TASK started=$E_STARTED label=$E_LABEL cmd=$E_CMD") else waiting=$(( waiting + 1 )) pos=$(( pos + 1 )) - wait_lines+=("wait seq=$E_SEQ pid=$E_PID task=$E_TASK position=$pos queued=$E_STARTED label=$E_LABEL cmd=$E_CMD") + wait_lines+=("wait seq=$E_SEQ pid=$E_PID home=$E_HOME task=$E_TASK position=$pos queued=$E_STARTED label=$E_LABEL cmd=$E_CMD") fi done < <(entry_files) printf 'ceiling=%s\n' "$slots" @@ -337,17 +401,75 @@ CMD_TEXT=$(one_line "$*") MY_PID=${BASHPID:-$$} MY_IDENTITY=$(identity_of "$MY_PID") MY_STARTED=$(date +%s) +MY_HOME=$(one_line "$FM_HOME") MY_SEQ= MY_ENTRY= +MY_ADMITTED=0 + +# The freshness bound for the cached host-pressure verdict, resolved ONCE here: +# two probe intervals, the same bound the heartbeat annotation uses. Resolving it +# per admission pass would re-fork the resolver every poll for no gain. This is a +# config read, not a probe, so it never samples the host afresh. +resolve_resource_bound() { + local interval=$RESOURCE_INTERVAL_OVERRIDE + if [ -z "$interval" ]; then + interval=$("$SCRIPT_DIR/fm-resource-check.sh" --interval 2>/dev/null || printf '') + fi + case "$interval" in ''|*[!0-9]*) interval=900 ;; esac + [ "$interval" -ge 1 ] || interval=900 + printf '%s\n' "$(( interval * 2 ))" +} +RESOURCE_STALE_BOUND=$(resolve_resource_bound) + +# True (0) only when the watcher's CACHED verdict is a FRESH `critical`. Reads +# the published cache and never samples afresh, honouring the sustained-sampling +# rule. Fails OPEN - absent, unreadable, stale, or non-critical all return 1 - so +# a home with no resource monitor is never wedged by this guard. +host_under_sustained_critical() { + local status age + [ -f "$RESOURCE_STATUS_FILE" ] && [ -r "$RESOURCE_STATUS_FILE" ] || return 1 + status=$(cat "$RESOURCE_STATUS_FILE" 2>/dev/null || true) + [ "$status" = critical ] || return 1 + [ "$RESOURCE_STALE_BOUND" -gt 0 ] || return 1 + age=$(fm_path_age "$RESOURCE_STATUS_FILE" 2>/dev/null || printf '%s' "$RESOURCE_STALE_BOUND") + case "$age" in ''|*[!0-9]*) return 1 ;; esac + [ "$age" -lt "$RESOURCE_STALE_BOUND" ] || return 1 + return 0 +} # --- register --------------------------------------------------------------- ensure_queue +# Emit exactly one release nudge when THIS run held a slot and a waiter is still +# queued, so firstmate wakes a crewmate parked on `paused: awaiting test slot`. +# Firstmate is the nudger, never the granter: the waiter re-acquires through +# ordinary admission. Lock-free on purpose - it runs from the exit trap, where +# re-entering the admission lock could deadlock against a lock this same process +# still holds, and it only needs to know whether ANY waiting record exists, which +# a lockless scan answers well enough for a best-effort nudge. A failed enqueue +# is swallowed: a missed nudge only delays a retry the waiter's own poll already +# covers, and must never turn a clean command exit into a failure. +# shellcheck disable=SC2329 # Invoked indirectly by cleanup() from the traps. +release_nudge() { + local f saw_waiter=0 + [ "$MY_ADMITTED" -eq 1 ] || return 0 + while IFS= read -r f; do + [ -n "$f" ] || continue + [ "$f" = "$MY_ENTRY" ] && continue + entry_read "$f" || continue + if [ "$E_STATE" = waiting ]; then saw_waiter=1; break; fi + done < <(entry_files) + [ "$saw_waiter" -eq 1 ] || return 0 + fm_wake_append check heavy-run-slot-free \ + "check: heavy-run slot freed, a waiter may retry" 2>/dev/null || true +} + # shellcheck disable=SC2329 # Invoked indirectly by the traps below. cleanup() { [ -n "$MY_ENTRY" ] && rm -f "$MY_ENTRY" lock_drop + release_nudge } trap cleanup EXIT # Until the command starts, a signalled requester should drop its own record @@ -400,8 +522,16 @@ try_admit() { done < <(entry_files) WAIT_AHEAD=$running if [ "$running" -lt "$SLOTS" ] && [ "$first_waiting" = "$MY_ENTRY" ]; then + # DELTA 4: a slot is free and this run is the oldest waiter, but do NOT start + # new heavy work into a host the watcher already calls sustained-critical. + # This reads the published cache only (no fresh sample) and fails open. + if host_under_sustained_critical; then + lock_drop + refuse_pressure "host is under sustained critical pressure ($RESOURCE_STATUS_FILE); append 'paused: awaiting test slot' and retry when it recovers" + fi if entry_write "$MY_ENTRY" running; then admitted=0 + MY_ADMITTED=1 else lock_drop return 3 diff --git a/docs/adr/0001-heavy-run-refuse-by-default-admission.md b/docs/adr/0001-heavy-run-refuse-by-default-admission.md new file mode 100644 index 0000000000..79fb0075c4 --- /dev/null +++ b/docs/adr/0001-heavy-run-refuse-by-default-admission.md @@ -0,0 +1,48 @@ +# ADR 0001: Heavy-run admission refuses by default, through a lease wrapper + +Date: 2026-08-05. +Status: accepted. +Context owner for the mechanism: [`bin/fm-heavy-run.sh`](../../bin/fm-heavy-run.sh)'s header. +Config owner: [`docs/configuration.md`](../configuration.md#heavy-run-serialization-configheavy-run-slots). + +## Context + +A fleet of crewmates shares one host. +The heavy cost is not a crewmate existing, it is a crewmate running a suite, a build, or a lint sweep: each of those is multiple gigabytes and transient. +The failure mode that motivated this control is a burst: several crewmates parked on captain decisions get unblocked at once, every one starts a suite in the same instant, and the host thrashes before the resource monitor takes its next reading. +Measured on a 10-core / 16 GB host, 13 concurrent runs drove load to 104 with swap at 83 percent, the unit suite went non-deterministic, and the watcher wedged. + +The resource monitor cannot catch this, because a reading is momentary and advisory while the burst needs a hard, stateful count held across the transition. +So heavy runs are serialized behind a concurrency ceiling, and the question this ADR settles is what happens to a run that arrives when no slot is free. + +Two shapes were available: + +- A long-running worker daemon that owns each child process and relays its output and exit status back over a transport. +- A lease wrapper where the requesting process itself takes a lease, runs its command in its own foreground, and drops the lease on exit. + +And two default postures for a full queue: + +- Block: the requester waits in a FIFO until a slot frees. +- Refuse: the requester exits with a prescriptive non-zero status and retries later, blocking only when it explicitly opts in with `--max-wait`. + +## Decision + +The mechanism is a lease wrapper, not a daemon. +The requesting crewmate's own process is the worker: `fm-heavy-run.sh --task -- `. +The command runs unchanged, its output streams straight through, and the wrapper exits with the command's own status. + +The default posture is refuse, not block. +When no slot is free the wrapper exits non-zero with a message naming the current holders and telling the crewmate to append `paused: awaiting test slot` and retry, or do non-test work in the meantime. +A crewmate that genuinely wants to block opts in with `--max-wait `, and even then the wait is bounded and refuses rather than running unserialized when it elapses. + +## Consequences + +- No component to keep alive, supervise, or recover after a reboot, and no second protocol to relay output and status: they are native to the requester's own shell. +- Release is structural, not disciplinary. A lease drops when the requester's process exits, including on an unclean death, because each admission pass reaps records whose process is gone or whose PID was reused. The wrapper never signals a process it did not start, which matters on a fleet where a blanket process kill has taken out live crewmate agents before. +- Refuse-by-default suits a harness where a long in-command block risks a shell timeout that fails worse than a clean refusal. A crewmate is never left hanging: it is told exactly what happened and what to do. +- The cost is that a refused crewmate must retry rather than being handed the slot the instant one frees. ADR 0003's release nudge softens that by waking a parked waiter, but the waiter still re-acquires through ordinary admission. Firstmate is the nudger, never the granter, and there is no FIFO ticket queue in this version. + +## Alternatives rejected + +- A daemon worker: rejected for the supervision, recovery, and relay-protocol cost, and for its own wedge risk when the worker dies holding a run. +- Block-by-default: rejected because a long in-command block fails badly under harness shell timeouts, and because a burst of unblocked crewmates all blocking at once is harder to observe than a burst all refusing loudly. diff --git a/docs/adr/0002-heavy-run-host-global-ledger.md b/docs/adr/0002-heavy-run-host-global-ledger.md new file mode 100644 index 0000000000..fad65c1043 --- /dev/null +++ b/docs/adr/0002-heavy-run-host-global-ledger.md @@ -0,0 +1,44 @@ +# ADR 0002: The heavy-run ledger lives at a host-global path, outside home isolation + +Date: 2026-08-05. +Status: accepted. +Context owner for the mechanism: [`bin/fm-heavy-run.sh`](../../bin/fm-heavy-run.sh)'s header. +Config owner: [`docs/configuration.md`](../configuration.md#heavy-run-serialization-configheavy-run-slots). +Builds on: [ADR 0001](0001-heavy-run-refuse-by-default-admission.md). + +## Context + +Firstmate keeps each operational home isolated. +A home has its own `state/`, `config/`, `data/`, and `projects/`, selected by `FM_HOME`, and the strong default everywhere in the fleet is that a home never reads or writes another home's state. +A running fleet is a primary home plus one or more secondmate homes, and those homes frequently share one physical host. + +The heavy-run cap protects the host, not the home. +Its whole purpose is to bound how many heavy runs execute on the machine at one instant. +The first landed version of the lease queue lived under a home's own `state/heavy-runs/`, which means each home enforced its own independent count of N slots. +On a host running a primary and two secondmate homes at N = 2, that is up to 6 heavy runs at once - three times the intended ceiling. +Per-home slots multiply across homes and defeat the control precisely when the fleet is busiest. + +## Decision + +The default ledger lives at a fixed host-global path outside any home: `${TMPDIR:-/tmp}/fm-heavy-runs-`, one directory per operating user. +Every home's heavy runs register in that one ledger and share one running count, so the ceiling is a property of the machine as the design always intended. + +Two supporting decisions keep this from eroding home isolation any further than necessary: + +- Each lease record carries a `home=` attribution field (the run's `FM_HOME`) alongside its task id, so `--status` and firstmate can see which home each shared-ledger run belongs to. Attribution is read-only reporting; it grants no home authority over another home's record beyond the identity-checked reap that already governs every record. +- The ceiling VALUE is resolved from one authoritative config so the homes sharing the ledger also share one cap. `FM_HEAVY_SLOTS_FILE` points at the primary home's `config/heavy-run-slots`; when it is unset or unreadable the resolver falls back to this home's own `config/heavy-run-slots`, then to the safe floor of 1. + +The per-user scoping (`-`) means a shared host with more than one operator does not cross-contaminate, and a world-writable `/tmp` cannot be used by another user to hijack this user's queue directory. +`FM_HEAVY_RUN_DIR` still overrides the default path: it is the test isolation seam, and the way to deliberately scope a queue to something other than the whole host. + +## Consequences + +- The cap once again means what it says: N heavy runs on the machine, not N per home. +- This is a deliberate, contained exception to home isolation. It is contained because the ledger holds only lease records, never a home's real state, and because attribution is reporting rather than authority. It is deliberate because the resource being protected is genuinely the host, which no single home owns. +- The reap contract is unchanged and still the only reaping action: a record whose process is gone or whose PID was reused is removed, and no process is ever signalled. That contract already tolerated records from arbitrary processes, so it needs nothing added to tolerate records from sibling homes. +- A misconfigured `FM_HEAVY_SLOTS_FILE` (pointing at a missing or unreadable file) does not fail the run: it falls back to the local config and then to 1, the safest value, matching the malformed-ceiling posture in ADR 0001. + +## Alternatives rejected + +- Keep the queue per-home and tell each home to set the same ceiling: rejected because it still multiplies the running count by the number of homes, and because a shared-count invariant enforced only by everyone independently configuring the same number is invisible the moment one home is misconfigured. +- Move agents themselves onto one shared home: rejected as a far larger change to the supervision fabric to solve a problem that a shared ledger solves directly. diff --git a/docs/configuration.md b/docs/configuration.md index 4179b9713f..caf43a3c2a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -495,11 +495,24 @@ bin/fm-heavy-run.sh --status ``` It prints `ceiling=`, `running=`, `waiting=`, then one line per run and per waiter in queue order. -The runner's state lives under `state/heavy-runs/`; these are runtime records, not files to edit by hand. +Each line also carries `home=`, the operational home that owns the run, because the ledger is shared across homes (below). +The runner's state lives in the ledger directory; these are runtime records, not files to edit by hand. -The queue is per operational home, because it lives under that home's `state/`. -A fleet whose secondmate homes share one physical host therefore gets one queue per home, not one for the machine. -Set the same ceiling in each home, or point every home at one queue by exporting `FM_HEAVY_RUN_DIR` to a shared directory, when the machine rather than the home is the thing being protected. +The ledger is host-global by default, not per home. +The machine is the resource being protected, and a fleet runs a primary home plus one or more secondmate homes on one host, so a per-home queue would give each home its own N slots and the host-wide count would be N times the number of homes - exactly the multiplication the cap exists to prevent. +The default ledger therefore lives at a fixed path outside any home, `${TMPDIR:-/tmp}/fm-heavy-runs-`, one directory per operating user so a shared host neither cross-contaminates nor can be hijacked through a world-writable `/tmp`. +Every home's runs share that one running count, and each record carries its owning home for attribution. +`FM_HEAVY_RUN_DIR` still overrides the path, which is how the tests isolate and how a queue can deliberately be scoped to something other than the whole host. +So that the homes sharing the ledger also share one ceiling, set `FM_HEAVY_SLOTS_FILE` to the authoritative `config/heavy-run-slots` (the primary home's); when it is unset or unreadable the resolver falls back to this home's own `config/heavy-run-slots`, then to `1`. +[ADR 0002](adr/0002-heavy-run-host-global-ledger.md) records this deliberate, contained exception to home isolation, and [ADR 0001](adr/0001-heavy-run-refuse-by-default-admission.md) records why admission refuses by default through a lease wrapper. + +Admission also reads the host-resource monitor's cached verdict as a second gate. +It is not coupled to the monitor's cadence: it reads only the word the resource probe already published in `state/.resource-status` and never samples the host afresh, so it honours the sustained-sampling rule rather than reacting to a momentary spike. +A free slot is refused (exit `76`) only while that cached verdict is a fresh `critical`, because starting new heavy work into a host that is already thrashing is the second failure mode this control exists to prevent. +It fails open: an absent, stale, unreadable, or non-critical verdict lets admission proceed normally, so a home with no resource monitor is never wedged by it. + +When a run that held a slot releases it and a waiter is still queued, the releasing process enqueues one `check heavy-run-slot-free` wake, so firstmate can nudge a crewmate parked on `paused: awaiting test slot` to retry. +Firstmate is the nudger, never the granter: the waiter still re-acquires through ordinary admission, and there is no separate FIFO ticket queue. ## Watcher cadence (config/watcher-cadence) @@ -689,8 +702,11 @@ FM_HOURLY_REVIEW_INTERVAL=3600 # seconds between hourly session-review passes; FM_HOURLY_CLEANUP_INTERVAL=3600 # seconds between hourly cleanup sweeps; same 0/malformed rules FM_RESOURCE_SWEEP_BUDGET=30 # seconds one sweep may spend on crew-liveness checks in total; 0 or malformed falls back to the default FM_RESOURCE_PROBE_TIMEOUT=5 # seconds allowed per crew-liveness check inside a sweep; 0 or malformed falls back to the default -FM_HEAVY_SLOTS= # heavy-run ceiling override; wins over config/heavy-run-slots, malformed or below-floor values fall back to 1 (see the heavy-run serialization section above) -FM_HEAVY_RUN_DIR= # alternate heavy-run queue dir, default state/heavy-runs; point several homes at one dir to cap a whole host +FM_HEAVY_SLOTS= # heavy-run ceiling override; wins over the authoritative and local config, malformed or below-floor values fall back to 1 (see the heavy-run serialization section above) +FM_HEAVY_SLOTS_FILE= # path to the authoritative heavy-run ceiling file (the primary home's config/heavy-run-slots); when set and readable it overrides this home's own config so every home sharing the host-global ledger resolves one cap, otherwise the local config is used +FM_HEAVY_RUN_DIR= # alternate heavy-run queue dir; the default is host-global ($TMPDIR/fm-heavy-runs-, one queue for the whole machine), and this override scopes the queue elsewhere (the tests use it to isolate) +FM_HEAVY_RESOURCE_STATUS= # path to the cached host-pressure verdict the admission guard reads, default state/.resource-status; a fresh `critical` verdict refuses a free slot (exit 76), absent/stale/non-critical fails open +FM_HEAVY_RESOURCE_INTERVAL= # override for the resource-probe interval used to bound the cached verdict's freshness (2 * interval); default resolves from bin/fm-resource-check.sh --interval, then 900 FM_HEAVY_POLL=2 # seconds between admission attempts while a heavy run is queued FM_HEAVY_NOTICE=30 # seconds between "still queued" notices printed by a waiting heavy run FM_HEAVY_LOCK_WAIT=30 # seconds a heavy run waits for the admission lock before refusing without running diff --git a/tests/fm-heavy-run.test.sh b/tests/fm-heavy-run.test.sh index 4d00224b45..477649d22c 100755 --- a/tests/fm-heavy-run.test.sh +++ b/tests/fm-heavy-run.test.sh @@ -22,6 +22,10 @@ HR="$ROOT/bin/fm-heavy-run.sh" export FM_STATE_OVERRIDE="$HOME_DIR/state" export FM_CONFIG_OVERRIDE="$HOME_DIR/config" +# The default ledger is now host-global (DELTA 2); pin it to this home's own dir +# so the suite is isolated from the real /tmp ledger and $QUEUE stays accurate. +# The one test that exercises the host-global default overrides this with env -u. +export FM_HEAVY_RUN_DIR="$HOME_DIR/state/heavy-runs" # Poll fast and notice often so waiting states are observable within a test. export FM_HEAVY_POLL=0.2 export FM_HEAVY_NOTICE=1 @@ -317,7 +321,158 @@ test_killed_requester_does_not_orphan_its_run() { pass "fm-heavy-run.sh: a terminated requester takes its run down and frees its slot" } +# --- DELTA 2: host-global ledger and home attribution ----------------------- + +test_default_ledger_is_host_global_not_per_home() { + # With FM_HEAVY_RUN_DIR unset, the queue must NOT live under this home's state, + # because a per-home queue multiplies slots across homes and defeats the cap. + local out global + reset_queue + global="$TMP_ROOT/global-ledger" + rm -rf "$global" + # Point the host-global default at a temp dir via TMPDIR, and drop the per-home + # override so the default path is exercised. + out=$(env -u FM_HEAVY_RUN_DIR -u FM_STATE_OVERRIDE FM_CONFIG_OVERRIDE="$HOME_DIR/config" \ + TMPDIR="$global" HOME="$HOME_DIR" "$HR" --status 2>/dev/null) + assert_contains "$out" "ceiling=" "--status must work against the host-global default" + [ -d "$global/fm-heavy-runs-$(id -u)/q" ] \ + || fail "the default ledger must live at the host-global \$TMPDIR/fm-heavy-runs-, not under a home" + pass "fm-heavy-run.sh: the default ledger is host-global, not per-home" +} + +test_record_carries_home_attribution() { + local holder status + reset_queue + printf '1\n' > "$HOME_DIR/config/heavy-run-slots" + FM_HOME="$HOME_DIR" "$HR" --task attributed -- sleep 4 & holder=$! + STRAYS+=("$holder") + wait_status running 1 10 || fail "the holding run never showed as running" + status=$("$HR" --status) + assert_contains "$status" "home=$HOME_DIR" "--status must attribute a running record to its home" + assert_contains "$status" "task=attributed" "--status must still name the task" + kill "$holder" 2>/dev/null + wait "$holder" 2>/dev/null || true + pass "fm-heavy-run.sh: a lease record carries its owning home for attribution" +} + +test_slots_pointer_overrides_local_ceiling() { + # FM_HEAVY_SLOTS_FILE (the authoritative/primary ceiling) must win over this + # home's own config, so homes sharing the host-global ledger share one cap. + local primary + reset_queue + primary="$TMP_ROOT/primary-heavy-run-slots" + printf '1\n' > "$HOME_DIR/config/heavy-run-slots" + printf '3\n' > "$primary" + [ "$(FM_HEAVY_SLOTS_FILE="$primary" "$HR" --slots 2>/dev/null)" = 3 ] \ + || fail "the authoritative ceiling pointer must override the local config" + # An unreadable/missing pointer falls back to the local config, never failing. + [ "$(FM_HEAVY_SLOTS_FILE="$TMP_ROOT/nonexistent" "$HR" --slots 2>/dev/null)" = 1 ] \ + || fail "a missing ceiling pointer must fall back to the local config" + pass "fm-heavy-run.sh: the authoritative ceiling pointer wins, and falls back safely" +} + +# --- DELTA 3: release-wake nudge -------------------------------------------- + +test_release_emits_a_nudge_when_a_waiter_is_queued() { + # A slot-holder that exits while a waiter is still queued must enqueue exactly + # one check wake, so firstmate nudges a crewmate parked on 'awaiting test slot'. + local queue_file holder waiter before after + reset_queue + export FM_HEAVY_RUN_DIR="$HOME_DIR/state/heavy-runs" + printf '1\n' > "$HOME_DIR/config/heavy-run-slots" + queue_file="$HOME_DIR/state/.wake-queue" + rm -f "$queue_file" + "$HR" --task nudge-holder -- sleep 2 & holder=$! + STRAYS+=("$holder") + wait_status running 1 10 || fail "the holding run never showed as running" + # A waiter that stays queued THROUGH the holder's exit: its record is present + # at the moment the holder releases, which is what triggers the nudge. It is + # then admitted and runs to completion. + "$HR" --task nudge-waiter --max-wait 20 -- true & waiter=$! + STRAYS+=("$waiter") + wait_status waiting 1 10 || fail "the waiter never showed as queued" + before=$(grep -c 'heavy-run-slot-free' "$queue_file" 2>/dev/null || printf '0') + wait "$holder" 2>/dev/null || true + wait "$waiter" 2>/dev/null || true + after=$(grep -c 'heavy-run-slot-free' "$queue_file" 2>/dev/null || printf '0') + [ "$after" -gt "$before" ] \ + || fail "releasing a slot with a waiter queued must enqueue a heavy-run-slot-free check wake" + pass "fm-heavy-run.sh: slot release nudges a queued waiter with a check wake" +} + +test_release_emits_no_nudge_with_no_waiter() { + # A lone run that never had a waiter must not spam the wake queue on exit. + local queue_file count + reset_queue + printf '1\n' > "$HOME_DIR/config/heavy-run-slots" + queue_file="$HOME_DIR/state/.wake-queue" + rm -f "$queue_file" + "$HR" --task lonely -- true + count=$(grep -c 'heavy-run-slot-free' "$queue_file" 2>/dev/null || printf '0') + [ "$count" -eq 0 ] \ + || fail "a run with no waiter behind it must not enqueue a release nudge" + pass "fm-heavy-run.sh: no waiter means no release nudge" +} + +# --- DELTA 4: cached-critical guard ----------------------------------------- + +test_free_slot_refused_under_sustained_critical() { + # A free slot must NOT be granted when the watcher's cached verdict is a fresh + # 'critical'. The guard reads the published cache; it never samples afresh. + local marker rc out + reset_queue + printf '2\n' > "$HOME_DIR/config/heavy-run-slots" + marker="$TMP_ROOT/critical-ran" + printf 'critical\n' > "$HOME_DIR/state/.resource-status" + out=$("$HR" --task under-pressure -- touch "$marker" 2>&1); rc=$? + expect_code 76 "$rc" "a fresh critical verdict must refuse a free slot with its own status" + assert_absent "$marker" "a critical refusal must never run the command" + assert_contains "$out" "sustained critical pressure" "the refusal must name the pressure" + pass "fm-heavy-run.sh: a free slot is refused under sustained critical pressure" +} + +test_pressure_guard_fails_open_when_status_absent_or_stale() { + # No cached verdict, or a stale one, must let admission proceed: a home with no + # resource monitor is never wedged by the guard. + local rc marker + reset_queue + printf '2\n' > "$HOME_DIR/config/heavy-run-slots" + rm -f "$HOME_DIR/state/.resource-status" + "$HR" --task no-monitor -- true; rc=$? + expect_code 0 "$rc" "an absent resource verdict must fail open, not refuse" + # A critical verdict older than the freshness bound (2 * interval) must be + # ignored. Force interval 1 so the bound is 2s, write critical, then age it + # past 2s before the run, so the guard sees it as stale and fails open. + marker="$TMP_ROOT/stale-critical-ran" + printf 'critical\n' > "$HOME_DIR/state/.resource-status" + sleep 3 + FM_HEAVY_RESOURCE_INTERVAL=1 "$HR" --task stale-crit -- touch "$marker"; rc=$? + expect_code 0 "$rc" "a stale critical verdict must fail open, not refuse" + assert_present "$marker" "a stale critical verdict must let the command run" + pass "fm-heavy-run.sh: the pressure guard fails open on an absent or stale verdict" +} + +test_symlinked_ledger_dir_is_refused() { + # A ledger root the operating uid does not own (here, a symlink standing in for + # an attacker pre-created dir on a sticky world-writable tmp) must be refused + # without running the command. + local target ledger marker rc out + target="$TMP_ROOT/attacker-ledger" + ledger="$TMP_ROOT/symlinked-ledger" + mkdir -p "$target" + rm -rf "$ledger" + ln -s "$target" "$ledger" + marker="$TMP_ROOT/symlink-ledger-ran" + out=$(FM_HEAVY_RUN_DIR="$ledger" "$HR" --task hijack -- touch "$marker" 2>&1); rc=$? + expect_code 69 "$rc" "a symlinked ledger root must be refused, not used" + assert_absent "$marker" "a refused ledger must never run the command" + assert_contains "$out" "$ledger" "the refusal must name the offending path" + rm -f "$ledger" + pass "fm-heavy-run.sh: a symlinked ledger directory is refused" +} + test_script_parses +test_symlinked_ledger_dir_is_refused test_help_includes_entire_header test_usage_error_runs_nothing test_passes_through_output_and_status @@ -331,3 +486,10 @@ test_dead_holder_does_not_wedge_the_queue test_reap_removes_the_record_but_never_the_process test_killed_waiter_does_not_wedge_the_queue test_killed_requester_does_not_orphan_its_run +test_default_ledger_is_host_global_not_per_home +test_record_carries_home_attribution +test_slots_pointer_overrides_local_ceiling +test_release_emits_a_nudge_when_a_waiter_is_queued +test_release_emits_no_nudge_with_no_waiter +test_free_slot_refused_under_sustained_critical +test_pressure_guard_fails_open_when_status_absent_or_stale