From 38d05614cbe0dd3272a8bad020a46d0a8d8b4b62 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:49:55 -0700 Subject: [PATCH 01/13] add secondmate restart helper --- AGENTS.md | 1 + bin/backends/herdr.sh | 33 ++++- bin/backends/tmux.sh | 10 ++ bin/fm-backend.sh | 31 ++++ bin/fm-restart.sh | 211 +++++++++++++++++++++++++++ bin/fm-send.sh | 4 +- tests/fm-backend-herdr.test.sh | 3 +- tests/fm-restart.test.sh | 259 +++++++++++++++++++++++++++++++++ 8 files changed, 547 insertions(+), 5 deletions(-) create mode 100755 bin/fm-restart.sh create mode 100755 tests/fm-restart.test.sh diff --git a/AGENTS.md b/AGENTS.md index 6fdaa675..86d09510 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -436,6 +436,7 @@ Route by the nature of the task, not just the project name. A project may appear in several `projects:` clone lists, so choose the secondmate whose natural-language scope actually fits the work, such as triage versus feature development. If the resolved project is `local-only`, keep the work with the main firstmate even when a secondmate scope sounds relevant. If a secondmate's scope fits, steer that secondmate with one concise instruction via `bin/fm-send.sh fm- ''` and let it run the normal lifecycle inside its own home. +When a live secondmate lane needs a context refresh, run `bin/fm-restart.sh ` instead of hand-closing tabs or hand-launching an agent; it asks the lane to stow and exit, respawns through `fm-spawn`, and handles backend cleanup ordering. The bare `fm-` target resolves through this home's `state/.meta`; pass an explicit backend target only when intentionally targeting an endpoint outside this firstmate home. A secondmate is itself a firstmate, so a request reaches it in its own chat, which you never read - the return channel that wakes you is its status file. So `fm-send` to a bare `fm-` whose meta is `kind=secondmate` automatically prepends a from-firstmate marker (`bin/fm-marker-lib.sh`); the secondmate recognizes it and returns its answer via its status file, or via a doc under its home plus a status pointer for a detailed response, never only in chat. diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 51835ca9..022f349d 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -550,15 +550,16 @@ fm_backend_herdr_send_literal() { # } # fm_backend_herdr_normalize_key: map firstmate's key vocabulary (Enter, -# Escape, C-c, as used by fm-send.sh --key and stuck-crewmate-recovery) onto +# Escape, C-c, and C-q as used by fm-send.sh --key and recovery paths) onto # herdr's `pane send-keys` names. Verified empirically: enter, escape/esc, and # both ctrl+c/C-c all work (case-insensitive on herdr's side, but normalize -# explicitly rather than relying on that). +# explicitly rather than relying on that). ctrl+q is used by grok's quit flow. fm_backend_herdr_normalize_key() { # case "$1" in Enter|enter) printf 'enter' ;; Escape|escape|Esc|esc) printf 'escape' ;; C-c|c-c|ctrl+c|Ctrl+C) printf 'ctrl+c' ;; + C-q|c-q|ctrl+q|Ctrl+Q) printf 'ctrl+q' ;; *) printf '%s' "$1" ;; esac } @@ -713,6 +714,34 @@ fm_backend_herdr_kill() { # fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane close "$FM_BACKEND_HERDR_PANE" >/dev/null 2>&1 || true } +# fm_backend_herdr_foreground_process: read the real foreground process list +# from herdr's process-info primitive. This deliberately excludes herdr's +# persisted agent metadata because an exited agent can leave stale agent labels +# behind while the foreground process has returned to a shell. +fm_backend_herdr_foreground_process() { # + local out + fm_backend_herdr_target_ready "$1" || return 0 + out=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane process-info --pane "$FM_BACKEND_HERDR_PANE" 2>/dev/null) || return 0 + printf '%s' "$out" | jq -r ' + .result.process_info.foreground_processes[]? | + [ + (.argv0 // empty), + (.argv[0] // empty), + (.cmdline // empty), + ((.argv // []) | join(" ")) + ] | .[] | select(. != "") + ' 2>/dev/null +} + +fm_backend_herdr_relabel_task() { # + local target=$1 new_label=$2 tab_id + fm_backend_herdr_target_ready "$target" || return 1 + tab_id=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane get "$FM_BACKEND_HERDR_PANE" 2>/dev/null \ + | jq -r '.result.pane.tab_id // empty' 2>/dev/null) + [ -n "$tab_id" ] || { echo "error: could not resolve herdr tab id for $target" >&2; return 1; } + fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" tab rename "$tab_id" "$new_label" >/dev/null +} + # fm_backend_herdr_busy_state: semantic busy state from herdr's native # agent-state detection (agent.get), the "first backend where fm_session_busy_state # gets real semantics" per the design report. working -> busy (actively diff --git a/bin/backends/tmux.sh b/bin/backends/tmux.sh index bf2f9674..d1cfd904 100644 --- a/bin/backends/tmux.sh +++ b/bin/backends/tmux.sh @@ -106,3 +106,13 @@ fm_backend_tmux_send_literal() { # fm_backend_tmux_kill() { # tmux kill-window -t "$1" 2>/dev/null || true } + +# fm_backend_tmux_foreground_process: expose tmux's structured current command +# for callers that need to know whether the launched harness is still running. +fm_backend_tmux_foreground_process() { # + tmux display-message -p -t "$1" '#{pane_current_command}' 2>/dev/null || true +} + +fm_backend_tmux_relabel_task() { # + tmux rename-window -t "$1" "$2" +} diff --git a/bin/fm-backend.sh b/bin/fm-backend.sh index ff0a9343..0b1f795f 100644 --- a/bin/fm-backend.sh +++ b/bin/fm-backend.sh @@ -628,3 +628,34 @@ fm_backend_target_exists() { # [expected-label] ;; esac } + +# fm_backend_foreground_process: best-effort structured read of the foreground +# process command(s) currently running inside TARGET. Empty output means no +# harness-like foreground process could be read. Callers must not parse pane +# chat to infer exit when this primitive is available. +fm_backend_foreground_process() { # [expected-label] + local backend=$1 + shift + fm_backend_source "$backend" || return 1 + case "$backend" in + tmux) fm_backend_tmux_foreground_process "$@" ;; + herdr) fm_backend_herdr_foreground_process "$@" ;; + zellij) printf '' ;; + orca) printf '' ;; + *) echo "error: no foreground-process implementation for backend '$backend'" >&2; return 1 ;; + esac +} + +# fm_backend_relabel_task: rename the backend task container that owns TARGET. +# Used before herdr restarts so fm-spawn's duplicate-label check can create a +# fresh fm- tab before the old tab is closed. +fm_backend_relabel_task() { # [expected-label] + local backend=$1 + shift + fm_backend_source "$backend" || return 1 + case "$backend" in + tmux) fm_backend_tmux_relabel_task "$@" ;; + herdr) fm_backend_herdr_relabel_task "$@" ;; + *) echo "error: no relabel implementation for backend '$backend'" >&2; return 1 ;; + esac +} diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh new file mode 100755 index 00000000..158af5d7 --- /dev/null +++ b/bin/fm-restart.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Refresh a persistent secondmate lane by asking it to stow and exit, waiting for +# the harness process to leave, then respawning it through fm-spawn bookkeeping. +# Usage: fm-restart.sh [--force] +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" + +# shellcheck source=bin/fm-backend.sh +. "$SCRIPT_DIR/fm-backend.sh" + +"$FM_ROOT/bin/fm-guard.sh" || true + +usage() { + echo "usage: fm-restart.sh [--force] " >&2 + exit 2 +} + +FORCE=0 +ID= +while [ "$#" -gt 0 ]; do + case "$1" in + --force) FORCE=1 ;; + -h|--help) usage ;; + --*) echo "error: unknown option $1" >&2; usage ;; + *) + [ -z "$ID" ] || usage + ID=$1 + ;; + esac + shift +done +[ -n "$ID" ] || usage + +TIMEOUT=${FM_RESTART_TIMEOUT:-120} +FORCE_TIMEOUT=${FM_RESTART_FORCE_TIMEOUT:-20} +POLL_INTERVAL=${FM_RESTART_POLL_INTERVAL:-1} +case "$TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac +case "$FORCE_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_FORCE_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac + +META="$STATE/$ID.meta" +[ -f "$META" ] || { echo "error: no meta for $ID at $META" >&2; exit 1; } + +KIND=$(fm_meta_get "$META" kind) +[ "$KIND" = secondmate ] || { echo "error: $ID is kind=${KIND:-ship}; only secondmate lanes are restarted (crewmates are torn down)" >&2; exit 1; } + +T=$(fm_meta_get "$META" window) +[ -n "$T" ] || { echo "error: no window= recorded in $META" >&2; exit 1; } +BACKEND=$(fm_backend_of_meta "$META") +HARNESS=$(fm_meta_get "$META" harness) +[ -n "$HARNESS" ] || HARNESS=unknown +EXPECTED_LABEL="fm-$ID" + +fm_backend_validate_spawn "$BACKEND" >/dev/null || exit 1 +fm_backend_source "$BACKEND" || exit 1 +case "$BACKEND" in + tmux|herdr) ;; + *) echo "error: fm-restart supports secondmate lanes on tmux/herdr only for now; $ID uses backend=$BACKEND" >&2; exit 1 ;; +esac + +harness_command() { + local h=$1 cmd + cmd=${h%% *} + cmd=${cmd##*/} + printf '%s' "$cmd" +} + +HARNESS_CMD=$(harness_command "$HARNESS") + +target_exists() { + fm_backend_target_exists "$BACKEND" "$T" "$EXPECTED_LABEL" >/dev/null 2>&1 +} + +harness_running() { + local processes cmd + cmd=$HARNESS_CMD + [ -n "$cmd" ] && [ "$cmd" != unknown ] || return 1 + processes=$(fm_backend_foreground_process "$BACKEND" "$T" "$EXPECTED_LABEL" 2>/dev/null || true) + [ -n "$processes" ] || return 1 + printf '%s\n' "$processes" | awk -v want="$cmd" ' + { + for (i = 1; i <= NF; i++) { + token = $i + sub(/^.*\//, "", token) + if (token == want) found = 1 + } + } + END { exit found ? 0 : 1 } + ' +} + +wait_for_exit() { # + local timeout=$1 deadline now + deadline=$(($(date +%s) + timeout)) + while :; do + ! target_exists && return 0 + ! harness_running && return 0 + now=$(date +%s) + [ "$now" -lt "$deadline" ] || return 1 + sleep "$POLL_INTERVAL" + done +} + +send_key_best_effort() { + "$SCRIPT_DIR/fm-send.sh" "$T" --key "$1" >/dev/null 2>&1 || true +} + +send_text_best_effort() { + "$SCRIPT_DIR/fm-send.sh" "$T" "$1" >/dev/null 2>&1 || true +} + +force_exit_sequence() { + case "$HARNESS_CMD" in + claude) + send_key_best_effort Escape + sleep 1 + send_text_best_effort /exit + ;; + codex) + send_key_best_effort Escape + sleep 1 + send_text_best_effort /quit + ;; + opencode) + send_key_best_effort Escape + sleep 0.5 + send_key_best_effort Escape + sleep 1 + send_text_best_effort /exit + ;; + pi) + send_key_best_effort Escape + sleep 1 + send_text_best_effort /quit + ;; + grok) + send_key_best_effort C-c + sleep 1 + send_key_best_effort C-q + sleep 0.2 + send_key_best_effort C-q + ;; + *) + send_key_best_effort C-c + ;; + esac +} + +respawn() { + "$SCRIPT_DIR/fm-spawn.sh" "$ID" --backend "$BACKEND" --secondmate +} + +cleanup_and_respawn() { + local archived_label out rc + case "$BACKEND" in + herdr) + if target_exists; then + archived_label="old-$EXPECTED_LABEL-$(date +%s)" + echo "restart: renaming old herdr tab to $archived_label" + fm_backend_relabel_task "$BACKEND" "$T" "$archived_label" "$EXPECTED_LABEL" \ + || { echo "error: could not rename old herdr tab for $ID; refusing to spawn a duplicate" >&2; exit 1; } + if out=$(respawn 2>&1); then + printf '%s\n' "$out" + else + rc=$? + fm_backend_relabel_task "$BACKEND" "$T" "$EXPECTED_LABEL" "$archived_label" >/dev/null 2>&1 || true + printf '%s\n' "$out" >&2 + echo "error: respawn failed; old herdr tab was left in place" >&2 + exit "$rc" + fi + fm_backend_kill "$BACKEND" "$T" >/dev/null 2>&1 || true + else + respawn + fi + ;; + *) + if target_exists; then + fm_backend_kill "$BACKEND" "$T" >/dev/null 2>&1 || true + fi + respawn + ;; + esac +} + +if target_exists && harness_running; then + echo "restart: asking $ID to stow and exit (timeout ${TIMEOUT}s)" + "$SCRIPT_DIR/fm-send.sh" "fm-$ID" "Stow any lane-local durable context if needed, then exit this agent process now. Do not start new work; firstmate will respawn this lane." >/dev/null + if ! wait_for_exit "$TIMEOUT"; then + if [ "$FORCE" -ne 1 ]; then + echo "error: $ID did not exit within ${TIMEOUT}s; rerun with --force to interrupt and restart it" >&2 + exit 1 + fi + echo "restart: $ID did not exit within ${TIMEOUT}s; forcing harness exit" + force_exit_sequence + if ! wait_for_exit "$FORCE_TIMEOUT"; then + echo "restart: force exit did not stop $ID within ${FORCE_TIMEOUT}s; closing old endpoint during cleanup" >&2 + fi + fi +else + if target_exists; then + echo "restart: $ID endpoint is live but no $HARNESS_CMD foreground process is running; respawning" + else + echo "restart: $ID endpoint is already dead; respawning" + fi +fi + +cleanup_and_respawn +echo "restart: $ID refreshed" diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 70e21529..7570536f 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -4,8 +4,8 @@ # may be a bare firstmate task name (fm-xyz), resolved through # this home's state/.meta, or an explicit backend target. # Special keys instead of text: fm-send.sh --key Enter -# Key support is backend-specific: tmux/herdr support Escape, Enter, and C-c; -# Orca currently supports Enter and C-c only, and rejects Escape. +# Key support is backend-specific: tmux/herdr support Escape, Enter, C-c, and +# C-q; Orca currently supports Enter and C-c only, and rejects Escape. # # Text submission is verified: the line is typed ONCE, then Enter is sent and # retried (Enter only, never retyped) until the target backend reports a diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 84be0469..0ecce79e 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -673,8 +673,9 @@ test_normalize_key() { [ "$(fm_backend_herdr_normalize_key Escape)" = escape ] || exit 1 [ "$(fm_backend_herdr_normalize_key C-c)" = ctrl+c ] || exit 1 [ "$(fm_backend_herdr_normalize_key ctrl+c)" = ctrl+c ] || exit 1 + [ "$(fm_backend_herdr_normalize_key C-q)" = ctrl+q ] || exit 1 ) || fail "fm_backend_herdr_normalize_key did not map firstmate's key vocabulary to herdr's verified names" - pass "fm_backend_herdr_normalize_key: Enter/Escape/C-c map to herdr's verified enter/escape/ctrl+c" + pass "fm_backend_herdr_normalize_key: Enter/Escape/C-c/C-q map to herdr's verified enter/escape/ctrl+c/ctrl+q" } # --- capture / send_key / kill / current_path -------------------------------- diff --git a/tests/fm-restart.test.sh b/tests/fm-restart.test.sh new file mode 100755 index 00000000..89cc176e --- /dev/null +++ b/tests/fm-restart.test.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# Tests for bin/fm-restart.sh's secondmate-only restart flow. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-restart-tests) + +make_restart_root() { # + local dir=$1 root="$1/root" bin="$1/root/bin" + mkdir -p "$bin" + cp "$ROOT/bin/fm-restart.sh" "$bin/fm-restart.sh" + chmod +x "$bin/fm-restart.sh" + ln -s "$ROOT/bin/fm-backend.sh" "$bin/fm-backend.sh" + ln -s "$ROOT/bin/fm-tmux-lib.sh" "$bin/fm-tmux-lib.sh" + ln -s "$ROOT/bin/backends" "$bin/backends" + cat > "$bin/fm-guard.sh" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + cat > "$bin/fm-send.sh" <<'SH' +#!/usr/bin/env bash +set -u +printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" +case "${1:-}:${2:-}" in + fm-lane:Stow*) printf 'zsh\n' > "$FM_FAKE_COMMAND_FILE" ;; +esac +exit 0 +SH + cat > "$bin/fm-spawn.sh" <<'SH' +#!/usr/bin/env bash +set -u +printf 'spawn:%s\n' "$*" >> "$FM_RESTART_LOG" +exit 0 +SH + chmod +x "$bin/fm-guard.sh" "$bin/fm-send.sh" "$bin/fm-spawn.sh" + printf '%s\n' "$root" +} + +make_fake_tmux() { # + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +set -u +printf 'tmux:%s\n' "$*" >> "$FM_RESTART_LOG" +case "${1:-}" in + display-message) + if [ "$(cat "$FM_FAKE_EXISTS_FILE" 2>/dev/null || echo 1)" != 1 ]; then + exit 1 + fi + case "$*" in + *pane_current_command*) cat "$FM_FAKE_COMMAND_FILE"; exit 0 ;; + *pane_id*) printf '%%1\n'; exit 0 ;; + esac + ;; + kill-window) + printf '0\n' > "$FM_FAKE_EXISTS_FILE" + exit 0 + ;; +esac +exit 0 +SH + chmod +x "$fb/tmux" + printf '%s\n' "$fb" +} + +make_fake_herdr() { # + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/herdr" <<'SH' +#!/usr/bin/env bash +set -u +args=() +for a in "$@"; do + [ "$a" = --session ] && break + args+=("$a") +done +printf 'herdr:%s\n' "${args[*]}" >> "$FM_RESTART_LOG" +cmd="${args[0]:-} ${args[1]:-}" +case "$cmd" in + "status --json") + printf '{"client":{"version":"0.7.1","protocol":14},"server":{"running":true}}\n' + ;; + "pane get") + [ "$(cat "$FM_FAKE_EXISTS_FILE" 2>/dev/null || echo 1)" = 1 ] || exit 1 + printf '{"result":{"pane":{"pane_id":"w1:p1","tab_id":"w1:t1"}}}\n' + ;; + "pane process-info") + [ "$(cat "$FM_FAKE_EXISTS_FILE" 2>/dev/null || echo 1)" = 1 ] || exit 1 + cmd_name=$(cat "$FM_FAKE_COMMAND_FILE") + printf '{"result":{"process_info":{"foreground_processes":[{"argv0":"%s","argv":["%s"]}]}}}\n' "$cmd_name" "$cmd_name" + ;; + "tab rename") + printf 'label:%s\n' "${args[3]:-}" > "$FM_FAKE_LABEL_FILE" + ;; + "pane close") + printf '0\n' > "$FM_FAKE_EXISTS_FILE" + ;; +esac +exit 0 +SH + chmod +x "$fb/herdr" + printf '%s\n' "$fb" +} + +new_case() { + local name=$1 dir + dir="$TMP_ROOT/$name" + mkdir -p "$dir/home/state" "$dir/home/data" "$dir/home/config" + printf '1\n' > "$dir/exists" + printf 'codex\n' > "$dir/command" + : > "$dir/log" + printf '%s\n' "$dir" +} + +write_lane_meta() { # + local dir=$1 kind=$2 backend=$3 window=$4 meta="$1/home/state/lane.meta" + fm_write_meta "$meta" \ + "window=$window" \ + "worktree=$dir/subhome" \ + "project=$dir/subhome" \ + "harness=codex" \ + "kind=$kind" \ + "mode=secondmate" \ + "home=$dir/subhome" \ + "projects=alpha" + [ "$backend" = tmux ] || printf 'backend=%s\n' "$backend" >> "$meta" +} + +run_restart() { # [args...] + local dir=$1 root=$2 fakebin=$3 + shift 3 + PATH="$fakebin:$PATH" \ + FM_ROOT_OVERRIDE="$root" \ + FM_HOME="$dir/home" \ + FM_RESTART_LOG="$dir/log" \ + FM_FAKE_EXISTS_FILE="$dir/exists" \ + FM_FAKE_COMMAND_FILE="$dir/command" \ + FM_FAKE_LABEL_FILE="$dir/label" \ + FM_RESTART_TIMEOUT="${FM_RESTART_TIMEOUT:-1}" \ + FM_RESTART_FORCE_TIMEOUT="${FM_RESTART_FORCE_TIMEOUT:-1}" \ + FM_RESTART_POLL_INTERVAL=1 \ + "$root/bin/fm-restart.sh" "$@" +} + +test_refuses_non_secondmate() { + local dir root fb out status + dir=$(new_case non-secondmate); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + write_lane_meta "$dir" ship tmux firstmate:fm-lane + out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "non-secondmate restart should fail" + assert_contains "$out" "only secondmate lanes are restarted" "non-secondmate refusal did not explain why" + pass "fm-restart: refuses non-secondmate ids" +} + +test_dead_lane_respawns_without_nudge() { + local dir root fb log + dir=$(new_case dead); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + printf '0\n' > "$dir/exists" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "dead lane respawn failed" + log=$(cat "$dir/log") + assert_contains "$log" "spawn:lane --backend tmux --secondmate" "dead lane did not respawn through fm-spawn" + assert_not_contains "$log" "send:fm-lane" "dead lane should not be nudged" + pass "fm-restart: dead lane respawns without nudge/wait" +} + +test_refuses_unsupported_backend() { + local dir root fb out status + dir=$(new_case unsupported-backend); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + write_lane_meta "$dir" secondmate zellij firstmate:1 + out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "unsupported backend restart should fail" + assert_contains "$out" "supports secondmate lanes on tmux/herdr only" "unsupported backend refusal did not explain the supported backends" + assert_not_contains "$(cat "$dir/log")" "spawn:" "unsupported backend must not respawn" + pass "fm-restart: refuses unsupported backends instead of guessing process state" +} + +test_tmux_live_lane_stows_exits_kills_then_respawns() { + local dir root fb log + dir=$(new_case tmux-live); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "tmux live restart failed" + log=$(cat "$dir/log") + assert_contains "$log" "send:fm-lane:Stow" "live lane was not asked to stow and exit via fm-send" + assert_contains "$log" "tmux:kill-window -t firstmate:fm-lane" "tmux old window was not killed before respawn" + assert_contains "$log" "spawn:lane --backend tmux --secondmate" "tmux lane did not respawn through fm-spawn" + case "$log" in + *"send:fm-lane:Stow"*$'\n'*"tmux:kill-window -t firstmate:fm-lane"*$'\n'*"spawn:lane --backend tmux --secondmate"*) ;; + *) fail "tmux restart order was not stow -> kill -> spawn"$'\n'"$log" ;; + esac + pass "fm-restart: live tmux lane stows/exits, kills old window, respawns" +} + +test_herdr_live_lane_renames_spawns_then_closes_old_tab() { + local dir root fb log + dir=$(new_case herdr-live); root=$(make_restart_root "$dir"); fb=$(make_fake_herdr "$dir") + write_lane_meta "$dir" secondmate herdr herdrtest:w1:p1 + run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "herdr live restart failed" + log=$(cat "$dir/log") + assert_contains "$log" "send:fm-lane:Stow" "herdr lane was not asked to stow and exit" + assert_contains "$log" "herdr:tab rename w1:t1 old-fm-lane-" "herdr old tab was not renamed out of fm-lane" + assert_contains "$log" "spawn:lane --backend herdr --secondmate" "herdr lane did not respawn through fm-spawn" + assert_contains "$log" "herdr:pane close w1:p1" "herdr old pane was not closed after respawn" + case "$log" in + *"herdr:tab rename w1:t1 old-fm-lane-"*$'\n'*"spawn:lane --backend herdr --secondmate"*$'\n'*"herdr:pane close w1:p1"*) ;; + *) fail "herdr restart order was not rename -> spawn -> close"$'\n'"$log" ;; + esac + pass "fm-restart: live herdr lane renames old tab, respawns, then closes old tab" +} + +test_timeout_aborts_without_force() { + local dir root fb out status + dir=$(new_case timeout); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + cat > "$root/bin/fm-send.sh" <<'SH' +#!/usr/bin/env bash +set -u +printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" +exit 0 +SH + chmod +x "$root/bin/fm-send.sh" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + FM_RESTART_TIMEOUT=0 out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "timeout without force should fail" + assert_contains "$out" "rerun with --force" "timeout did not explain --force recovery" + assert_not_contains "$(cat "$dir/log")" "spawn:" "timeout without force must not respawn" + pass "fm-restart: timeout aborts honestly without --force" +} + +test_force_uses_exit_sequence_then_respawns() { + local dir root fb log + dir=$(new_case force); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + cat > "$root/bin/fm-send.sh" <<'SH' +#!/usr/bin/env bash +set -u +printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" +exit 0 +SH + chmod +x "$root/bin/fm-send.sh" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + FM_RESTART_TIMEOUT=0 FM_RESTART_FORCE_TIMEOUT=0 run_restart "$dir" "$root" "$fb" --force lane >/dev/null \ + || fail "forced restart failed" + log=$(cat "$dir/log") + assert_contains "$log" "send:firstmate:fm-lane:--key" "force did not send an interrupt key through fm-send" + assert_contains "$log" "send:firstmate:fm-lane:/quit" "codex force did not send /quit to the explicit backend target" + assert_contains "$log" "tmux:kill-window -t firstmate:fm-lane" "force did not close old endpoint" + assert_contains "$log" "spawn:lane --backend tmux --secondmate" "force did not respawn" + pass "fm-restart: --force uses harness exit sequence, then closes and respawns" +} + +test_refuses_non_secondmate +test_dead_lane_respawns_without_nudge +test_refuses_unsupported_backend +test_tmux_live_lane_stows_exits_kills_then_respawns +test_herdr_live_lane_renames_spawns_then_closes_old_tab +test_timeout_aborts_without_force +test_force_uses_exit_sequence_then_respawns From 746657d6899c15144b6ddb7440773d6205f5f728 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:22:18 -0700 Subject: [PATCH 02/13] no-mistakes(review): Captain, harden secondmate restart cleanup --- bin/backends/tmux.sh | 35 ++++++++++++++++++++++++++- bin/fm-restart.sh | 18 ++++++++++++++ tests/fm-restart.test.sh | 52 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/bin/backends/tmux.sh b/bin/backends/tmux.sh index d1cfd904..d5d31eb0 100644 --- a/bin/backends/tmux.sh +++ b/bin/backends/tmux.sh @@ -110,7 +110,40 @@ fm_backend_tmux_kill() { # # fm_backend_tmux_foreground_process: expose tmux's structured current command # for callers that need to know whether the launched harness is still running. fm_backend_tmux_foreground_process() { # - tmux display-message -p -t "$1" '#{pane_current_command}' 2>/dev/null || true + local target=$1 pane_pid processes + pane_pid=$(tmux display-message -p -t "$target" '#{pane_pid}' 2>/dev/null || true) + if [ -n "$pane_pid" ]; then + processes=$(ps -axo pid=,ppid=,stat=,command= 2>/dev/null | awk -v root="$pane_pid" ' + { + pid = $1 + ppid = $2 + stat = $3 + command = $0 + sub(/^[[:space:]]*[0-9]+[[:space:]]+[0-9]+[[:space:]]+[^[:space:]]+[[:space:]]+/, "", command) + parent[pid] = ppid + state[pid] = stat + cmd[pid] = command + } + END { + child[root] = 1 + changed = 1 + while (changed) { + changed = 0 + for (pid in parent) { + if (!child[pid] && child[parent[pid]]) { + child[pid] = 1 + changed = 1 + } + } + } + for (pid in cmd) { + if (pid != root && child[pid] && state[pid] !~ /^Z/) print cmd[pid] + } + } + ' || true) + [ -z "$processes" ] || { printf '%s\n' "$processes"; return 0; } + fi + tmux display-message -p -t "$target" '#{pane_current_command}' 2>/dev/null || true } fm_backend_tmux_relabel_task() { # diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh index 158af5d7..eeb1b00e 100755 --- a/bin/fm-restart.sh +++ b/bin/fm-restart.sh @@ -37,9 +37,11 @@ done TIMEOUT=${FM_RESTART_TIMEOUT:-120} FORCE_TIMEOUT=${FM_RESTART_FORCE_TIMEOUT:-20} +CLEANUP_TIMEOUT=${FM_RESTART_CLEANUP_TIMEOUT:-5} POLL_INTERVAL=${FM_RESTART_POLL_INTERVAL:-1} case "$TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac case "$FORCE_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_FORCE_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac +case "$CLEANUP_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_CLEANUP_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac META="$STATE/$ID.meta" [ -f "$META" ] || { echo "error: no meta for $ID at $META" >&2; exit 1; } @@ -85,6 +87,7 @@ harness_running() { for (i = 1; i <= NF; i++) { token = $i sub(/^.*\//, "", token) + sub(/\.(cjs|mjs|js)$/, "", token) if (token == want) found = 1 } } @@ -104,6 +107,17 @@ wait_for_exit() { # done } +wait_for_target_gone() { # + local timeout=$1 deadline now + deadline=$(($(date +%s) + timeout)) + while :; do + ! target_exists && return 0 + now=$(date +%s) + [ "$now" -lt "$deadline" ] || return 1 + sleep "$POLL_INTERVAL" + done +} + send_key_best_effort() { "$SCRIPT_DIR/fm-send.sh" "$T" --key "$1" >/dev/null 2>&1 || true } @@ -172,6 +186,10 @@ cleanup_and_respawn() { exit "$rc" fi fm_backend_kill "$BACKEND" "$T" >/dev/null 2>&1 || true + if ! wait_for_target_gone "$CLEANUP_TIMEOUT"; then + echo "error: old herdr endpoint $T for $ID still exists after cleanup; close it manually before treating the lane as refreshed" >&2 + exit 1 + fi else respawn fi diff --git a/tests/fm-restart.test.sh b/tests/fm-restart.test.sh index 89cc176e..5387f64c 100755 --- a/tests/fm-restart.test.sh +++ b/tests/fm-restart.test.sh @@ -24,7 +24,10 @@ SH set -u printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" case "${1:-}:${2:-}" in - fm-lane:Stow*) printf 'zsh\n' > "$FM_FAKE_COMMAND_FILE" ;; + fm-lane:Stow*) + printf 'zsh\n' > "$FM_FAKE_COMMAND_FILE" + printf 'zsh\n' > "$FM_FAKE_PROCESS_ARGV_FILE" + ;; esac exit 0 SH @@ -51,6 +54,7 @@ case "${1:-}" in exit 1 fi case "$*" in + *pane_pid*) printf '100\n'; exit 0 ;; *pane_current_command*) cat "$FM_FAKE_COMMAND_FILE"; exit 0 ;; *pane_id*) printf '%%1\n'; exit 0 ;; esac @@ -62,7 +66,17 @@ case "${1:-}" in esac exit 0 SH - chmod +x "$fb/tmux" + cat > "$fb/ps" <<'SH' +#!/usr/bin/env bash +set -u +if [ "${1:-}" = -axo ]; then + printf '100 1 Ss zsh\n' + printf '101 100 S %s\n' "$(cat "$FM_FAKE_PROCESS_ARGV_FILE" 2>/dev/null || cat "$FM_FAKE_COMMAND_FILE")" + exit 0 +fi +exit 1 +SH + chmod +x "$fb/tmux" "$fb/ps" printf '%s\n' "$fb" } @@ -96,7 +110,7 @@ case "$cmd" in printf 'label:%s\n' "${args[3]:-}" > "$FM_FAKE_LABEL_FILE" ;; "pane close") - printf '0\n' > "$FM_FAKE_EXISTS_FILE" + [ -f "$FM_FAKE_CLOSE_STICKS_FILE" ] || printf '0\n' > "$FM_FAKE_EXISTS_FILE" ;; esac exit 0 @@ -111,6 +125,7 @@ new_case() { mkdir -p "$dir/home/state" "$dir/home/data" "$dir/home/config" printf '1\n' > "$dir/exists" printf 'codex\n' > "$dir/command" + printf 'codex\n' > "$dir/process-argv" : > "$dir/log" printf '%s\n' "$dir" } @@ -138,9 +153,12 @@ run_restart() { # [args...] FM_RESTART_LOG="$dir/log" \ FM_FAKE_EXISTS_FILE="$dir/exists" \ FM_FAKE_COMMAND_FILE="$dir/command" \ + FM_FAKE_PROCESS_ARGV_FILE="$dir/process-argv" \ FM_FAKE_LABEL_FILE="$dir/label" \ + FM_FAKE_CLOSE_STICKS_FILE="$dir/close-sticks" \ FM_RESTART_TIMEOUT="${FM_RESTART_TIMEOUT:-1}" \ FM_RESTART_FORCE_TIMEOUT="${FM_RESTART_FORCE_TIMEOUT:-1}" \ + FM_RESTART_CLEANUP_TIMEOUT="${FM_RESTART_CLEANUP_TIMEOUT:-1}" \ FM_RESTART_POLL_INTERVAL=1 \ "$root/bin/fm-restart.sh" "$@" } @@ -194,6 +212,19 @@ test_tmux_live_lane_stows_exits_kills_then_respawns() { pass "fm-restart: live tmux lane stows/exits, kills old window, respawns" } +test_tmux_node_wrapped_codex_stows_before_respawn() { + local dir root fb log + dir=$(new_case tmux-node-codex); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + printf 'node\n' > "$dir/command" + printf 'node /opt/homebrew/bin/codex.js\n' > "$dir/process-argv" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "tmux node-wrapped codex restart failed" + log=$(cat "$dir/log") + assert_contains "$log" "send:fm-lane:Stow" "node-wrapped codex lane was not recognized as the harness process" + assert_contains "$log" "spawn:lane --backend tmux --secondmate" "node-wrapped codex lane did not respawn" + pass "fm-restart: tmux detects node-wrapped codex from pane process argv" +} + test_herdr_live_lane_renames_spawns_then_closes_old_tab() { local dir root fb log dir=$(new_case herdr-live); root=$(make_restart_root "$dir"); fb=$(make_fake_herdr "$dir") @@ -211,6 +242,19 @@ test_herdr_live_lane_renames_spawns_then_closes_old_tab() { pass "fm-restart: live herdr lane renames old tab, respawns, then closes old tab" } +test_herdr_cleanup_failure_refuses_success() { + local dir root fb out status + dir=$(new_case herdr-close-sticks); root=$(make_restart_root "$dir"); fb=$(make_fake_herdr "$dir") + touch "$dir/close-sticks" + write_lane_meta "$dir" secondmate herdr herdrtest:w1:p1 + FM_RESTART_CLEANUP_TIMEOUT=0 out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "herdr restart should fail when the old pane remains after cleanup" + assert_contains "$out" "old herdr endpoint herdrtest:w1:p1 for lane still exists after cleanup" "herdr cleanup failure did not surface manual cleanup" + assert_contains "$(cat "$dir/log")" "spawn:lane --backend herdr --secondmate" "herdr cleanup check should happen after replacement spawn" + assert_not_contains "$out" "restart: lane refreshed" "herdr cleanup failure must not report refreshed" + pass "fm-restart: herdr cleanup verifies the old endpoint is gone before success" +} + test_timeout_aborts_without_force() { local dir root fb out status dir=$(new_case timeout); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") @@ -254,6 +298,8 @@ test_refuses_non_secondmate test_dead_lane_respawns_without_nudge test_refuses_unsupported_backend test_tmux_live_lane_stows_exits_kills_then_respawns +test_tmux_node_wrapped_codex_stows_before_respawn test_herdr_live_lane_renames_spawns_then_closes_old_tab +test_herdr_cleanup_failure_refuses_success test_timeout_aborts_without_force test_force_uses_exit_sequence_then_respawns From ec02070cca70bc157b4634aed2560987ed0337d1 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:27:08 -0700 Subject: [PATCH 03/13] no-mistakes(review): Captain, scope herdr liveness checks --- tests/fm-backend-herdr.test.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 0ecce79e..f29a3afe 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -281,6 +281,22 @@ test_cli_helper_sets_env_and_appends_trailing_session_flag() { pass "fm_backend_herdr_cli: sets HERDR_SESSION AND appends a trailing --session flag on every call" } +test_target_exists_uses_session_scoped_herdr_cli() { + local dir log resp fb + dir="$TMP_ROOT/target-exists"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" + fb=$(make_herdr_fakebin "$dir") + PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + bash -c '. "$0/bin/fm-backend.sh"; fm_backend_target_exists herdr fmtest:w1:p2' "$ROOT" + expect_code 0 $? "fm_backend_target_exists should succeed for a fake live herdr pane" + assert_contains "$(cat "$log")" "HERDR_SESSION=fmtest"$'\x1f''pane'$'\x1f''get'$'\x1f''w1:p2' \ + "target_exists did not query the requested herdr pane" + assert_contains "$(cat "$log")" $'\x1f''pane'$'\x1f''get'$'\x1f''w1:p2'$'\x1f''--session'$'\x1f''fmtest' \ + "target_exists did not route the herdr pane check through an explicit --session flag" + assert_not_contains "$(cat "$log")" $'\x1f''server' \ + "target_exists must remain a passive liveness read and not start a herdr server" + pass "fm_backend_target_exists: herdr liveness checks use explicit session targeting without autostart" +} + # --- container_ensure / create_task ------------------------------------------ test_container_ensure_starts_server_and_workspace() { @@ -1284,6 +1300,7 @@ test_workspace_label_secondmate_marker_trims_whitespace test_workspace_label_empty_marker_falls_back_to_primary test_workspace_label_different_secondmates_get_different_labels test_cli_helper_sets_env_and_appends_trailing_session_flag +test_target_exists_uses_session_scoped_herdr_cli test_container_ensure_starts_server_and_workspace test_container_ensure_reuses_existing_workspace test_container_ensure_creates_with_no_focus_flag From 83063e6b726ce4c485f1f299be3a9eb74aa7bcdd Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:36:37 -0700 Subject: [PATCH 04/13] fix secondmate restart exit routing --- AGENTS.md | 2 +- bin/fm-restart.sh | 42 +++++++++++++++++++++++++++++----------- tests/fm-restart.test.sh | 18 +++++++++++------ 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 86d09510..b6ae0365 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -436,7 +436,7 @@ Route by the nature of the task, not just the project name. A project may appear in several `projects:` clone lists, so choose the secondmate whose natural-language scope actually fits the work, such as triage versus feature development. If the resolved project is `local-only`, keep the work with the main firstmate even when a secondmate scope sounds relevant. If a secondmate's scope fits, steer that secondmate with one concise instruction via `bin/fm-send.sh fm- ''` and let it run the normal lifecycle inside its own home. -When a live secondmate lane needs a context refresh, run `bin/fm-restart.sh ` instead of hand-closing tabs or hand-launching an agent; it asks the lane to stow and exit, respawns through `fm-spawn`, and handles backend cleanup ordering. +When a live secondmate lane needs a context refresh, run `bin/fm-restart.sh ` instead of hand-closing tabs or hand-launching an agent; it asks the lane to stow, sends the harness exit command directly to the raw endpoint, verifies the harness process is gone, respawns through `fm-spawn`, and handles backend cleanup ordering. The bare `fm-` target resolves through this home's `state/.meta`; pass an explicit backend target only when intentionally targeting an endpoint outside this firstmate home. A secondmate is itself a firstmate, so a request reaches it in its own chat, which you never read - the return channel that wakes you is its status file. So `fm-send` to a bare `fm-` whose meta is `kind=secondmate` automatically prepends a from-firstmate marker (`bin/fm-marker-lib.sh`); the secondmate recognizes it and returns its answer via its status file, or via a doc under its home plus a status pointer for a detailed response, never only in chat. diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh index eeb1b00e..40a20ef3 100755 --- a/bin/fm-restart.sh +++ b/bin/fm-restart.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Refresh a persistent secondmate lane by asking it to stow and exit, waiting for -# the harness process to leave, then respawning it through fm-spawn bookkeeping. +# Refresh a persistent secondmate lane by asking it to stow, sending the harness +# exit command directly to the raw endpoint, waiting for the harness process to +# leave, then respawning it through fm-spawn bookkeeping. # Usage: fm-restart.sh [--force] set -eu @@ -38,10 +39,12 @@ done TIMEOUT=${FM_RESTART_TIMEOUT:-120} FORCE_TIMEOUT=${FM_RESTART_FORCE_TIMEOUT:-20} CLEANUP_TIMEOUT=${FM_RESTART_CLEANUP_TIMEOUT:-5} +STOW_SETTLE=${FM_RESTART_STOW_SETTLE:-2} POLL_INTERVAL=${FM_RESTART_POLL_INTERVAL:-1} case "$TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac case "$FORCE_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_FORCE_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac case "$CLEANUP_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_CLEANUP_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac +case "$STOW_SETTLE" in ''|*[!0-9]*) echo "error: FM_RESTART_STOW_SETTLE must be whole seconds" >&2; exit 2 ;; esac META="$STATE/$ID.meta" [ -f "$META" ] || { echo "error: no meta for $ID at $META" >&2; exit 1; } @@ -131,31 +134,29 @@ force_exit_sequence() { claude) send_key_best_effort Escape sleep 1 - send_text_best_effort /exit + graceful_exit_sequence ;; codex) send_key_best_effort Escape sleep 1 - send_text_best_effort /quit + graceful_exit_sequence ;; opencode) send_key_best_effort Escape sleep 0.5 send_key_best_effort Escape sleep 1 - send_text_best_effort /exit + graceful_exit_sequence ;; pi) send_key_best_effort Escape sleep 1 - send_text_best_effort /quit + graceful_exit_sequence ;; grok) send_key_best_effort C-c sleep 1 - send_key_best_effort C-q - sleep 0.2 - send_key_best_effort C-q + graceful_exit_sequence ;; *) send_key_best_effort C-c @@ -163,6 +164,23 @@ force_exit_sequence() { esac } +graceful_exit_sequence() { + # Use the explicit backend target ($T), not fm-. A bare secondmate target + # gets the from-firstmate marker prepended by fm-send, which turns slash + # commands into plain chat. The agent also cannot execute harness slash + # commands on firstmate's behalf, so firstmate must send the command itself. + case "$HARNESS_CMD" in + claude|opencode) send_text_best_effort /exit ;; + codex|pi) send_text_best_effort /quit ;; + grok) + send_key_best_effort C-q + sleep 0.2 + send_key_best_effort C-q + ;; + *) send_key_best_effort C-c ;; + esac +} + respawn() { "$SCRIPT_DIR/fm-spawn.sh" "$ID" --backend "$BACKEND" --secondmate } @@ -204,8 +222,10 @@ cleanup_and_respawn() { } if target_exists && harness_running; then - echo "restart: asking $ID to stow and exit (timeout ${TIMEOUT}s)" - "$SCRIPT_DIR/fm-send.sh" "fm-$ID" "Stow any lane-local durable context if needed, then exit this agent process now. Do not start new work; firstmate will respawn this lane." >/dev/null + echo "restart: asking $ID to stow, then sending raw harness exit (timeout ${TIMEOUT}s)" + "$SCRIPT_DIR/fm-send.sh" "fm-$ID" "Stow any lane-local durable context now. Do not start new work; firstmate will refresh this lane." >/dev/null + [ "$STOW_SETTLE" = 0 ] || sleep "$STOW_SETTLE" + graceful_exit_sequence if ! wait_for_exit "$TIMEOUT"; then if [ "$FORCE" -ne 1 ]; then echo "error: $ID did not exit within ${TIMEOUT}s; rerun with --force to interrupt and restart it" >&2 diff --git a/tests/fm-restart.test.sh b/tests/fm-restart.test.sh index 5387f64c..13014e32 100755 --- a/tests/fm-restart.test.sh +++ b/tests/fm-restart.test.sh @@ -24,7 +24,8 @@ SH set -u printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" case "${1:-}:${2:-}" in - fm-lane:Stow*) + fm-lane:Stow*) ;; + *:/quit|*:/exit) printf 'zsh\n' > "$FM_FAKE_COMMAND_FILE" printf 'zsh\n' > "$FM_FAKE_PROCESS_ARGV_FILE" ;; @@ -159,6 +160,7 @@ run_restart() { # [args...] FM_RESTART_TIMEOUT="${FM_RESTART_TIMEOUT:-1}" \ FM_RESTART_FORCE_TIMEOUT="${FM_RESTART_FORCE_TIMEOUT:-1}" \ FM_RESTART_CLEANUP_TIMEOUT="${FM_RESTART_CLEANUP_TIMEOUT:-1}" \ + FM_RESTART_STOW_SETTLE="${FM_RESTART_STOW_SETTLE:-0}" \ FM_RESTART_POLL_INTERVAL=1 \ "$root/bin/fm-restart.sh" "$@" } @@ -202,12 +204,14 @@ test_tmux_live_lane_stows_exits_kills_then_respawns() { write_lane_meta "$dir" secondmate tmux firstmate:fm-lane run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "tmux live restart failed" log=$(cat "$dir/log") - assert_contains "$log" "send:fm-lane:Stow" "live lane was not asked to stow and exit via fm-send" + assert_contains "$log" "send:fm-lane:Stow" "live lane was not asked to stow via marked fm-send" + assert_not_contains "$log" "exit this agent" "marked stow nudge must not ask the agent to exit itself" + assert_contains "$log" "send:firstmate:fm-lane:/quit" "live lane exit command was not sent unmarked to the raw target" assert_contains "$log" "tmux:kill-window -t firstmate:fm-lane" "tmux old window was not killed before respawn" assert_contains "$log" "spawn:lane --backend tmux --secondmate" "tmux lane did not respawn through fm-spawn" case "$log" in - *"send:fm-lane:Stow"*$'\n'*"tmux:kill-window -t firstmate:fm-lane"*$'\n'*"spawn:lane --backend tmux --secondmate"*) ;; - *) fail "tmux restart order was not stow -> kill -> spawn"$'\n'"$log" ;; + *"send:fm-lane:Stow"*$'\n'*"send:firstmate:fm-lane:/quit"*$'\n'*"tmux:kill-window -t firstmate:fm-lane"*$'\n'*"spawn:lane --backend tmux --secondmate"*) ;; + *) fail "tmux restart order was not marked stow -> raw exit -> kill -> spawn"$'\n'"$log" ;; esac pass "fm-restart: live tmux lane stows/exits, kills old window, respawns" } @@ -221,6 +225,7 @@ test_tmux_node_wrapped_codex_stows_before_respawn() { run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "tmux node-wrapped codex restart failed" log=$(cat "$dir/log") assert_contains "$log" "send:fm-lane:Stow" "node-wrapped codex lane was not recognized as the harness process" + assert_contains "$log" "send:firstmate:fm-lane:/quit" "node-wrapped codex lane exit command was not sent to raw target" assert_contains "$log" "spawn:lane --backend tmux --secondmate" "node-wrapped codex lane did not respawn" pass "fm-restart: tmux detects node-wrapped codex from pane process argv" } @@ -232,12 +237,13 @@ test_herdr_live_lane_renames_spawns_then_closes_old_tab() { run_restart "$dir" "$root" "$fb" lane >/dev/null || fail "herdr live restart failed" log=$(cat "$dir/log") assert_contains "$log" "send:fm-lane:Stow" "herdr lane was not asked to stow and exit" + assert_contains "$log" "send:herdrtest:w1:p1:/quit" "herdr lane exit command was not sent unmarked to the raw target" assert_contains "$log" "herdr:tab rename w1:t1 old-fm-lane-" "herdr old tab was not renamed out of fm-lane" assert_contains "$log" "spawn:lane --backend herdr --secondmate" "herdr lane did not respawn through fm-spawn" assert_contains "$log" "herdr:pane close w1:p1" "herdr old pane was not closed after respawn" case "$log" in - *"herdr:tab rename w1:t1 old-fm-lane-"*$'\n'*"spawn:lane --backend herdr --secondmate"*$'\n'*"herdr:pane close w1:p1"*) ;; - *) fail "herdr restart order was not rename -> spawn -> close"$'\n'"$log" ;; + *"send:fm-lane:Stow"*$'\n'*"send:herdrtest:w1:p1:/quit"*$'\n'*"herdr:tab rename w1:t1 old-fm-lane-"*$'\n'*"spawn:lane --backend herdr --secondmate"*$'\n'*"herdr:pane close w1:p1"*) ;; + *) fail "herdr restart order was not marked stow -> raw exit -> rename -> spawn -> close"$'\n'"$log" ;; esac pass "fm-restart: live herdr lane renames old tab, respawns, then closes old tab" } From 838e19b9c839fa4f5ff771f55d783c8a8fe21180 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:46:18 -0700 Subject: [PATCH 05/13] no-mistakes(review): Captain, harden secondmate restart safety --- AGENTS.md | 2 +- bin/backends/herdr.sh | 21 ++++++++-- bin/fm-backend.sh | 14 +++++-- bin/fm-restart.sh | 63 +++++++++++++++++++++------- tests/fm-restart.test.sh | 89 +++++++++++++++++++++++++++++++++++----- 5 files changed, 158 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b6ae0365..54c5c094 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -436,7 +436,7 @@ Route by the nature of the task, not just the project name. A project may appear in several `projects:` clone lists, so choose the secondmate whose natural-language scope actually fits the work, such as triage versus feature development. If the resolved project is `local-only`, keep the work with the main firstmate even when a secondmate scope sounds relevant. If a secondmate's scope fits, steer that secondmate with one concise instruction via `bin/fm-send.sh fm- ''` and let it run the normal lifecycle inside its own home. -When a live secondmate lane needs a context refresh, run `bin/fm-restart.sh ` instead of hand-closing tabs or hand-launching an agent; it asks the lane to stow, sends the harness exit command directly to the raw endpoint, verifies the harness process is gone, respawns through `fm-spawn`, and handles backend cleanup ordering. +When a live secondmate lane needs a context refresh, run `bin/fm-restart.sh ` instead of hand-closing tabs or hand-launching an agent; it asks the lane to stow, waits for `stowed: restart-ready` in the lane status unless `--skip-stow` is explicitly passed, sends the harness exit command directly to the raw endpoint, verifies the harness process is gone, respawns through `fm-spawn`, and handles backend cleanup ordering. The bare `fm-` target resolves through this home's `state/.meta`; pass an explicit backend target only when intentionally targeting an endpoint outside this firstmate home. A secondmate is itself a firstmate, so a request reaches it in its own chat, which you never read - the return channel that wakes you is its status file. So `fm-send` to a bare `fm-` whose meta is `kind=secondmate` automatically prepends a from-firstmate marker (`bin/fm-marker-lib.sh`); the secondmate recognizes it and returns its answer via its status file, or via a doc under its home plus a status pointer for a detailed response, never only in chat. diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 022f349d..c1e4fb9b 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -513,6 +513,18 @@ fm_backend_herdr_target_ready() { # fm_backend_herdr_server_ensure "$FM_BACKEND_HERDR_SESSION" || return 1 } +fm_backend_herdr_target_label_matches() { # + local target=$1 expected_label=$2 out tab_id label + [ -n "$expected_label" ] || return 0 + fm_backend_herdr_parse_target "$target" || return 1 + out=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane get "$FM_BACKEND_HERDR_PANE" 2>/dev/null) || return 1 + tab_id=$(printf '%s' "$out" | jq -r '.result.pane.tab_id // empty' 2>/dev/null) + [ -n "$tab_id" ] || return 1 + label=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" tab list 2>/dev/null \ + | jq -r --arg tab "$tab_id" '.result.tabs[]? | select(.tab_id == $tab) | .label' 2>/dev/null | head -1) + [ "$label" = "$expected_label" ] +} + # fm_backend_herdr_current_path: the live FOREGROUND process's cwd, or empty on # any error. Mirrors tmux's pane_current_path poll used for worktree-path # discovery after `treehouse get`. @@ -709,8 +721,9 @@ fm_backend_herdr_send_text_submit() { # # fm_backend_herdr_kill: remove the task's pane, best-effort (mirrors # tmux-kill-window's `|| true` contract). Verified: closing a tab's only pane # closes the tab too, so a separate tab close is unnecessary. -fm_backend_herdr_kill() { # +fm_backend_herdr_kill() { # [expected-label] fm_backend_herdr_target_ready "$1" || return 0 + fm_backend_herdr_target_label_matches "$1" "${2:-}" || return 1 fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane close "$FM_BACKEND_HERDR_PANE" >/dev/null 2>&1 || true } @@ -718,9 +731,10 @@ fm_backend_herdr_kill() { # # from herdr's process-info primitive. This deliberately excludes herdr's # persisted agent metadata because an exited agent can leave stale agent labels # behind while the foreground process has returned to a shell. -fm_backend_herdr_foreground_process() { # +fm_backend_herdr_foreground_process() { # [expected-label] local out fm_backend_herdr_target_ready "$1" || return 0 + fm_backend_herdr_target_label_matches "$1" "${2:-}" || return 0 out=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane process-info --pane "$FM_BACKEND_HERDR_PANE" 2>/dev/null) || return 0 printf '%s' "$out" | jq -r ' .result.process_info.foreground_processes[]? | @@ -733,9 +747,10 @@ fm_backend_herdr_foreground_process() { # ' 2>/dev/null } -fm_backend_herdr_relabel_task() { # +fm_backend_herdr_relabel_task() { # [expected-label] local target=$1 new_label=$2 tab_id fm_backend_herdr_target_ready "$target" || return 1 + fm_backend_herdr_target_label_matches "$target" "${3:-}" || return 1 tab_id=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane get "$FM_BACKEND_HERDR_PANE" 2>/dev/null \ | jq -r '.result.pane.tab_id // empty' 2>/dev/null) [ -n "$tab_id" ] || { echo "error: could not resolve herdr tab id for $target" >&2; return 1; } diff --git a/bin/fm-backend.sh b/bin/fm-backend.sh index 0b1f795f..a618f124 100644 --- a/bin/fm-backend.sh +++ b/bin/fm-backend.sh @@ -502,7 +502,7 @@ fm_backend_send_text_submit() { # +fm_backend_kill() { # [expected-label] local backend=$1 shift fm_backend_source "$backend" || return 1 @@ -591,7 +591,7 @@ fm_backend_composer_state() { # -> empty|pending|unknown # primitive so callers that only need a fast alive/dead read (recovery # digests, the session-start fleet digest) do not re-derive it inline. fm_backend_target_exists() { # [expected-label] - local backend=$1 target=$2 expected_label=${3:-} session pane + local backend=$1 target=$2 expected_label=${3:-} session pane out tab_id label case "$backend" in tmux) tmux display-message -p -t "$target" '#{pane_id}' >/dev/null 2>&1 @@ -609,7 +609,15 @@ fm_backend_target_exists() { # [expected-label] # flag on top, so this check is correctly scoped even when the caller's # own ambient session (e.g. the primary firstmate's default session) is # a DIFFERENT one than the target's. - fm_backend_herdr_cli "$session" pane get "$pane" >/dev/null 2>&1 + out=$(fm_backend_herdr_cli "$session" pane get "$pane" 2>/dev/null) || return 1 + if [ -n "$expected_label" ]; then + tab_id=$(printf '%s' "$out" | jq -r '.result.pane.tab_id // empty' 2>/dev/null) + [ -n "$tab_id" ] || return 1 + label=$(fm_backend_herdr_cli "$session" tab list 2>/dev/null \ + | jq -r --arg tab "$tab_id" '.result.tabs[]? | select(.tab_id == $tab) | .label' 2>/dev/null | head -1) + [ "$label" = "$expected_label" ] || return 1 + fi + return 0 ;; zellij) fm_backend_source zellij || return 1 diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh index 40a20ef3..2b534c66 100755 --- a/bin/fm-restart.sh +++ b/bin/fm-restart.sh @@ -2,7 +2,7 @@ # Refresh a persistent secondmate lane by asking it to stow, sending the harness # exit command directly to the raw endpoint, waiting for the harness process to # leave, then respawning it through fm-spawn bookkeeping. -# Usage: fm-restart.sh [--force] +# Usage: fm-restart.sh [--force] [--skip-stow] set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -16,15 +16,17 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" "$FM_ROOT/bin/fm-guard.sh" || true usage() { - echo "usage: fm-restart.sh [--force] " >&2 + echo "usage: fm-restart.sh [--force] [--skip-stow] " >&2 exit 2 } FORCE=0 +SKIP_STOW=0 ID= while [ "$#" -gt 0 ]; do case "$1" in --force) FORCE=1 ;; + --skip-stow) SKIP_STOW=1 ;; -h|--help) usage ;; --*) echo "error: unknown option $1" >&2; usage ;; *) @@ -39,15 +41,16 @@ done TIMEOUT=${FM_RESTART_TIMEOUT:-120} FORCE_TIMEOUT=${FM_RESTART_FORCE_TIMEOUT:-20} CLEANUP_TIMEOUT=${FM_RESTART_CLEANUP_TIMEOUT:-5} -STOW_SETTLE=${FM_RESTART_STOW_SETTLE:-2} +STOW_TIMEOUT=${FM_RESTART_STOW_TIMEOUT:-${FM_RESTART_STOW_SETTLE:-120}} POLL_INTERVAL=${FM_RESTART_POLL_INTERVAL:-1} case "$TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac case "$FORCE_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_FORCE_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac case "$CLEANUP_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_CLEANUP_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac -case "$STOW_SETTLE" in ''|*[!0-9]*) echo "error: FM_RESTART_STOW_SETTLE must be whole seconds" >&2; exit 2 ;; esac +case "$STOW_TIMEOUT" in ''|*[!0-9]*) echo "error: FM_RESTART_STOW_TIMEOUT must be whole seconds" >&2; exit 2 ;; esac META="$STATE/$ID.meta" [ -f "$META" ] || { echo "error: no meta for $ID at $META" >&2; exit 1; } +STATUS_FILE="$STATE/$ID.status" KIND=$(fm_meta_get "$META" kind) [ "$KIND" = secondmate ] || { echo "error: $ID is kind=${KIND:-ship}; only secondmate lanes are restarted (crewmates are torn down)" >&2; exit 1; } @@ -76,7 +79,11 @@ harness_command() { HARNESS_CMD=$(harness_command "$HARNESS") target_exists() { - fm_backend_target_exists "$BACKEND" "$T" "$EXPECTED_LABEL" >/dev/null 2>&1 + target_exists_with_label "$EXPECTED_LABEL" +} + +target_exists_with_label() { + fm_backend_target_exists "$BACKEND" "$T" "$1" >/dev/null 2>&1 } harness_running() { @@ -110,11 +117,29 @@ wait_for_exit() { # done } -wait_for_target_gone() { # - local timeout=$1 deadline now +wait_for_target_gone() { # [expected-label] + local timeout=$1 expected_label=${2:-$EXPECTED_LABEL} deadline now deadline=$(($(date +%s) + timeout)) while :; do - ! target_exists && return 0 + ! target_exists_with_label "$expected_label" && return 0 + now=$(date +%s) + [ "$now" -lt "$deadline" ] || return 1 + sleep "$POLL_INTERVAL" + done +} + +status_line_count() { + [ -f "$STATUS_FILE" ] || { printf '0'; return 0; } + wc -l < "$STATUS_FILE" | tr -d '[:space:]' +} + +wait_for_stow_signal() { # + local before=$1 timeout=$2 deadline now + deadline=$(($(date +%s) + timeout)) + while :; do + if [ -f "$STATUS_FILE" ] && awk -v before="$before" 'NR > before && /^stowed: restart-ready([[:space:]]|$)/ { found = 1 } END { exit found ? 0 : 1 }' "$STATUS_FILE"; then + return 0 + fi now=$(date +%s) [ "$now" -lt "$deadline" ] || return 1 sleep "$POLL_INTERVAL" @@ -122,10 +147,12 @@ wait_for_target_gone() { # } send_key_best_effort() { + target_exists || { echo "error: $ID target $T no longer belongs to $EXPECTED_LABEL; aborting before sending exit input" >&2; exit 1; } "$SCRIPT_DIR/fm-send.sh" "$T" --key "$1" >/dev/null 2>&1 || true } send_text_best_effort() { + target_exists || { echo "error: $ID target $T no longer belongs to $EXPECTED_LABEL; aborting before sending exit input" >&2; exit 1; } "$SCRIPT_DIR/fm-send.sh" "$T" "$1" >/dev/null 2>&1 || true } @@ -203,8 +230,8 @@ cleanup_and_respawn() { echo "error: respawn failed; old herdr tab was left in place" >&2 exit "$rc" fi - fm_backend_kill "$BACKEND" "$T" >/dev/null 2>&1 || true - if ! wait_for_target_gone "$CLEANUP_TIMEOUT"; then + fm_backend_kill "$BACKEND" "$T" "$archived_label" >/dev/null 2>&1 || true + if ! wait_for_target_gone "$CLEANUP_TIMEOUT" "$archived_label"; then echo "error: old herdr endpoint $T for $ID still exists after cleanup; close it manually before treating the lane as refreshed" >&2 exit 1 fi @@ -214,7 +241,7 @@ cleanup_and_respawn() { ;; *) if target_exists; then - fm_backend_kill "$BACKEND" "$T" >/dev/null 2>&1 || true + fm_backend_kill "$BACKEND" "$T" "$EXPECTED_LABEL" >/dev/null 2>&1 || true fi respawn ;; @@ -222,9 +249,17 @@ cleanup_and_respawn() { } if target_exists && harness_running; then - echo "restart: asking $ID to stow, then sending raw harness exit (timeout ${TIMEOUT}s)" - "$SCRIPT_DIR/fm-send.sh" "fm-$ID" "Stow any lane-local durable context now. Do not start new work; firstmate will refresh this lane." >/dev/null - [ "$STOW_SETTLE" = 0 ] || sleep "$STOW_SETTLE" + if [ "$SKIP_STOW" -eq 1 ]; then + echo "restart: skipping stow confirmation for $ID, then sending raw harness exit (timeout ${TIMEOUT}s)" + else + before_stow=$(status_line_count) + echo "restart: asking $ID to stow; waiting for status 'stowed: restart-ready' (timeout ${STOW_TIMEOUT}s)" + "$SCRIPT_DIR/fm-send.sh" "fm-$ID" "Stow any lane-local durable context now. Do not start new work; firstmate will refresh this lane. When stow is complete, append exactly 'stowed: restart-ready' to your status." >/dev/null + if ! wait_for_stow_signal "$before_stow" "$STOW_TIMEOUT"; then + echo "error: $ID did not confirm stow within ${STOW_TIMEOUT}s; aborting before exit. Rerun with --skip-stow only if the lane is already stowed or safe to restart." >&2 + exit 1 + fi + fi graceful_exit_sequence if ! wait_for_exit "$TIMEOUT"; then if [ "$FORCE" -ne 1 ]; then diff --git a/tests/fm-restart.test.sh b/tests/fm-restart.test.sh index 13014e32..e82f554f 100755 --- a/tests/fm-restart.test.sh +++ b/tests/fm-restart.test.sh @@ -24,7 +24,7 @@ SH set -u printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" case "${1:-}:${2:-}" in - fm-lane:Stow*) ;; + fm-lane:Stow*) printf 'stowed: restart-ready\n' >> "$FM_HOME/state/lane.status" ;; *:/quit|*:/exit) printf 'zsh\n' > "$FM_FAKE_COMMAND_FILE" printf 'zsh\n' > "$FM_FAKE_PROCESS_ARGV_FILE" @@ -102,13 +102,16 @@ case "$cmd" in [ "$(cat "$FM_FAKE_EXISTS_FILE" 2>/dev/null || echo 1)" = 1 ] || exit 1 printf '{"result":{"pane":{"pane_id":"w1:p1","tab_id":"w1:t1"}}}\n' ;; + "tab list") + printf '{"result":{"tabs":[{"tab_id":"w1:t1","label":"%s","workspace_id":"w1"}]}}\n' "$(cat "$FM_FAKE_LABEL_FILE" 2>/dev/null || echo fm-lane)" + ;; "pane process-info") [ "$(cat "$FM_FAKE_EXISTS_FILE" 2>/dev/null || echo 1)" = 1 ] || exit 1 cmd_name=$(cat "$FM_FAKE_COMMAND_FILE") printf '{"result":{"process_info":{"foreground_processes":[{"argv0":"%s","argv":["%s"]}]}}}\n' "$cmd_name" "$cmd_name" ;; "tab rename") - printf 'label:%s\n' "${args[3]:-}" > "$FM_FAKE_LABEL_FILE" + printf '%s\n' "${args[3]:-}" > "$FM_FAKE_LABEL_FILE" ;; "pane close") [ -f "$FM_FAKE_CLOSE_STICKS_FILE" ] || printf '0\n' > "$FM_FAKE_EXISTS_FILE" @@ -127,6 +130,7 @@ new_case() { printf '1\n' > "$dir/exists" printf 'codex\n' > "$dir/command" printf 'codex\n' > "$dir/process-argv" + printf 'fm-lane\n' > "$dir/label" : > "$dir/log" printf '%s\n' "$dir" } @@ -261,22 +265,69 @@ test_herdr_cleanup_failure_refuses_success() { pass "fm-restart: herdr cleanup verifies the old endpoint is gone before success" } -test_timeout_aborts_without_force() { +test_stow_timeout_aborts_before_exit() { + local dir root fb out status + dir=$(new_case stow-timeout); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + cat > "$root/bin/fm-send.sh" <<'SH' +#!/usr/bin/env bash +set -u +printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" +exit 0 +SH + chmod +x "$root/bin/fm-send.sh" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + FM_RESTART_STOW_TIMEOUT=0 out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "stow timeout should fail" + assert_contains "$out" "did not confirm stow" "stow timeout did not explain missing status signal" + assert_not_contains "$(cat "$dir/log")" "send:firstmate:fm-lane:/quit" "stow timeout must not send raw exit" + assert_not_contains "$(cat "$dir/log")" "spawn:" "stow timeout must not respawn" + pass "fm-restart: stow timeout aborts before raw exit" +} + +test_skip_stow_bypasses_stow_signal() { + local dir root fb log + dir=$(new_case skip-stow); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + cat > "$root/bin/fm-send.sh" <<'SH' +#!/usr/bin/env bash +set -u +printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" +case "${1:-}:${2:-}" in + *:/quit|*:/exit) + printf 'zsh\n' > "$FM_FAKE_COMMAND_FILE" + printf 'zsh\n' > "$FM_FAKE_PROCESS_ARGV_FILE" + ;; +esac +exit 0 +SH + chmod +x "$root/bin/fm-send.sh" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + run_restart "$dir" "$root" "$fb" --skip-stow lane >/dev/null || fail "skip-stow restart failed" + log=$(cat "$dir/log") + assert_not_contains "$log" "send:fm-lane:Stow" "--skip-stow should not ask the lane to stow" + assert_contains "$log" "send:firstmate:fm-lane:/quit" "--skip-stow should still send raw exit" + assert_contains "$log" "spawn:lane --backend tmux --secondmate" "--skip-stow should respawn" + pass "fm-restart: --skip-stow is the explicit stow escape hatch" +} + +test_exit_timeout_aborts_without_force_after_stow() { local dir root fb out status - dir=$(new_case timeout); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + dir=$(new_case exit-timeout); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") cat > "$root/bin/fm-send.sh" <<'SH' #!/usr/bin/env bash set -u printf 'send:%s:%s\n' "${1:-}" "${2:-}" >> "$FM_RESTART_LOG" +case "${1:-}:${2:-}" in + fm-lane:Stow*) printf 'stowed: restart-ready\n' >> "$FM_HOME/state/lane.status" ;; +esac exit 0 SH chmod +x "$root/bin/fm-send.sh" write_lane_meta "$dir" secondmate tmux firstmate:fm-lane FM_RESTART_TIMEOUT=0 out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? - [ "$status" -ne 0 ] || fail "timeout without force should fail" - assert_contains "$out" "rerun with --force" "timeout did not explain --force recovery" - assert_not_contains "$(cat "$dir/log")" "spawn:" "timeout without force must not respawn" - pass "fm-restart: timeout aborts honestly without --force" + [ "$status" -ne 0 ] || fail "exit timeout without force should fail" + assert_contains "$out" "rerun with --force" "exit timeout did not explain --force recovery" + assert_not_contains "$(cat "$dir/log")" "spawn:" "exit timeout without force must not respawn" + pass "fm-restart: exit timeout aborts honestly without --force after stow" } test_force_uses_exit_sequence_then_respawns() { @@ -290,7 +341,7 @@ exit 0 SH chmod +x "$root/bin/fm-send.sh" write_lane_meta "$dir" secondmate tmux firstmate:fm-lane - FM_RESTART_TIMEOUT=0 FM_RESTART_FORCE_TIMEOUT=0 run_restart "$dir" "$root" "$fb" --force lane >/dev/null \ + FM_RESTART_TIMEOUT=0 FM_RESTART_FORCE_TIMEOUT=0 run_restart "$dir" "$root" "$fb" --skip-stow --force lane >/dev/null \ || fail "forced restart failed" log=$(cat "$dir/log") assert_contains "$log" "send:firstmate:fm-lane:--key" "force did not send an interrupt key through fm-send" @@ -300,6 +351,21 @@ SH pass "fm-restart: --force uses harness exit sequence, then closes and respawns" } +test_herdr_stale_label_refuses_restart() { + local dir root fb out status log + dir=$(new_case herdr-stale-label); root=$(make_restart_root "$dir"); fb=$(make_fake_herdr "$dir") + printf 'fm-other\n' > "$dir/label" + write_lane_meta "$dir" secondmate herdr herdrtest:w1:p1 + out=$(run_restart "$dir" "$root" "$fb" --skip-stow lane 2>&1); status=$? + [ "$status" -eq 0 ] || fail "stale herdr target should be treated as a dead lane and respawn cleanly, got: $out" + log=$(cat "$dir/log") + assert_not_contains "$log" "send:herdrtest:w1:p1:/quit" "stale herdr target must not receive raw exit" + assert_not_contains "$log" "herdr:tab rename" "stale herdr target must not be renamed" + assert_not_contains "$log" "herdr:pane close" "stale herdr target must not be closed" + assert_contains "$log" "spawn:lane --backend herdr --secondmate" "stale herdr lane should still respawn" + pass "fm-restart: stale herdr pane labels are not controlled" +} + test_refuses_non_secondmate test_dead_lane_respawns_without_nudge test_refuses_unsupported_backend @@ -307,5 +373,8 @@ test_tmux_live_lane_stows_exits_kills_then_respawns test_tmux_node_wrapped_codex_stows_before_respawn test_herdr_live_lane_renames_spawns_then_closes_old_tab test_herdr_cleanup_failure_refuses_success -test_timeout_aborts_without_force +test_stow_timeout_aborts_before_exit +test_skip_stow_bypasses_stow_signal +test_exit_timeout_aborts_without_force_after_stow test_force_uses_exit_sequence_then_respawns +test_herdr_stale_label_refuses_restart From d0ea4cb2691aefe1d48fc42a28ebeb611abba134 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:07:54 -0700 Subject: [PATCH 06/13] no-mistakes(test): Model herdr liveness label lookup --- tests/fm-session-start.test.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index a5acede4..50fea982 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -124,17 +124,22 @@ SH } # make_fake_herdr : `herdr pane get ` succeeds only -# for the given pane id - the exact primitive fm_backend_target_exists uses -# for a herdr endpoint liveness read. No version/server-start calls: a -# liveness check must never auto-start a server (fm-backend.sh's contract). +# for the given pane id, and `herdr tab list` reports that pane's task label. +# No version/server-start calls: a liveness check must never auto-start a +# server (fm-backend.sh's contract). make_fake_herdr() { local fakebin=$1 live=$2 cat > "$fakebin/herdr" < Date: Fri, 3 Jul 2026 20:22:45 -0700 Subject: [PATCH 07/13] no-mistakes(review): Captain, guard uncertain restart process state --- bin/fm-restart.sh | 39 +++++++++++++++++++++++++++------------ tests/fm-restart.test.sh | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh index 2b534c66..b3c6bf26 100755 --- a/bin/fm-restart.sh +++ b/bin/fm-restart.sh @@ -86,13 +86,13 @@ target_exists_with_label() { fm_backend_target_exists "$BACKEND" "$T" "$1" >/dev/null 2>&1 } -harness_running() { +harness_process_state() { local processes cmd cmd=$HARNESS_CMD - [ -n "$cmd" ] && [ "$cmd" != unknown ] || return 1 + [ -n "$cmd" ] && [ "$cmd" != unknown ] || { printf 'unknown'; return 0; } processes=$(fm_backend_foreground_process "$BACKEND" "$T" "$EXPECTED_LABEL" 2>/dev/null || true) - [ -n "$processes" ] || return 1 - printf '%s\n' "$processes" | awk -v want="$cmd" ' + [ -n "$processes" ] || { printf 'unknown'; return 0; } + if printf '%s\n' "$processes" | awk -v want="$cmd" ' { for (i = 1; i <= NF; i++) { token = $i @@ -102,7 +102,11 @@ harness_running() { } } END { exit found ? 0 : 1 } - ' + '; then + printf 'running' + return 0 + fi + printf 'not-running' } wait_for_exit() { # @@ -110,7 +114,10 @@ wait_for_exit() { # deadline=$(($(date +%s) + timeout)) while :; do ! target_exists && return 0 - ! harness_running && return 0 + case "$(harness_process_state)" in + not-running) return 0 ;; + running|unknown) ;; + esac now=$(date +%s) [ "$now" -lt "$deadline" ] || return 1 sleep "$POLL_INTERVAL" @@ -248,7 +255,19 @@ cleanup_and_respawn() { esac } -if target_exists && harness_running; then +if target_exists; then + PROCESS_STATE=$(harness_process_state) + if [ "$PROCESS_STATE" != running ] && [ "$SKIP_STOW" -ne 1 ]; then + case "$PROCESS_STATE" in + unknown) + echo "error: $ID endpoint is live, but the $HARNESS_CMD foreground process state is unknown; aborting before stow/exit/cleanup. Rerun with --skip-stow only if the lane is already stowed or safe to restart." >&2 + ;; + *) + echo "error: $ID endpoint is live, but the foreground process does not match $HARNESS_CMD; aborting before stow/exit/cleanup. Rerun with --skip-stow only if the lane is already stowed or safe to restart." >&2 + ;; + esac + exit 1 + fi if [ "$SKIP_STOW" -eq 1 ]; then echo "restart: skipping stow confirmation for $ID, then sending raw harness exit (timeout ${TIMEOUT}s)" else @@ -273,11 +292,7 @@ if target_exists && harness_running; then fi fi else - if target_exists; then - echo "restart: $ID endpoint is live but no $HARNESS_CMD foreground process is running; respawning" - else - echo "restart: $ID endpoint is already dead; respawning" - fi + echo "restart: $ID endpoint is already dead; respawning" fi cleanup_and_respawn diff --git a/tests/fm-restart.test.sh b/tests/fm-restart.test.sh index e82f554f..36e4d7d1 100755 --- a/tests/fm-restart.test.sh +++ b/tests/fm-restart.test.sh @@ -202,6 +202,40 @@ test_refuses_unsupported_backend() { pass "fm-restart: refuses unsupported backends instead of guessing process state" } +test_live_lane_with_unknown_process_state_aborts_before_stow() { + local dir root fb out status log + dir=$(new_case unknown-process); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + : > "$dir/command" + : > "$dir/process-argv" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "unknown live process state should fail" + assert_contains "$out" "foreground process state is unknown" "unknown process state did not explain the safety abort" + log=$(cat "$dir/log") + assert_not_contains "$log" "send:fm-lane:Stow" "unknown process state must not ask for stow through an uncertain endpoint" + assert_not_contains "$log" "send:firstmate:fm-lane:/quit" "unknown process state must not send raw exit" + assert_not_contains "$log" "tmux:kill-window" "unknown process state must not clean up" + assert_not_contains "$log" "spawn:" "unknown process state must not respawn" + pass "fm-restart: live lane with unknown process state aborts before stow" +} + +test_live_lane_with_mismatched_process_aborts_before_cleanup() { + local dir root fb out status log + dir=$(new_case mismatched-process); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") + printf 'zsh\n' > "$dir/command" + printf 'zsh\n' > "$dir/process-argv" + write_lane_meta "$dir" secondmate tmux firstmate:fm-lane + out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + [ "$status" -ne 0 ] || fail "mismatched live process should fail without --skip-stow" + assert_contains "$out" "foreground process does not match codex" "mismatched process did not explain the safety abort" + log=$(cat "$dir/log") + assert_not_contains "$log" "send:fm-lane:Stow" "mismatched process state must not ask for stow" + assert_not_contains "$log" "send:firstmate:fm-lane:/quit" "mismatched process state must not send raw exit" + assert_not_contains "$log" "tmux:kill-window" "mismatched process state must not clean up" + assert_not_contains "$log" "spawn:" "mismatched process state must not respawn" + pass "fm-restart: live lane with mismatched process aborts before cleanup" +} + test_tmux_live_lane_stows_exits_kills_then_respawns() { local dir root fb log dir=$(new_case tmux-live); root=$(make_restart_root "$dir"); fb=$(make_fake_tmux "$dir") @@ -369,6 +403,8 @@ test_herdr_stale_label_refuses_restart() { test_refuses_non_secondmate test_dead_lane_respawns_without_nudge test_refuses_unsupported_backend +test_live_lane_with_unknown_process_state_aborts_before_stow +test_live_lane_with_mismatched_process_aborts_before_cleanup test_tmux_live_lane_stows_exits_kills_then_respawns test_tmux_node_wrapped_codex_stows_before_respawn test_herdr_live_lane_renames_spawns_then_closes_old_tab From fc23377a90c520a2f031e0024042b5f44b8cc5e4 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:40:37 -0700 Subject: [PATCH 08/13] no-mistakes(document): Captain, sync restart docs --- CONTRIBUTING.md | 3 ++- README.md | 21 ++++++++++++++++++++- docs/architecture.md | 8 ++++++-- docs/configuration.md | 6 ++++++ docs/herdr-backend.md | 7 +++++-- docs/scripts.md | 7 ++++--- 6 files changed, 43 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e066fa8..32c1e143 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,13 +96,14 @@ tests/fm-secondmate-sync.test.sh # local-HEAD secondmate sync, no-fetch tests/fm-secondmate-harness.test.sh # secondmate-vs-crewmate harness resolution, optional secondmate model/effort pins, primary-to-secondmate config inheritance, and config-push tests tests/fm-secondmate-lifecycle-e2e.test.sh # persistent secondmate routing, seeding, backlog handoff, spawn, recovery, teardown, and FM_HOME flow tests tests/fm-secondmate-safety.test.sh # secondmate home safety, idle charter, handoff validation, and teardown boundary tests +tests/fm-restart.test.sh # secondmate lane refresh helper: kind refusal, stow gating, --skip-stow, raw harness exit routing, force handling, dead-lane respawn, and backend cleanup tests/fm-teardown.test.sh # fm-teardown.sh landed-work safety and reminder checks: fork-remote allow, squash/content landings, dirty and unlanded refusals, PR-head metadata, no-pr= branch discovery, tasks-axi/manual backlog reminder, --force override tests/fm-review-diff.test.sh # fm-review-diff.sh authoritative review diff coverage: recorded pr_head=, fetched refs/pull//head, no-pr local branch behavior, and warning fallback tests/fm-pr-merge.test.sh # fm-pr-merge.sh records pr= and available pr_head= before merging, parses PR URLs into gh-axi number/--repo calls, defaults to squash, preserves explicit merge methods, rejects malformed URLs and repo overrides, and propagates real merge failures tests/fm-crew-state.test.sh # fm-crew-state.sh current-state reconciliation: run-step authority including closed panes, stale needs-decision/blocked superseded by a resumed run, genuine-parked, cross-branch runs-list attribution, pane/status-log fallback, scout skip, torn-down/missing-meta graceful tests/fm-backend.test.sh # runtime-backend abstraction: fm-backend.sh selection/meta/dispatch helpers, shell-portable sourced backend matching, and old-vs-new fake-tool command-log conformance for fm-send/fm-peek/fm-spawn/fm-teardown tests/fm-backend-tmux-smoke.test.sh # real (private-socket) tmux smoke test for the tmux adapter: create/duplicate-refuse, send text + Enter, send literal + key, bounded capture, live-window resolve, kill -tests/fm-backend-herdr.test.sh # fake herdr CLI unit tests for the experimental herdr adapter, including version/tool gates, target parsing, send/capture, structural composer-state verification, slash-submit retry regression coverage, native busy state, per-home workspace-label resolution, default-tab prune safety, restored-layout husk replacement, and verified CLI bug workarounds +tests/fm-backend-herdr.test.sh # fake herdr CLI unit tests for the experimental herdr adapter, including version/tool gates, target parsing, send/capture, structural composer-state verification, slash-submit retry regression coverage, native busy state, foreground-process and label-guard reads, per-home workspace-label resolution, default-tab prune safety, restored-layout husk replacement, and verified CLI bug workarounds tests/fm-backend-herdr-smoke.test.sh # real herdr adapter smoke test, skipped when herdr or jq is unavailable, using an isolated throwaway HERDR_SESSION and guarded session cleanup, including live-agent duplicate refusal and no-agent husk replacement tests/fm-backend-autodetect-smoke.test.sh # real herdr auto-detection smoke test, skipped when herdr, jq, or treehouse is unavailable, using the same guarded session cleanup tests/fm-backend-herdr-workspace-per-home-e2e.test.sh # mandatory isolated E2E for workspace-per-home: primary and secondmate-shaped homes, a crewmate spawned from a secondmate home, teardown, list-live recovery diff --git a/README.md b/README.md index d6d31cd0..c904edb0 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,26 @@ Setup guides for tmux (the default) and every other supported backend (herdr, ze You chat with the first mate. It routes each request to a crewmate in its own session endpoint and git worktree, supervises the fleet with a zero-token event-driven watcher, and brings you finished PRs, approved local merges, or investigation reports. -Optional secondmates extend this to persistent domain supervisors, dispatch profiles let you steer which harness handles which task, and an opt-in X mode lets the same fleet answer public mentions. +Persistent secondmate homes are linked firstmate worktrees; locked session start syncs live ones and secondmate launch syncs the target home to the primary default-branch commit without fetching from origin when it is safe. +Crewmate dispatch can stay on a static `config/crew-harness` or use optional natural-language profiles in local `config/crew-dispatch.json` to choose a per-task harness, model, and effort. +When that profile file exists, crewmate and scout spawns must pass the resolved harness explicitly so `config/crew-harness` is not used as an unnoticed bypass. +Secondmate launch can use a separate local `config/secondmate-harness`, whose first non-empty, non-comment line is parsed as ` [] []` to durably pin that secondmate's launch profile. +The runtime session-provider backend is selected from explicit `--backend`, `FM_BACKEND`, local `config/backend`, runtime auto-detection from `$TMUX` or `HERDR_ENV=1`, then the hard `tmux` default. +`tmux` is the verified reference backend. +`herdr`, `zellij`, and `orca` are experimental task-spawn backends; herdr can also be auto-detected, while zellij and orca are selected only explicitly. +Unlike tmux/herdr/zellij, Orca provides the task worktree as well as the terminal endpoint. +Secondmate homes inherit the primary's declared local config, including `config/crew-dispatch.json`, `config/crew-harness`, and `config/backlog-backend`, at launch, during the locked session-start bootstrap step, or during an explicit `bin/fm-config-push.sh` run, so their own crewmates, dispatch profiles, and backlog backend use the primary settings. +When a routed request goes to a secondmate, firstmate marks it so the answer returns through status or a document pointer; direct typing into that secondmate window stays conversational. +When a live secondmate lane needs a clean context refresh, `bin/fm-restart.sh ` asks it to stow, waits for `stowed: restart-ready` unless `--skip-stow` is explicitly passed, exits the harness through the raw backend endpoint, verifies the process is gone, then respawns it through normal secondmate launch bookkeeping. +A presence-gated sub-supervisor (`/afk`) can self-handle routine events and batch only what matters while you step away. +An opt-in X mode can also use the watcher check path to answer your public `@myfirstmate` mentions and act on normal reversible mention requests from the current fleet state, with `FMX_DRY_RUN` available to test the poll -> compose -> would-post loop without publishing. +The relay routes only the owner's own mentions to that owner's firstmate home; parent-thread context may still include other public accounts. +The token is standing authorization for those autonomous replies and eligible lifecycle actions; destructive, irreversible, or security-sensitive asks are flagged for trusted-channel confirmation instead of being executed from a public mention. +Requests that finish immediately get one public-safe outcome reply. +Requests that spawn longer-running work get an acknowledgement first, a task link in local state, and one completion follow-up within the relay's 24h window when that task lands, reports, or fails. +It preserves parent-tweet context for conversational replies and dismisses pure acknowledgments at the relay without posting. +Replies can attach one local image with `--image ` when there is a visual artifact; long replies split into bounded numbered threads when needed, with the image attached only to the opener tweet. +When firstmate works on itself, spawn-time isolation checks and a primary-checkout tangle alarm keep the operating checkout on its default branch and stop a crewmate that did not land in a separate worktree. Full architecture - the supervision engine, worktree isolation, secondmates, dispatch profiles, project modes, optional X mode, fleet sync, and self-update - is in [docs/architecture.md](docs/architecture.md). diff --git a/docs/architecture.md b/docs/architecture.md index 05972d29..d18e12e1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,7 +44,7 @@ Stalled escalation delivery raises `state/.subsuper-inject-wedged` after `FM_MAX ## Runtime session backends The runtime backend is the session-provider layer below firstmate's scripts. -It owns task endpoint creation, bounded capture, text/key sends, current-path reads for spawn-time worktree discovery when the backend does not create the worktree itself, live-window fallback lookup, and endpoint teardown. +It owns task endpoint creation, bounded capture, text/key sends, current-path reads for spawn-time worktree discovery when the backend does not create the worktree itself, live-window fallback lookup, foreground-process reads for restart safety where supported, task relabeling where supported, and endpoint teardown. `bin/fm-backend.sh` centralizes backend selection, `state/.meta` helpers, selector resolution, and operation dispatch; `bin/backends/tmux.sh` is the verified reference adapter ([`docs/tmux-backend.md`](tmux-backend.md)), and `bin/backends/herdr.sh` (P2), `bin/backends/zellij.sh` (P3), `bin/backends/orca.sh` (P4), and `bin/backends/cmux.sh` (P5) are experimental task-spawn adapters. New spawns select a backend from `--backend`, then `FM_BACKEND`, then local `config/backend`, then runtime auto-detection from `$TMUX`, `HERDR_ENV=1`, or cmux runtime signals, then default `tmux`. Runtime auto-detection is innermost-first: `$TMUX` wins over `HERDR_ENV=1`, which wins over cmux's primary `CMUX_WORKSPACE_ID` marker and documented fallback signals; auto-detected herdr or cmux prints a one-time opt-out notice, auto-detected tmux stays silent, and zellij and orca are never auto-detected (only explicit selection). @@ -104,6 +104,9 @@ The same project may appear in multiple secondmate homes when their scopes diffe Secondmates are idle by default: after startup recovery reconciles only work already in their own home, an empty queue waits silently for routed tasks, and they never self-initiate surveys or audits. Bare `fm-send.sh fm-` requests to a live `kind=secondmate` are prefixed with the from-firstmate marker from `bin/fm-marker-lib.sh`, so the secondmate returns terse answers through status lines and detailed answers through docs plus status pointers instead of replying only in its own chat. Explicit backend-target sends and direct human typing stay unmarked, so captain intervention in a secondmate pane remains conversational. +When a live secondmate lane needs a context refresh, use `bin/fm-restart.sh ` instead of hand-closing panes or hand-launching a replacement. +It accepts only `kind=secondmate` meta records, asks the lane to stow through the marked `fm-send.sh fm-` path, requires a new `stowed: restart-ready` status line before exit unless `--skip-stow` is explicitly passed, sends the harness exit command directly to the raw backend endpoint so it is handled by the harness rather than by chat, verifies the harness process or endpoint is gone, and then respawns through `fm-spawn.sh --secondmate`. +On herdr it also verifies the recorded pane still belongs to the expected `fm-` tab, renames the old tab before spawning the replacement, and closes the old tab after the respawn succeeds. After seeding a secondmate, `fm-backlog-handoff.sh` moves already-judged in-scope queued items from the main backlog into that secondmate home so the domain queue starts in the right place. Idle secondmate panes are healthy; teardown is explicit and refuses while the secondmate home has in-flight work unless the captain has approved discard with `--force`. @@ -203,7 +206,8 @@ The mechanics are owned by the `/updatefirstmate` skill and firstmate's operatin Fleet state lives in each task's session-provider backend (tmux by hard default, herdr or cmux when selected or auto-detected, zellij/orca when explicitly selected), no-mistakes run records, status event logs, local markdown under `data/` including `data/captain.md` and `data/learnings.md`, and persistent secondmate homes. For herdr, respawning after a server-restored layout closes and replaces confirmed no-agent or dead task-tab husks instead of requiring manual tab cleanup. -Use `/stow` before an intentional reset when the conversation may hold durable knowledge that has not yet been written to disk; after that, the next firstmate session can reconcile and carry on. +Use `/stow` before an intentional reset when the conversation may hold durable knowledge that has not yet been written to disk; for a live secondmate lane refresh, prefer `bin/fm-restart.sh ` so stow confirmation, raw harness exit, backend cleanup, and respawn happen in one guarded path. +After that, the next firstmate session can reconcile and carry on. ## Development notes diff --git a/docs/configuration.md b/docs/configuration.md index 516475f8..9435d3ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -265,6 +265,12 @@ GROK_HOME= # optional Grok config home for firstmate's global grok FM_SEND_RETRIES=3 # fm-send Enter-retry attempts after typing the line once FM_SEND_SLEEP=0.4 # seconds between fm-send submit checks FM_SEND_SETTLE=1 # seconds fm-send waits after a successful text submit; 0 disables +FM_RESTART_TIMEOUT=120 # seconds fm-restart waits for a stowed secondmate harness to exit gracefully +FM_RESTART_FORCE_TIMEOUT=20 # seconds fm-restart waits after --force interrupt/exit input +FM_RESTART_CLEANUP_TIMEOUT=5 # seconds fm-restart waits for the old backend endpoint to disappear after cleanup +FM_RESTART_STOW_TIMEOUT=120 # seconds fm-restart waits for a new 'stowed: restart-ready' status signal +FM_RESTART_STOW_SETTLE=120 # legacy fallback for FM_RESTART_STOW_TIMEOUT when the latter is unset +FM_RESTART_POLL_INTERVAL=1 # seconds between fm-restart stow/exit/cleanup polls # sub-supervisor (bin/fm-supervise-daemon.sh); presence-gated via /afk FM_SUPERVISOR_BACKEND= # optional supervisor pane backend override; tmux/herdr only, otherwise detects $TMUX_PANE then HERDR_ENV/HERDR_PANE_ID before tmux fallback FM_SUPERVISOR_TARGET= # optional supervisor pane target override; tmux target or herdr :, otherwise auto-detected diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 21645fc3..f5931dac 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -181,10 +181,13 @@ Herdr tasks additionally record: | Duplicate task check | `herdr tab list --workspace `, match by `.label` | Herdr does NOT enforce tab-label uniqueness itself; two tabs can share a label. The adapter's own duplicate check is required. | | Send literal (unsubmitted) | `herdr pane send-text ` | Does NOT auto-submit, contrary to the original design addendum's guess. Verified directly: a unique marker sent this way sits unexecuted in the composer until a separate Enter. Behaves exactly like tmux's `send-keys -l`. | | Send + submit atomically | `herdr pane run ` | Runs and submits a command in one call; used for the two fixed spawn-time commands (`treehouse get`, the `GOTMPDIR` export) exactly where tmux used one `send-keys ... Enter` call. | -| Send key | `herdr pane send-keys ` | Verified names: `enter`, `escape` (alias `esc`), `ctrl+c` (aliases `C-c`, `c-c`). `ctrl+c` verified to interrupt a running foreground process immediately. | +| Send key | `herdr pane send-keys ` | Verified names: `enter`, `escape` (alias `esc`), `ctrl+c` (aliases `C-c`, `c-c`), and `ctrl+q` (aliases `C-q`, `c-q`). `ctrl+c` verified to interrupt a running foreground process immediately; `ctrl+q` is used for grok's harness quit flow. | | Bounded capture | `herdr pane read --source recent --lines N` | See "Verified bug" below - N is never passed through directly. | | Busy state | `herdr agent get ` -> `.result.agent.agent_status` | Verified live against an interactive `claude` session: reports `working` while generating, `done` once idle. Mapped: `working` -> busy; `idle`/`done` -> idle; `blocked` -> idle (surfaced like a stale pane, not suppressed as busy - a blocked agent is stuck waiting on the human, not grinding); anything else -> unknown (the cue for the shared tail-regex fallback). | -| Kill | `herdr pane close ` | Closing a tab's only (root) pane also closes the tab - no separate tab-close call needed for this adapter's one-pane-per-tab shape. Best-effort: closing an already-closed pane exits non-zero, matching tmux's `kill-window \|\| true` contract. Teardown itself only ever closes the task's own pane/tab, never the workspace - but closing a workspace's LAST tab (verified real-herdr behavior) deletes the workspace as a side effect, so a home's own workspace persists only while at least one task tab remains; see "Workspace lifecycle" above. | +| Foreground process | `herdr pane process-info --pane ` -> `.result.process_info.foreground_processes[]` | Used by `fm-restart.sh` to verify the recorded harness process is actually still running or has exited. This deliberately ignores persisted herdr agent metadata because restored or exited panes can keep stale agent labels after the foreground process has returned to a shell. | +| Target label guard | `herdr pane get ` -> tab id, then `herdr tab list` -> label | Used by restart and cleanup paths before control actions that target a recorded pane. A mismatched label means the stored pane no longer belongs to the expected `fm-` tab, so firstmate refuses to send exit input or close the endpoint. | +| Relabel task tab | `herdr tab rename ` | Used by `fm-restart.sh` before respawning a live secondmate lane: the old `fm-` tab is renamed to an archival label so `fm-spawn.sh --secondmate` can create the fresh `fm-` tab first, then close the old tab afterward. | +| Kill | `herdr pane close ` | Closing a tab's only (root) pane also closes the tab - no separate tab-close call needed for this adapter's one-pane-per-tab shape. Best-effort for normal teardown: closing an already-closed pane exits non-zero, matching tmux's `kill-window \|\| true` contract. When an expected label is provided, the adapter verifies the pane still belongs to that tab label before closing. Teardown itself only ever closes the task's own pane/tab, never the workspace - but closing a workspace's LAST tab (verified real-herdr behavior) deletes the workspace as a side effect, so a home's own workspace persists only while at least one task tab remains; see "Workspace lifecycle" above. | | Default-tab prune (create_task, first task in a fresh workspace only) | `herdr workspace create`'s own response (`.result.tab.tab_id`) identifies the seeded tab; `herdr tab list` + `herdr agent get ` re-verify it; `herdr pane close ` closes exactly that tab id | `herdr workspace create` seeds the new workspace with one auto-created default tab (label `1`, id captured straight from the create response) firstmate never uses. `fm_backend_herdr_create_task` closes EXACTLY that captured tab id right after creating the first real task tab in a freshly created workspace - never right after `workspace create` itself (see Kill row), and never re-derived from a tab's label or the workspace's tab count at create_task time (see "Default-tab prune" above for the created-vs-adopted safety gate and the 2026-07-02 incident it fixes). Best-effort; an ADOPTED workspace (not freshly created by this same call) is never a prune candidate at all. | | Recovery / list-live | `herdr tab list --workspace `, filter labels starting with `fm-` | Label-based, never trusts a stored id blindly - see "ID stability" below. `` is always THIS home's own workspace (`fm_backend_herdr_workspace_find`), so recovery never sees a sibling home's tabs. | | Workspace create / tab create (focus) | `herdr workspace create --no-focus`, `herdr tab create --no-focus` | Verified: neither focuses by default once a workspace already exists in the session, matching pre-P3 (flagless) behavior; `--no-focus` is passed anyway for defense in depth, since the very first workspace ever created in a brand-new session focuses regardless of the flag. `--focus` was separately verified to reliably focus, confirming the flag has real effect. | diff --git a/docs/scripts.md b/docs/scripts.md index b8aa8aad..a3a87162 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -17,14 +17,15 @@ If you have changed away from the firstmate home in an interactive shell, invoke | `fm-turnend-guard.sh` | Claude Code Stop hook, primary-scoped only: blocks (exit 2, exact reason) a primary turn end when work is in flight without a live identity-matched watcher lock and fresh beacon, using Claude Code's own `stop_hook_active` field so it never blocks twice in one turn (docs/turnend-guard.md) | | `fm-home-seed.sh` | Lease/provision a secondmate home transactionally, clone projects, initialize gates, and maintain `data/secondmates.md` | | `fm-spawn.sh` | Spawn one task, several `id=repo` pairs, or a persistent secondmate with `--secondmate`; accepts concrete `--harness`, `--model`, `--effort`, and `--backend` axes; ship/scout spawns require an explicit resolved harness when dispatch profiles are active and an isolated worktree, install per-harness turn-end signaling, and secondmate spawns resolve the secondmate harness plus optional `config/secondmate-harness` model/effort tokens, locally sync the home, propagate declared inheritable config, land herdr tabs in the target home's workspace, land home-scoped zellij tabs in the selected shared zellij session, land cmux workspaces in the shared cmux app, or create Orca worktrees/terminals before launch | -| `fm-backend.sh` | Runtime session-provider backend selector with explicit/env/config/runtime auto-detection precedence, meta helper, selector resolver, spawn-capability validation, operation dispatcher, and shell-portable backend-name membership for bash-sourced scripts or zsh-sourced diagnostics; defaults absent `backend=` meta to `tmux`; `fm_backend_target_exists` is a cheap read-only alive/dead endpoint check that never starts a server or session; `fm_backend_composer_state` exposes backend composer checks for guarded submit paths | +| `fm-backend.sh` | Runtime session-provider backend selector with explicit/env/config/runtime auto-detection precedence, meta helper, selector resolver, spawn-capability validation, operation dispatcher, and shell-portable backend-name membership for bash-sourced scripts or zsh-sourced diagnostics; defaults absent `backend=` meta to `tmux`; exposes cheap read-only target-exists checks, foreground-process reads, task relabeling where an adapter supports them, and backend composer checks for guarded submit paths | | `fm-backend-hometag-lib.sh` | Shared home-tag derivation for zellij tab titles and cmux workspace titles, using the active `FM_HOME` label plus a short hash of the resolved `FM_ROOT` path | -| `backends/tmux.sh` | Verified tmux session-provider adapter used by `fm-backend.sh`; owns create, send, capture, current-path, live-window, and kill primitives | -| `backends/herdr.sh` | Experimental herdr session-provider adapter used by `fm-backend.sh`; owns version/tool gating, per-home workspace/tab creation, created-vs-adopted default-tab prune safety, restored-layout husk respawn replacement, session-scoped CLI calls, send with structural composer-state verification, capture, native busy-state, current-path, label-based live discovery, and kill primitives | +| `backends/tmux.sh` | Verified tmux session-provider adapter used by `fm-backend.sh`; owns create, send, capture, current-path, live-window, foreground-process, relabel, and kill primitives | +| `backends/herdr.sh` | Experimental herdr session-provider adapter used by `fm-backend.sh`; owns version/tool gating, per-home workspace/tab creation, created-vs-adopted default-tab prune safety, restored-layout husk respawn replacement, session-scoped CLI calls, send with structural composer-state verification, capture, native busy-state, foreground-process, current-path, label-checked target safety, label-based live discovery, relabel, and kill primitives | | `backends/zellij.sh` | Experimental zellij session-provider adapter used by `fm-backend.sh`; owns version/tool gating, one-session/tab-per-task creation with home-scoped titles, session-scoped CLI calls, send, capture, active current-path probing, label-checked target safety, scoped live discovery, legacy-title fallback, and tab cleanup primitives | | `backends/orca.sh` | Experimental Orca backend used by `fm-backend.sh`; owns repo registration, worktree creation/removal, terminal creation, capture, send text, Enter/Ctrl-C interrupt keys, and close; Escape is unsupported | | `backends/cmux.sh` | Experimental cmux session-provider adapter used by `fm-backend.sh`; owns version/tool/socket-access gating, home-scoped workspace creation, current-path probing, title-based recovery, send/capture, structural composer-state verification, Enter/Escape/Ctrl-C keys, and workspace cleanup primitives | | `fm-config-push.sh` | Config-only mid-session push of declared inheritable local config into live secondmate homes; reports each item as pushed, unchanged, skipped, or error without fast-forwarding tracked files or nudging agents | +| `fm-restart.sh` | Refresh one persistent secondmate lane on tmux or herdr: accepts only recorded `kind=secondmate` tasks, asks for a `stowed: restart-ready` status signal unless `--skip-stow` is passed, sends the harness exit command to the raw backend endpoint, verifies exit with backend process/target primitives, respawns through `fm-spawn.sh --secondmate`, and cleans up the old endpoint; `--force` adds a bounded interrupt/exit sequence after confirmed stow | | `fm-project-mode.sh` | Resolve a project's delivery mode and `+yolo` flag from `data/projects.md` | | `fm-merge-local.sh` | Fast-forward a `local-only` project's local default branch after approval | | `fm-review-diff.sh` | Review a crewmate branch or recorded PR head against the authoritative base, warning and falling back to the local branch when an expected PR head cannot be resolved, with optional `--stat` output | From 9c074e5539a826cbfeb24de9c60336513109336e Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:44:27 -0700 Subject: [PATCH 09/13] no-mistakes(lint): Captain, lint clean --- tests/fm-restart.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fm-restart.test.sh b/tests/fm-restart.test.sh index 36e4d7d1..230608bf 100755 --- a/tests/fm-restart.test.sh +++ b/tests/fm-restart.test.sh @@ -310,7 +310,7 @@ exit 0 SH chmod +x "$root/bin/fm-send.sh" write_lane_meta "$dir" secondmate tmux firstmate:fm-lane - FM_RESTART_STOW_TIMEOUT=0 out=$(run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? + out=$(FM_RESTART_STOW_TIMEOUT=0 run_restart "$dir" "$root" "$fb" lane 2>&1); status=$? [ "$status" -ne 0 ] || fail "stow timeout should fail" assert_contains "$out" "did not confirm stow" "stow timeout did not explain missing status signal" assert_not_contains "$(cat "$dir/log")" "send:firstmate:fm-lane:/quit" "stow timeout must not send raw exit" From d55387231a40f651aceacba64d64b02865d62252 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:07:09 -0700 Subject: [PATCH 10/13] no-mistakes: apply CI fixes --- bin/fm-restart.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh index b3c6bf26..4c7ce012 100755 --- a/bin/fm-restart.sh +++ b/bin/fm-restart.sh @@ -89,7 +89,10 @@ target_exists_with_label() { harness_process_state() { local processes cmd cmd=$HARNESS_CMD - [ -n "$cmd" ] && [ "$cmd" != unknown ] || { printf 'unknown'; return 0; } + if [ -z "$cmd" ] || [ "$cmd" = unknown ]; then + printf 'unknown' + return 0 + fi processes=$(fm_backend_foreground_process "$BACKEND" "$T" "$EXPECTED_LABEL" 2>/dev/null || true) [ -n "$processes" ] || { printf 'unknown'; return 0; } if printf '%s\n' "$processes" | awk -v want="$cmd" ' From 7d0947198b8f83b57a5ada5fac0d4f35608fc5ec Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Sun, 5 Jul 2026 10:30:18 -0700 Subject: [PATCH 11/13] test: isolate gotmp teardown fake root --- tests/fm-gotmp.test.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/fm-gotmp.test.sh b/tests/fm-gotmp.test.sh index 8ea2a70c..5ae1a400 100755 --- a/tests/fm-gotmp.test.sh +++ b/tests/fm-gotmp.test.sh @@ -119,6 +119,11 @@ test_spawn_contract_and_mkdir_pattern() { # --- fm-teardown side (real subprocess) --- +run_fake_teardown() { # + env -u FM_HOME -u FM_ROOT_OVERRIDE -u FM_STATE_OVERRIDE -u FM_DATA_OVERRIDE -u FM_CONFIG_OVERRIDE \ + bash "$1/bin/fm-teardown.sh" "$2" +} + test_teardown_removes_tasktmp_dir() { local id=td-rm-z2 local task_tmp="$TMP_ROOT/fm-$id" @@ -129,7 +134,7 @@ test_teardown_removes_tasktmp_dir() { # Sanity: dir + contents exist before teardown. [ -d "$task_tmp/gotmp" ] || fail "precondition: gotmp missing before teardown" # Run the REAL teardown against the fake root. - bash "$fake/bin/fm-teardown.sh" "$id" >/dev/null 2>&1 \ + run_fake_teardown "$fake" "$id" >/dev/null 2>&1 \ || fail "teardown exited non-zero with a valid tasktmp" [ ! -e "$task_tmp" ] \ || fail "teardown did not remove the tasktmp dir ($task_tmp still exists)" @@ -169,7 +174,7 @@ kind=ship mode=no-mistakes yolo=off META - bash "$fake/bin/fm-teardown.sh" "$id" >/dev/null 2>&1 \ + run_fake_teardown "$fake" "$id" >/dev/null 2>&1 \ || fail "teardown exited non-zero when tasktmp= was absent" pass "fm-teardown skips gracefully when tasktmp= is absent (backward compat)" } @@ -182,7 +187,7 @@ test_teardown_skips_gracefully_when_dir_missing() { [ ! -e "$task_tmp" ] || fail "precondition: task_tmp should not exist yet" local fake fake=$(make_fake_root "$id" "$task_tmp") - bash "$fake/bin/fm-teardown.sh" "$id" >/dev/null 2>&1 \ + run_fake_teardown "$fake" "$id" >/dev/null 2>&1 \ || fail "teardown exited non-zero when tasktmp dir was missing" [ ! -e "$task_tmp" ] || fail "teardown created/left the tasktmp dir unexpectedly" pass "fm-teardown skips gracefully when tasktmp= points to a nonexistent dir" From f1ca7ca257e81d0d3cd682958c154df3b9a47abc Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Sun, 5 Jul 2026 10:37:13 -0700 Subject: [PATCH 12/13] fix: reject unsupported restart backends early --- bin/fm-restart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fm-restart.sh b/bin/fm-restart.sh index 4c7ce012..2f82f0c6 100755 --- a/bin/fm-restart.sh +++ b/bin/fm-restart.sh @@ -63,11 +63,11 @@ HARNESS=$(fm_meta_get "$META" harness) EXPECTED_LABEL="fm-$ID" fm_backend_validate_spawn "$BACKEND" >/dev/null || exit 1 -fm_backend_source "$BACKEND" || exit 1 case "$BACKEND" in tmux|herdr) ;; *) echo "error: fm-restart supports secondmate lanes on tmux/herdr only for now; $ID uses backend=$BACKEND" >&2; exit 1 ;; esac +fm_backend_source "$BACKEND" || exit 1 harness_command() { local h=$1 cmd From 137a228ff2fb61e95d1a10380c57e33be649f053 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Sun, 5 Jul 2026 10:46:26 -0700 Subject: [PATCH 13/13] test: isolate turnend guard hook env --- tests/fm-turnend-guard.test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/fm-turnend-guard.test.sh b/tests/fm-turnend-guard.test.sh index 5edbc3ac..7e040bdb 100755 --- a/tests/fm-turnend-guard.test.sh +++ b/tests/fm-turnend-guard.test.sh @@ -127,7 +127,9 @@ make_crewmate_worktree_dir() { run_hook() { local dir=$1 stop_active=$2 - printf '{"stop_hook_active":%s}' "$stop_active" | bash "$dir/bin/fm-turnend-guard.sh" 2>&1 + printf '{"stop_hook_active":%s}' "$stop_active" \ + | env -u FM_HOME -u FM_ROOT_OVERRIDE -u FM_STATE_OVERRIDE -u FM_DATA_OVERRIDE -u FM_CONFIG_OVERRIDE \ + bash "$dir/bin/fm-turnend-guard.sh" 2>&1 } nonexistent_pid() {