diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e066fa8..f669f3d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ for test_script in tests/*.test.sh; do bash "$test_script"; done # behavior te tests/fm-wake-queue.test.sh # durable wake queue losslessness, catch-up, double-drain, duplicate-collapse, and drain liveness guard tests tests/fm-watcher-lock.test.sh # watcher singleton, lock-race, watch-arm liveness, and guard-warning tests tests/fm-turnend-guard.test.sh # shared supervision predicate plus Claude Stop-hook scoping, loop guard, fail-open, and live watcher health tests -tests/fm-watch-triage.test.sh # always-on watcher triage: benign absorb, actionable surface, stale status-log override, wedge threshold, heartbeat backstop, and afk one-shot coherence +tests/fm-watch-triage.test.sh # always-on watcher triage: benign absorb, actionable surface, stale status-log override, herdr semantic stale dedup, wedge threshold, heartbeat backstop, and afk one-shot coherence tests/fm-daemon.test.sh # sub-supervisor classifier, /afk presence-gating, max-defer, composer, and fm-send submit tests tests/fm-send-settle.test.sh # fm-send post-submit settle pause, tuning, disable, and --key bypass tests tests/fm-send-popup-settle.test.sh # fm-send pre-Enter popup-settle selection for slash commands and codex $skill invocations diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 554f3d21..542ec3dd 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -158,23 +158,52 @@ hash_pane() { if command -v md5 >/dev/null 2>&1; then md5 -q; else md5sum | cut -d' ' -f1; fi } -# window_is_busy: 0 (busy) iff the task's harness is actively working. Prefers -# a backend's native semantic busy state (fm_backend_busy_state - herdr's -# agent.get; herdr-addendum "busy state" row, "the first backend where -# fm_session_busy_state gets real semantics"); falls back to the existing -# pane-tail regex ONLY when the backend reports unknown (tmux always does, so -# its path is unchanged byte-for-byte). is the same bounded capture -# already read for hashing, so this adds no extra backend calls on the -# regex-fallback path. -window_is_busy() { # - local w=$1 tail40=$2 bs - bs=$(fm_backend_busy_state "$(window_backend "$w")" "$w" 2>/dev/null) - case "$bs" in +# busy_state_is_busy: 0 (busy) iff the task's harness is actively working, from +# an ALREADY-RESOLVED backend busy state (busy|idle|unknown) plus the +# capture. A backend that reports a semantic state (herdr's agent.get - +# the herdr-addendum "busy state" row) decides directly; unknown (tmux always, +# and herdr when it cannot read the agent) falls back to the last 6 non-blank +# lines of the pane tail matched against the busy regex, so the tmux path is +# byte-identical. The stale loop resolves the backend busy state ONCE and shares +# it with stale_signal below, so herdr's agent.get is not called twice per pane +# per poll. +busy_state_is_busy() { # + case "$1" in busy) return 0 ;; idle) return 1 ;; *) - printf '%s' "$tail40" | grep -v '^[[:space:]]*$' | tail -6 | grep -qiE "$BUSY_REGEX" + printf '%s' "$2" | grep -v '^[[:space:]]*$' | tail -6 | grep -qiE "$BUSY_REGEX" + ;; + esac +} + +# stale_signal: the value the stale-dedup logic hashes for a window given its +# ALREADY-RESOLVED backend busy state and capture. This is the +# herdr stale-echo-churn fix. On a backend with a native SETTLED agent state +# (herdr reports idle for idle/done/blocked), the signal is that semantic state, +# NOT the pane content - so a settled pane that merely REPAINTS its volatile +# harness UI (claude's timer-driven recap line, rotating tips, animated +# "for Xm Ys" summaries, transient slash-completion menus) keeps ONE signal and +# is surfaced once, instead of minting a fresh stale hash every few minutes. +# An idle verdict is treated as settled only when the same last-6-nonblank-line +# BUSY_REGEX corroboration used by the fallback path does not see the busy banner, +# because docs/herdr-backend.md records that herdr can report idle during a long +# foreground tool call while the pane still renders "esc to interrupt". +# Content normalization cannot fix repaint churn robustly, because volatile rows +# also shift a variable-length prefix through the tail-N hash window. For tmux +# (busy state always unknown), busy/unknown herdr panes, and idle herdr panes +# that still look busy, the signal is the pane-content hash exactly as before, +# so the proven tmux path is unchanged byte-for-byte. +stale_signal() { # + case "$1" in + idle) + if printf '%s' "$2" | grep -v '^[[:space:]]*$' | tail -6 | grep -qiE "$BUSY_REGEX"; then + printf '%s' "$2" | hash_pane + else + printf 'agentstate:idle' + fi ;; + *) printf '%s' "$2" | hash_pane ;; esac } @@ -460,8 +489,14 @@ EOF # A secondmate idling on its own watcher is healthy. Its parent supervises # it through status writes and heartbeats, not pane-idle staleness. [ "$(window_kind "$w")" = secondmate ] && continue - tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null) || continue - h=$(printf '%s' "$tail40" | hash_pane) + backend=$(window_backend "$w") + tail40=$(fm_backend_capture "$backend" "$w" 40 "$(window_label "$w")" 2>/dev/null) || continue + # Resolve the backend busy state ONCE and derive the stale-dedup signal from + # it: for a settled herdr pane the signal is the semantic agent state (so + # volatile-UI repaints do not churn a new stale hash), else the pane-content + # hash exactly as before (tmux path unchanged). + bs=$(fm_backend_busy_state "$backend" "$w" 2>/dev/null) + h=$(stale_signal "$bs" "$tail40") key=$(printf '%s' "$w" | tr ':/.' '___') hf="$STATE/.hash-$key" cf="$STATE/.count-$key" @@ -475,7 +510,7 @@ EOF # else the last 6 non-blank lines only (the TUI footer area, where every # verified harness renders its busy indicator) so busy-looking strings # in displayed content cannot suppress stale detection. - if [ "$n" -ge 2 ] && ! window_is_busy "$w" "$tail40"; then + if [ "$n" -ge 2 ] && ! busy_state_is_busy "$bs" "$tail40"; then # The pane is idle/stale at hash $h. Triage decides whether this wakes # firstmate. Detection itself is unchanged from above. if afk_present; then @@ -557,7 +592,7 @@ EOF else printf '%s' "$h" > "$hf" echo 0 > "$cf" - # Pane content changed: the crew is active again, so reset the escalation timer. + rm -f "$sf" rm -f "$ssf" fi done < <(recorded_windows) diff --git a/docs/architecture.md b/docs/architecture.md index 05972d29..6fae24a6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,7 +50,8 @@ New spawns select a backend from `--backend`, then `FM_BACKEND`, then local `con Runtime auto-detection is innermost-first: `$TMUX` wins over `HERDR_ENV=1`, which wins over cmux's primary `CMUX_WORKSPACE_ID` marker and documented fallback signals; auto-detected herdr or cmux prints a one-time opt-out notice, auto-detected tmux stays silent, and zellij and orca are never auto-detected (only explicit selection). Unknown backend names fail loudly. For compatibility, default tmux tasks do not write `backend=tmux`; every reader treats a missing `backend=` field as `tmux`. -`fm-watch.sh` polls each window's backend for a busy state: tmux, zellij, orca, and cmux have no native primitive and always report unknown, preserving the original pane-tail-regex detection unchanged; herdr's `agent.get` semantic state (working/idle/done/blocked) is consulted first for stale detection, with unknown native states falling back to the same regex. +`fm-watch.sh` polls each window's backend for a busy state: tmux, zellij, orca, and cmux have no native primitive and always report unknown, preserving the original pane-tail-regex detection unchanged. +Herdr's `agent.get` semantic state (working/idle/done/blocked) is consulted first; a native `busy` verdict suppresses stale detection, a settled `idle` verdict with no rendered busy banner dedupes stale wakes on the semantic `agentstate:idle` signal instead of volatile pane content, and unknown or still-busy-looking panes fall back to the same pane hash and regex path as tmux. That poll loop is the default event source for backends with no native push events, so this stays an extraction of the abstraction rather than a watcher rewrite. Herdr is experimental and can be selected explicitly or by runtime auto-detection: treehouse remains the worktree provider for it exactly as it is for tmux (herdr is a session provider only), and its full verification - the container shape decision, created-vs-adopted default-tab prune safety, restored-layout husk respawn idempotency, verified CLI facts, a verified small-`--lines` capture bug and its workaround, and known gaps - is recorded in `docs/herdr-backend.md`. Herdr's container shape is workspace-per-home plus tab-per-task: the primary home uses workspace label `firstmate`, secondmate homes use `2ndmate-`, and recovery/list-live scopes to the current `FM_HOME`'s workspace. diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 21645fc3..ea1a6de6 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -218,6 +218,19 @@ Together, those gaps let a genuinely still-working herdr crew read as not provab The cross-branch attribution fallback now uses the real `no-mistakes runs` command, and the watcher checks provably-working evidence before a stale status-log verb can make a stale pane terminal. This does not mask a genuinely human-blocked agent (a permission dialog, not mid-tool-call): that pane does not render the busy banner, so the corroboration still correctly reports not-busy for it. +## Watcher stale-dedup: semantic signal for a settled pane + +`bin/fm-watch.sh`'s stale detection hashes a bounded pane capture and surfaces a fresh wake whenever a settled pane produces a new stable hash. +On herdr this churned: an idle or done `claude` pane repaints timer-driven volatile UI even when the crew has not changed - the `※ recap:` line toggling on and off, rotating `Tip:` lines, the animated `✻ …ed for Xm Ys` summary, and transient slash-completion menus - so the capture's hash flips every few minutes and re-wakes the supervisor for a crew that is genuinely done. +Verified live against the real herdr 0.7.1 backend on 2026-07-05 by capturing `herdr pane read --source recent --lines 200` for every workspace pane every 20 seconds for ~7 minutes and correlating with each pane's `agent get` state: a `done` pane produced 4 distinct hashes over the window and an `idle` pane 8, each one a spurious stale re-wake, while a plain post-exit shell pane stayed at 1. +Line-stripping the volatile rows before hashing does not converge, because those rows also change the total line count, which shifts the tail-N hash window over a variable-length prefix (measured: stripping still left 3 distinct hashes across 4 `done`-pane samples). + +The fix keys the stale-dedup signal on the native SETTLED agent state instead of the pane content, but only after corroborating that the idle pane no longer renders the busy banner described in the 2026-07-02 gap above. +When `fm_backend_busy_state` reports `idle` (herdr's `idle`/`done`/`blocked`, per the "Busy state" row above) and the last 6 non-blank pane lines do not match `BUSY_REGEX`, the watcher hashes the constant `agentstate:idle` rather than the capture, so a settled pane that merely repaints keeps one signal and is surfaced once; a genuine state change still mints a new signal and surfaces once. +tmux (busy state always `unknown`), a herdr pane whose state is `working`/`unknown`, and an `idle` herdr pane that still displays the busy banner keep the pane-content hash exactly as before, so the proven default path is unchanged byte-for-byte. +The same resolved busy state feeds both this signal and the existing busy-suppression check, so herdr's `agent.get` is read once per pane per poll, not twice. +This is a distinct axis from the 2026-07-02 incident above (a still-working crew misread as not-working); here a correctly-settled crew was surfaced repeatedly rather than once. + ## Slash/`$` autocomplete popup hazard (confirmed, same mitigation as tmux) Typing `/mem` into a live `claude` composer inside a herdr pane and reading the pane back within 0.1 seconds already shows the full autocomplete popup. diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index a5acede4..05aa7254 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -257,7 +257,9 @@ EOF make_fake_toolchain "$fakebin" make_fake_ps_claude "$fakebin" # Force a MISSING diagnostic line so the bootstrap section is non-trivial. - rm -f "$fakebin/node" + # Use a firstmate wrapper that exists only in fakebin; node may exist in the + # fixed BASE_PATH on developer and CI hosts. + rm -f "$fakebin/gh-axi" printf 'window=fm-sess:w1\nkind=ship\n' > "$home/state/task-a.meta" @@ -280,7 +282,7 @@ EOF [ "$context_line" -lt "$fleet_line" ] || fail "CONTEXT did not precede FLEET STATE" [ "$fleet_line" -lt "$next_line" ] || fail "FLEET STATE did not precede NEXT STEP" - missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: node' | head -1 | cut -d: -f1) + missing_line=$(printf '%s\n' "$out" | grep -n 'MISSING: gh-axi' | head -1 | cut -d: -f1) [ -n "$missing_line" ] || fail "MISSING diagnostic did not appear at all" [ "$missing_line" -lt "$fleet_line" ] || fail "actionable MISSING diagnostic was buried after the bulk fleet-state digest" @@ -400,7 +402,7 @@ $rec EOF make_fake_toolchain "$fakebin" make_fake_ps_claude "$fakebin" - rm -f "$fakebin/node" + rm -f "$fakebin/gh-axi" append_wake "$home/state" signal task-z "needs-decision: pick a library" @@ -409,7 +411,7 @@ EOF # fm-lock.sh's own exact success text. assert_contains "$out" "lock acquired: harness pid" "fm-lock.sh's real output did not appear (composition, not reimplementation)" # fm-bootstrap.sh's own exact MISSING-tool line format. - assert_contains "$out" "MISSING: node (install:" "fm-bootstrap.sh's real detect line did not appear verbatim" + assert_contains "$out" "MISSING: gh-axi (install:" "fm-bootstrap.sh's real detect line did not appear verbatim" # fm-wake-drain.sh's real drained record (raw tab-separated queue line). assert_contains "$out" "$(printf 'signal\ttask-z\tneeds-decision: pick a library')" "fm-wake-drain.sh's real drained record did not appear" diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 6d25b148..3fa6f166 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -8,9 +8,10 @@ # provably-working no-verb wakes absorbed (no exit, no queue entry, suppressor # advanced, beacon fresh), stopped-crew no-verb wakes surfaced (queue + exit), # provably-working stale panes absorbed-then-escalated past the threshold, -# terminal-looking stale status lines overridden by an active run, the heartbeat -# backstop fail-safe, and afk coherence (no double-triage while the away-mode -# daemon owns supervision). +# terminal-looking stale status lines overridden by an active run, settled herdr +# panes deduped by semantic agent state rather than volatile repainting content, +# the heartbeat backstop fail-safe, and afk coherence (no double-triage while the +# away-mode daemon owns supervision). # # Daemon-side classification/injection lives in fm-daemon.test.sh; watcher/lock # liveness in fm-watcher-lock.test.sh; the durable-queue safety matrix in @@ -318,6 +319,160 @@ test_terminal_stale_surfaced() { pass "a stale pane sitting on a terminal status is surfaced (queue + exit)" } +# --- herdr semantic stale signal (stale-echo-churn fix) --------------------- +# A settled (idle/done) herdr pane repaints volatile harness UI on a timer (the +# recap line, rotating tips, animated "for Xm Ys" summaries, slash menus), so +# the stale-dedup signal must be the SEMANTIC agent state, not the volatile pane +# hash - otherwise each repaint mints a fresh stale hash and re-wakes the +# supervisor for a crew that has not changed. + +# make_herdr_stale_fakebin: a minimal `herdr` stub answering the calls the +# watcher's herdr capture + busy-state reads make, with a canned agent status +# and an arbitrary pane body (the body is deliberately irrelevant here - the +# whole point is that the signal ignores it). +make_herdr_stale_fakebin() { # + local fb=$1 agent=$2 body=$3 + mkdir -p "$fb" + cat > "$fb/herdr" < + local fb=$1 + mkdir -p "$fb" + cat > "$fb/herdr" <<'SH' +#!/usr/bin/env bash +set -u +dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +agent=$(cat "$dir/agent-status" 2>/dev/null || printf idle) +case "${1:-} ${2:-}" in + "status --json") printf '{"client":{"protocol":14},"server":{"running":true}}\n' ;; + "agent get") printf '{"result":{"agent":{"agent_status":"%s"}}}\n' "$agent" ;; + "pane read") cat "$dir/pane-body" 2>/dev/null || true ;; + "pane get") printf '{"result":{"pane":{"pane_id":"p"}}}\n' ;; + *) : ;; +esac +exit 0 +SH + chmod +x "$fb/herdr" +} + +test_herdr_settled_stale_signal_is_semantic() { + local dir state fakebin out drain_out window key sig pid + command -v jq >/dev/null 2>&1 || { pass "herdr stale-signal test skipped (jq not installed, required by the herdr adapter)"; return; } + dir=$(make_case herdr-stale-signal); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out" + window="default:w1:p2" + fm_write_meta "$state/hs.meta" "window=$window" "backend=herdr" "kind=ship" + printf 'done: PR https://example.test/pr/9\n' > "$state/hs.status" + sig=$(seen_sig "$state/hs.status"); printf '%s' "$sig" > "$state/.seen-hs_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + # Pre-seed the pane as already-seen at the SEMANTIC signal so the next poll + # confirms it stably stale (n>=2). The fake herdr's pane body is arbitrary and + # would churn a content hash on every repaint; the signal must ignore it. + printf 'agentstate:idle' > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + make_herdr_stale_fakebin "$fakebin" idle "volatile recap line that would churn a content hash" + PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "watcher did not surface a settled herdr stale pane" + grep -Fx "stale: $window" "$out" >/dev/null || fail "watcher did not print the herdr stale wake" + [ "$(cat "$state/.stale-$key" 2>/dev/null)" = "agentstate:idle" ] \ + || fail "stale signal was not the semantic agent state (got '$(cat "$state/.stale-$key" 2>/dev/null)')" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the herdr stale failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "herdr stale was not queued" + pass "herdr settled pane: stale-dedup signal is the semantic agent state, not the volatile pane hash" +} + +test_herdr_idle_busy_banner_uses_pane_hash() { + local dir state fakebin out window key sig pid current + command -v jq >/dev/null 2>&1 || { pass "herdr idle-busy stale-signal test skipped (jq not installed, required by the herdr adapter)"; return; } + dir=$(make_case herdr-idle-busy-signal); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out" + window="default:w1:p2" + fm_write_meta "$state/hs.meta" "window=$window" "backend=herdr" "kind=ship" + printf 'done: PR https://example.test/pr/9\n' > "$state/hs.status" + sig=$(seen_sig "$state/hs.status"); printf '%s' "$sig" > "$state/.seen-hs_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf 'agentstate:idle' > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + printf 'agentstate:idle' > "$state/.stale-$key" + make_herdr_dynamic_stale_fakebin "$fakebin" + printf 'idle\n' > "$fakebin/agent-status" + printf 'esc to interrupt\nrunning long tool output 1\n' > "$fakebin/pane-body" + export FM_FAKE_CREW_STATE='state: done · source: run-step · complete' + watch_bg "$state" "$fakebin" "$out" + pid=$! + for _ in $(seq 1 40); do + current=$(cat "$state/.hash-$key" 2>/dev/null || true) + [ -n "$current" ] && [ "$current" != "agentstate:idle" ] && break + sleep 0.1 + done + if [ -z "${current:-}" ] || [ "$current" = "agentstate:idle" ]; then + reap "$pid" + fail "idle pane with busy banner did not use pane-content hash" + fi + [ ! -e "$state/.stale-$key" ] || { reap "$pid"; fail "idle pane with busy banner did not clear stale suppressor"; } + printf 'esc to interrupt\nrunning long tool output 2\n' > "$fakebin/pane-body" + if ! wait_live "$pid" 30; then + fail "idle pane with changing busy-banner output surfaced stale: $(cat "$out")" + fi + reap "$pid" + unset FM_FAKE_CREW_STATE + pass "herdr idle pane: busy banner keeps stale signal on pane-content hash" +} + +test_herdr_settled_stale_resurfaces_after_busy_transition() { + local dir state fakebin out drain_out window key sig pid i current + command -v jq >/dev/null 2>&1 || { pass "herdr stale-resurface test skipped (jq not installed, required by the herdr adapter)"; return; } + dir=$(make_case herdr-stale-resurface); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out" + window="default:w1:p2" + fm_write_meta "$state/hs.meta" "window=$window" "backend=herdr" "kind=ship" + printf 'done: PR https://example.test/pr/9\n' > "$state/hs.status" + sig=$(seen_sig "$state/hs.status"); printf '%s' "$sig" > "$state/.seen-hs_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf 'agentstate:idle' > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + printf 'agentstate:idle' > "$state/.stale-$key" + make_herdr_dynamic_stale_fakebin "$fakebin" + printf 'working\n' > "$fakebin/agent-status" + printf 'busy render\n' > "$fakebin/pane-body" + export FM_FAKE_CREW_STATE='state: done · source: run-step · complete' + watch_bg "$state" "$fakebin" "$out" + pid=$! + for i in $(seq 1 40); do + current=$(cat "$state/.hash-$key" 2>/dev/null || true) + [ -n "$current" ] && [ "$current" != "agentstate:idle" ] && break + sleep 0.1 + done + if [ -z "${current:-}" ] || [ "$current" = "agentstate:idle" ]; then + reap "$pid" + fail "watcher did not record the intervening busy signal" + fi + [ ! -e "$state/.stale-$key" ] || { reap "$pid"; fail "busy transition did not clear the stale suppressor"; } + printf 'idle\n' > "$fakebin/agent-status" + printf 'settled again\n' > "$fakebin/pane-body" + wait_for_exit "$pid" 40 || fail "watcher did not resurface a settled herdr pane after intervening work" + grep -Fx "stale: $window" "$out" >/dev/null || fail "watcher did not print the resurfaced herdr stale wake" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the resurfaced herdr stale failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "resurfaced herdr stale was not queued" + unset FM_FAKE_CREW_STATE + pass "herdr settled pane: stale suppressor resets after intervening work" +} + # --- stale pane, STALE terminal status overridden by an active run: absorbed --- # Regression for the 2026-07 herdr false-surface incidents: a crew's own status # log gets no new entry once firstmate hands it to a no-mistakes validation @@ -655,6 +810,9 @@ test_turn_ended_not_working_surfaced test_working_note_not_working_surfaced test_actionable_signal_surfaced test_terminal_stale_surfaced +test_herdr_settled_stale_signal_is_semantic +test_herdr_idle_busy_banner_uses_pane_hash +test_herdr_settled_stale_resurfaces_after_busy_transition test_stale_terminal_status_overridden_by_active_run test_nonterminal_stale_provably_working_absorbed_then_escalated test_nonterminal_stale_not_working_surfaced