Canonicalize monitor-watchdog.sh and fix over-firing (STALE_SECS/flock/fail-closed)

.agents/skills/monitor-tick/SKILL.md

@@ -1152,6 +1152,20 @@ against a node that is in real-time sync with age=2s).
    ```bash
    ARCHIVE_DIR="$HOME/data/$MONITOR_SESSION_ID/metrics/archive"
    mkdir -p "$ARCHIVE_DIR"
+
+   # Serialize this archive critical section against a concurrent HEADLESS
+   # tick launched by the crontab watchdog (scripts/monitor-watchdog.sh) on
+   # the shared whole-tick lock. Without it, an in-session tick and a
+   # watchdog-launched tick interleave their scrape->rotate->archive sequences
+   # and cross-contaminate a snapshot's current.prom/prev.prom + metadata.env
+   # (#3757 comment 5100384841 / #3789). The watchdog holds this same lock for
+   # its whole run, so if it is mid-flight this flock -w 30 may time out; when
+   # it does we SKIP writing this one snapshot and continue the tick — a missed
+   # snapshot is harmless, a corrupt one is not. Wrap ONLY this archive step
+   # (do NOT extend the lock over the status-comment/publish path — #3789).
+   TICK_LOCK="$HOME/data/monitor-tick.lock"
+   (
+   flock -w 30 8 || { echo "check-12 archive: tick lock busy after 30s — skipping this snapshot" >&2; exit 0; }
    TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)
    SNAP_TMP="$ARCHIVE_DIR/${TIMESTAMP}.tmp"
    SNAP_FINAL="$ARCHIVE_DIR/${TIMESTAMP}"
@@ -1199,6 +1213,7 @@ against a node that is in real-time sync with age=2s).
    # Clean up orphaned .tmp dirs from crashed prior ticks
    find "$ARCHIVE_DIR" -maxdepth 1 -name '*.tmp' -type d -mmin +5 \
      -exec rm -rf {} + 2>/dev/null || true
+   ) 8>"$TICK_LOCK"
    ```
 
 8. **Weekly alarm regression replay** — replay the archived metrics history

.claude/skills/monitor-tick/SKILL.md

@@ -1340,6 +1340,20 @@ tracks node health, and this signal is not one.
    ```bash
    ARCHIVE_DIR="$HOME/data/$MONITOR_SESSION_ID/metrics/archive"
    mkdir -p "$ARCHIVE_DIR"
+
+   # Serialize this archive critical section against a concurrent HEADLESS
+   # tick launched by the crontab watchdog (scripts/monitor-watchdog.sh) on
+   # the shared whole-tick lock. Without it, an in-session tick and a
+   # watchdog-launched tick interleave their scrape->rotate->archive sequences
+   # and cross-contaminate a snapshot's current.prom/prev.prom + metadata.env
+   # (#3757 comment 5100384841 / #3789). The watchdog holds this same lock for
+   # its whole run, so if it is mid-flight this flock -w 30 may time out; when
+   # it does we SKIP writing this one snapshot and continue the tick — a missed
+   # snapshot is harmless, a corrupt one is not. Wrap ONLY this archive step
+   # (do NOT extend the lock over the status-comment/publish path — #3789).
+   TICK_LOCK="$HOME/data/monitor-tick.lock"
+   (
+   flock -w 30 8 || { echo "check-12 archive: tick lock busy after 30s — skipping this snapshot" >&2; exit 0; }
    TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)
    SNAP_TMP="$ARCHIVE_DIR/${TIMESTAMP}.tmp"
    SNAP_FINAL="$ARCHIVE_DIR/${TIMESTAMP}"
@@ -1383,6 +1397,7 @@ tracks node health, and this signal is not one.
    # Clean up orphaned .tmp dirs from crashed prior ticks
    find "$ARCHIVE_DIR" -maxdepth 1 -name '*.tmp' -type d -mmin +5 \
      -exec rm -rf {} + 2>/dev/null || true
+   ) 8>"$TICK_LOCK"
    ```
 
 8. **Weekly alarm regression replay** — replay the archived metrics history

.github/workflows/ci.yml

@@ -181,6 +181,15 @@ jobs:
       - name: Run project-loop skill snippet harness
         run: bash scripts/test-project-loop-skill-snippets.sh
 
+  monitor-watchdog:
+    name: Monitor Watchdog
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run monitor-watchdog decision + wrapper harness
+        run: bash scripts/test-monitor-watchdog.sh
+
   ssc-henyey-mixed-mission-scripts:
     name: Henyey Mixed-Image SSC Scripts
     runs-on: ubuntu-latest

scripts/lib/monitor-watchdog-decisions.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+#
+# Pure decision logic for the monitor-loop crontab watchdog
+# (scripts/monitor-watchdog.sh).
+#
+# Mirrors the testability pattern of scripts/lib/monitor-decisions.sh: the
+# watchdog wrapper owns all I/O (locks, launching the headless tick, writing
+# the refire marker, logging); the two functions here are pure and fully unit-
+# testable (scripts/test-monitor-watchdog.sh).
+#
+# Requires: Bash 3.2+, GNU/Linux (GNU `date -d`, `tail`, `grep -oE`, `sed -E`).
+# Does NOT set shell options (set -e, -u) — callers control strictness.
+# Idempotent: safe to source multiple times.
+#
+
+[[ -n "${_MONITOR_WATCHDOG_DECISIONS_LOADED:-}" ]] && return 0
+_MONITOR_WATCHDOG_DECISIONS_LOADED=1
+
+# ─────────────────────────────────────────────────────────────────────────────
+# watchdog_last_epoch HIST_FILE
+#
+# Resolve the completion time of the last tick recorded in HIST_FILE (a JSONL
+# tick history; the timestamp lives on the LAST line under key `ts` OR
+# `timestamp`).
+#
+# Echoes EXACTLY one token on stdout:
+#   MISSING      — HIST_FILE does not exist. A dark/bootstrapping loop; the
+#                  caller SHOULD fire (this is the one fail-OPEN case).
+#   PARSE_ERROR  — HIST_FILE exists but its last line has no parseable
+#                  `ts`/`timestamp` value (not JSON, missing key, or a value
+#                  GNU `date` cannot interpret). The caller MUST fail CLOSED:
+#                  do NOT fire and do NOT write the refire marker (#3789). An
+#                  unparseable tail is NOT evidence the loop is dead — treating
+#                  it as epoch 0 (the old behavior) fired unconditionally and
+#                  produced a 56,590-year "staleness" (#3757).
+#   <integer>    — epoch seconds of the last completed tick.
+#
+# Dependency-light on purpose: uses only `tail`/`grep`/`sed`/`date` — never
+# python3 or jq. A missing interpreter on cron's minimal PATH must not be able
+# to silently blind the watchdog (dark-monitoring incident 2026-07-22).
+#
+# Accepts both `ts` (henyey monitor loop) and `timestamp` (alternate producers)
+# keys; the first of either found on the last line wins.
+#
+# Returns: 0 always.
+# ─────────────────────────────────────────────────────────────────────────────
+watchdog_last_epoch() {
+  local hist="$1"
+  [ -f "$hist" ] || { printf 'MISSING\n'; return 0; }
+
+  local line ts epoch
+  line=$(tail -n 1 "$hist" 2>/dev/null)
+
+  # Isolate the first "ts":"..." or "timestamp":"..." pair, then strip the
+  # key/colon/quote scaffolding. Value may itself contain colons (ISO-8601
+  # time), so peel the fixed prefix/suffix rather than splitting on ':'.
+  ts=$(printf '%s' "$line" \
+        | grep -oE '"(ts|timestamp)"[[:space:]]*:[[:space:]]*"[^"]*"' \
+        | head -n 1 \
+        | sed -E 's/^"[^"]*"[[:space:]]*:[[:space:]]*"//; s/"$//')
+
+  if [ -z "$ts" ]; then
+    printf 'PARSE_ERROR\n'
+    return 0
+  fi
+
+  epoch=$(date -u -d "$ts" +%s 2>/dev/null)
+  case "$epoch" in
+    ''|*[!0-9]*) printf 'PARSE_ERROR\n'; return 0 ;;
+  esac
+
+  printf '%s\n' "$epoch"
+  return 0
+}
+
+# ─────────────────────────────────────────────────────────────────────────────
+# watchdog_should_fire NOW LAST_TOKEN STALE_SECS LAST_FIRE REFIRE_COOLDOWN
+#
+# Pure staleness + cooldown decision. Reads and writes NO files; all times are
+# passed in as epoch seconds so the whole matrix is deterministically testable.
+#
+#   NOW              — current epoch seconds.
+#   LAST_TOKEN       — output of watchdog_last_epoch (an epoch, MISSING, or
+#                      PARSE_ERROR).
+#   STALE_SECS       — fire only if the last completed tick is older than this.
+#   LAST_FIRE        — epoch of the last watchdog firing (0 if never / unknown).
+#   REFIRE_COOLDOWN  — never fire twice within this many seconds.
+#
+# Echoes EXACTLY one line "VERDICT REASON" on stdout:
+#   fire no-history  — no history file: dark loop / bootstrap (fail open).
+#   fire stale       — last tick older than STALE_SECS and cooldown elapsed.
+#   skip fresh       — last tick within STALE_SECS.
+#   skip cooldown    — stale, but a firing happened within REFIRE_COOLDOWN.
+#   skip parse-error — history tail unparseable: FAIL CLOSED (do not fire,
+#                      do not write the marker).
+#
+# Returns: 0 always.
+# ─────────────────────────────────────────────────────────────────────────────
+watchdog_should_fire() {
+  local now="$1" last_token="$2" stale_secs="$3" last_fire="$4" cooldown="$5"
+
+  # Fail closed on an unparseable tail — highest precedence.
+  if [ "$last_token" = "PARSE_ERROR" ]; then
+    printf 'skip parse-error\n'
+    return 0
+  fi
+
+  # Staleness.
+  local stale=no
+  if [ "$last_token" = "MISSING" ]; then
+    stale=yes
+  elif [ $(( now - last_token )) -gt "$stale_secs" ]; then
+    stale=yes
+  fi
+
+  if [ "$stale" = no ]; then
+    printf 'skip fresh\n'
+    return 0
+  fi
+
+  # Stale — apply the refire cooldown.
+  case "$last_fire" in ''|*[!0-9]*) last_fire=0 ;; esac
+  if [ "$last_fire" -gt 0 ] && [ $(( now - last_fire )) -lt "$cooldown" ]; then
+    printf 'skip cooldown\n'
+    return 0
+  fi
+
+  if [ "$last_token" = "MISSING" ]; then
+    printf 'fire no-history\n'
+  else
+    printf 'fire stale\n'
+  fi
+  return 0
+}

scripts/monitor-watchdog.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Monitor-loop watchdog — crontab backstop for the in-session /monitor-tick loop.
+#
+# The primary loop runs inside an interactive Claude Code session via
+# ScheduleWakeup (~20 min cadence). That loop can silently die (session
+# compaction, consumed wakeups, session exit). Audit 2026-07-16 found 571
+# dead-hours across 64 days (33 gaps >1h). This script runs from crontab
+# every 15 min and launches a headless tick ONLY when the loop looks dead.
+#
+# Canonical copy: scripts/monitor-watchdog.sh in the henyey repo (this file).
+# Live copy: /home/tomer/data/monitor-watchdog.sh (what crontab executes,
+# decoupled from repo checkout state — deploy by copying this file there).
+#
+# Install: */15 * * * * /home/tomer/data/monitor-watchdog.sh
+#
+# Decision logic (staleness, cooldown, tail parsing) lives in the pure,
+# unit-tested lib scripts/lib/monitor-watchdog-decisions.sh (mirrors the
+# monitor-decisions.sh testability pattern). This wrapper owns all I/O: the
+# self-serialization lock, the shared whole-tick lock, the refire marker, the
+# log, and the headless launch. Covered by scripts/test-monitor-watchdog.sh.
+#
+# Staleness rule (corrected — #3789): the watchdog measures COMPLETION-to-now,
+# so it fires whenever `wakeup + tick_duration > STALE_SECS`. STALE_SECS must
+# therefore satisfy `expected_wakeup <= STALE_SECS - expected_tick_duration`,
+# NOT merely "keep wakeups under STALE_SECS". With the loop's observed ~30-min
+# idle cadence plus multi-minute ticks, the old 1800s bar fired on healthy but
+# slow cycles (#3757: the watchdog fired a duplicate headless tick on nearly
+# every cron cycle). STALE_SECS=4200 (70 min) gives ~40 min of headroom.
+set -u
+
+# cron runs with a minimal PATH (/usr/bin:/bin) that omits ~/.local/bin, where
+# the `claude` launcher lives. Without this the headless tick fails with
+# "timeout: failed to run command 'claude': No such file or directory" and the
+# watchdog silently no-ops every 15 min (dark-monitoring incident 2026-07-22).
+export PATH="/home/tomer/.local/bin:$PATH"
+
+# Source the pure decision lib relative to this script (works regardless of the
+# cwd cron invokes us from).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/lib/monitor-watchdog-decisions.sh
+source "$SCRIPT_DIR/lib/monitor-watchdog-decisions.sh"
+
+# ── Config — production defaults, all env-overridable (the test harness and any
+# alternate deployment retarget these without editing the script). ────────────
+CLAUDE_BIN="${CLAUDE_BIN:-/home/tomer/.local/bin/claude}"
+CLAUDE_MODEL="${WATCHDOG_CLAUDE_MODEL:-claude-opus-4-8}"
+DATA="${WATCHDOG_DATA:-/home/tomer/data}"
+ENV_FILE="${WATCHDOG_ENV_FILE:-$DATA/monitor-loop.env}"
+REPO="${WATCHDOG_REPO:-/home/tomer/henyey-1}"
+
+# SESSION_ID: prefer an explicit MONITOR_SESSION_ID, else read it from
+# monitor-loop.env, else the historical production default. Keeps the watchdog
+# and the loop pointed at the same session dir even across loop restarts.
+SESSION_ID="${MONITOR_SESSION_ID:-}"
+if [ -z "$SESSION_ID" ] && [ -f "$ENV_FILE" ]; then
+  SESSION_ID=$(grep -E '^MONITOR_SESSION_ID=' "$ENV_FILE" 2>/dev/null | tail -n 1 | cut -d= -f2-)
+fi
+SESSION_ID="${SESSION_ID:-74535976}"
+
+SESS="$DATA/$SESSION_ID"
+HIST="${WATCHDOG_HIST:-$SESS/tick-history.jsonl}"
+LOCK="${WATCHDOG_LOCK:-$DATA/monitor-watchdog.lock}"        # self-serialization
+TICK_LOCK="${WATCHDOG_TICK_LOCK:-$DATA/monitor-tick.lock}"  # shared whole-tick
+MARKER="${WATCHDOG_MARKER:-$DATA/monitor-watchdog.lastfire}"
+LOG="${WATCHDOG_LOG:-$DATA/monitor-watchdog.log}"
+
+STALE_SECS="${WATCHDOG_STALE_SECS:-4200}"        # fire when last COMPLETED tick
+                                                 # is older than 70 min
+REFIRE_COOLDOWN="${WATCHDOG_REFIRE_COOLDOWN:-1800}"  # >= 1 firing / 30 min
+TICK_TIMEOUT="${WATCHDOG_TICK_TIMEOUT:-1500}"    # cap a headless tick at 25 min
+
+mkdir -p "$DATA" 2>/dev/null || true
+
+log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$1" >> "$LOG"; }
+
+# ── Self-serialize watchdog instances (fd 9). ────────────────────────────────
+exec 9>"$LOCK"
+flock -n 9 || exit 0
+
+now=$(date -u +%s)
+
+# ── Decide (pure). ───────────────────────────────────────────────────────────
+last_token=$(watchdog_last_epoch "$HIST")
+
+last_fire=0
+if [ -f "$MARKER" ]; then
+  last_fire=$(cat "$MARKER" 2>/dev/null || echo 0)
+  case "$last_fire" in ''|*[!0-9]*) last_fire=0 ;; esac
+fi
+
+decision=$(watchdog_should_fire "$now" "$last_token" "$STALE_SECS" "$last_fire" "$REFIRE_COOLDOWN")
+verdict="${decision%% *}"
+reason="${decision#* }"
+
+if [ "$verdict" != "fire" ]; then
+  # Fail CLOSED on an unparseable tail: log LOUDLY (an unparseable history is a
+  # real anomaly an operator should see) and exit WITHOUT writing the marker
+  # (#3789). fresh/cooldown skips are the quiet common case.
+  if [ "$reason" = "parse-error" ]; then
+    log "WARNING: history tail unparseable ($HIST) — failing CLOSED, NOT firing. Inspect the loop's liveness manually."
+  fi
+  exit 0
+fi
+
+# ── In-flight guard (fd 8): take the shared whole-tick lock. If an in-session
+# tick (which takes the same lock around its metrics scrape/archive critical
+# section) or a prior headless firing holds it, a tick is in flight — skip.
+# Held across the headless launch below so the child inherits it and the
+# in-session check-12 serializes behind us. This REPLACES the old .alive-mtime
+# heuristic, closing the interleaving race that corrupted archive metadata.env
+# (#3757 / #3789). ─────────────────────────────────────────────────────────────
+exec 8>"$TICK_LOCK"
+if ! flock -n 8; then
+  log "skipped: tick in flight"
+  exit 0
+fi
+
+# Committed to firing — record the marker (only reached past the fail-closed
+# and in-flight guards, so a PARSE_ERROR skip never writes it).
+printf '%s\n' "$now" > "$MARKER"
+
+# Keep the log bounded (~5 MB).
+if [ -f "$LOG" ] && [ "$(stat -c %s "$LOG" 2>/dev/null || echo 0)" -gt 5242880 ]; then
+  tail -c 1048576 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
+fi
+
+log "watchdog: loop stale (reason=$reason, last=$last_token, now=$now) — launching headless tick"
+cd "$REPO" || { log "watchdog: repo $REPO missing — aborting firing"; exit 1; }
+timeout "$TICK_TIMEOUT" "$CLAUDE_BIN" --model "$CLAUDE_MODEL" --dangerously-skip-permissions \
+  -p '/monitor-tick' >> "$LOG" 2>&1
+rc=$?
+log "watchdog: headless tick exit=$rc"