feat(ops): heartbeat pings for the worker and bot, and a tunnel readiness check

.env.example

@@ -67,6 +67,16 @@ REDIS_URL=redis://localhost:6379
 ALERT_TELEGRAM_CHAT_ID=
 DIGEST_HOUR_UTC=8
 
+# Dead-man's switch: ping URLs from an external monitoring service (any
+# healthchecks.io-style "expect a request every N minutes" check). The worker
+# pings only while every loop is fresh and Redis answers; the bot only while
+# its Telegram long-poll succeeds. Unset = no pings. CLOUDFLARED_READY_URL
+# lets the worker alert when the tunnel loses its edge connections; set it to
+# http://cloudflared:2000/ready in the compose deployment.
+HEARTBEAT_URL_WORKER=
+HEARTBEAT_URL_BOT=
+CLOUDFLARED_READY_URL=
+
 # Worker only: max ms to wait for in-flight webhook batches on SIGTERM before
 # forcing close. Tune to ~2x observed p95 delivery time.
 GRACEFUL_SHUTDOWN_TIMEOUT_MS=10000

bot/index.js

@@ -9,6 +9,34 @@ const apiBase = `https://api.telegram.org/bot${botToken}`;
 const longPollSeconds = 30;
 let offset = 0;
 
+// Dead-man's switch: ping HEARTBEAT_URL once a minute, but only while the
+// long-poll keeps succeeding. A poll is healthy even when it returns no
+// updates; what matters is that Telegram answered. No URL, no pings.
+const heartbeatUrl = process.env.HEARTBEAT_URL || "";
+const HEARTBEAT_INTERVAL_MS = 60 * 1000;
+const POLL_STALE_MS = 2 * (longPollSeconds + 5) * 1000;
+let lastPollOkAt = 0;
+
+function pollIsFresh() {
+  return Date.now() - lastPollOkAt < POLL_STALE_MS;
+}
+
+async function pingHeartbeat() {
+  if (!heartbeatUrl) return;
+  if (!pollIsFresh()) {
+    logEvent("warn", "heartbeat_withheld", { lastPollAgeMs: lastPollOkAt ? Date.now() - lastPollOkAt : null });
+    return;
+  }
+  try {
+    const res = await fetch(heartbeatUrl, { signal: AbortSignal.timeout(5000) });
+    if (!res.ok) logEvent("warn", "heartbeat_ping_rejected", { status: res.status });
+  } catch (err) {
+    logEvent("warn", "heartbeat_ping_failed", { error: err });
+  }
+}
+
+if (heartbeatUrl) setInterval(pingHeartbeat, HEARTBEAT_INTERVAL_MS);
+
 // Structured logging, inlined because the bot image ships only index.js and
 // cannot import the worker's logger. Same JSON-line shape as worker-log.js so
 // one log pipeline reads every process.
@@ -77,7 +105,12 @@ async function startStatusMonitor() {
   }).catch(err => logEvent("warn", "pin_status_failed", { error: err }));
 
   setInterval(async () => {
-    const updatedText = `server status: UP\nStarted at: ${startTime}\nLast checked: ${new Date().toISOString()}`;
+    // The pinned message used to say UP unconditionally; now it reflects
+    // whether the long-poll is actually succeeding.
+    const state = pollIsFresh()
+      ? "UP"
+      : `DEGRADED (last successful poll ${lastPollOkAt ? Math.round((Date.now() - lastPollOkAt) / 1000) + "s ago" : "never"})`;
+    const updatedText = `server status: ${state}\nStarted at: ${startTime}\nLast checked: ${new Date().toISOString()}`;
     await editMessage(analyticsChatId, messageId, updatedText);
   }, 10 * 60 * 1000);
 }
@@ -118,6 +151,7 @@ async function getUpdates() {
       await sleep(1000);
       return [];
     }
+    lastPollOkAt = Date.now();
     return data.result;
   } catch (err) {
     logEvent("warn", "get_updates_error", { error: err });

docker-compose.yml

@@ -128,6 +128,7 @@ services:
       # accepts a decision only from this id, so an unset value silently
       # rejects every tap. Mandatory here rather than defaulted.
       BEARER_ADMIN_TELEGRAM_ID: ${BEARER_ADMIN_TELEGRAM_ID:?set BEARER_ADMIN_TELEGRAM_ID}
+      HEARTBEAT_URL: ${HEARTBEAT_URL_BOT:-}
       TELEGRAM_ANALYTICS_CHAT_ID: ${TELEGRAM_ANALYTICS_CHAT_ID:-}
       TELEGRAM_ANALYTICS_THREAD_ID: ${TELEGRAM_ANALYTICS_THREAD_ID:-}
     depends_on:
@@ -189,6 +190,8 @@ services:
       BEARER_ADMIN_TELEGRAM_ID: ${BEARER_ADMIN_TELEGRAM_ID:-}
       GRACEFUL_SHUTDOWN_TIMEOUT_MS: ${GRACEFUL_SHUTDOWN_TIMEOUT_MS:-10000}
       DIGEST_HOUR_UTC: ${DIGEST_HOUR_UTC:-8}
+      HEARTBEAT_URL: ${HEARTBEAT_URL_WORKER:-}
+      CLOUDFLARED_READY_URL: ${CLOUDFLARED_READY_URL:-}
       NODE_ENV: production
     depends_on:
       redis:

docs/deployment.md

@@ -139,8 +139,50 @@ absence is itself the alert. `docker compose exec -T worker node worker.js
 runbook's "is the channel alive" check.
 
 These alerts are sent from the host. When the host, the tunnel, or the
-worker is down nothing can send them, so they do not replace an external
-uptime monitor.
+worker is down nothing can send them, so they do not replace the external
+monitor described next.
+
+## Monitoring
+
+Two kinds of check, both run by a monitoring service outside the host (any
+vendor that offers HTTP probes and "expect a ping every N minutes" checks,
+delivering to the same Telegram chat if you like):
+
+HTTP probes, from outside, through the tunnel:
+
+- `GET https://auth.bneck.com/api/health/ready`, expect 200 and a body
+  containing `"ok":true`, every 1 to 2 minutes. This is the end-to-end check:
+  tunnel, app, Postgres and Redis.
+- `GET https://auth.bneck.com/.well-known/openid-configuration`, expect 200
+  and `"issuer":"https://auth.bneck.com"`, every 5 minutes. Catches a
+  misconfigured base URL that the readiness probe would not.
+- Optional: `GET https://auth.bneck.com/oauth/jwks`, expect the active
+  `kid`, every 15 minutes, to catch a signing-key regression.
+
+Cloudflare's bot protection may block a vendor's probes; if the checks fail
+with 403 while the site works, add a WAF skip rule for `/api/health/*` and
+the discovery path scoped to the vendor's user agent or IP ranges.
+
+Heartbeats (dead-man's switches), pinged from inside:
+
+- `HEARTBEAT_URL_WORKER`: the worker pings once a minute, but only while
+  every background loop has completed without error inside its tolerance
+  and Redis answers. Configure the check for a 1 minute period and a 3
+  minute grace. When it withholds the ping it also logs
+  `heartbeat_withheld` and sends a `worker_unhealthy` alert, so the two
+  signals corroborate each other.
+- `HEARTBEAT_URL_BOT`: the bot pings once a minute while its Telegram
+  long-poll keeps succeeding. Same period and grace. The pinned status
+  message in the analytics chat also reads DEGRADED with the age of the last
+  good poll when it is stale.
+- `CLOUDFLARED_READY_URL` (`http://cloudflared:2000/ready`): not a heartbeat,
+  but the worker checks it every minute and alerts `cloudflared_not_ready`
+  while the tunnel has no edge connection. The HTTP probes above are the
+  tunnel's dead-man's switch; this only gets the news out faster while the
+  host can still speak.
+
+The daily digest (above) is the slowest of the switches: its absence means
+the worker, the database, or the alert channel has been dead for up to a day.
 
 ## Backup and restore
 

tests/unit/worker-heartbeat.test.ts

@@ -0,0 +1,63 @@
+import { createRequire } from 'node:module';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const requireCjs = createRequire(import.meta.url);
+const worker = requireCjs('../../worker.js') as {
+  LOOP_TOLERANCE_MS: Record<string, number>;
+  markLoopOk: (loop: string, at?: number) => void;
+  staleLoops: (now?: number, since?: number) => string[];
+  _resetLoopStateForTests: () => void;
+  pingHeartbeat: (url: string, fetchImpl?: (url: string, init: RequestInit) => Promise<{ ok: boolean; status?: number }>) => Promise<boolean>;
+};
+
+const LOOPS = Object.keys(worker.LOOP_TOLERANCE_MS);
+const T0 = Date.UTC(2026, 8, 6, 12, 0, 0);
+
+describe('worker heartbeat: loop freshness', () => {
+  beforeEach(() => {
+    for (const loop of LOOPS) worker.markLoopOk(loop, T0);
+  });
+
+  it('reports no stale loops right after every loop completed', () => {
+    expect(worker.staleLoops(T0 + 1000)).toEqual([]);
+  });
+
+  it('flags a loop once its tolerance has passed, and only that loop', () => {
+    worker.markLoopOk('webhook_delivery', T0 - worker.LOOP_TOLERANCE_MS.webhook_delivery - 1);
+    expect(worker.staleLoops(T0)).toEqual(['webhook_delivery']);
+  });
+
+  it('counts a loop that never completed from process start', () => {
+    worker._resetLoopStateForTests();
+    const olderThanEveryTolerance = T0 - worker.LOOP_TOLERANCE_MS.hygiene - 1;
+    expect(worker.staleLoops(T0, olderThanEveryTolerance)).toEqual(LOOPS);
+    expect(worker.staleLoops(T0, T0)).toEqual([]);
+  });
+
+  it('tolerances are a little over two periods of each loop', () => {
+    expect(worker.LOOP_TOLERANCE_MS.webhook_delivery).toBeGreaterThan(2 * 1000);
+    expect(worker.LOOP_TOLERANCE_MS.activation_expiry).toBeGreaterThan(2 * 60_000);
+    expect(worker.LOOP_TOLERANCE_MS.hygiene).toBeGreaterThan(60 * 60_000);
+  });
+});
+
+describe('worker heartbeat: ping', () => {
+  it('is a no-op without a URL', async () => {
+    const fetchImpl = vi.fn();
+    expect(await worker.pingHeartbeat('', fetchImpl)).toBe(false);
+    expect(fetchImpl).not.toHaveBeenCalled();
+  });
+
+  it('GETs the URL with a timeout and reports success', async () => {
+    const fetchImpl = vi.fn(async () => ({ ok: true }));
+    expect(await worker.pingHeartbeat('https://ping.example/abc', fetchImpl)).toBe(true);
+    const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit];
+    expect(url).toBe('https://ping.example/abc');
+    expect(init.signal).toBeInstanceOf(AbortSignal);
+  });
+
+  it('never throws: a rejected or failed ping is reported as false', async () => {
+    expect(await worker.pingHeartbeat('https://ping.example/abc', async () => ({ ok: false, status: 503 }))).toBe(false);
+    expect(await worker.pingHeartbeat('https://ping.example/abc', async () => { throw new Error('ECONNRESET'); })).toBe(false);
+  });
+});