feat(bot): add a /status command that names the failing layer

.env.example

@@ -76,6 +76,9 @@ DIGEST_HOUR_UTC=8
 HEARTBEAT_URL_WORKER=
 HEARTBEAT_URL_BOT=
 CLOUDFLARED_READY_URL=
+# The monitor Worker's /status URL (same token as the ping URLs); the bot's
+# /status command reads it. Unset = the command reports the host view only.
+MONITOR_STATUS_URL=
 
 # Worker only: max ms to wait for in-flight webhook batches on SIGTERM before
 # forcing close. Tune to ~2x observed p95 delivery time.

.github/workflows/security.yml

@@ -58,6 +58,7 @@ jobs:
       - run: npm run build
       - run: docker build --target runner -t auth-runner:ci .
       - run: docker build --target worker .
+      - run: docker build -t auth-bot:ci ./bot && docker run --rm auth-bot:ci node -e "import('./status.js')"
       # The image that ships is the one under test: nonce CSP, HSTS, the
       # standalone server and migrate-on-boot only exist in production mode.
       # --network host lets the container reach the service containers on

bot/Dockerfile

@@ -2,6 +2,6 @@ FROM node:22-alpine
 WORKDIR /app
 ENV NODE_ENV=production
 ENV NODE_OPTIONS=--max-old-space-size=128
-COPY package.json index.js ./
+COPY package.json index.js status.js ./
 USER node
 CMD ["node", "index.js"]

bot/README.md

@@ -21,6 +21,25 @@ Long-polls Telegram (`getUpdates`), so it does not need a public URL. Only the a
 
 The bot username (`TELEGRAM_BOT_USERNAME`) is only needed by the auth service to build the start link.
 
+## Optional env
+
+- `HEARTBEAT_URL` - pinged once a minute while the long-poll succeeds (see `docs/deployment.md`, Monitoring)
+- `ALERT_TELEGRAM_CHAT_ID` - the chat in which `/status` is answered; the admin's private chat always works
+- `MONITOR_STATUS_URL` - the external monitor's `/status?token=...` URL, read by `/status`
+- `CLOUDFLARED_READY_URL` - cloudflared's `/ready` endpoint, read by `/status`
+- `TELEGRAM_ANALYTICS_CHAT_ID` / `TELEGRAM_ANALYTICS_THREAD_ID` - where the pinned status message lives
+
+## /status
+
+`/status` in the alert chat (or from the admin in a private chat) replies with a
+layer-by-layer breakdown: Cloudflare's own status page, the tunnel's edge
+connections, the app's readiness (Postgres, Redis), the worker and bot
+heartbeats as the external monitor sees them, and the public probe. The first
+line says UP, DEGRADED or DOWN and the second names the layer to blame. No reply
+means the bot itself, or the whole host, is down; the external monitor's last
+DOWN message in the chat says which. `node index.js --status` prints the same
+report from a shell (`docker compose exec bot node index.js --status`).
+
 ## Run
 
 ```

bot/index.js

@@ -1,9 +1,14 @@
+import { diagnose, formatStatus, gatherStatus, isStatusCommand, mayRequestStatus } from "./status.js";
+
 const botToken = required("TELEGRAM_BOT_TOKEN");
 const webhookSecret = required("TELEGRAM_BOT_WEBHOOK_SECRET");
 const authBaseUrl = process.env.AUTH_INTERNAL_URL || "http://localhost:3000";
 const bearerAdminTelegramId = required("BEARER_ADMIN_TELEGRAM_ID");
 const analyticsChatId = process.env.TELEGRAM_ANALYTICS_CHAT_ID;
 const analyticsThreadId = process.env.TELEGRAM_ANALYTICS_THREAD_ID;
+const alertChatId = process.env.ALERT_TELEGRAM_CHAT_ID;
+const monitorStatusUrl = process.env.MONITOR_STATUS_URL || "";
+const tunnelReadyUrl = process.env.CLOUDFLARED_READY_URL || "";
 const apiBase = `https://api.telegram.org/bot${botToken}`;
 
 const longPollSeconds = 30;
@@ -35,8 +40,6 @@ async function pingHeartbeat() {
   }
 }
 
-if (heartbeatUrl) setInterval(pingHeartbeat, HEARTBEAT_INTERVAL_MS);
-
 // Structured logging, inlined because the bot image ships only index.js and
 // cannot import the worker's logger. Same JSON-line shape as worker-log.js so
 // one log pipeline reads every process.
@@ -55,11 +58,26 @@ function logEvent(level, msg, metadata) {
 }
 
 // main() is a floating promise: a throw from the startup status monitor used
-// to surface as an unstructured unhandled rejection.
-main().catch(err => {
-  logEvent("error", "bot_crashed", { error: err });
-  process.exit(1);
-});
+// to surface as an unstructured unhandled rejection. `--status` prints the
+// breakdown the /status command sends and exits without ever polling, so it
+// can run beside the live bot from a shell on the host.
+if (process.argv.includes("--status")) {
+  statusReport(false).then(
+    text => {
+      process.stdout.write(text + "\n");
+      process.exit(0);
+    },
+    err => {
+      logEvent("error", "status_failed", { error: err });
+      process.exit(1);
+    },
+  );
+} else {
+  main().catch(err => {
+    logEvent("error", "bot_crashed", { error: err });
+    process.exit(1);
+  });
+}
 
 process.on("unhandledRejection", err => {
   logEvent("error", "unhandled_rejection", { error: err });
@@ -116,6 +134,7 @@ async function startStatusMonitor() {
 }
 
 async function main() {
+  if (heartbeatUrl) setInterval(pingHeartbeat, HEARTBEAT_INTERVAL_MS);
   await startStatusMonitor();
   while (true) {
     const updates = await getUpdates();
@@ -171,6 +190,14 @@ async function handleUpdate(update) {
     return;
   }
 
+  if (isStatusCommand(message.text)) {
+    if (mayRequestStatus(message, { alertChatId, adminTelegramId: bearerAdminTelegramId })) {
+      const threadId = message.is_topic_message ? message.message_thread_id : undefined;
+      await reply(message.chat.id, await statusReport(), threadId);
+    }
+    return;
+  }
+
   const match = message.text.match(/^\/start(?:\s+(\S+))?/);
   if (!match) {
     return;
@@ -386,11 +413,16 @@ async function callVerify(startToken, from) {
   return { message: "Verification service is unavailable. Try again in a moment." };
 }
 
-async function reply(chatId, text) {
+async function statusReport(viaTelegram = true) {
+  const report = await gatherStatus({ appUrl: authBaseUrl, tunnelReadyUrl, monitorStatusUrl });
+  return formatStatus(report, diagnose(report), { viaTelegram });
+}
+
+async function reply(chatId, text, threadId) {
   await fetch(`${apiBase}/sendMessage`, {
     method: "POST",
     headers: { "content-type": "application/json" },
-    body: JSON.stringify({ chat_id: chatId, text }),
+    body: JSON.stringify({ chat_id: chatId, message_thread_id: threadId, text }),
   }).catch(err => logEvent("warn", "send_message_error", { error: err }));
 }
 

bot/status.js

@@ -0,0 +1,172 @@
+// The /status command: probe each layer between a user and the service and
+// say which one failed, the way Cloudflare's error page points at the
+// browser, the edge, or the origin. Pure apart from the injected fetch, so it
+// is unit-testable; index.js supplies the URLs.
+
+const PROBE_TIMEOUT_MS = 5_000;
+const CLOUDFLARE_STATUS_URL = "https://www.cloudflarestatus.com/api/v2/status.json";
+
+export function isStatusCommand(text) {
+  return /^\/status(?:@\w+)?(?:\s|$)/.test(text || "");
+}
+
+// Only the alert group, or the admin in a private chat, may ask: the reply
+// names internal components and the bot is reachable by anyone on Telegram.
+export function mayRequestStatus(message, { alertChatId, adminTelegramId }) {
+  const chatId = String(message.chat?.id ?? "");
+  if (alertChatId && chatId === String(alertChatId)) return true;
+  return message.chat?.type === "private" && String(message.from?.id ?? "") === String(adminTelegramId);
+}
+
+async function probeJson(fetchImpl, url) {
+  const res = await fetchImpl(url, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
+  let body = null;
+  try {
+    body = await res.json();
+  } catch {
+    body = null;
+  }
+  return { status: res.status, body };
+}
+
+async function probeApp(fetchImpl, appUrl) {
+  try {
+    const { status, body } = await probeJson(fetchImpl, `${appUrl}/api/health/ready`);
+    const parts = Object.entries(body?.checks ?? {}).map(([name, up]) => `${name} ${up ? "ok" : "down"}`);
+    const detail = parts.length > 0 ? parts.join(", ") : "no detail";
+    return {
+      ok: status === 200 && body?.ok === true,
+      detail: status === 200 ? detail : `${status}, ${detail}`,
+    };
+  } catch (err) {
+    return { ok: false, detail: `unreachable (${err.message})` };
+  }
+}
+
+async function probeTunnel(fetchImpl, readyUrl) {
+  if (!readyUrl) return { ok: null, detail: "not configured" };
+  try {
+    const { status, body } = await probeJson(fetchImpl, readyUrl);
+    const connections = Number(body?.readyConnections ?? 0);
+    return {
+      ok: status === 200 && connections > 0,
+      detail: `${connections} edge connection${connections === 1 ? "" : "s"}`,
+    };
+  } catch (err) {
+    return { ok: false, detail: `unreachable (${err.message})` };
+  }
+}
+
+const NO_EXTERNAL = { checks: {}, heartbeats: {} };
+
+async function probeExternal(fetchImpl, statusUrl) {
+  if (!statusUrl) return { ok: null, detail: "not configured", ...NO_EXTERNAL };
+  try {
+    const { status, body } = await probeJson(fetchImpl, statusUrl);
+    if (status !== 200 || !body) return { ok: false, detail: `status ${status}`, ...NO_EXTERNAL };
+    const checks = Object.fromEntries((body.checks ?? []).map(c => [c.name, c]));
+    // Ages come from the Worker's clock, so a skewed host clock cannot make
+    // a fresh heartbeat look stale.
+    const heartbeats = Object.fromEntries(
+      (body.heartbeats ?? []).map(h => [h.name, Math.max(0, body.now - h.last_seen_at)]),
+    );
+    return { ok: true, detail: "reachable", checks, heartbeats };
+  } catch (err) {
+    return { ok: false, detail: `unreachable (${err.message})`, ...NO_EXTERNAL };
+  }
+}
+
+async function probeCloudflare(fetchImpl) {
+  try {
+    const { status, body } = await probeJson(fetchImpl, CLOUDFLARE_STATUS_URL);
+    const indicator = body?.status?.indicator;
+    if (status !== 200 || !indicator) return { ok: null, detail: "status page unreachable" };
+    return { ok: indicator === "none", detail: body.status.description || indicator };
+  } catch {
+    return { ok: null, detail: "status page unreachable" };
+  }
+}
+
+export async function gatherStatus({ appUrl, tunnelReadyUrl, monitorStatusUrl, fetchImpl = fetch }) {
+  const [app, tunnel, external, cloudflare] = await Promise.all([
+    probeApp(fetchImpl, appUrl),
+    probeTunnel(fetchImpl, tunnelReadyUrl),
+    probeExternal(fetchImpl, monitorStatusUrl),
+    probeCloudflare(fetchImpl),
+  ]);
+  return { app, tunnel, external, cloudflare };
+}
+
+// Walk the request path from the origin outwards and blame the first broken
+// layer. The external checks come last: they see every layer at once, so
+// they only add information when the inner ones are healthy.
+export function diagnose(report) {
+  const { app, tunnel, external, cloudflare } = report;
+  const { http_ready: ready, http_discovery: discovery, heartbeat_worker: worker, heartbeat_bot: bot } = external.checks;
+
+  if (!app.ok) return { verdict: "DOWN", where: `host (app): ${app.detail}` };
+  if (tunnel.ok === false) return { verdict: "DOWN", where: `Cloudflare tunnel: cloudflared ${tunnel.detail}` };
+  if (ready?.status === "down") {
+    const incident = cloudflare.ok === false ? `; Cloudflare reports ${cloudflare.detail}` : "";
+    return {
+      verdict: "DOWN",
+      where: `Cloudflare edge: origin and tunnel are healthy but the public probe fails (${ready.detail})${incident}`,
+    };
+  }
+  if (worker?.status === "down") return { verdict: "DEGRADED", where: `host (worker): heartbeat ${worker.detail}` };
+  if (bot?.status === "down") {
+    return { verdict: "DEGRADED", where: `host (bot): heartbeat ${bot.detail} although the bot answers; check HEARTBEAT_URL_BOT` };
+  }
+  if (discovery?.status === "down") return { verdict: "DEGRADED", where: `host (app config): discovery ${discovery.detail}` };
+  if (external.ok !== true) return { verdict: "UP", where: `unverified from outside: external monitor ${external.detail}` };
+  return { verdict: "UP", where: null };
+}
+
+function age(ms) {
+  return ms === undefined ? "never" : `${Math.round(ms / 1000)}s ago`;
+}
+
+function mark(ok) {
+  return ok === null ? "unknown" : ok ? "ok" : "DOWN";
+}
+
+function clock(ms) {
+  return new Date(ms).toISOString().slice(11, 16) + " UTC";
+}
+
+function heartbeatLine(label, check, ageMs) {
+  const state = check ? mark(check.status === "up") : "unknown";
+  return `${label}: ${state} - heartbeat ${age(ageMs)}`;
+}
+
+function publicProbeLine(external) {
+  if (external.ok !== true) return `Public probe: unknown - ${external.detail}`;
+  const { http_ready: ready, http_discovery: discovery } = external.checks;
+  if (!ready) return "Public probe: unknown - no checks recorded yet";
+  const state = ready.status === "up" ? "ok" : `DOWN since ${clock(ready.since)}`;
+  const parts = [`ready ${ready.detail}`];
+  if (discovery) parts.push(`discovery ${discovery.status === "up" ? "ok" : discovery.detail}`);
+  return `Public probe: ${state} - ${parts.join(", ")}`;
+}
+
+export function formatStatus(report, diagnosis, { now = Date.now(), viaTelegram = true } = {}) {
+  const { app, tunnel, external, cloudflare } = report;
+  const lines = [
+    `Status: ${diagnosis.verdict}`,
+    `Where: ${diagnosis.where ?? "nothing is down"}`,
+    "",
+    viaTelegram ? "You -> Telegram -> bot: ok (this reply)" : "You -> Telegram -> bot: not exercised (shell run)",
+    `Cloudflare: ${cloudflare.ok === false ? "incident" : mark(cloudflare.ok)} - ${cloudflare.detail}`,
+    `Tunnel (cloudflared): ${mark(tunnel.ok)} - ${tunnel.detail}`,
+    `Host app: ${mark(app.ok)} - ${app.detail}`,
+    heartbeatLine("Host worker", external.checks.heartbeat_worker, external.heartbeats.worker),
+    heartbeatLine("Host bot", external.checks.heartbeat_bot, external.heartbeats.bot),
+    publicProbeLine(external),
+    "",
+    `Checked ${clock(now)}.`,
+  ];
+  if (diagnosis.verdict === "UP") {
+    lines[lines.length - 1] += " If the site still fails for you, the problem is on your side (DNS, network, browser).";
+  }
+  return lines.join("\n");
+}

docker-compose.yml

@@ -129,6 +129,11 @@ services:
       # rejects every tap. Mandatory here rather than defaulted.
       BEARER_ADMIN_TELEGRAM_ID: ${BEARER_ADMIN_TELEGRAM_ID:?set BEARER_ADMIN_TELEGRAM_ID}
       HEARTBEAT_URL: ${HEARTBEAT_URL_BOT:-}
+      # /status answers only in this chat (or the admin's private chat) and
+      # reads the external monitor through MONITOR_STATUS_URL.
+      ALERT_TELEGRAM_CHAT_ID: ${ALERT_TELEGRAM_CHAT_ID:-}
+      MONITOR_STATUS_URL: ${MONITOR_STATUS_URL:-}
+      CLOUDFLARED_READY_URL: ${CLOUDFLARED_READY_URL:-}
       TELEGRAM_ANALYTICS_CHAT_ID: ${TELEGRAM_ANALYTICS_CHAT_ID:-}
       TELEGRAM_ANALYTICS_THREAD_ID: ${TELEGRAM_ANALYTICS_THREAD_ID:-}
     depends_on:

docs/deployment.md

@@ -110,7 +110,10 @@ webhook batch drain (up to `GRACEFUL_SHUTDOWN_TIMEOUT_MS`, default 10s), then
 closes the Postgres pool and Redis connection before exiting. The Next.js app
 drains in-flight requests on `SIGTERM` itself. Both sides are restart-safe: a
 hard kill mid-delivery is recovered via `webhook_deliveries.next_attempt_at`,
-so `docker compose up -d --build app worker` is safe at any time.
+so `docker compose up -d --build app worker` is safe at any time. To rebuild
+only the worker or the bot, add `--no-deps`: without it Compose also builds
+and recreates the app they depend on, which costs a short window of 502s
+at the edge.
 
 ## Operator alerts
 
@@ -171,10 +174,19 @@ prints the id but does not write it into the config; paste it into
 
 `wrangler deploy` prints the Worker URL; the heartbeat URLs for the env file
 are `https://<worker-url>/ping/worker?token=<PING_TOKEN>` and
-`.../ping/bot?token=<PING_TOKEN>`. The token is compared in constant time
-and never logged. `TELEGRAM_BOT_TOKEN` now has a second consumer (see the
-runbook's rotation matrix). Any other ping-URL monitoring service works in
-its place; the checks below are what it must implement.
+`.../ping/bot?token=<PING_TOKEN>`, and `MONITOR_STATUS_URL` is
+`https://<worker-url>/status?token=<PING_TOKEN>`. The token is compared in
+constant time and never logged. `TELEGRAM_BOT_TOKEN` now has a second
+consumer (see the runbook's rotation matrix). Any other ping-URL monitoring
+service works in its place; the checks below are what it must implement.
+
+`/status` in the alert chat asks the bot for a layer-by-layer breakdown
+(Cloudflare's status page, cloudflared's edge connections, the app's
+readiness, the heartbeats as the Worker sees them, the public probe) with a
+verdict and the layer to blame, so an outage can be placed at the host, the
+tunnel, the edge, or the caller's own side. It needs `ALERT_TELEGRAM_CHAT_ID`,
+`MONITOR_STATUS_URL` and `CLOUDFLARED_READY_URL` on the bot service. No reply
+means the bot or the host is down; the Worker's own DOWN message says which.
 
 Two kinds of check: