From 1953c5fc3e12aa1f36fbda2b31a7ab855f2ba4a5 Mon Sep 17 00:00:00 2001 From: Matthew Demidoff Date: Sun, 6 Sep 2026 09:14:54 +0200 Subject: [PATCH 1/2] feat(bot): add a /status command that names the failing layer In the alert chat (or from the admin in private), /status probes each layer between a user and the service and blames the first broken one: the app's readiness from inside, cloudflared's edge connections, the public probes and heartbeats as the external monitor sees them, and Cloudflare's own status page. The Worker gains a token-gated GET /status that returns its recorded state. node index.js --status prints the same report from a shell. --- .env.example | 3 + .github/workflows/security.yml | 1 + bot/Dockerfile | 2 +- bot/README.md | 19 ++++ bot/index.js | 50 +++++++-- bot/status.js | 172 +++++++++++++++++++++++++++++ docker-compose.yml | 5 + docs/deployment.md | 17 ++- monitor/src/index.ts | 38 +++++-- runbooks/oncall.md | 6 +- tests/unit/bot-status.test.ts | 192 +++++++++++++++++++++++++++++++++ 11 files changed, 482 insertions(+), 23 deletions(-) create mode 100644 bot/status.js create mode 100644 tests/unit/bot-status.test.ts diff --git a/.env.example b/.env.example index 77c4b3c..42fa52e 100644 --- a/.env.example +++ b/.env.example @@ -76,6 +76,9 @@ DIGEST_HOUR_UTC=8 HEARTBEAT_URL_WORKER= HEARTBEAT_URL_BOT= CLOUDFLARED_READY_URL= +# The monitor Worker's /status URL (same token as the ping URLs); the bot's +# /status command reads it. Unset = the command reports the host view only. +MONITOR_STATUS_URL= # Worker only: max ms to wait for in-flight webhook batches on SIGTERM before # forcing close. Tune to ~2x observed p95 delivery time. diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 31eaf01..ea9668f 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -58,6 +58,7 @@ jobs: - run: npm run build - run: docker build --target runner -t auth-runner:ci . - run: docker build --target worker . + - run: docker build -t auth-bot:ci ./bot && docker run --rm auth-bot:ci node -e "import('./status.js')" # The image that ships is the one under test: nonce CSP, HSTS, the # standalone server and migrate-on-boot only exist in production mode. # --network host lets the container reach the service containers on diff --git a/bot/Dockerfile b/bot/Dockerfile index c1be1f2..ae30613 100644 --- a/bot/Dockerfile +++ b/bot/Dockerfile @@ -2,6 +2,6 @@ FROM node:22-alpine WORKDIR /app ENV NODE_ENV=production ENV NODE_OPTIONS=--max-old-space-size=128 -COPY package.json index.js ./ +COPY package.json index.js status.js ./ USER node CMD ["node", "index.js"] diff --git a/bot/README.md b/bot/README.md index 32eb391..6176d20 100644 --- a/bot/README.md +++ b/bot/README.md @@ -21,6 +21,25 @@ Long-polls Telegram (`getUpdates`), so it does not need a public URL. Only the a The bot username (`TELEGRAM_BOT_USERNAME`) is only needed by the auth service to build the start link. +## Optional env + +- `HEARTBEAT_URL` - pinged once a minute while the long-poll succeeds (see `docs/deployment.md`, Monitoring) +- `ALERT_TELEGRAM_CHAT_ID` - the chat in which `/status` is answered; the admin's private chat always works +- `MONITOR_STATUS_URL` - the external monitor's `/status?token=...` URL, read by `/status` +- `CLOUDFLARED_READY_URL` - cloudflared's `/ready` endpoint, read by `/status` +- `TELEGRAM_ANALYTICS_CHAT_ID` / `TELEGRAM_ANALYTICS_THREAD_ID` - where the pinned status message lives + +## /status + +`/status` in the alert chat (or from the admin in a private chat) replies with a +layer-by-layer breakdown: Cloudflare's own status page, the tunnel's edge +connections, the app's readiness (Postgres, Redis), the worker and bot +heartbeats as the external monitor sees them, and the public probe. The first +line says UP, DEGRADED or DOWN and the second names the layer to blame. No reply +means the bot itself, or the whole host, is down; the external monitor's last +DOWN message in the chat says which. `node index.js --status` prints the same +report from a shell (`docker compose exec bot node index.js --status`). + ## Run ``` diff --git a/bot/index.js b/bot/index.js index a331f64..2556636 100644 --- a/bot/index.js +++ b/bot/index.js @@ -1,9 +1,14 @@ +import { diagnose, formatStatus, gatherStatus, isStatusCommand, mayRequestStatus } from "./status.js"; + const botToken = required("TELEGRAM_BOT_TOKEN"); const webhookSecret = required("TELEGRAM_BOT_WEBHOOK_SECRET"); const authBaseUrl = process.env.AUTH_INTERNAL_URL || "http://localhost:3000"; const bearerAdminTelegramId = required("BEARER_ADMIN_TELEGRAM_ID"); const analyticsChatId = process.env.TELEGRAM_ANALYTICS_CHAT_ID; const analyticsThreadId = process.env.TELEGRAM_ANALYTICS_THREAD_ID; +const alertChatId = process.env.ALERT_TELEGRAM_CHAT_ID; +const monitorStatusUrl = process.env.MONITOR_STATUS_URL || ""; +const tunnelReadyUrl = process.env.CLOUDFLARED_READY_URL || ""; const apiBase = `https://api.telegram.org/bot${botToken}`; const longPollSeconds = 30; @@ -35,8 +40,6 @@ async function pingHeartbeat() { } } -if (heartbeatUrl) setInterval(pingHeartbeat, HEARTBEAT_INTERVAL_MS); - // Structured logging, inlined because the bot image ships only index.js and // cannot import the worker's logger. Same JSON-line shape as worker-log.js so // one log pipeline reads every process. @@ -55,11 +58,26 @@ function logEvent(level, msg, metadata) { } // main() is a floating promise: a throw from the startup status monitor used -// to surface as an unstructured unhandled rejection. -main().catch(err => { - logEvent("error", "bot_crashed", { error: err }); - process.exit(1); -}); +// to surface as an unstructured unhandled rejection. `--status` prints the +// breakdown the /status command sends and exits without ever polling, so it +// can run beside the live bot from a shell on the host. +if (process.argv.includes("--status")) { + statusReport(false).then( + text => { + process.stdout.write(text + "\n"); + process.exit(0); + }, + err => { + logEvent("error", "status_failed", { error: err }); + process.exit(1); + }, + ); +} else { + main().catch(err => { + logEvent("error", "bot_crashed", { error: err }); + process.exit(1); + }); +} process.on("unhandledRejection", err => { logEvent("error", "unhandled_rejection", { error: err }); @@ -116,6 +134,7 @@ async function startStatusMonitor() { } async function main() { + if (heartbeatUrl) setInterval(pingHeartbeat, HEARTBEAT_INTERVAL_MS); await startStatusMonitor(); while (true) { const updates = await getUpdates(); @@ -171,6 +190,14 @@ async function handleUpdate(update) { return; } + if (isStatusCommand(message.text)) { + if (mayRequestStatus(message, { alertChatId, adminTelegramId: bearerAdminTelegramId })) { + const threadId = message.is_topic_message ? message.message_thread_id : undefined; + await reply(message.chat.id, await statusReport(), threadId); + } + return; + } + const match = message.text.match(/^\/start(?:\s+(\S+))?/); if (!match) { return; @@ -386,11 +413,16 @@ async function callVerify(startToken, from) { return { message: "Verification service is unavailable. Try again in a moment." }; } -async function reply(chatId, text) { +async function statusReport(viaTelegram = true) { + const report = await gatherStatus({ appUrl: authBaseUrl, tunnelReadyUrl, monitorStatusUrl }); + return formatStatus(report, diagnose(report), { viaTelegram }); +} + +async function reply(chatId, text, threadId) { await fetch(`${apiBase}/sendMessage`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ chat_id: chatId, text }), + body: JSON.stringify({ chat_id: chatId, message_thread_id: threadId, text }), }).catch(err => logEvent("warn", "send_message_error", { error: err })); } diff --git a/bot/status.js b/bot/status.js new file mode 100644 index 0000000..4eb28b5 --- /dev/null +++ b/bot/status.js @@ -0,0 +1,172 @@ +// The /status command: probe each layer between a user and the service and +// say which one failed, the way Cloudflare's error page points at the +// browser, the edge, or the origin. Pure apart from the injected fetch, so it +// is unit-testable; index.js supplies the URLs. + +const PROBE_TIMEOUT_MS = 5_000; +const CLOUDFLARE_STATUS_URL = "https://www.cloudflarestatus.com/api/v2/status.json"; + +export function isStatusCommand(text) { + return /^\/status(?:@\w+)?(?:\s|$)/.test(text || ""); +} + +// Only the alert group, or the admin in a private chat, may ask: the reply +// names internal components and the bot is reachable by anyone on Telegram. +export function mayRequestStatus(message, { alertChatId, adminTelegramId }) { + const chatId = String(message.chat?.id ?? ""); + if (alertChatId && chatId === String(alertChatId)) return true; + return message.chat?.type === "private" && String(message.from?.id ?? "") === String(adminTelegramId); +} + +async function probeJson(fetchImpl, url) { + const res = await fetchImpl(url, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) }); + let body = null; + try { + body = await res.json(); + } catch { + body = null; + } + return { status: res.status, body }; +} + +async function probeApp(fetchImpl, appUrl) { + try { + const { status, body } = await probeJson(fetchImpl, `${appUrl}/api/health/ready`); + const parts = Object.entries(body?.checks ?? {}).map(([name, up]) => `${name} ${up ? "ok" : "down"}`); + const detail = parts.length > 0 ? parts.join(", ") : "no detail"; + return { + ok: status === 200 && body?.ok === true, + detail: status === 200 ? detail : `${status}, ${detail}`, + }; + } catch (err) { + return { ok: false, detail: `unreachable (${err.message})` }; + } +} + +async function probeTunnel(fetchImpl, readyUrl) { + if (!readyUrl) return { ok: null, detail: "not configured" }; + try { + const { status, body } = await probeJson(fetchImpl, readyUrl); + const connections = Number(body?.readyConnections ?? 0); + return { + ok: status === 200 && connections > 0, + detail: `${connections} edge connection${connections === 1 ? "" : "s"}`, + }; + } catch (err) { + return { ok: false, detail: `unreachable (${err.message})` }; + } +} + +const NO_EXTERNAL = { checks: {}, heartbeats: {} }; + +async function probeExternal(fetchImpl, statusUrl) { + if (!statusUrl) return { ok: null, detail: "not configured", ...NO_EXTERNAL }; + try { + const { status, body } = await probeJson(fetchImpl, statusUrl); + if (status !== 200 || !body) return { ok: false, detail: `status ${status}`, ...NO_EXTERNAL }; + const checks = Object.fromEntries((body.checks ?? []).map(c => [c.name, c])); + // Ages come from the Worker's clock, so a skewed host clock cannot make + // a fresh heartbeat look stale. + const heartbeats = Object.fromEntries( + (body.heartbeats ?? []).map(h => [h.name, Math.max(0, body.now - h.last_seen_at)]), + ); + return { ok: true, detail: "reachable", checks, heartbeats }; + } catch (err) { + return { ok: false, detail: `unreachable (${err.message})`, ...NO_EXTERNAL }; + } +} + +async function probeCloudflare(fetchImpl) { + try { + const { status, body } = await probeJson(fetchImpl, CLOUDFLARE_STATUS_URL); + const indicator = body?.status?.indicator; + if (status !== 200 || !indicator) return { ok: null, detail: "status page unreachable" }; + return { ok: indicator === "none", detail: body.status.description || indicator }; + } catch { + return { ok: null, detail: "status page unreachable" }; + } +} + +export async function gatherStatus({ appUrl, tunnelReadyUrl, monitorStatusUrl, fetchImpl = fetch }) { + const [app, tunnel, external, cloudflare] = await Promise.all([ + probeApp(fetchImpl, appUrl), + probeTunnel(fetchImpl, tunnelReadyUrl), + probeExternal(fetchImpl, monitorStatusUrl), + probeCloudflare(fetchImpl), + ]); + return { app, tunnel, external, cloudflare }; +} + +// Walk the request path from the origin outwards and blame the first broken +// layer. The external checks come last: they see every layer at once, so +// they only add information when the inner ones are healthy. +export function diagnose(report) { + const { app, tunnel, external, cloudflare } = report; + const { http_ready: ready, http_discovery: discovery, heartbeat_worker: worker, heartbeat_bot: bot } = external.checks; + + if (!app.ok) return { verdict: "DOWN", where: `host (app): ${app.detail}` }; + if (tunnel.ok === false) return { verdict: "DOWN", where: `Cloudflare tunnel: cloudflared ${tunnel.detail}` }; + if (ready?.status === "down") { + const incident = cloudflare.ok === false ? `; Cloudflare reports ${cloudflare.detail}` : ""; + return { + verdict: "DOWN", + where: `Cloudflare edge: origin and tunnel are healthy but the public probe fails (${ready.detail})${incident}`, + }; + } + if (worker?.status === "down") return { verdict: "DEGRADED", where: `host (worker): heartbeat ${worker.detail}` }; + if (bot?.status === "down") { + return { verdict: "DEGRADED", where: `host (bot): heartbeat ${bot.detail} although the bot answers; check HEARTBEAT_URL_BOT` }; + } + if (discovery?.status === "down") return { verdict: "DEGRADED", where: `host (app config): discovery ${discovery.detail}` }; + if (external.ok !== true) return { verdict: "UP", where: `unverified from outside: external monitor ${external.detail}` }; + return { verdict: "UP", where: null }; +} + +function age(ms) { + return ms === undefined ? "never" : `${Math.round(ms / 1000)}s ago`; +} + +function mark(ok) { + return ok === null ? "unknown" : ok ? "ok" : "DOWN"; +} + +function clock(ms) { + return new Date(ms).toISOString().slice(11, 16) + " UTC"; +} + +function heartbeatLine(label, check, ageMs) { + const state = check ? mark(check.status === "up") : "unknown"; + return `${label}: ${state} - heartbeat ${age(ageMs)}`; +} + +function publicProbeLine(external) { + if (external.ok !== true) return `Public probe: unknown - ${external.detail}`; + const { http_ready: ready, http_discovery: discovery } = external.checks; + if (!ready) return "Public probe: unknown - no checks recorded yet"; + const state = ready.status === "up" ? "ok" : `DOWN since ${clock(ready.since)}`; + const parts = [`ready ${ready.detail}`]; + if (discovery) parts.push(`discovery ${discovery.status === "up" ? "ok" : discovery.detail}`); + return `Public probe: ${state} - ${parts.join(", ")}`; +} + +export function formatStatus(report, diagnosis, { now = Date.now(), viaTelegram = true } = {}) { + const { app, tunnel, external, cloudflare } = report; + const lines = [ + `Status: ${diagnosis.verdict}`, + `Where: ${diagnosis.where ?? "nothing is down"}`, + "", + viaTelegram ? "You -> Telegram -> bot: ok (this reply)" : "You -> Telegram -> bot: not exercised (shell run)", + `Cloudflare: ${cloudflare.ok === false ? "incident" : mark(cloudflare.ok)} - ${cloudflare.detail}`, + `Tunnel (cloudflared): ${mark(tunnel.ok)} - ${tunnel.detail}`, + `Host app: ${mark(app.ok)} - ${app.detail}`, + heartbeatLine("Host worker", external.checks.heartbeat_worker, external.heartbeats.worker), + heartbeatLine("Host bot", external.checks.heartbeat_bot, external.heartbeats.bot), + publicProbeLine(external), + "", + `Checked ${clock(now)}.`, + ]; + if (diagnosis.verdict === "UP") { + lines[lines.length - 1] += " If the site still fails for you, the problem is on your side (DNS, network, browser)."; + } + return lines.join("\n"); +} diff --git a/docker-compose.yml b/docker-compose.yml index 4728f4f..e76b6e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,6 +129,11 @@ services: # rejects every tap. Mandatory here rather than defaulted. BEARER_ADMIN_TELEGRAM_ID: ${BEARER_ADMIN_TELEGRAM_ID:?set BEARER_ADMIN_TELEGRAM_ID} HEARTBEAT_URL: ${HEARTBEAT_URL_BOT:-} + # /status answers only in this chat (or the admin's private chat) and + # reads the external monitor through MONITOR_STATUS_URL. + ALERT_TELEGRAM_CHAT_ID: ${ALERT_TELEGRAM_CHAT_ID:-} + MONITOR_STATUS_URL: ${MONITOR_STATUS_URL:-} + CLOUDFLARED_READY_URL: ${CLOUDFLARED_READY_URL:-} TELEGRAM_ANALYTICS_CHAT_ID: ${TELEGRAM_ANALYTICS_CHAT_ID:-} TELEGRAM_ANALYTICS_THREAD_ID: ${TELEGRAM_ANALYTICS_THREAD_ID:-} depends_on: diff --git a/docs/deployment.md b/docs/deployment.md index b31e163..36e2078 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -171,10 +171,19 @@ prints the id but does not write it into the config; paste it into `wrangler deploy` prints the Worker URL; the heartbeat URLs for the env file are `https:///ping/worker?token=` and -`.../ping/bot?token=`. The token is compared in constant time -and never logged. `TELEGRAM_BOT_TOKEN` now has a second consumer (see the -runbook's rotation matrix). Any other ping-URL monitoring service works in -its place; the checks below are what it must implement. +`.../ping/bot?token=`, and `MONITOR_STATUS_URL` is +`https:///status?token=`. The token is compared in +constant time and never logged. `TELEGRAM_BOT_TOKEN` now has a second +consumer (see the runbook's rotation matrix). Any other ping-URL monitoring +service works in its place; the checks below are what it must implement. + +`/status` in the alert chat asks the bot for a layer-by-layer breakdown +(Cloudflare's status page, cloudflared's edge connections, the app's +readiness, the heartbeats as the Worker sees them, the public probe) with a +verdict and the layer to blame, so an outage can be placed at the host, the +tunnel, the edge, or the caller's own side. It needs `ALERT_TELEGRAM_CHAT_ID`, +`MONITOR_STATUS_URL` and `CLOUDFLARED_READY_URL` on the bot service. No reply +means the bot or the host is down; the Worker's own DOWN message says which. Two kinds of check: diff --git a/monitor/src/index.ts b/monitor/src/index.ts index cbae2a6..1eef7e4 100644 --- a/monitor/src/index.ts +++ b/monitor/src/index.ts @@ -26,6 +26,8 @@ declare global { } } +type HeartbeatRow = { name: string; last_seen_at: number }; + const PROBE_TIMEOUT_MS = 10_000; const TELEGRAM_TIMEOUT_MS = 5_000; const encoder = new TextEncoder(); @@ -51,10 +53,14 @@ function isHeartbeatSource(value: string): value is HeartbeatSource { return (HEARTBEAT_SOURCES as readonly string[]).includes(value); } -async function handlePing(request: Request, env: Env): Promise { +// Both routes hide behind the same token and answer 404 to anything else, so +// an unauthenticated caller cannot tell the endpoints from the void. +async function handleRequest(request: Request, env: Env): Promise { const url = new URL(request.url); - const match = /^\/ping\/([a-z]+)$/.exec(url.pathname); - if (request.method !== "GET" || !match || !isHeartbeatSource(match[1])) { + const ping = /^\/ping\/([a-z]+)$/.exec(url.pathname); + const source = ping && isHeartbeatSource(ping[1]) ? ping[1] : null; + const status = url.pathname === "/status"; + if (request.method !== "GET" || (!status && !source)) { return new Response("not found", { status: 404 }); } if (!env.PING_TOKEN) { @@ -65,15 +71,33 @@ async function handlePing(request: Request, env: Env): Promise { if (!(await tokenMatches(token, env.PING_TOKEN))) { return new Response("not found", { status: 404 }); } + return source ? handlePing(source, env) : handleStatus(env); +} + +async function handlePing(source: HeartbeatSource, env: Env): Promise { await env.DB.prepare( `insert into heartbeats (name, last_seen_at) values (?1, ?2) on conflict (name) do update set last_seen_at = excluded.last_seen_at`, ) - .bind(match[1], Date.now()) + .bind(source, Date.now()) .run(); return new Response(null, { status: 204 }); } +// The current state as the cron last wrote it, for the bot's /status command. +// `now` is this clock so the reader can age the heartbeats without trusting +// its own. +async function handleStatus(env: Env): Promise { + const [checks, heartbeats] = await Promise.all([ + env.DB.prepare("select name, status, since, failures, detail from checks").all(), + env.DB.prepare("select name, last_seen_at from heartbeats").all(), + ]); + return Response.json( + { now: Date.now(), checks: checks.results, heartbeats: heartbeats.results }, + { headers: { "cache-control": "no-store" } }, + ); +} + // Our own endpoints return small JSON documents; reading them whole is fine. async function probe(url: string): Promise { try { @@ -97,8 +121,6 @@ async function sendTelegram(env: Env, text: string): Promise { if (!res.ok) throw new Error(`telegram sendMessage failed: ${res.status}`); } -type HeartbeatRow = { name: string; last_seen_at: number }; - async function runChecks(env: Env, now: number): Promise { const base = env.TARGET_BASE_URL; const graceMs = Number(env.HEARTBEAT_GRACE_SECONDS) * 1000; @@ -169,9 +191,9 @@ async function runChecks(env: Env, now: number): Promise { export default { async fetch(request, env): Promise { try { - return await handlePing(request, env); + return await handleRequest(request, env); } catch (err) { - log("error", "ping_failed", { error: err instanceof Error ? err.message : String(err) }); + log("error", "request_failed", { error: err instanceof Error ? err.message : String(err) }); return new Response("error", { status: 500 }); } }, diff --git a/runbooks/oncall.md b/runbooks/oncall.md index 13a3622..c359013 100644 --- a/runbooks/oncall.md +++ b/runbooks/oncall.md @@ -7,6 +7,10 @@ stack is Docker Compose on a single host behind a Cloudflare Tunnel: ## First moves +Type `/status` in the alert chat first: the bot answers with which layer is +broken (host, tunnel, Cloudflare edge, or nothing). No reply within a few +seconds means the bot or the whole host is down. Then, on the host: + ```sh docker compose ps # what is up / restarting curl -s localhost:3000/api/health/ready # readiness (needs to run on the host net) @@ -80,7 +84,7 @@ changed), which is a short outage you should schedule, not trip over. | `CLOUDFLARED_TOKEN` | `cloudflared` | `up -d --no-deps cloudflared` | seconds of tunnel outage | second tunnel + DNS cutover; rarely worth it | | `RESEND_API_KEY` | `app` | `up -d --no-deps app` | verification emails fail | create new, deploy, delete old | | `OAUTH_DYNAMIC_REGISTRATION_TOKEN` | `app`, DCR clients | `up -d --no-deps app` | DCR calls 401 until clients update | none | -| `PING_TOKEN` (monitor Worker secret) | monitor Worker, `worker` and `bot` via `HEARTBEAT_URL_*` | `wrangler secret put PING_TOKEN`, new URLs in the env file, `up -d --no-deps worker bot` | pings rejected (404) until both sides agree; the monitor reports the heartbeats down after the grace period | none; do both within the grace period | +| `PING_TOKEN` (monitor Worker secret) | monitor Worker, `worker` and `bot` via `HEARTBEAT_URL_*`, `bot` via `MONITOR_STATUS_URL` | `wrangler secret put PING_TOKEN`, new URLs in the env file, `up -d --no-deps worker bot` | pings rejected (404) until both sides agree; the monitor reports the heartbeats down after the grace period | none; do both within the grace period | `BEARER_ADMIN_TELEGRAM_ID` and `ALERT_TELEGRAM_CHAT_ID` are identifiers, not secrets, but changing them takes the same restart sets as `TELEGRAM_BOT_TOKEN`. diff --git a/tests/unit/bot-status.test.ts b/tests/unit/bot-status.test.ts new file mode 100644 index 0000000..2f2be04 --- /dev/null +++ b/tests/unit/bot-status.test.ts @@ -0,0 +1,192 @@ +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +type Probe = { ok: boolean | null; detail: string }; +type Report = { + app: Probe; + tunnel: Probe; + external: Probe & { checks: Record; heartbeats: Record }; + cloudflare: Probe; +}; +type Diagnosis = { verdict: 'UP' | 'DEGRADED' | 'DOWN'; where: string | null }; +type StatusModule = { + isStatusCommand: (text: string | undefined) => boolean; + mayRequestStatus: (message: unknown, gate: { alertChatId?: string; adminTelegramId: string }) => boolean; + gatherStatus: (opts: { + appUrl: string; + tunnelReadyUrl: string; + monitorStatusUrl: string; + fetchImpl: (url: string) => Promise; + }) => Promise; + diagnose: (report: Report) => Diagnosis; + formatStatus: (report: Report, diagnosis: Diagnosis, opts?: { now?: number; viaTelegram?: boolean }) => string; +}; + +// The bot ships as an untyped ES module; load it by URL the way the worker +// tests use createRequire, so the root typecheck does not need allowJs. +const status = (await import(pathToFileURL(path.resolve(process.cwd(), 'bot/status.js')).href)) as StatusModule; + +const NOW = Date.UTC(2026, 8, 6, 12, 0, 0); +const APP = 'http://app:3000'; +const TUNNEL = 'http://cloudflared:2000/ready'; +const MONITOR = 'https://monitor.example/status?token=secret'; + +function reply(statusCode: number, body: unknown) { + return new Response(JSON.stringify(body), { status: statusCode, headers: { 'content-type': 'application/json' } }); +} + +function externalBody(overrides: Partial> = {}, botAgeMs = 30_000) { + const check = (name: string, fallback: { status: string; detail: string }) => ({ + name, + since: NOW - 3_600_000, + failures: 0, + ...fallback, + ...(overrides[name] ?? {}), + }); + return { + now: NOW, + checks: [ + check('http_ready', { status: 'up', detail: '200 ok' }), + check('http_discovery', { status: 'up', detail: '200 issuer ok' }), + check('heartbeat_worker', { status: 'up', detail: 'last seen 40s ago' }), + check('heartbeat_bot', { status: 'up', detail: 'last seen 30s ago' }), + ], + heartbeats: [ + { name: 'worker', last_seen_at: NOW - 40_000 }, + { name: 'bot', last_seen_at: NOW - botAgeMs }, + ], + }; +} + +type Answers = Partial Promise>>; + +function fakeFetch(answers: Answers) { + const healthy: Required = { + app: async () => reply(200, { ok: true, checks: { postgres: true, redis: true } }), + tunnel: async () => reply(200, { status: 200, readyConnections: 4 }), + monitor: async () => reply(200, externalBody()), + cloudflare: async () => reply(200, { status: { indicator: 'none', description: 'All Systems Operational' } }), + }; + const merged = { ...healthy, ...answers }; + return async (url: string) => { + if (url.startsWith(APP)) return merged.app(); + if (url === TUNNEL) return merged.tunnel(); + if (url === MONITOR) return merged.monitor(); + if (url.includes('cloudflarestatus.com')) return merged.cloudflare(); + throw new Error(`unexpected url ${url}`); + }; +} + +async function run(answers: Answers = {}, monitorStatusUrl = MONITOR) { + const report = await status.gatherStatus({ + appUrl: APP, + tunnelReadyUrl: TUNNEL, + monitorStatusUrl, + fetchImpl: fakeFetch(answers), + }); + return { report, diagnosis: status.diagnose(report) }; +} + +describe('/status command gate', () => { + it('matches the bare and the addressed command only', () => { + expect(status.isStatusCommand('/status')).toBe(true); + expect(status.isStatusCommand('/status@auth_bot')).toBe(true); + expect(status.isStatusCommand('/status please')).toBe(true); + expect(status.isStatusCommand('/statusx')).toBe(false); + expect(status.isStatusCommand('/start abc')).toBe(false); + expect(status.isStatusCommand(undefined)).toBe(false); + }); + + it('answers in the alert chat or to the admin in private, nowhere else', () => { + const gate = { alertChatId: '-100123', adminTelegramId: '42' }; + expect(status.mayRequestStatus({ chat: { id: -100123, type: 'supergroup' }, from: { id: 7 } }, gate)).toBe(true); + expect(status.mayRequestStatus({ chat: { id: 42, type: 'private' }, from: { id: 42 } }, gate)).toBe(true); + expect(status.mayRequestStatus({ chat: { id: 8, type: 'private' }, from: { id: 8 } }, gate)).toBe(false); + expect(status.mayRequestStatus({ chat: { id: -100999, type: 'supergroup' }, from: { id: 42 } }, gate)).toBe(false); + expect(status.mayRequestStatus({ chat: { id: -100999, type: 'supergroup' }, from: { id: 42 } }, { adminTelegramId: '42' })).toBe(false); + }); +}); + +describe('/status diagnosis', () => { + it('is UP with nothing to blame when every layer answers', async () => { + const { report, diagnosis } = await run(); + expect(diagnosis).toEqual({ verdict: 'UP', where: null }); + const text = status.formatStatus(report, diagnosis, { now: NOW }); + expect(text).toContain('Status: UP\nWhere: nothing is down'); + expect(text).toContain('Tunnel (cloudflared): ok - 4 edge connections'); + expect(text).toContain('Host app: ok - postgres ok, redis ok'); + expect(text).toContain('Host worker: ok - heartbeat 40s ago'); + expect(text).toContain('Public probe: ok - ready 200 ok, discovery ok'); + expect(text).toContain('Checked 12:00 UTC. If the site still fails for you'); + expect(text).not.toContain('secret'); + }); + + it('blames the host when the app reports a dead dependency', async () => { + const { report, diagnosis } = await run({ + app: async () => reply(503, { ok: false, checks: { postgres: false, redis: true }, failed: ['postgres'] }), + }); + expect(diagnosis).toEqual({ verdict: 'DOWN', where: 'host (app): 503, postgres down, redis ok' }); + expect(status.formatStatus(report, diagnosis, { now: NOW })).toContain('Host app: DOWN - 503, postgres down, redis ok'); + }); + + it('blames the host when the app does not answer at all', async () => { + const { diagnosis } = await run({ + app: async () => { + throw new Error('fetch failed'); + }, + }); + expect(diagnosis.verdict).toBe('DOWN'); + expect(diagnosis.where).toBe('host (app): unreachable (fetch failed)'); + }); + + it('blames the tunnel when cloudflared has no edge connection', async () => { + const { diagnosis } = await run({ tunnel: async () => reply(503, { status: 503, readyConnections: 0 }) }); + expect(diagnosis).toEqual({ verdict: 'DOWN', where: 'Cloudflare tunnel: cloudflared 0 edge connections' }); + }); + + it('blames the edge when origin and tunnel are fine but the public probe fails', async () => { + const { report, diagnosis } = await run({ + monitor: async () => reply(200, externalBody({ http_ready: { status: 'down', detail: 'status 502' } })), + cloudflare: async () => reply(200, { status: { indicator: 'major', description: 'Partial System Outage' } }), + }); + expect(diagnosis.verdict).toBe('DOWN'); + expect(diagnosis.where).toBe( + 'Cloudflare edge: origin and tunnel are healthy but the public probe fails (status 502); Cloudflare reports Partial System Outage', + ); + const text = status.formatStatus(report, diagnosis, { now: NOW }); + expect(text).toContain('Cloudflare: incident - Partial System Outage'); + expect(text).toContain('Public probe: DOWN since 11:00 UTC - ready status 502, discovery ok'); + }); + + it('is DEGRADED when the worker heartbeat is stale', async () => { + const { diagnosis } = await run({ + monitor: async () => reply(200, externalBody({ heartbeat_worker: { status: 'down', detail: 'last seen 400s ago' } })), + }); + expect(diagnosis).toEqual({ verdict: 'DEGRADED', where: 'host (worker): heartbeat last seen 400s ago' }); + }); + + it('stays UP but says so when the external monitor cannot be reached', async () => { + const { report, diagnosis } = await run({ + monitor: async () => { + throw new Error('fetch failed'); + }, + }); + expect(diagnosis).toEqual({ verdict: 'UP', where: 'unverified from outside: external monitor unreachable (fetch failed)' }); + const text = status.formatStatus(report, diagnosis, { now: NOW }); + expect(text).toContain('Host worker: unknown - heartbeat never'); + expect(text).toContain('Public probe: unknown - unreachable (fetch failed)'); + }); + + it('treats an unconfigured monitor URL as not configured, not as an outage', async () => { + const { diagnosis } = await run({}, ''); + expect(diagnosis).toEqual({ verdict: 'UP', where: 'unverified from outside: external monitor not configured' }); + }); + + it('labels a shell run so the Telegram leg is not claimed', async () => { + const { report, diagnosis } = await run(); + expect(status.formatStatus(report, diagnosis, { now: NOW, viaTelegram: false })).toContain( + 'You -> Telegram -> bot: not exercised (shell run)', + ); + }); +}); From 3efbd84f767aea282a5d2b85ecea337b3c200e25 Mon Sep 17 00:00:00 2001 From: Matthew Demidoff Date: Sun, 6 Sep 2026 09:16:30 +0200 Subject: [PATCH 2/2] docs(deployment): rebuild the worker or bot with --no-deps --- docs/deployment.md | 5 ++++- runbooks/oncall.md | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 36e2078..e5a25ff 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -110,7 +110,10 @@ webhook batch drain (up to `GRACEFUL_SHUTDOWN_TIMEOUT_MS`, default 10s), then closes the Postgres pool and Redis connection before exiting. The Next.js app drains in-flight requests on `SIGTERM` itself. Both sides are restart-safe: a hard kill mid-delivery is recovered via `webhook_deliveries.next_attempt_at`, -so `docker compose up -d --build app worker` is safe at any time. +so `docker compose up -d --build app worker` is safe at any time. To rebuild +only the worker or the bot, add `--no-deps`: without it Compose also builds +and recreates the app they depend on, which costs a short window of 502s +at the edge. ## Operator alerts diff --git a/runbooks/oncall.md b/runbooks/oncall.md index c359013..28d3515 100644 --- a/runbooks/oncall.md +++ b/runbooks/oncall.md @@ -147,6 +147,7 @@ alarm from any external probe. ```sh docker compose up -d --build app worker # rebuild + rolling restart, migrations run on app start +docker compose up -d --build --no-deps worker bot # rebuild those two without recreating app docker compose restart worker # graceful (SIGTERM) restart, drains in-flight deliveries ```