diff --git a/.github/workflows/uptime.yml b/.github/workflows/uptime.yml index d8dc4014b..eb0f06626 100644 --- a/.github/workflows/uptime.yml +++ b/.github/workflows/uptime.yml @@ -26,6 +26,13 @@ jobs: - name: Run checks id: uptime + env: + # Mainnet is only checked once this is actually set — see + # docs/RUNBOOK.md "Uptime monitoring". No default: an unset value + # here must mean "not configured", never "check testnet again". + UPTIME_MAINNET_API_URL: ${{ vars.UPTIME_MAINNET_API_URL }} + UPTIME_MAINNET_WEB_URL: ${{ vars.UPTIME_MAINNET_WEB_URL }} + UPTIME_MAINNET_SYNTHETIC_CHECK: ${{ vars.UPTIME_MAINNET_SYNTHETIC_CHECK }} run: | node scripts/uptime-check.mjs | tee /tmp/uptime.log { diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index d85af825c..dea9baf54 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -68,6 +68,40 @@ prerequisite, not a preference: in a per-process `Map`. The persisted replay table still works; only the concurrent-duplicate guard is lost, and it guards a money endpoint. +## Uptime monitoring + +`scripts/uptime-check.mjs` (`.github/workflows/uptime.yml`) checks every +configured environment on one schedule, each with its own history series and +its own section in `docs/STATUS.md` — a healthy testnet can never stand in +for an unmonitored mainnet (issue 8.8). + +**Testnet is always checked**, with the same defaults and unprefixed target +ids (`api` / `web` / `synthetic`) this script has always used — +`https://quay-api.onrender.com` / `https://quay-web.vercel.app`, overridable +via `UPTIME_API_URL` / `UPTIME_WEB_URL`. + +**Mainnet is checked only once you configure it — there is no default of any +kind.** Set these as repository **Variables** (Settings → Secrets and +variables → Actions → Variables — they're plain hostnames, not secrets): + +| Variable | Required | What it does | +|---|---|---| +| `UPTIME_MAINNET_API_URL` | to watch mainnet at all | e.g. `https://quay-api-mainnet.onrender.com` (`render.mainnet.yaml`'s `quay-api-mainnet`). Unset means mainnet is skipped entirely, not silently checked against the testnet URL. | +| `UPTIME_MAINNET_WEB_URL` | optional | Only set this if a dedicated mainnet web deployment exists. `render.mainnet.yaml` declares no web service today, so leave unset until one does. | +| `UPTIME_MAINNET_SYNTHETIC_CHECK` | optional, default off | Set to `1` to also POST a throwaway `/links` synthetic check against mainnet, same as testnet already does. Left off by default: it would write a real row into the production database on every successful run, and unlike testnet, `POST /links` there has no scoped-credential story yet — see issue #163 (least-privilege API key for this check) before turning it on. | + +Once `UPTIME_MAINNET_API_URL` is set, the next run adds a `## Mainnet` +section to `docs/STATUS.md` and starts filing incidents titled +`🔴 Uptime: Mainnet — API is down` (the environment name is always in the +title and body — see `renderStatusMd`/`buildTargets` in the script) instead of +the ambiguous `🔴 Uptime: API is down` a pre-8.8 reader might mistake for +testnet. + +**The scheduled run itself is still disabled** (`.github/workflows/uptime.yml` +only has `workflow_dispatch`, no `schedule` — see `TODO.md` §5). Re-enabling +it and setting the variables above are both owner actions: this doc only +covers what to set once you do. + ## Deploy Render deploys `apps/api` as a single always-on Docker web service (starter diff --git a/scripts/uptime-check.mjs b/scripts/uptime-check.mjs index 59c012f39..de7379df2 100644 --- a/scripts/uptime-check.mjs +++ b/scripts/uptime-check.mjs @@ -1,9 +1,22 @@ #!/usr/bin/env node -// Uptime + synthetic check for the live demo. Run standalone (`pnpm sweep`) or -// on a schedule (.github/workflows/uptime.yml), which also persists history to -// docs/uptime-state.json and regenerates docs/STATUS.md + the README badges. +// Uptime + synthetic check, multi-environment (issue 8.8). Run standalone +// (`pnpm sweep`) or on a schedule (.github/workflows/uptime.yml), which also +// persists history to docs/uptime-state.json and regenerates docs/STATUS.md + +// the README badges. // // No external monitoring service required: this is the whole check. +// +// Environments: testnet is always checked, using the same env vars and target +// ids (`api` / `web` / `synthetic`) this script has always used — existing +// history and badge files keep working with no migration. Mainnet is checked +// only once UPTIME_MAINNET_API_URL is actually set (a repo Actions variable, +// see docs/RUNBOOK.md): unlike testnet there is no default of any kind, on +// purpose — render.mainnet.yaml's own guidance is that a default here would +// silently mean "the testnet sandbox", and a mainnet outage that goes +// unreported because the checker quietly monitored the wrong service is worse +// than one that's honestly unconfigured. Mainnet's targets are prefixed +// (`mainnet-api` / `mainnet-web` / `mainnet-synthetic`) so they get their own +// history series and never collide with testnet's. import { readFile, writeFile, mkdir } from "node:fs/promises"; import { fileURLToPath } from "node:url"; @@ -14,17 +27,89 @@ const root = resolve(here, ".."); const statePath = resolve(root, "docs/uptime-state.json"); const statusPath = resolve(root, "docs/STATUS.md"); -const API_URL = process.env.UPTIME_API_URL ?? "https://quay-api.onrender.com"; -const WEB_URL = process.env.UPTIME_WEB_URL ?? "https://quay-web.vercel.app"; const HISTORY_DAYS = 90; const FETCH_TIMEOUT_MS = 15000; const FAILURE_THRESHOLD = 2; -const TARGETS = [ - { id: "api", label: "API", check: () => checkGet(`${API_URL}/health`) }, - { id: "web", label: "Web dashboard", check: () => checkGet(WEB_URL) }, - { id: "synthetic", label: "Create-link (synthetic)", check: () => checkSyntheticLink(API_URL) }, -]; +function envUrl(vars, name) { + const v = vars[name]; + return v && v.trim() ? v.trim() : null; +} + +/** + * The environments this run checks. `apiUrl: null` means "not configured" — + * filtered out below rather than checked against a guessed URL. + */ +export function buildEnvironments(vars = process.env) { + return [ + { + id: "testnet", + label: "Testnet", + // Back-compat: UPTIME_API_URL / UPTIME_WEB_URL are the original, + // unprefixed names this script has always read; always defaults to the + // public testnet deploy so `pnpm sweep` works with zero setup. + apiUrl: envUrl(vars, "UPTIME_TESTNET_API_URL") ?? envUrl(vars, "UPTIME_API_URL") ?? "https://quay-api.onrender.com", + webUrl: envUrl(vars, "UPTIME_TESTNET_WEB_URL") ?? envUrl(vars, "UPTIME_WEB_URL") ?? "https://quay-web.vercel.app", + // Leaves a tiny throwaway link behind on every successful run (title + // "uptime-check") — proves the public write path works. Known + // trade-off for a demo-scale DB; see checkSyntheticLink. + syntheticLink: true, + prefixIds: false, + }, + { + id: "mainnet", + label: "Mainnet", + apiUrl: envUrl(vars, "UPTIME_MAINNET_API_URL"), + webUrl: envUrl(vars, "UPTIME_MAINNET_WEB_URL"), + // Off unless explicitly opted into: this would leave a throwaway row in + // the REAL production database on every successful run, and — unlike + // testnet — POST /links there has no authorization story yet (issue + // #163 tracks a least-privilege API key for this). Don't hit live + // infrastructure with an unauthenticated write until that lands. + syntheticLink: envUrl(vars, "UPTIME_MAINNET_SYNTHETIC_CHECK") === "1", + prefixIds: true, + }, + ]; +} + +/** Environments that actually have an API URL configured — the rest are skipped, not guessed. */ +export function activeEnvironments(environments) { + return environments.filter((env) => env.apiUrl); +} + +/** Per-environment targets: API (always), web (if configured), synthetic-link (if enabled). */ +export function buildTargets(environments) { + const targets = []; + for (const env of activeEnvironments(environments)) { + const prefix = env.prefixIds ? `${env.id}-` : ""; + targets.push({ + id: `${prefix}api`, + kind: "API", + label: `${env.label} — API`, + env, + check: () => checkGet(`${env.apiUrl}/health`), + }); + if (env.webUrl) { + targets.push({ + id: `${prefix}web`, + kind: "Web dashboard", + label: `${env.label} — Web dashboard`, + env, + check: () => checkGet(env.webUrl), + }); + } + if (env.syntheticLink) { + targets.push({ + id: `${prefix}synthetic`, + kind: "Create-link (synthetic)", + label: `${env.label} — Create-link (synthetic)`, + env, + check: () => checkSyntheticLink(env.apiUrl), + }); + } + } + return targets; +} async function checkGet(url) { const res = await fetchWithTimeout(url, { method: "GET" }); @@ -72,7 +157,7 @@ function emptyTargetState() { } /** Mutates `state.targets[id]` with this run's result; returns { justFailed, justRecovered }. */ -function recordResult(state, id, ok, message) { +export function recordResult(state, id, ok, message) { const t = (state.targets[id] ??= emptyTargetState()); const day = todayUTC(); @@ -97,7 +182,7 @@ function recordResult(state, id, ok, message) { return { justFailed: !wasFailing && isFailing, justRecovered: wasFailing && ok }; } -function uptimePct(history, today) { +export function uptimePct(history, today) { const days = today ? [...history, { up: today.up, down: today.down }] : history; const totals = days.reduce((acc, d) => ({ up: acc.up + d.up, down: acc.down + d.down }), { up: 0, down: 0 }); const total = totals.up + totals.down; @@ -116,39 +201,49 @@ function renderBadge(targetState, id) { }; } -function renderStatusMd(state) { +/** Grouped by environment, so a green testnet section can never stand in for a missing mainnet one. */ +export function renderStatusMd(state, environments) { + const targets = buildTargets(environments); const lines = [ "# Status", "", "Generated by `.github/workflows/uptime.yml` (every 5 minutes) — do not edit by hand.", "", ]; - for (const target of TARGETS) { - const t = state.targets[target.id]; - if (!t) continue; - const pct = uptimePct(t.history, t.today); - lines.push(`## ${target.label}`); + for (const env of activeEnvironments(environments)) { + const envTargets = targets.filter((t) => t.env === env); + if (!envTargets.some((t) => state.targets[t.id])) continue; // never checked yet — nothing to report + lines.push(`## ${env.label}`); lines.push(""); - lines.push(`- Status: **${t.lastStatus === "up" ? "🟢 up" : "🔴 down"}** (last checked ${t.lastCheckedAt})`); - lines.push(`- Uptime (last ${HISTORY_DAYS} days): **${pct.toFixed(2)}%**`); - if (t.lastError) lines.push(`- Last error: \`${t.lastError}\``); - lines.push(""); - lines.push("| Date | Up | Down |"); - lines.push("| --- | --- | --- |"); - const rows = [...t.history, t.today].filter(Boolean).slice(-HISTORY_DAYS); - for (const row of rows.slice().reverse()) { - lines.push(`| ${row.date} | ${row.up} | ${row.down} |`); + for (const target of envTargets) { + const t = state.targets[target.id]; + if (!t) continue; + const pct = uptimePct(t.history, t.today); + lines.push(`### ${target.kind}`); + lines.push(""); + lines.push(`- Status: **${t.lastStatus === "up" ? "🟢 up" : "🔴 down"}** (last checked ${t.lastCheckedAt})`); + lines.push(`- Uptime (last ${HISTORY_DAYS} days): **${pct.toFixed(2)}%**`); + if (t.lastError) lines.push(`- Last error: \`${t.lastError}\``); + lines.push(""); + lines.push("| Date | Up | Down |"); + lines.push("| --- | --- | --- |"); + const rows = [...t.history, t.today].filter(Boolean).slice(-HISTORY_DAYS); + for (const row of rows.slice().reverse()) { + lines.push(`| ${row.date} | ${row.up} | ${row.down} |`); + } + lines.push(""); } - lines.push(""); } return lines.join("\n"); } async function main() { + const environments = buildEnvironments(process.env); + const targets = buildTargets(environments); const state = await loadState(); const events = []; - for (const target of TARGETS) { + for (const target of targets) { let ok = true; let message = null; try { @@ -165,22 +260,31 @@ async function main() { await mkdir(dirname(statePath), { recursive: true }); await writeFile(statePath, JSON.stringify(state, null, 2) + "\n"); - await writeFile(statusPath, renderStatusMd(state) + "\n"); + await writeFile(statusPath, renderStatusMd(state, environments) + "\n"); - for (const target of TARGETS) { + for (const target of targets) { const badgePath = resolve(root, `docs/uptime-badge-${target.id}.json`); await writeFile(badgePath, JSON.stringify(renderBadge(state.targets[target.id], target.id), null, 2) + "\n"); } if (events.length > 0) { - console.log("EVENTS_JSON=" + JSON.stringify(events.map((e) => ({ type: e.type, target: e.target.id, label: e.target.label })))); + console.log( + "EVENTS_JSON=" + + JSON.stringify(events.map((e) => ({ type: e.type, target: e.target.id, label: e.target.label }))), + ); } - const anyDown = TARGETS.some((t) => state.targets[t.id]?.consecutiveFailures >= FAILURE_THRESHOLD); + const anyDown = targets.some((t) => state.targets[t.id]?.consecutiveFailures >= FAILURE_THRESHOLD); if (anyDown && process.env.UPTIME_STRICT_EXIT === "1") process.exitCode = 1; } -main().catch((err) => { - console.error("[uptime] fatal:", err); - process.exitCode = 1; -}); +function isCliInvocation() { + return process.argv[1] !== undefined && fileURLToPath(import.meta.url) === process.argv[1]; +} + +if (isCliInvocation()) { + main().catch((err) => { + console.error("[uptime] fatal:", err); + process.exitCode = 1; + }); +} diff --git a/scripts/uptime-check.test.ts b/scripts/uptime-check.test.ts new file mode 100644 index 000000000..71b8291ca --- /dev/null +++ b/scripts/uptime-check.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; +import { activeEnvironments, buildEnvironments, buildTargets, recordResult, renderStatusMd, uptimePct } from "./uptime-check.mjs"; + +describe("buildEnvironments", () => { + it("testnet always defaults to the public testnet deploy, unprefixed", () => { + const [testnet] = buildEnvironments({}); + expect(testnet.id).toBe("testnet"); + expect(testnet.apiUrl).toBe("https://quay-api.onrender.com"); + expect(testnet.webUrl).toBe("https://quay-web.vercel.app"); + expect(testnet.syntheticLink).toBe(true); + expect(testnet.prefixIds).toBe(false); + }); + + it("testnet honors the original UPTIME_API_URL / UPTIME_WEB_URL var names", () => { + const [testnet] = buildEnvironments({ + UPTIME_API_URL: "https://custom-api.example", + UPTIME_WEB_URL: "https://custom-web.example", + }); + expect(testnet.apiUrl).toBe("https://custom-api.example"); + expect(testnet.webUrl).toBe("https://custom-web.example"); + }); + + it("mainnet has no URL default of any kind — unset means unconfigured, not guessed", () => { + const [, mainnet] = buildEnvironments({}); + expect(mainnet.id).toBe("mainnet"); + expect(mainnet.apiUrl).toBeNull(); + expect(mainnet.webUrl).toBeNull(); + }); + + it("mainnet picks up its URLs once configured, and prefixes its target ids", () => { + const [, mainnet] = buildEnvironments({ + UPTIME_MAINNET_API_URL: "https://quay-api-mainnet.onrender.com", + UPTIME_MAINNET_WEB_URL: "https://quay-web-mainnet.example", + }); + expect(mainnet.apiUrl).toBe("https://quay-api-mainnet.onrender.com"); + expect(mainnet.webUrl).toBe("https://quay-web-mainnet.example"); + expect(mainnet.prefixIds).toBe(true); + }); + + it("mainnet's synthetic-link check stays off unless explicitly opted into", () => { + const [, withoutOptIn] = buildEnvironments({ UPTIME_MAINNET_API_URL: "https://mainnet.example" }); + expect(withoutOptIn.syntheticLink).toBe(false); + + const [, withOptIn] = buildEnvironments({ + UPTIME_MAINNET_API_URL: "https://mainnet.example", + UPTIME_MAINNET_SYNTHETIC_CHECK: "1", + }); + expect(withOptIn.syntheticLink).toBe(true); + }); +}); + +describe("activeEnvironments", () => { + it("drops any environment with no API URL configured", () => { + const environments = buildEnvironments({}); + expect(activeEnvironments(environments).map((e) => e.id)).toEqual(["testnet"]); + }); + + it("includes mainnet once its API URL is set", () => { + const environments = buildEnvironments({ UPTIME_MAINNET_API_URL: "https://mainnet.example" }); + expect(activeEnvironments(environments).map((e) => e.id)).toEqual(["testnet", "mainnet"]); + }); +}); + +describe("buildTargets", () => { + it("testnet keeps its original, unprefixed target ids (back-compat with existing history/badges)", () => { + const environments = buildEnvironments({}); + const ids = buildTargets(environments).map((t) => t.id); + expect(ids).toEqual(["api", "web", "synthetic"]); + }); + + it("mainnet's target ids are prefixed and never collide with testnet's", () => { + const environments = buildEnvironments({ + UPTIME_MAINNET_API_URL: "https://mainnet.example", + UPTIME_MAINNET_WEB_URL: "https://mainnet-web.example", + UPTIME_MAINNET_SYNTHETIC_CHECK: "1", + }); + const ids = buildTargets(environments).map((t) => t.id); + expect(ids).toEqual(["api", "web", "synthetic", "mainnet-api", "mainnet-web", "mainnet-synthetic"]); + }); + + it("omits the web target for an environment with no web URL, and the synthetic target when disabled", () => { + const environments = buildEnvironments({ UPTIME_MAINNET_API_URL: "https://mainnet.example" }); + const mainnetIds = buildTargets(environments) + .filter((t) => t.env.id === "mainnet") + .map((t) => t.id); + expect(mainnetIds).toEqual(["mainnet-api"]); + }); + + it("labels every target with its environment, so an issue title can never be ambiguous about which one", () => { + const environments = buildEnvironments({ UPTIME_MAINNET_API_URL: "https://mainnet.example" }); + const targets = buildTargets(environments); + expect(targets.find((t) => t.id === "api")?.label).toBe("Testnet — API"); + expect(targets.find((t) => t.id === "mainnet-api")?.label).toBe("Mainnet — API"); + }); +}); + +describe("recordResult / uptimePct", () => { + it("tracks consecutive failures per target id independently", () => { + const state = { targets: {} }; + recordResult(state, "testnet-api", false, "boom"); + const { justFailed } = recordResult(state, "testnet-api", false, "boom"); + expect(justFailed).toBe(true); + // A different id (e.g. mainnet's own "api") must not share this counter. + expect(state.targets["mainnet-api"]).toBeUndefined(); + }); + + it("uptimePct is 100 with no data, and reflects a mixed today", () => { + expect(uptimePct([], null)).toBe(100); + expect(uptimePct([], { up: 3, down: 1 })).toBe(75); + }); +}); + +describe("renderStatusMd", () => { + it("only reports environments that have actually been checked", () => { + const environments = buildEnvironments({ UPTIME_MAINNET_API_URL: "https://mainnet.example" }); + const state = { targets: {} }; + recordResult(state, "api", true, null); + + const md = renderStatusMd(state, environments); + expect(md).toContain("## Testnet"); + expect(md).not.toContain("## Mainnet"); + }); + + it("gives each environment its own section, with per-kind subsections underneath", () => { + const environments = buildEnvironments({ UPTIME_MAINNET_API_URL: "https://mainnet.example" }); + const state = { targets: {} }; + recordResult(state, "api", true, null); + recordResult(state, "mainnet-api", false, "connection refused"); + + const md = renderStatusMd(state, environments); + const testnetIdx = md.indexOf("## Testnet"); + const mainnetIdx = md.indexOf("## Mainnet"); + expect(testnetIdx).toBeGreaterThanOrEqual(0); + expect(mainnetIdx).toBeGreaterThan(testnetIdx); + expect(md.indexOf("### API", testnetIdx)).toBeLessThan(mainnetIdx); + expect(md).toContain("🔴 down"); + expect(md).toContain("connection refused"); + }); +});