feat: website health check cron (standalone GH Actions workflow)

.github/workflows/healthcheck.yml

@@ -0,0 +1,101 @@
+# Website health check
+#
+# Standalone scheduled probe of the production site, independent of ci.yml/cd.yml.
+# Every 12h (and on demand) it curls the public, header-gated liveness endpoint
+# https://usegofin.com/healthz, which returns 200 only when the Cloudflare edge,
+# tunnel, shell, gateway, and all four downstream services (auth, expense,
+# finance, datarights) are healthy; 503 (or unreachable) otherwise. On failure
+# the job goes red AND posts exactly one Discord alert. Success is silent.
+#
+# Cloudflare gate: a WAF Custom Rule (action = Skip) bypasses Under Attack Mode for
+#   http.request.uri.path eq "/healthz"
+#     and http.request.headers["x-health-token"][0] eq <HEALTHCHECK_TOKEN>
+# The token value lives only in the GH secret HEALTHCHECK_TOKEN and in the
+# Cloudflare rule expression (never committed). Without the header the same
+# request receives a 403 challenge, so this probe is the only automated client
+# that can reach the origin.
+#
+# Secrets (repo-level GH Actions):
+#   HEALTHCHECK_TOKEN   - edge bypass header value (matches the Cloudflare rule)
+#   DISCORD_WEBHOOK_URL - failure-alert webhook (reuses the Alertmanager webhook)
+
+name: Healthcheck
+
+on:
+  schedule:
+    # Every 12 hours (UTC, best-effort per GitHub's scheduler).
+    - cron: "0 */12 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: healthcheck
+  cancel-in-progress: false
+
+jobs:
+  healthcheck:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Probe production /healthz
+        env:
+          HEALTHCHECK_TOKEN: ${{ secrets.HEALTHCHECK_TOKEN }}
+        run: |
+          set -euo pipefail
+          url="https://usegofin.com/healthz"
+          attempts=3
+          delay=10
+          http_code="000"
+
+          # Bounded retry so a single transient blip (slow deploy, brief network
+          # hiccup) does not raise a false alarm. Success short-circuits.
+          for i in $(seq 1 "$attempts"); do
+            http_code=$(curl -sS -m 15 \
+              -H "X-Health-Token: ${HEALTHCHECK_TOKEN}" \
+              -o /tmp/healthz_body \
+              -w "%{http_code}" \
+              "$url") || true
+            echo "Attempt ${i}/${attempts}: HTTP ${http_code}"
+            if [ "$http_code" = "200" ]; then
+              echo "healthz OK"
+              exit 0
+            fi
+            if [ "$i" -lt "$attempts" ]; then
+              sleep "$delay"
+            fi
+          done
+
+          body="$(cat /tmp/healthz_body 2>/dev/null || true)"
+          echo "healthz FAILED after ${attempts} attempts (last HTTP ${http_code})"
+          echo "Response body: ${body}"
+
+          # Hand the failure detail to the Discord step via the environment file.
+          {
+            echo "HEALTHCHECK_CODE=${http_code}"
+            echo "HEALTHCHECK_BODY<<HEALTHZ_EOF"
+            echo "${body}"
+            echo "HEALTHZ_EOF"
+          } >> "$GITHUB_ENV"
+          exit 1
+
+      - name: Notify Discord on failure
+        if: failure()
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+        run: |
+          set -euo pipefail
+          run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          content=$(printf ':rotating_light: gofin production healthcheck FAILED\nRepo: %s\nStatus: %s\nBody: %s\nRun: %s' \
+            "${GITHUB_REPOSITORY}" \
+            "${HEALTHCHECK_CODE:-unreachable}" \
+            "${HEALTHCHECK_BODY:-<none>}" \
+            "${run_url}")
+          # jq builds and escapes the JSON payload so newlines/quotes in the
+          # relayed body cannot break it.
+          payload=$(jq -n --arg content "$content" '{content: $content}')
+          curl -sS -m 15 -X POST \
+            -H "Content-Type: application/json" \
+            -d "$payload" \
+            "$DISCORD_WEBHOOK_URL"

frontend/apps/shell/server/__tests__/healthz.test.ts

@@ -0,0 +1,118 @@
+import { describe, it, expect, vi } from "vitest";
+import type { Response } from "express";
+import { createHealthzHandler } from "../healthz";
+
+const GATEWAY_URL = "http://gateway:8080";
+
+// Captures what the handler wrote to the express Response. The handler only
+// touches status/type/send/json, so a minimal chainable stub stands in for the
+// full express Response (one boundary cast).
+interface CapturedResponse {
+  status: number;
+  contentType?: string;
+  body?: string | Record<string, unknown>;
+}
+
+function createMockRes(captured: CapturedResponse): Response {
+  const res = {
+    status(code: number) {
+      captured.status = code;
+      return res;
+    },
+    type(value: string) {
+      captured.contentType = value;
+      return res;
+    },
+    send(payload: string) {
+      captured.body = payload;
+      return res;
+    },
+    json(payload: Record<string, unknown>) {
+      captured.body = payload;
+      return res;
+    },
+  };
+  return res as unknown as Response;
+}
+
+async function invokeHandler(fetchFn: typeof fetch): Promise<CapturedResponse> {
+  const captured: CapturedResponse = { status: 0 };
+  const handler = createHealthzHandler({
+    gatewayUrl: GATEWAY_URL,
+    fetchFn,
+    timeoutMs: 5000,
+  });
+  // The handler ignores the request; a bare object stands in for it.
+  await handler(
+    {} as never,
+    createMockRes(captured),
+    (() => {}) as never,
+  );
+  return captured;
+}
+
+describe("createHealthzHandler", () => {
+  it("relays 200 and the gateway body when /readyz is healthy", async () => {
+    const fetchFn = vi.fn(async () => ({
+      ok: true,
+      status: 200,
+      text: async () => '{"status":"ok"}',
+    })) as unknown as typeof fetch;
+
+    const captured = await invokeHandler(fetchFn);
+
+    expect(fetchFn).toHaveBeenCalledWith(
+      `${GATEWAY_URL}/readyz`,
+      expect.objectContaining({ signal: expect.any(AbortSignal) }),
+    );
+    expect(captured.status).toBe(200);
+    expect(captured.contentType).toBe("application/json");
+    expect(captured.body).toBe('{"status":"ok"}');
+  });
+
+  it("relays 503 and the gateway body naming the failing service", async () => {
+    const gatewayBody =
+      '{"status":"unhealthy","services":{"expense":"unreachable"}}';
+    const fetchFn = vi.fn(async () => ({
+      ok: false,
+      status: 503,
+      text: async () => gatewayBody,
+    })) as unknown as typeof fetch;
+
+    const captured = await invokeHandler(fetchFn);
+
+    expect(captured.status).toBe(503);
+    expect(captured.contentType).toBe("application/json");
+    expect(captured.body).toBe(gatewayBody);
+  });
+
+  it("returns 503 gateway_unreachable when fetch rejects", async () => {
+    const fetchFn = vi.fn(async () => {
+      throw new Error("ECONNREFUSED");
+    }) as unknown as typeof fetch;
+
+    const captured = await invokeHandler(fetchFn);
+
+    expect(captured.status).toBe(503);
+    expect(captured.body).toEqual({
+      status: "unhealthy",
+      reason: "gateway_unreachable",
+    });
+  });
+
+  it("returns 503 gateway_unreachable when the request times out", async () => {
+    // AbortSignal.timeout aborts with a DOMException; the handler must treat a
+    // timeout the same as any other failure and never hang.
+    const fetchFn = vi.fn(async () => {
+      throw new DOMException("The operation timed out.", "TimeoutError");
+    }) as unknown as typeof fetch;
+
+    const captured = await invokeHandler(fetchFn);
+
+    expect(captured.status).toBe(503);
+    expect(captured.body).toEqual({
+      status: "unhealthy",
+      reason: "gateway_unreachable",
+    });
+  });
+});

frontend/apps/shell/server/app.ts

@@ -4,6 +4,7 @@ import express from "express";
 import { createProxyMiddleware } from "http-proxy-middleware";
 import path from "path";
 import { fileURLToPath } from "url";
+import { createHealthzHandler } from "./healthz";
 
 const API_GATEWAY_URL =
   process.env.API_GATEWAY_URL || "http://localhost:8080";
@@ -51,6 +52,19 @@ app.use(
   }),
 );
 
+// Public liveness endpoint for the external health-check cron. Mirrors the
+// gateway's /readyz aggregate (200 when all backend services are healthy, else
+// 503) and never hangs on a slow gateway. Mounted after the /api proxy and
+// before the SSR catch-all so it is not proxied or swallowed by SSR.
+app.get(
+  "/healthz",
+  createHealthzHandler({
+    gatewayUrl: API_GATEWAY_URL,
+    fetchFn: fetch,
+    timeoutMs: 5000,
+  }),
+);
+
 // SSR: React Router handles all non-API routes
 app.use(
   createRequestHandler({

frontend/apps/shell/server/healthz.ts

@@ -0,0 +1,39 @@
+import type { RequestHandler } from "express";
+
+/**
+ * Dependencies for {@link createHealthzHandler}. Injected so the handler is unit
+ * testable without booting SSR or a real gateway.
+ */
+interface HealthzDeps {
+  gatewayUrl: string;
+  fetchFn: typeof fetch;
+  timeoutMs: number;
+}
+
+/**
+ * createHealthzHandler builds the shell's public /healthz liveness handler.
+ *
+ * It mirrors the gateway's /readyz aggregate: 200 when the gateway reports every
+ * backend service healthy, 503 otherwise. The gateway's JSON body is relayed
+ * verbatim so CI logs show which downstream service failed. A hung or
+ * unreachable gateway is bounded by AbortSignal.timeout, so the endpoint always
+ * responds (503) rather than hanging.
+ */
+export const createHealthzHandler =
+  ({ gatewayUrl, fetchFn, timeoutMs }: HealthzDeps): RequestHandler =>
+  async (_req, res) => {
+    try {
+      const response = await fetchFn(`${gatewayUrl}/readyz`, {
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+      const body = await response.text();
+      res
+        .status(response.ok ? 200 : 503)
+        .type("application/json")
+        .send(body);
+    } catch {
+      res
+        .status(503)
+        .json({ status: "unhealthy", reason: "gateway_unreachable" });
+    }
+  };

frontend/apps/shell/vitest.config.ts

@@ -6,7 +6,7 @@ export default defineConfig({
     environment: "jsdom",
     globals: true,
     setupFiles: ["./app/__tests__/setup.ts"],
-    include: ["app/__tests__/**/*.test.{ts,tsx}", "app/features/**/__tests__/**/*.test.{ts,tsx}"],
+    include: ["app/__tests__/**/*.test.{ts,tsx}", "app/features/**/__tests__/**/*.test.{ts,tsx}", "server/**/__tests__/**/*.test.{ts,tsx}"],
     coverage: {
       provider: "istanbul",
       reporter: ["text", "lcov"],

services/gateway/internal/access/control.go

@@ -132,12 +132,12 @@ func AccessControl(validator TokenValidator, resolve func(method, path string) s
 }
 
 // GatewayResolve is the resolver the gateway injects into AccessControl. It
-// classifies the two gateway-native endpoints (/health, /metrics) as Public
-// and delegates every /api route to the shared registry resolver. Keeping this
-// composition in the gateway is why services/access never needs to know about
-// gateway-owned routes.
+// classifies the gateway-native endpoints (/health, /metrics, /readyz) as
+// Public and delegates every /api route to the shared registry resolver.
+// Keeping this composition in the gateway is why services/access never needs to
+// know about gateway-owned routes.
 func GatewayResolve(method, path string) sharedaccess.Level {
-	if path == "/health" || path == "/metrics" {
+	if path == "/health" || path == "/metrics" || path == "/readyz" {
 		return sharedaccess.Public
 	}
 	return sharedaccess.Resolve(method, path)

services/gateway/internal/access/control_test.go

@@ -486,6 +486,7 @@ func TestGatewayResolve(t *testing.T) {
 	}{
 		{"health is public", http.MethodGet, "/health", sharedaccess.Public},
 		{"metrics is public", http.MethodGet, "/metrics", sharedaccess.Public},
+		{"readyz is public", http.MethodGet, "/readyz", sharedaccess.Public},
 		{"a real personal route keeps its level", http.MethodGet, "/api/finance/periods", sharedaccess.Personal},
 		{"a real admin route keeps its level", http.MethodGet, "/api/admin/users", sharedaccess.Admin},
 		{"an unknown path falls to deny", http.MethodGet, "/api/unknown", sharedaccess.Deny},