diff --git a/.github/workflows/healthcheck.yml b/.github/workflows/healthcheck.yml new file mode 100644 index 00000000..76f2f4d2 --- /dev/null +++ b/.github/workflows/healthcheck.yml @@ -0,0 +1,101 @@ +# Website health check +# +# Standalone scheduled probe of the production site, independent of ci.yml/cd.yml. +# Every 12h (and on demand) it curls the public, header-gated liveness endpoint +# https://usegofin.com/healthz, which returns 200 only when the Cloudflare edge, +# tunnel, shell, gateway, and all four downstream services (auth, expense, +# finance, datarights) are healthy; 503 (or unreachable) otherwise. On failure +# the job goes red AND posts exactly one Discord alert. Success is silent. +# +# Cloudflare gate: a WAF Custom Rule (action = Skip) bypasses Under Attack Mode for +# http.request.uri.path eq "/healthz" +# and http.request.headers["x-health-token"][0] eq +# The token value lives only in the GH secret HEALTHCHECK_TOKEN and in the +# Cloudflare rule expression (never committed). Without the header the same +# request receives a 403 challenge, so this probe is the only automated client +# that can reach the origin. +# +# Secrets (repo-level GH Actions): +# HEALTHCHECK_TOKEN - edge bypass header value (matches the Cloudflare rule) +# DISCORD_WEBHOOK_URL - failure-alert webhook (reuses the Alertmanager webhook) + +name: Healthcheck + +on: + schedule: + # Every 12 hours (UTC, best-effort per GitHub's scheduler). + - cron: "0 */12 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: healthcheck + cancel-in-progress: false + +jobs: + healthcheck: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Probe production /healthz + env: + HEALTHCHECK_TOKEN: ${{ secrets.HEALTHCHECK_TOKEN }} + run: | + set -euo pipefail + url="https://usegofin.com/healthz" + attempts=3 + delay=10 + http_code="000" + + # Bounded retry so a single transient blip (slow deploy, brief network + # hiccup) does not raise a false alarm. Success short-circuits. + for i in $(seq 1 "$attempts"); do + http_code=$(curl -sS -m 15 \ + -H "X-Health-Token: ${HEALTHCHECK_TOKEN}" \ + -o /tmp/healthz_body \ + -w "%{http_code}" \ + "$url") || true + echo "Attempt ${i}/${attempts}: HTTP ${http_code}" + if [ "$http_code" = "200" ]; then + echo "healthz OK" + exit 0 + fi + if [ "$i" -lt "$attempts" ]; then + sleep "$delay" + fi + done + + body="$(cat /tmp/healthz_body 2>/dev/null || true)" + echo "healthz FAILED after ${attempts} attempts (last HTTP ${http_code})" + echo "Response body: ${body}" + + # Hand the failure detail to the Discord step via the environment file. + { + echo "HEALTHCHECK_CODE=${http_code}" + echo "HEALTHCHECK_BODY<> "$GITHUB_ENV" + exit 1 + + - name: Notify Discord on failure + if: failure() + env: + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + run: | + set -euo pipefail + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + content=$(printf ':rotating_light: gofin production healthcheck FAILED\nRepo: %s\nStatus: %s\nBody: %s\nRun: %s' \ + "${GITHUB_REPOSITORY}" \ + "${HEALTHCHECK_CODE:-unreachable}" \ + "${HEALTHCHECK_BODY:-}" \ + "${run_url}") + # jq builds and escapes the JSON payload so newlines/quotes in the + # relayed body cannot break it. + payload=$(jq -n --arg content "$content" '{content: $content}') + curl -sS -m 15 -X POST \ + -H "Content-Type: application/json" \ + -d "$payload" \ + "$DISCORD_WEBHOOK_URL" diff --git a/frontend/apps/shell/server/__tests__/healthz.test.ts b/frontend/apps/shell/server/__tests__/healthz.test.ts new file mode 100644 index 00000000..c799af58 --- /dev/null +++ b/frontend/apps/shell/server/__tests__/healthz.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect, vi } from "vitest"; +import type { Response } from "express"; +import { createHealthzHandler } from "../healthz"; + +const GATEWAY_URL = "http://gateway:8080"; + +// Captures what the handler wrote to the express Response. The handler only +// touches status/type/send/json, so a minimal chainable stub stands in for the +// full express Response (one boundary cast). +interface CapturedResponse { + status: number; + contentType?: string; + body?: string | Record; +} + +function createMockRes(captured: CapturedResponse): Response { + const res = { + status(code: number) { + captured.status = code; + return res; + }, + type(value: string) { + captured.contentType = value; + return res; + }, + send(payload: string) { + captured.body = payload; + return res; + }, + json(payload: Record) { + captured.body = payload; + return res; + }, + }; + return res as unknown as Response; +} + +async function invokeHandler(fetchFn: typeof fetch): Promise { + const captured: CapturedResponse = { status: 0 }; + const handler = createHealthzHandler({ + gatewayUrl: GATEWAY_URL, + fetchFn, + timeoutMs: 5000, + }); + // The handler ignores the request; a bare object stands in for it. + await handler( + {} as never, + createMockRes(captured), + (() => {}) as never, + ); + return captured; +} + +describe("createHealthzHandler", () => { + it("relays 200 and the gateway body when /readyz is healthy", async () => { + const fetchFn = vi.fn(async () => ({ + ok: true, + status: 200, + text: async () => '{"status":"ok"}', + })) as unknown as typeof fetch; + + const captured = await invokeHandler(fetchFn); + + expect(fetchFn).toHaveBeenCalledWith( + `${GATEWAY_URL}/readyz`, + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + expect(captured.status).toBe(200); + expect(captured.contentType).toBe("application/json"); + expect(captured.body).toBe('{"status":"ok"}'); + }); + + it("relays 503 and the gateway body naming the failing service", async () => { + const gatewayBody = + '{"status":"unhealthy","services":{"expense":"unreachable"}}'; + const fetchFn = vi.fn(async () => ({ + ok: false, + status: 503, + text: async () => gatewayBody, + })) as unknown as typeof fetch; + + const captured = await invokeHandler(fetchFn); + + expect(captured.status).toBe(503); + expect(captured.contentType).toBe("application/json"); + expect(captured.body).toBe(gatewayBody); + }); + + it("returns 503 gateway_unreachable when fetch rejects", async () => { + const fetchFn = vi.fn(async () => { + throw new Error("ECONNREFUSED"); + }) as unknown as typeof fetch; + + const captured = await invokeHandler(fetchFn); + + expect(captured.status).toBe(503); + expect(captured.body).toEqual({ + status: "unhealthy", + reason: "gateway_unreachable", + }); + }); + + it("returns 503 gateway_unreachable when the request times out", async () => { + // AbortSignal.timeout aborts with a DOMException; the handler must treat a + // timeout the same as any other failure and never hang. + const fetchFn = vi.fn(async () => { + throw new DOMException("The operation timed out.", "TimeoutError"); + }) as unknown as typeof fetch; + + const captured = await invokeHandler(fetchFn); + + expect(captured.status).toBe(503); + expect(captured.body).toEqual({ + status: "unhealthy", + reason: "gateway_unreachable", + }); + }); +}); diff --git a/frontend/apps/shell/server/app.ts b/frontend/apps/shell/server/app.ts index 8d26c197..1fc8c8f2 100644 --- a/frontend/apps/shell/server/app.ts +++ b/frontend/apps/shell/server/app.ts @@ -4,6 +4,7 @@ import express from "express"; import { createProxyMiddleware } from "http-proxy-middleware"; import path from "path"; import { fileURLToPath } from "url"; +import { createHealthzHandler } from "./healthz"; const API_GATEWAY_URL = process.env.API_GATEWAY_URL || "http://localhost:8080"; @@ -51,6 +52,19 @@ app.use( }), ); +// Public liveness endpoint for the external health-check cron. Mirrors the +// gateway's /readyz aggregate (200 when all backend services are healthy, else +// 503) and never hangs on a slow gateway. Mounted after the /api proxy and +// before the SSR catch-all so it is not proxied or swallowed by SSR. +app.get( + "/healthz", + createHealthzHandler({ + gatewayUrl: API_GATEWAY_URL, + fetchFn: fetch, + timeoutMs: 5000, + }), +); + // SSR: React Router handles all non-API routes app.use( createRequestHandler({ diff --git a/frontend/apps/shell/server/healthz.ts b/frontend/apps/shell/server/healthz.ts new file mode 100644 index 00000000..cef7f7d0 --- /dev/null +++ b/frontend/apps/shell/server/healthz.ts @@ -0,0 +1,39 @@ +import type { RequestHandler } from "express"; + +/** + * Dependencies for {@link createHealthzHandler}. Injected so the handler is unit + * testable without booting SSR or a real gateway. + */ +interface HealthzDeps { + gatewayUrl: string; + fetchFn: typeof fetch; + timeoutMs: number; +} + +/** + * createHealthzHandler builds the shell's public /healthz liveness handler. + * + * It mirrors the gateway's /readyz aggregate: 200 when the gateway reports every + * backend service healthy, 503 otherwise. The gateway's JSON body is relayed + * verbatim so CI logs show which downstream service failed. A hung or + * unreachable gateway is bounded by AbortSignal.timeout, so the endpoint always + * responds (503) rather than hanging. + */ +export const createHealthzHandler = + ({ gatewayUrl, fetchFn, timeoutMs }: HealthzDeps): RequestHandler => + async (_req, res) => { + try { + const response = await fetchFn(`${gatewayUrl}/readyz`, { + signal: AbortSignal.timeout(timeoutMs), + }); + const body = await response.text(); + res + .status(response.ok ? 200 : 503) + .type("application/json") + .send(body); + } catch { + res + .status(503) + .json({ status: "unhealthy", reason: "gateway_unreachable" }); + } + }; diff --git a/frontend/apps/shell/vitest.config.ts b/frontend/apps/shell/vitest.config.ts index 3741a83c..7967ffd8 100644 --- a/frontend/apps/shell/vitest.config.ts +++ b/frontend/apps/shell/vitest.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ environment: "jsdom", globals: true, setupFiles: ["./app/__tests__/setup.ts"], - include: ["app/__tests__/**/*.test.{ts,tsx}", "app/features/**/__tests__/**/*.test.{ts,tsx}"], + include: ["app/__tests__/**/*.test.{ts,tsx}", "app/features/**/__tests__/**/*.test.{ts,tsx}", "server/**/__tests__/**/*.test.{ts,tsx}"], coverage: { provider: "istanbul", reporter: ["text", "lcov"], diff --git a/services/gateway/internal/access/control.go b/services/gateway/internal/access/control.go index b19631fb..8632b941 100644 --- a/services/gateway/internal/access/control.go +++ b/services/gateway/internal/access/control.go @@ -132,12 +132,12 @@ func AccessControl(validator TokenValidator, resolve func(method, path string) s } // GatewayResolve is the resolver the gateway injects into AccessControl. It -// classifies the two gateway-native endpoints (/health, /metrics) as Public -// and delegates every /api route to the shared registry resolver. Keeping this -// composition in the gateway is why services/access never needs to know about -// gateway-owned routes. +// classifies the gateway-native endpoints (/health, /metrics, /readyz) as +// Public and delegates every /api route to the shared registry resolver. +// Keeping this composition in the gateway is why services/access never needs to +// know about gateway-owned routes. func GatewayResolve(method, path string) sharedaccess.Level { - if path == "/health" || path == "/metrics" { + if path == "/health" || path == "/metrics" || path == "/readyz" { return sharedaccess.Public } return sharedaccess.Resolve(method, path) diff --git a/services/gateway/internal/access/control_test.go b/services/gateway/internal/access/control_test.go index 036cb81c..e8f3c911 100644 --- a/services/gateway/internal/access/control_test.go +++ b/services/gateway/internal/access/control_test.go @@ -486,6 +486,7 @@ func TestGatewayResolve(t *testing.T) { }{ {"health is public", http.MethodGet, "/health", sharedaccess.Public}, {"metrics is public", http.MethodGet, "/metrics", sharedaccess.Public}, + {"readyz is public", http.MethodGet, "/readyz", sharedaccess.Public}, {"a real personal route keeps its level", http.MethodGet, "/api/finance/periods", sharedaccess.Personal}, {"a real admin route keeps its level", http.MethodGet, "/api/admin/users", sharedaccess.Admin}, {"an unknown path falls to deny", http.MethodGet, "/api/unknown", sharedaccess.Deny}, diff --git a/services/gateway/internal/readiness/readiness.go b/services/gateway/internal/readiness/readiness.go new file mode 100644 index 00000000..adec574a --- /dev/null +++ b/services/gateway/internal/readiness/readiness.go @@ -0,0 +1,143 @@ +// Package readiness aggregates the liveness of the downstream services behind +// the gateway. It backs the gateway-native GET /readyz endpoint, which reports +// 200 only when every downstream service's shallow /health returns 200, and 503 +// (naming the unhealthy/unreachable services) otherwise. +// +// It is deliberately separate from the gateway's own /health: /health is the +// shallow container-liveness signal wired to Docker HEALTHCHECK, whereas /readyz +// fans out to the downstream services so a single probe can prove the whole +// backend is reachable. Coupling the two would risk container restart loops on a +// transient downstream blip. +package readiness + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/gin-gonic/gin" +) + +// Per-service status values reported in a Result. A concrete non-200 status is +// reported as "status_" (e.g. "status_503") so the body distinguishes a +// service that answered unhealthily from one that could not be reached at all. +const ( + statusOK = "ok" + statusUnreachable = "unreachable" +) + +// Checker probes a fixed set of downstream services by issuing a concurrent GET +// {baseURL}/health to each, bounded by a per-probe timeout. It is a deep module: +// callers build it once with NewChecker and call Check; the fan-out, timeout, +// and result classification are hidden behind a single method. +type Checker struct { + client *http.Client + services map[string]string // service name -> base URL + timeout time.Duration +} + +// NewChecker builds a Checker over the given service name -> base URL map. The +// keys are the canonical service names (auth|expense|finance|datarights) so the +// 503 body and logs name the same service the gateway proxies to. The injected +// client and timeout keep Check a pure function of its inputs and testable +// against httptest downstreams. +func NewChecker(client *http.Client, services map[string]string, timeout time.Duration) *Checker { + return &Checker{client: client, services: services, timeout: timeout} +} + +// Result is the aggregate outcome of a readiness probe. Services maps each +// service name to its status ("ok" | "unreachable" | "status_"). +type Result struct { + Healthy bool + Services map[string]string +} + +// Check probes every configured service concurrently and returns once all +// probes settle. Healthy is true only when every service's /health returned +// 200. The fan-out is concurrent so total latency is bounded by the slowest +// single probe (the per-probe timeout), not their sum. +func (c *Checker) Check(ctx context.Context) Result { + type probeResult struct { + name string + status string + ok bool + } + + results := make(chan probeResult, len(c.services)) + var waitGroup sync.WaitGroup + for name, baseURL := range c.services { + waitGroup.Add(1) + go func(name, baseURL string) { + defer waitGroup.Done() + status, ok := c.probe(ctx, baseURL) + results <- probeResult{name: name, status: status, ok: ok} + }(name, baseURL) + } + waitGroup.Wait() + close(results) + + statuses := make(map[string]string, len(c.services)) + healthy := true + for result := range results { + statuses[result.name] = result.status + if !result.ok { + healthy = false + } + } + return Result{Healthy: healthy, Services: statuses} +} + +// probe issues a single GET {baseURL}/health bounded by the checker's timeout. +// A transport error or an unbuildable request is "unreachable"; any non-200 +// response is reported as "status_". +func (c *Checker) probe(ctx context.Context, baseURL string) (string, bool) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/health", nil) + if err != nil { + return statusUnreachable, false + } + + resp, err := c.client.Do(req) + if err != nil { + return statusUnreachable, false + } + // Drain then close so the underlying connection can be reused (keep-alive). + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + if resp.StatusCode == http.StatusOK { + return statusOK, true + } + return fmt.Sprintf("status_%d", resp.StatusCode), false +} + +// Handler adapts a Checker into the gin handler for GET /readyz: 200 +// {"status":"ok"} when every downstream is healthy, else 503 +// {"status":"unhealthy","services":{...}} naming each service's state. On a 503 +// it emits a structured warning naming the per-service statuses so the failure +// is diagnosable from logs alone. +func Handler(checker *Checker, logger *slog.Logger) gin.HandlerFunc { + return func(c *gin.Context) { + result := checker.Check(c.Request.Context()) + if result.Healthy { + c.JSON(http.StatusOK, gin.H{"status": "ok"}) + return + } + + logger.Warn("readiness check failed", + slog.Any("services", result.Services), + ) + c.JSON(http.StatusServiceUnavailable, gin.H{ + "status": "unhealthy", + "services": result.Services, + }) + } +} diff --git a/services/gateway/internal/readiness/readiness_test.go b/services/gateway/internal/readiness/readiness_test.go new file mode 100644 index 00000000..36b535cd --- /dev/null +++ b/services/gateway/internal/readiness/readiness_test.go @@ -0,0 +1,207 @@ +package readiness_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ItsThompson/gofin/services/gateway/internal/readiness" +) + +func init() { + gin.SetMode(gin.TestMode) +} + +func silentLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// newHealthServer returns an httptest server whose /health responds with the +// given status code. Any other path 404s, so a probe hitting the wrong path +// surfaces as unhealthy rather than silently passing. +func newHealthServer(t *testing.T, status int) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/health" { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(status) + })) + t.Cleanup(server.Close) + return server +} + +func newChecker(services map[string]string, timeout time.Duration) *readiness.Checker { + return readiness.NewChecker(&http.Client{}, services, timeout) +} + +func TestChecker_AllHealthy_ReportsHealthy(t *testing.T) { + auth := newHealthServer(t, http.StatusOK) + expense := newHealthServer(t, http.StatusOK) + + checker := newChecker(map[string]string{ + "auth": auth.URL, + "expense": expense.URL, + }, 2*time.Second) + + result := checker.Check(context.Background()) + + assert.True(t, result.Healthy) + assert.Equal(t, map[string]string{"auth": "ok", "expense": "ok"}, result.Services) +} + +func TestChecker_OneUnhealthy_ReportsUnhealthyAndNamesService(t *testing.T) { + auth := newHealthServer(t, http.StatusOK) + expense := newHealthServer(t, http.StatusServiceUnavailable) + + checker := newChecker(map[string]string{ + "auth": auth.URL, + "expense": expense.URL, + }, 2*time.Second) + + result := checker.Check(context.Background()) + + assert.False(t, result.Healthy) + assert.Equal(t, "ok", result.Services["auth"]) + assert.Equal(t, "status_503", result.Services["expense"], "the failing service must be named with its status") +} + +func TestChecker_Unreachable_ReportsUnreachable(t *testing.T) { + auth := newHealthServer(t, http.StatusOK) + // A closed server's address is unreachable: the probe must classify it as + // such rather than hang or panic. + dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + deadURL := dead.URL + dead.Close() + + checker := newChecker(map[string]string{ + "auth": auth.URL, + "expense": deadURL, + }, 2*time.Second) + + result := checker.Check(context.Background()) + + assert.False(t, result.Healthy) + assert.Equal(t, "ok", result.Services["auth"]) + assert.Equal(t, "unreachable", result.Services["expense"]) +} + +func TestChecker_Timeout_ReportsUnreachableWithinBound(t *testing.T) { + auth := newHealthServer(t, http.StatusOK) + // A downstream whose /health hangs past the probe timeout must be bounded: + // Check returns within the timeout (plus slack) and marks it unreachable. The + // stub honours request-context cancellation so it unblocks the moment the + // probe aborts, keeping t.Cleanup's Close() from waiting out the full sleep. + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-time.After(2 * time.Second): + w.WriteHeader(http.StatusOK) + case <-r.Context().Done(): + } + })) + t.Cleanup(slow.Close) + + checker := newChecker(map[string]string{ + "auth": auth.URL, + "expense": slow.URL, + }, 100*time.Millisecond) + + start := time.Now() + result := checker.Check(context.Background()) + elapsed := time.Since(start) + + assert.False(t, result.Healthy) + assert.Equal(t, "unreachable", result.Services["expense"]) + assert.Less(t, elapsed, time.Second, "a hung downstream must not block past the per-probe timeout") +} + +// TestChecker_ProbesConcurrently proves the fan-out is concurrent: several slow +// downstreams settle in roughly one timeout window, not the sum of them. +func TestChecker_ProbesConcurrently(t *testing.T) { + var inFlight, maxInFlight int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + current := atomic.AddInt32(&inFlight, 1) + for { + observed := atomic.LoadInt32(&maxInFlight) + if current <= observed || atomic.CompareAndSwapInt32(&maxInFlight, observed, current) { + break + } + } + time.Sleep(150 * time.Millisecond) + atomic.AddInt32(&inFlight, -1) + w.WriteHeader(http.StatusOK) + }) + + services := map[string]string{} + for _, name := range []string{"auth", "expense", "finance", "datarights"} { + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + services[name] = server.URL + } + + checker := newChecker(services, 2*time.Second) + + start := time.Now() + result := checker.Check(context.Background()) + elapsed := time.Since(start) + + require.True(t, result.Healthy) + assert.Less(t, elapsed, 500*time.Millisecond, "four 150ms probes run serially would exceed 500ms") + assert.Equal(t, int32(4), atomic.LoadInt32(&maxInFlight), "all four probes should be in flight at once") +} + +// serveReadyz wires readiness.Handler behind a gin engine and returns the +// recorded response for GET /readyz. +func serveReadyz(checker *readiness.Checker) *httptest.ResponseRecorder { + engine := gin.New() + engine.GET("/readyz", readiness.Handler(checker, silentLogger())) + req := httptest.NewRequest(http.MethodGet, "/readyz", nil) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + return rec +} + +func TestHandler_AllHealthy_Returns200OK(t *testing.T) { + auth := newHealthServer(t, http.StatusOK) + expense := newHealthServer(t, http.StatusOK) + + checker := newChecker(map[string]string{ + "auth": auth.URL, + "expense": expense.URL, + }, 2*time.Second) + + rec := serveReadyz(checker) + + assert.Equal(t, http.StatusOK, rec.Code) + assert.JSONEq(t, `{"status":"ok"}`, rec.Body.String()) +} + +func TestHandler_OneDown_Returns503NamingService(t *testing.T) { + auth := newHealthServer(t, http.StatusOK) + expense := newHealthServer(t, http.StatusServiceUnavailable) + + checker := newChecker(map[string]string{ + "auth": auth.URL, + "expense": expense.URL, + }, 2*time.Second) + + rec := serveReadyz(checker) + + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.JSONEq(t, + `{"status":"unhealthy","services":{"auth":"ok","expense":"status_503"}}`, + rec.Body.String(), + ) +} diff --git a/services/gateway/internal/router/router.go b/services/gateway/internal/router/router.go index 4499637b..ebe617b8 100644 --- a/services/gateway/internal/router/router.go +++ b/services/gateway/internal/router/router.go @@ -5,6 +5,7 @@ import ( "log/slog" "net/http" "net/url" + "time" "github.com/gin-gonic/gin" @@ -12,9 +13,15 @@ import ( "github.com/ItsThompson/gofin/services/gateway/internal/access" "github.com/ItsThompson/gofin/services/gateway/internal/middleware" "github.com/ItsThompson/gofin/services/gateway/internal/proxy" + "github.com/ItsThompson/gofin/services/gateway/internal/readiness" "github.com/ItsThompson/gofin/services/metrics" ) +// readinessTimeout bounds each downstream /health probe issued by /readyz. It +// mirrors the healthcheck lib's 2s Docker HEALTHCHECK timeout so /readyz and +// container liveness share the same notion of "slow enough to be down". +const readinessTimeout = 2 * time.Second + // ServiceURLs holds the parsed downstream service URLs. type ServiceURLs struct { AuthREST *url.URL @@ -59,13 +66,30 @@ func New( c.JSON(http.StatusOK, gin.H{"status": "ok"}) }) - // Build one reverse-proxy handler per downstream service, keyed by the - // service name used in the shared Registry and ProxyPrefixes. - proxies := map[string]http.Handler{ - "auth": proxy.NewServiceProxy(serviceURLs.AuthREST, logger), - "expense": proxy.NewServiceProxy(serviceURLs.ExpenseREST, logger), - "finance": proxy.NewServiceProxy(serviceURLs.FinanceREST, logger), - "datarights": proxy.NewServiceProxy(serviceURLs.DatarightsREST, logger), + // The canonical service-name -> URL map is the single source of truth that + // both the reverse-proxy wiring and the readiness fan-out derive from, so the + // two can never drift: onboarding a downstream is one edit (in serviceURLMap), + // and both /api proxying and /readyz pick it up together. + services := serviceURLMap(serviceURLs) + + // Readiness aggregate (Public via GatewayResolve). Unlike the shallow + // /health above, /readyz fans out to every downstream service's /health so a + // single probe proves the whole backend is reachable. It probes exactly the + // services the gateway proxies to (both derive from the shared map above), so + // the 503 body and logs name the service the gateway actually routes to. + readinessTargets := make(map[string]string, len(services)) + for name, serviceURL := range services { + readinessTargets[name] = serviceURL.String() + } + checker := readiness.NewChecker(&http.Client{}, readinessTargets, readinessTimeout) + engine.GET("/readyz", readiness.Handler(checker, logger)) + + // Build one reverse-proxy handler per downstream service from the same shared + // map, keyed by the service name used in the shared Registry and + // ProxyPrefixes. + proxies := make(map[string]http.Handler, len(services)) + for name, serviceURL := range services { + proxies[name] = proxy.NewServiceProxy(serviceURL, logger) } // Derive the proxy wiring from the injected prefix inventory so onboarding a @@ -97,3 +121,18 @@ func ginWrapHandler(handler http.Handler) gin.HandlerFunc { handler.ServeHTTP(c.Writer, c.Request) } } + +// serviceURLMap is the canonical service-name -> URL map for the downstream +// services. Both the reverse-proxy wiring and the readiness fan-out derive from +// it, so they share one source of truth: adding a downstream is a single edit +// here, and because the prefix->proxy wiring panics for a service with no +// handler, a proxied service that /readyz fails to probe is impossible by +// construction. +func serviceURLMap(serviceURLs *ServiceURLs) map[string]*url.URL { + return map[string]*url.URL{ + "auth": serviceURLs.AuthREST, + "expense": serviceURLs.ExpenseREST, + "finance": serviceURLs.FinanceREST, + "datarights": serviceURLs.DatarightsREST, + } +} diff --git a/services/gateway/internal/router/router_test.go b/services/gateway/internal/router/router_test.go index ffbc2a09..14934ebc 100644 --- a/services/gateway/internal/router/router_test.go +++ b/services/gateway/internal/router/router_test.go @@ -314,6 +314,61 @@ func TestRouter_HealthEndpoint(t *testing.T) { assert.Contains(t, body, `"status":"ok"`) } +// TestRouter_ReadyzEndpoint_AllHealthy_Returns200 pins the happy path: with +// every downstream answering /health with 200, /readyz aggregates to 200 +// {"status":"ok"} and requires no cookie (Public via GatewayResolve). +func TestRouter_ReadyzEndpoint_AllHealthy_Returns200(t *testing.T) { + doRequest := setupGateway(t, userValidator()) + + resp, body := doRequest(http.MethodGet, "/readyz", nil) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Contains(t, body, `"status":"ok"`) +} + +// TestRouter_ReadyzEndpoint_DownstreamUnhealthy_Returns503NamingService proves +// /readyz aggregates downstream health and names the failing service in a 503. +// Built directly (not via setupGateway) so one downstream can fail its /health +// while the others stay healthy. +func TestRouter_ReadyzEndpoint_DownstreamUnhealthy_Returns503NamingService(t *testing.T) { + newHealthy := func() *url.URL { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + u, _ := url.Parse(server.URL) + return u + } + + unhealthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(unhealthy.Close) + expenseURL, _ := url.Parse(unhealthy.URL) + + engine := router.New(userValidator(), &router.ServiceURLs{ + AuthREST: newHealthy(), + ExpenseREST: expenseURL, + FinanceREST: newHealthy(), + DatarightsREST: newHealthy(), + }, sharedaccess.Prefixes(), newSilentLogger(), false) + + gatewayServer := httptest.NewServer(engine) + t.Cleanup(gatewayServer.Close) + + resp, err := gatewayServer.Client().Get(gatewayServer.URL + "/readyz") + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + assert.Contains(t, string(body), `"status":"unhealthy"`) + assert.Contains(t, string(body), "expense", "the 503 body must name the unhealthy service") +} + // TestRouter_New_PanicsOnPrefixWithNoProxy pins the data-driven wiring's // fail-fast contract: because router.New derives its proxy groups from the // injected prefix inventory, a prefix naming a service with no proxy handler