Add structured observability foundation

app/api/health/route.test.ts

@@ -0,0 +1,26 @@
+import { afterEach, describe, expect, it } from "vitest"
+
+import { GET } from "@/app/api/health/route"
+
+const originalMongoUri = process.env.MONGODB_URI
+const originalJwtSecret = process.env.JWT_SECRET
+
+afterEach(() => {
+  process.env.MONGODB_URI = originalMongoUri
+  process.env.JWT_SECRET = originalJwtSecret
+})
+
+describe("health endpoint", () => {
+  it("changes readiness without exposing configuration", async () => {
+    process.env.MONGODB_URI = "mongodb://private-host/chainmove"
+    process.env.JWT_SECRET = "not-for-response"
+    const ready = await GET()
+    expect(ready.status).toBe(200)
+    expect(await ready.json()).toEqual({ status: "ready", checks: { configuration: "ok" } })
+
+    delete process.env.JWT_SECRET
+    const degraded = await GET()
+    expect(degraded.status).toBe(503)
+    expect(JSON.stringify(await degraded.json())).not.toContain("private-host")
+  })
+})

app/api/health/route.ts

@@ -0,0 +1,19 @@
+import { NextResponse } from "next/server"
+
+import { newCorrelationId } from "@/lib/api/errors"
+import { logger } from "@/lib/observability/logger"
+
+export const dynamic = "force-dynamic"
+
+/** Safe liveness/readiness signal: deliberately omits connection strings and configuration values. */
+export async function GET() {
+  const correlationId = newCorrelationId()
+  const requiredConfigPresent = Boolean(process.env.MONGODB_URI && process.env.JWT_SECRET)
+  const status = requiredConfigPresent ? "ready" : "degraded"
+  logger.info({ event: "health.check", correlationId, status })
+
+  return NextResponse.json(
+    { status, checks: { configuration: requiredConfigPresent ? "ok" : "missing" } },
+    { status: requiredConfigPresent ? 200 : 503, headers: { "X-Correlation-Id": correlationId } },
+  )
+}

docs/observability.md

@@ -0,0 +1,22 @@
+# Observability
+
+Every API response includes `X-Correlation-Id`. Send that value in a subsequent
+`X-Correlation-Id` request header to trace retries, webhooks, and background work.
+Structured JSON logs use `timestamp`, `level`, `service`, `event`,
+`correlationId`, `operationId`, `method`, `status`, and `durationMs`. Business
+identifiers may be added as `transactionId`, `eventId`, `poolId`, or `jobId`.
+
+`lib/observability/logger.ts` recursively redacts authorization and cookie values,
+API keys, tokens, secrets, KYC fields, account/routing numbers, cards, and provider
+payloads. Do not log a full request body; pass only allow-listed business IDs.
+Logging is best-effort and cannot fail a request.
+
+Metric names are low-cardinality: `http.requests`, `http.errors`, and
+`http.duration`. Alert on elevated `http.errors:5xx`, sustained p95
+`http.duration`, webhook failures, database failures, idempotency conflicts, and
+job retry exhaustion. To investigate an incident, search the correlation ID, then
+follow events ordered by timestamp; never paste redacted values back into tickets.
+
+`GET /api/health` reports only application configuration readiness and has no
+connection strings, secrets, or topology details. A `503` means configuration is
+incomplete; it is safe for load balancer readiness checks.

lib/api/route-handler.ts

@@ -18,6 +18,8 @@ import {
   type ApiErrorEnvelope,
 } from "@/lib/api/errors"
 import { assertNoForbiddenFields, assertNoRawDocuments } from "@/lib/api/serialization"
+import { logger } from "@/lib/observability/logger"
+import { incrementMetric, recordLatency } from "@/lib/observability/metrics"
 import {
   API_VERSION_HEADER,
   deprecationHeaders,
@@ -120,6 +122,7 @@ export function defineRoute<
 >(definition: RouteDefinition<TParamsSchema, TQuerySchema, TBodySchema, TResponseSchema, TAuth>) {
   return async function routeHandler(request: Request, nextContext: NextRouteContext): Promise<Response> {
     const correlationId = resolveCorrelationId(request)
+    const startedAt = performance.now()
     const extraHeaders: Record<string, string> = {}
     let successStatus = definition.successStatus ?? (definition.method === "POST" ? 201 : 200)
     let version: ApiVersion | undefined
@@ -165,22 +168,49 @@ export function defineRoute<
       })
 
       if (auth.shouldRefreshSession && auth.user) {
+        logRequest(definition, request, correlationId, successStatus, startedAt)
         return withSessionRefresh(response, auth.user)
       }
 
+      logRequest(definition, request, correlationId, successStatus, startedAt)
       return response
     } catch (error) {
-      return errorResponse(error, {
+      const response = errorResponse(error, {
         correlationId,
         operationId: definition.operationId,
         method: definition.method,
         version,
         headers: definition.deprecation ? deprecationHeaders(definition.deprecation) : {},
       })
+      logRequest(definition, request, correlationId, response.status, startedAt)
+      return response
     }
   }
 }
 
+function logRequest(
+  definition: { operationId: string; method: HttpMethod },
+  request: Request,
+  correlationId: string,
+  status: number,
+  startedAt: number,
+) {
+  const durationMs = Math.round((performance.now() - startedAt) * 100) / 100
+  const outcome = status >= 500 ? "5xx" : status >= 400 ? "4xx" : "success"
+  incrementMetric("http.requests", outcome)
+  if (status >= 500) incrementMetric("http.errors", "5xx")
+  recordLatency("http.duration", durationMs)
+  logger.info({
+    event: "http.request.completed",
+    correlationId,
+    operationId: definition.operationId,
+    method: definition.method,
+    route: new URL(request.url).pathname,
+    status,
+    durationMs,
+  })
+}
+
 /** Sentinel a handler can return to emit `204 No Content`. */
 export class NoContent {}
 
@@ -456,11 +486,11 @@ function logApiError(
   // 5xx means the server is at fault and an operator needs the stack; 4xx is
   // routine client behaviour and stays at debug volume.
   if (error.status >= 500) {
-    console.error("API_ERROR", detail, (error as { cause?: unknown }).cause ?? error)
+    logger.error({ event: "api.error", ...detail, error: (error as { cause?: unknown }).cause ?? error })
     return
   }
 
   if (process.env.NODE_ENV === "development") {
-    console.warn("API_CLIENT_ERROR", detail)
+    logger.debug({ event: "api.client_error", ...detail })
   }
 }

lib/dbConnect.ts

@@ -1,4 +1,6 @@
 import mongoose from "mongoose"
+import { logger } from "@/lib/observability/logger"
+import { incrementMetric, recordLatency } from "@/lib/observability/metrics"
 
 type MongooseCache = {
   conn: typeof mongoose | null
@@ -35,8 +37,33 @@ async function dbConnect() {
     })
   }
 
-  cached.conn = await cached.promise
+  const startedAt = performance.now()
+  try {
+    cached.conn = await cached.promise
+    recordLatency("database.connect", performance.now() - startedAt)
+    incrementMetric("database.operations", "success")
+  } catch (error) {
+    cached.promise = null
+    incrementMetric("database.failures", "connect")
+    logger.error({ event: "database.connect.failed", error })
+    throw error
+  }
   return cached.conn
 }
 
+// Mongoose's debug hook is intentionally enabled only outside production. It
+// captures query shape without logging values that can be personal or financial
+// data; explicit explain plans remain a local investigation tool.
+if (process.env.NODE_ENV !== "production") {
+  mongoose.set("debug", (collection: string, method: string, query: unknown) => {
+    incrementMetric("database.operations", method)
+    logger.debug({
+      event: "database.query",
+      collection,
+      method,
+      queryShape: Object.keys((query || {}) as Record<string, unknown>),
+    })
+  })
+}
+
 export default dbConnect

lib/observability/logger.test.ts

@@ -0,0 +1,21 @@
+import { describe, expect, it } from "vitest"
+
+import { redact } from "@/lib/observability/logger"
+
+describe("observability redaction", () => {
+  it("redacts nested secrets, KYC data, and provider payloads", () => {
+    expect(
+      redact({ authorization: "Bearer secret", nested: { apiKey: "key", kycDocument: { bvn: "123" } }, events: [{ accountNumber: "456" }] }),
+    ).toEqual({
+      authorization: "[REDACTED]",
+      nested: { apiKey: "[REDACTED]", kycDocument: "[REDACTED]" },
+      events: [{ accountNumber: "[REDACTED]" }],
+    })
+  })
+
+  it("serializes errors without arbitrary properties", () => {
+    const error = new Error("provider unavailable")
+    ;(error as Error & { token?: string }).token = "do-not-log"
+    expect(redact({ error })).toMatchObject({ error: { name: "Error", message: "provider unavailable" } })
+  })
+})

lib/observability/logger.ts

@@ -0,0 +1,66 @@
+export type LogLevel = "debug" | "info" | "warn" | "error"
+
+const REDACTED = "[REDACTED]"
+const SENSITIVE_KEY = /(?:authorization|cookie|api[_-]?key|token|secret|password|passcode|kyc|account(?:number)?|routing|provider.*payload|card|cvv|ssn|bvn|nin)/i
+const MAX_DEPTH = 8
+
+export interface LogContext {
+  correlationId?: string
+  operationId?: string
+  event?: string
+  [key: string]: unknown
+}
+
+/**
+ * Redacts values by key at every nesting level before they leave process memory.
+ * It deliberately preserves field names so operators can diagnose payload shape
+ * without exposing the underlying financial or identity data.
+ */
+export function redact(value: unknown, depth = 0, seen = new WeakSet<object>()): unknown {
+  if (value == null || typeof value !== "object") return value
+  if (depth >= MAX_DEPTH) return "[TRUNCATED]"
+  if (seen.has(value as object)) return "[CIRCULAR]"
+  seen.add(value as object)
+
+  if (value instanceof Error) {
+    return {
+      name: value.name,
+      message: SENSITIVE_KEY.test(value.message) ? REDACTED : value.message,
+      ...(value.stack ? { stack: value.stack.split("\n").slice(0, 12).join("\n") } : {}),
+      ...(value.cause !== undefined ? { cause: redact(value.cause, depth + 1, seen) } : {}),
+    }
+  }
+
+  if (Array.isArray(value)) return value.map((item) => redact(item, depth + 1, seen))
+
+  return Object.fromEntries(
+    Object.entries(value as Record<string, unknown>).map(([key, item]) => [
+      key,
+      SENSITIVE_KEY.test(key) ? REDACTED : redact(item, depth + 1, seen),
+    ]),
+  )
+}
+
+function shouldLog(level: LogLevel) {
+  const configured = process.env.LOG_LEVEL || (process.env.NODE_ENV === "production" ? "info" : "debug")
+  return ["debug", "info", "warn", "error"].indexOf(level) >= ["debug", "info", "warn", "error"].indexOf(configured)
+}
+
+/** JSON-lines logger. Logging is best-effort: a broken log destination cannot interrupt payments. */
+export function log(level: LogLevel, context: LogContext = {}) {
+  if (!shouldLog(level)) return
+  try {
+    process.stdout.write(
+      `${JSON.stringify({ timestamp: new Date().toISOString(), level, service: "chainmove-api", ...redact(context) })}\n`,
+    )
+  } catch {
+    // Observability must never become an availability dependency.
+  }
+}
+
+export const logger = {
+  debug: (context: LogContext) => log("debug", context),
+  info: (context: LogContext) => log("info", context),
+  warn: (context: LogContext) => log("warn", context),
+  error: (context: LogContext) => log("error", context),
+}

lib/observability/metrics.ts

@@ -0,0 +1,22 @@
+type MetricKey = `${string}:${string}`
+const counters = new Map<MetricKey, number>()
+const timings = new Map<MetricKey, number[]>()
+
+/** Metric names are stable, low-cardinality names suitable for a future exporter. */
+export function incrementMetric(name: string, outcome = "success") {
+  const key: MetricKey = `${name}:${outcome}`
+  counters.set(key, (counters.get(key) || 0) + 1)
+}
+
+export function recordLatency(name: string, milliseconds: number) {
+  const key: MetricKey = `${name}:ms`
+  const values = timings.get(key) || []
+  // Bound development memory while retaining a representative recent window.
+  if (values.length >= 1_000) values.shift()
+  values.push(milliseconds)
+  timings.set(key, values)
+}
+
+export function metricSnapshot() {
+  return { counters: Object.fromEntries(counters), timings: Object.fromEntries(timings) }
+}