feat(backend): expand health checks with liveness, readiness, dependency probes (#353)

backend/src/api/routes/agents.ts

@@ -21,6 +21,15 @@ export interface AgentsRouterOptions {
 
 const DEFAULT_HEALTH_TIMEOUT_MS = 3_000;
 
+// Mirrors the RegisterAgentRequest schema documented in api/docs.ts.
+const RegisterAgentSchema = z.object({
+  agentId: z.string().min(1),
+  capabilities: z.array(z.string()).min(1),
+  pricingXLM: z.number().min(0.001),
+  endpoint: z.string().url(),
+  stellarPublicKey: z.string().regex(/^G[A-Z2-7]{55}$/),
+});
+
 export function createAgentsRouter(options: AgentsRouterOptions = {}): Router {
   const router = Router();
   const config = getConfig();

backend/src/api/routes/health.test.ts

@@ -135,9 +135,40 @@ describe("GET /health/deep", () => {
   });
 });
 
+// ── GET /health/live ──────────────────────────────────────────────────────────
+
+describe("GET /health/live", () => {
+  const app = buildApp();
+
+  it("returns the same shape as GET /health", async () => {
+    const res = await request(app).get("/health/live");
+    expect(res.status).toBe(200);
+    expect(res.body.status).toBe("ok");
+    expect(typeof res.body.uptime).toBe("number");
+    expect(typeof res.body.version).toBe("string");
+    expect(typeof res.body.stellarNetwork).toBe("string");
+  });
+});
+
 // ── GET /health/ready ─────────────────────────────────────────────────────────
 
 describe("GET /health/ready", () => {
+  let fetchSpy: jest.SpyInstance;
+  let wsSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    fetchSpy = jest.spyOn(global, "fetch" as any).mockResolvedValue({ ok: true } as Response);
+    const { metricsService } = require("../../services/metrics");
+    wsSpy = jest
+      .spyOn(metricsService, "getWebSocketStatus")
+      .mockReturnValue({ status: "unknown", error: "WebSocket server not attached" });
+  });
+
+  afterEach(() => {
+    fetchSpy.mockRestore();
+    wsSpy.mockRestore();
+  });
+
   it("returns 200 or 500 with structured checks response", async () => {
     // The /health/ready endpoint performs dynamic imports of DB modules and
     // runs SELECT 1 against them. In the test environment without real DB files
@@ -152,11 +183,56 @@ describe("GET /health/ready", () => {
     expect(["ok", "error"]).toContain(res.body.status);
   });
 
-  it("checks object always has tasks and payments keys", async () => {
+  it("checks object has tasks, payments, queue, venice, horizon, and websocket keys", async () => {
     const app = buildApp();
     const res = await request(app).get("/health/ready");
     expect(res.body.checks).toHaveProperty("tasks");
     expect(res.body.checks).toHaveProperty("payments");
+    expect(res.body.checks).toHaveProperty("queue");
+    expect(res.body.checks).toHaveProperty("venice");
+    expect(res.body.checks).toHaveProperty("horizon");
+    expect(res.body.checks).toHaveProperty("websocket");
+  });
+
+  it("fails readiness when Venice is unreachable, even if the databases are fine", async () => {
+    fetchSpy.mockImplementation((url: string) => {
+      if (url.includes("venice.ai")) return Promise.reject(new Error("network error"));
+      return Promise.resolve({ ok: true } as Response);
+    });
+    const app = buildApp();
+    const res = await request(app).get("/health/ready");
+    expect(res.status).toBe(500);
+    expect(res.body.status).toBe("error");
+    expect(res.body.checks.venice).toBe("error");
+  });
+
+  it("fails readiness when Stellar Horizon is unreachable", async () => {
+    fetchSpy.mockImplementation((url: string) => {
+      if (url.includes("venice.ai")) return Promise.resolve({ ok: true } as Response);
+      return Promise.reject(new Error("network error"));
+    });
+    const app = buildApp();
+    const res = await request(app).get("/health/ready");
+    expect(res.status).toBe(500);
+    expect(res.body.checks.horizon).toBe("error");
+  });
+
+  it("reports websocket as unknown (not a failure) when no probe is attached", async () => {
+    const app = buildApp();
+    const res = await request(app).get("/health/ready");
+    expect(res.body.checks.websocket).toBe("unknown");
+    // An "unknown" websocket alone must not drag down an otherwise-ready service.
+    if (res.body.checks.tasks === "ok" && res.body.checks.payments === "ok" && res.body.checks.queue === "ok") {
+      expect(res.status).toBe(200);
+    }
+  });
+
+  it("fails readiness when the websocket probe reports it is not listening", async () => {
+    wsSpy.mockReturnValue({ status: "unreachable", error: "not listening" });
+    const app = buildApp();
+    const res = await request(app).get("/health/ready");
+    expect(res.body.checks.websocket).toBe("error");
+    expect(res.status).toBe(500);
   });
 });
 

backend/src/api/routes/health.ts

@@ -25,7 +25,24 @@ router.get("/", cachedRoute("health"), (_req: Request, res: Response) => {
     version: config.NPM_PACKAGE_VERSION,
     stellarNetwork: config.STELLAR_NETWORK,
   });
-});
+}
+
+router.get("/", livenessHandler);
+
+/**
+ * @openapi
+ * /health/live:
+ *   get:
+ *     summary: Basic liveness check
+ *     operationId: getLive
+ *     description: Alias for `GET /health` — process-only liveness, no dependency checks.
+ *     tags: [Health]
+ *     security: []
+ *     responses:
+ *       200:
+ *         description: Service is up
+ */
+router.get("/live", livenessHandler);
 
 router.get("/deep", cachedRoute("health"), async (_req: Request, res: Response) => {
   const config = getConfig();
@@ -47,14 +64,19 @@ router.get("/deep", cachedRoute("health"), async (_req: Request, res: Response)
 });
 
 router.get("/ready", async (_req: Request, res: Response) => {
-  const checks: Record<string, "ok" | "error"> = {
+  const checks: Record<string, "ok" | "error" | "unknown"> = {
     tasks: "ok",
     payments: "ok",
+    queue: "ok",
+    venice: "ok",
+    horizon: "ok",
+    websocket: "ok",
   };
 
   try {
     const tasksModule = await import("../../db/tasks.js");
     const paymentsModule = await import("../../db/index.js");
+    const queueModule = await import("../../queue/jobStore.js");
 
     try {
       const taskDb = (tasksModule.getTaskDb as Function)();
@@ -73,13 +95,44 @@ router.get("/ready", async (_req: Request, res: Response) => {
     } finally {
       (paymentsModule.closeDb as Function)();
     }
+
+    try {
+      const jobDb = (queueModule.getJobDb as Function)();
+      jobDb.prepare("SELECT 1").get();
+    } catch (error) {
+      (checks as any).queue = "error";
+    } finally {
+      (queueModule.closeJobDb as Function)();
+    }
   } catch (error) {
     res.status(500).json({ status: "error", checks, error: String(error) });
     return;
   }
 
-  const allOk = Object.values(checks).every((status) => status === "ok");
-  res.status(allOk ? 200 : 500).json({ status: allOk ? "ok" : "error", checks });
+  const config = getConfig();
+  const timeoutMs = config.HEALTH_PROBE_TIMEOUT_MS;
+  const [veniceStatus, horizonStatus] = await Promise.all([
+    checkVenice(config.VENICE_API_KEY, timeoutMs),
+    checkHorizon(config.STELLAR_HORIZON_URL, timeoutMs),
+  ]);
+  checks.venice = veniceStatus === "ok" ? "ok" : "error";
+  checks.horizon = horizonStatus === "ok" ? "ok" : "error";
+
+  const websocketStatus = metricsService.getWebSocketStatus();
+  checks.websocket =
+    websocketStatus.status === "unknown"
+      ? "unknown"
+      : websocketStatus.status === "ok"
+        ? "ok"
+        : "error";
+
+  // A missing WebSocket probe ("unknown") is a valid configuration — the
+  // stream layer may simply not be attached — so it alone does not fail
+  // readiness. A probe that *is* attached and reports "error" (not
+  // listening) does, same as every other dependency.
+  const failing = Object.values(checks).filter((status) => status === "error");
+  const ready = failing.length === 0;
+  res.status(ready ? 200 : 500).json({ status: ready ? "ok" : "error", checks });
 });
 
 router.get("/dashboard", adminAuthMiddleware, async (req: Request, res: Response) => {
@@ -108,7 +161,7 @@ router.get("/traces/:traceId", (req: Request, res: Response) => {
   res.json(trace);
 });
 
-async function checkVenice(apiKey: string): Promise<"ok" | "unreachable"> {
+async function checkVenice(apiKey: string, timeoutMs = 5000): Promise<"ok" | "unreachable"> {
   try {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), 5_000);
@@ -123,7 +176,7 @@ async function checkVenice(apiKey: string): Promise<"ok" | "unreachable"> {
   }
 }
 
-async function checkHorizon(url: string): Promise<"ok" | "unreachable"> {
+async function checkHorizon(url: string, timeoutMs = 5000): Promise<"ok" | "unreachable"> {
   try {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), 5_000);

backend/src/api/routes/stream.ts

@@ -15,6 +15,22 @@ const STREAM_PATH = /^\/tasks\/([^/?]+)\/stream(?:\?.*)?$/;
 
 const logger = createLogger({ module: 'ws-stream' });
 
+/**
+ * Every WebSocketServer created by attachTaskStream(), tracked so
+ * getStreamConnectionCount() can report a live total across all of them
+ * (normally just one, per HTTP server) for the health/metrics dashboard.
+ */
+const activeStreamServers = new Set<WebSocketServer>();
+
+/** Total connected WebSocket clients across every attached stream server. */
+export function getStreamConnectionCount(): number {
+  let count = 0;
+  for (const wss of activeStreamServers) {
+    count += wss.clients.size;
+  }
+  return count;
+}
+
 // ---------------------------------------------------------------------------
 // Wire-format normalisation
 // ---------------------------------------------------------------------------

backend/src/services/metrics.ts

@@ -467,6 +467,15 @@ export class MetricsService {
     this.webSocketProbe = probe;
   }
 
+  /**
+   * Current WebSocket reachability, without a full dashboard collection.
+   * Used by GET /health/ready. See {@link checkWebSocket} for the "no probe
+   * registered" -> `unknown` semantics.
+   */
+  getWebSocketStatus(): DependencyStatus {
+    return checkWebSocket(this.webSocketProbe);
+  }
+
   /** Begin observing GC pauses. Safe to call more than once. */
   startGcObserver(): void {
     if (this.gcObserver) return;