feat: fail-fast config validation and deep readiness probe

.github/workflows/ci.yml

@@ -52,6 +52,11 @@ jobs:
         ports:
           - 27017:27017
 
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -70,21 +75,8 @@ jobs:
           find . -name "*.js" -not -path "./node_modules/*" -print0 \
             | xargs -0 -n1 node --check
 
-      - name: Boot server and hit /health
-        run: |
-          node server.js &
-          SERVER_PID=$!
-          for i in $(seq 1 30); do
-            if curl -sf http://localhost:5000/health > /dev/null; then
-              echo "Server is up and healthy"
-              kill $SERVER_PID
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "Server failed to become healthy in time"
-          kill $SERVER_PID || true
-          exit 1
+      - name: Start server
+        run: node server.js &
         env:
           NODE_ENV: test
           PORT: 5000
@@ -94,3 +86,23 @@ jobs:
           CLOUDINARY_CLOUD_NAME: ci_test_cloud
           CLOUDINARY_API_KEY: ci_test_key
           CLOUDINARY_API_SECRET: ci_test_secret_that_should_not_leak
+          REDIS_HOST: localhost
+          REDIS_PORT: 6379
+
+      - name: Wait for readiness
+        run: |
+          for i in {1..30}; do
+            STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/readyz)
+            if [ "$STATUS" = "200" ]; then
+              echo "Server is ready"
+              exit 0
+            fi
+            echo "Attempt $i: /readyz returned $STATUS, waiting..."
+            sleep 2
+          done
+          echo "Server failed to become ready"
+          curl -v http://localhost:5000/readyz
+          exit 1
+
+      - name: Assert liveness still works
+        run: curl -sf http://localhost:5000/livez

app.js

@@ -4,6 +4,7 @@ import cookieParser from "cookie-parser";
 import compression from "compression";
 import dotenv from "dotenv";
 import crypto from "crypto";
+import mongoose from "mongoose";
 import "./src/jobs/handlers.js";
 
 // Load env vars, except in tests where test/jest.setup.js has already loaded
@@ -17,6 +18,7 @@ import connectDB from "./src/config/db.js";
 import validateEnv from "./src/config/validateEnv.js";
 import logger from "./src/config/logger.js";
 import { registry, metricsMiddleware, observeHttpDuration } from "./src/config/metrics.js";
+import { isRedisReady } from "./src/config/redis.js";
 
 import {
   helmetMiddleware,
@@ -67,6 +69,27 @@ if (process.env.NODE_ENV !== "test") {
 
 const app = express();
 
+// ========================================
+// READINESS STATE
+// ========================================
+
+let isReady = true;
+
+/**
+ * Set readiness state for the application
+ * Called during shutdown to signal to load balancers to stop sending new requests
+ */
+export const setReadiness = (ready) => {
+  isReady = ready;
+};
+
+/**
+ * Get current readiness state
+ */
+export const getReadiness = () => {
+  return isReady;
+};
+
 app.set("trust proxy", 1);
 
 // ======================
@@ -164,14 +187,62 @@ app.get("/", (req, res) => {
   });
 });
 
-app.get("/health", (req, res) => {
-  res.json({
-    success: true,
-    message: "pong",
+// ========================================
+// HEALTH CHECK ENDPOINTS
+// ========================================
+
+/**
+ * Liveness probe — cheap, just proves event loop is alive
+ * Should return 200 as long as the process hasn't crashed
+ * Even during shutdown, this should remain responsive
+ */
+app.get("/livez", (req, res) => {
+  res.status(200).json({
+    status: "ok",
+    uptime: process.uptime(),
     timestamp: new Date().toISOString(),
   });
 });
 
+/**
+ * Readiness probe — checks real dependency state
+ * Returns 200 only when ready to accept traffic
+ * Returns 503 during shutdown or when dependencies are down
+ */
+app.get("/readyz", async (req, res) => {
+  // Check readiness flag (set to false during shutdown)
+  if (!isReady) {
+    return res.status(503).json({
+      status: "not_ready",
+      reason: "shutting_down",
+      dependencies: {
+        mongo: "unknown",
+        redis: "unknown",
+      },
+      timestamp: new Date().toISOString(),
+    });
+  }
+
+  // Check MongoDB readyState: 1 = connected
+  const mongoReady = mongoose.connection.readyState === 1;
+
+  // Check Redis using existing isRedisReady()
+  const redisReady = isRedisReady();
+
+  const allReady = mongoReady && redisReady;
+
+  const status = {
+    status: allReady ? "ready" : "not_ready",
+    dependencies: {
+      mongo: mongoReady ? "up" : "down",
+      redis: redisReady ? "up" : "down",
+    },
+    timestamp: new Date().toISOString(),
+  };
+
+  return res.status(allReady ? 200 : 503).json(status);
+});
+
 // SEP-1 stellar.toml — must be outside /api rate limiter
 app.use("/.well-known", wellKnownRoutes);
 

server.js

@@ -1,4 +1,4 @@
-import app from "./app.js";
+import app, { setReadiness } from "./app.js";
 import logger from "./src/config/logger.js";
 import { initRedis, closeRedis } from "./src/config/redis.js";
 import { startJobs, stopJobs } from "./src/jobs/queue.js";
@@ -39,6 +39,13 @@ if (process.env.INGESTION_WORKER_ENABLED === "true") {
 const gracefulShutdown = async (signal) => {
   logger.info(`${signal} received. Starting graceful shutdown...`);
 
+  // FIRST: signal not ready so load balancer stops sending new requests
+  setReadiness(false);
+
+  // Give load balancer time to drain (adjust to your LB health check interval)
+  // Typical health check intervals are 5-10 seconds
+  await new Promise((resolve) => setTimeout(resolve, 5000));
+
   server.close(async () => {
     logger.info("HTTP server closed");