Backend reliability hardening: resumable Teleporter, metric-service dedup, health/shutdown, dead-code cleanup

README.md

@@ -6,7 +6,6 @@ A Node.js backend service for the L1Beat, providing API endpoints for Avalanche
 
 - **Chain Data**: Fetch and store information about Avalanche chains
 - **Validator Data**: Track validators for each chain
-- **TVL Tracking**: Historical and current TVL data for Avalanche
 - **TPS Metrics**: Track transactions per second for each chain and the entire network
 - **Caching**: In-memory caching for improved performance
 - **Structured Logging**: Comprehensive logging system
@@ -15,7 +14,7 @@ A Node.js backend service for the L1Beat, providing API endpoints for Avalanche
 ## Tech Stack
 
 - **Node.js** and **Express**: Backend framework
-- **MongoDB**: Database for storing chain, validator, TVL, and TPS data
+- **MongoDB**: Database for storing chain, validator, and TPS data
 - **Mongoose**: MongoDB object modeling
 - **Winston**: Structured logging
 - **Helmet**: Security headers
@@ -31,11 +30,6 @@ A Node.js backend service for the L1Beat, providing API endpoints for Avalanche
 - `GET /api/chains/:chainId`: Get a specific chain by ID
 - `GET /api/chains/:chainId/validators`: Get validators for a specific chain
 
-### TVL Endpoints
-
-- `GET /api/tvl/history`: Get historical TVL data
-- `GET /api/tvl/health`: Check TVL data health
-
 ### TPS Endpoints
 
 - `GET /api/chains/:chainId/tps/history`: Get TPS history for a specific chain
@@ -89,46 +83,41 @@ A Node.js backend service for the L1Beat, providing API endpoints for Avalanche
 
 ### Production Deployment
 
-For production deployment, set `NODE_ENV=production` and ensure all environment variables are properly configured.
-
-#### Deploying to Vercel
-
-This application is configured for deployment on Vercel. To deploy:
+The application runs as a long-lived Node.js process on DigitalOcean (it relies
+on in-process `node-cron` jobs and long-running background updates, so it must
+run as a persistent process — not a serverless function).
 
-1. Install the Vercel CLI:
-   ```
-   npm install -g vercel
-   ```
-
-2. Create a `.env.production` file with your production environment variables:
+1. Set the production environment variables (e.g. via a `.env` file or the
+   process environment):
    ```
    NODE_ENV=production
    PROD_MONGODB_URI=your_production_mongodb_uri
    ADMIN_API_KEY=your_production_admin_key
    UPDATE_API_KEY=your_production_update_key
    ```
 
-3. Run the deployment script:
+2. Install dependencies and start the server:
    ```
-   ./deploy.sh
+   npm ci
+   npm start
    ```
 
-Alternatively, you can deploy directly from the Vercel dashboard by connecting your GitHub repository.
+Run it under a process manager (e.g. PM2 or a systemd unit) so it restarts on
+crash, and place it behind a reverse proxy / load balancer (the app sets
+`trust proxy` in production). On `SIGTERM`/`SIGINT` the server shuts down
+gracefully, draining in-flight requests and closing the MongoDB connection.
 
 ## Scheduled Tasks
 
 The application runs several scheduled tasks:
 
-- TVL updates: Every 30 minutes
 - Chain and TPS updates: Every hour
-- TPS verification: Every 15 minutes
 
 ## Caching
 
 The application implements in-memory caching for frequently accessed data:
 
 - Chain data: 5 minutes
-- TVL history: 15 minutes
 - TPS data: 5 minutes
 
 ## Security
@@ -166,9 +155,6 @@ The following environment variables are required for the application to function
 - `GLACIER_VALIDATORS_ENDPOINT` - Endpoint for validators (default: /networks/mainnet/validators)
 - `GLACIER_L1VALIDATORS_ENDPOINT` - Endpoint for L1Validators (default: /networks/mainnet/l1Validators)
 
-- `DEFILLAMA_API_BASE` - Base URL for the DefiLlama API
-- `DEFILLAMA_API_TIMEOUT` - Timeout for DefiLlama API requests in milliseconds (default: 30000)
-
 - `METRICS_API_BASE` - Base URL for the Metrics API
 - `METRICS_API_TIMEOUT` - Timeout for Metrics API requests in milliseconds (default: 30000)
 - `METRICS_RATE_LIMIT` - Rate limit for Metrics API requests per minute (default: 20)

package.json

@@ -8,10 +8,12 @@
     "dev": "NODE_ENV=development nodemon src/app.js",
     "seed-authors": "node src/scripts/seedAuthors.js",
     "populate-tokens": "node scripts/populate-native-tokens.js",
-    "vercel-build": "echo hello",
     "test": "cross-env NODE_ENV=test jest --verbose --detectOpenHandles",
     "test:watch": "cross-env NODE_ENV=test jest --watch"
   },
+  "nodemonConfig": {
+    "ignore": ["logs/", "data/", "*.log"]
+  },
   "keywords": [],
   "author": "",
   "license": "ISC",

src/app.js

@@ -5,11 +5,11 @@ const cron = require('node-cron');
 const helmet = require('helmet');
 const rateLimit = require('express-rate-limit');
 const pLimit = require('p-limit');
+const mongoose = require('mongoose');
+const axios = require('axios');
 const config = require('./config/config');
 const connectDB = require('./config/db');
 const chainRoutes = require('./routes/chainRoutes');
-const fetchAndUpdateData = require('./utils/fetchGlacierData');
-const chainDataService = require('./services/chainDataService');
 const Chain = require('./models/chain');
 const chainService = require('./services/chainService');
 const tpsRoutes = require('./routes/tpsRoutes');
@@ -54,11 +54,9 @@ process.on('uncaughtException', (error) => {
 
 const app = express();
 
-// Check if we're running on Vercel
-const isVercel = process.env.VERCEL === '1';
-
-// Trust proxy when running on Vercel or other cloud platforms
-if (isVercel || config.isProduction) {
+// Trust proxy in production (the app runs behind a reverse proxy / load
+// balancer on DigitalOcean).
+if (config.isProduction) {
   logger.info('Running behind a proxy, setting trust proxy to true');
   app.set('trust proxy', 1);
 }
@@ -130,8 +128,89 @@ app.use(cors({
 app.use(express.json({ limit: '1mb' }));
 
 // Health check endpoint - MUST be before DB connection for deployment health checks
+// Liveness: is the process up and serving? Always 200 while running. MongoDB
+// state is reported for visibility but does not affect liveness. Kept cheap
+// (no external I/O) so uptime checks can hit it frequently.
 app.get('/health', (req, res) => {
-  res.status(200).json({ status: 'ok' });
+  res.status(200).json({
+    status: 'ok',
+    uptime: process.uptime(),
+    timestamp: new Date().toISOString(),
+    dependencies: {
+      mongodb: mongoose.connection.readyState === 1 ? 'connected' : 'disconnected'
+    }
+  });
+});
+
+// Readiness: should this instance receive traffic right now? Returns 503 when
+// MongoDB is not connected so a load balancer can drain it. Still cheap — only
+// inspects the local connection state, no external I/O.
+app.get('/health/ready', (req, res) => {
+  const mongoConnected = mongoose.connection.readyState === 1;
+  res.status(mongoConnected ? 200 : 503).json({
+    status: mongoConnected ? 'ok' : 'not_ready',
+    timestamp: new Date().toISOString(),
+    dependencies: {
+      mongodb: mongoConnected ? 'connected' : 'disconnected'
+    }
+  });
+});
+
+// Deep dependency probe. Actively pings external APIs, so it is intentionally
+// kept off the main /health path to avoid latency and burning rate limits on
+// every uptime check. Call this on demand or from a low-frequency monitor.
+// Cache the deep probe briefly so this endpoint can't be used to hammer the
+// upstream APIs (and burn the rate budget the cron jobs depend on): the outbound
+// probes run at most once per TTL regardless of request volume.
+const DEP_HEALTH_TTL_MS = 30 * 1000;
+let depHealthCache = null; // { expiresAt, statusCode, body }
+
+app.get('/health/dependencies', async (req, res) => {
+  if (depHealthCache && depHealthCache.expiresAt > Date.now()) {
+    return res.status(depHealthCache.statusCode).json({ ...depHealthCache.body, cached: true });
+  }
+
+  const probe = async (name, url) => {
+    if (!url) return { name, status: 'unconfigured' };
+    const startedAt = Date.now();
+    try {
+      await axios.get(url, {
+        timeout: 5000,
+        // Any HTTP response means the host is reachable; we only care about
+        // connectivity here, not the specific status code.
+        validateStatus: () => true
+      });
+      return { name, status: 'reachable', latencyMs: Date.now() - startedAt };
+    } catch (error) {
+      return { name, status: 'unreachable', error: error.message };
+    }
+  };
+
+  const mongoConnected = mongoose.connection.readyState === 1;
+  const [glacier, metrics] = await Promise.all([
+    probe('glacier', config.api && config.api.glacier && config.api.glacier.baseUrl),
+    probe('metrics', config.api && config.api.metrics && config.api.metrics.baseUrl)
+  ]);
+
+  // 'unconfigured' counts as unhealthy: GLACIER_API_BASE / METRICS_API_BASE are
+  // required, so a missing base URL is a real misconfiguration, not "fine".
+  const healthy = mongoConnected &&
+    glacier.status === 'reachable' &&
+    metrics.status === 'reachable';
+
+  const body = {
+    status: healthy ? 'ok' : 'degraded',
+    timestamp: new Date().toISOString(),
+    dependencies: {
+      mongodb: mongoConnected ? 'connected' : 'disconnected',
+      glacier,
+      metrics
+    }
+  };
+  const statusCode = healthy ? 200 : 503;
+
+  depHealthCache = { expiresAt: Date.now() + DEP_HEALTH_TTL_MS, statusCode, body };
+  res.status(statusCode).json(body);
 });
 
 // Single initialization point for data updates
@@ -637,12 +716,12 @@ if (missingEnvVars.length > 0) {
   // Still allow the server to start (for development convenience)
 }
 
-// For Vercel, we need to export the app
+// Export the app so tests (supertest) can mount it without binding a port.
 module.exports = app;
 
-// Only listen if not running on Vercel or in test mode
+// Only bind a port outside of tests (in tests supertest drives the app directly).
 const isTest = process.env.NODE_ENV === 'test';
-if (!isVercel && !isTest) {
+if (!isTest) {
   const server = app.listen(PORT, () => {
     logger.info(`Server running on port ${PORT}`, {
       environment: config.env,
@@ -656,4 +735,36 @@ if (!isVercel && !isTest) {
   server.on('error', (error) => {
     logger.error('Server error:', { error: error.message, stack: error.stack });
   });
+
+  // Graceful shutdown: stop accepting new connections, then close the DB
+  // connection so in-flight work can drain before the process exits.
+  let shuttingDown = false;
+  const gracefulShutdown = (signal) => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    logger.info(`Received ${signal}, shutting down gracefully...`);
+
+    // Hard limit so a hung connection can't block shutdown indefinitely.
+    const forceExit = setTimeout(() => {
+      logger.error('Graceful shutdown timed out, forcing exit');
+      process.exit(1);
+    }, 10000);
+    forceExit.unref();
+
+    server.close(async () => {
+      logger.info('HTTP server closed, no longer accepting connections');
+      try {
+        await mongoose.connection.close(false);
+        logger.info('MongoDB connection closed');
+      } catch (error) {
+        logger.error('Error closing MongoDB connection:', { error: error.message });
+      } finally {
+        clearTimeout(forceExit);
+        process.exit(0);
+      }
+    });
+  };
+
+  process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
+  process.on('SIGINT', () => gracefulShutdown('SIGINT'));
 }

src/models/teleporterMessage.js

@@ -79,6 +79,14 @@ const teleporterUpdateStateSchema = new mongoose.Schema({
         required: true,
         default: Date.now
     },
+    // Anchor for resumable multi-day fetches: the fixed "end of window" the
+    // per-day windows are measured back from. Captured when a weekly run first
+    // starts so that resuming hours later still produces a consistent 7-day
+    // dataset (rather than a window that slides with wall-clock time).
+    referenceEndTime: {
+        type: Date,
+        default: null
+    },
     // Progress information
     progress: {
         currentDay: {