Add load testing scenarios for webhook delivery under concurrent pressure

README.md

@@ -391,6 +391,18 @@ Retrieves recent immutable audit trail records for mutating admin actions (`POST
 
 Mutating admin requests are intercepted by `auditLogMiddleware` and recorded asynchronously upon response completion. Sensitive keys (`password`, `secret`, `apiKey`, `token`, `signature`, `privateKey`, `seed`) are deeply redacted before persistence.
 
+### `GET /admin/webhooks/health`
+Aggregates webhook delivery health so operators can spot broken merchant integrations.
+- **Query Parameters:**
+  - `username` (optional) – Scope the aggregates to one merchant's webhooks.
+- **Headers:** `x-api-key` (required) – must match `ADMIN_API_KEY`.
+- **Returns:** JSON object with `success: true`, a `summary` (`total`, `healthy`, `failing`, `successRate24h`), and `failingOver24h` — the webhooks that have been failing continuously for more than 24 hours.
+- **Status Codes:**
+  - `200 OK`: Health snapshot retrieved successfully.
+  - `401 Unauthorized`: Missing or invalid API key.
+
+A webhook is "failing" while its `failingSince` timestamp is set (cleared on the next successful delivery). `successRate24h` is the share of webhooks with a delivery attempt in the last 24h that are currently healthy; it is `null` when nothing has been active in that window.
+
 ### `GET /metrics`
 
 Prometheus scrape endpoint, served in the Prometheus text format. Exempt from the

stellar-payment-platform/.gitignore

@@ -12,3 +12,6 @@ data/*.db-wal
 
 # Rotating log files (#294)
 logs/
+
+# Generated webhook load-test keypairs (see scripts/seed-webhook-load-test-users.js)
+scripts/.load-test-fixtures/

stellar-payment-platform/artillery-webhooks.yml

@@ -0,0 +1,64 @@
+config:
+  target: "{{ $processEnvironment.TARGET_URL || 'http://localhost:5000' }}"
+  processor: "./scripts/webhook-load-processor.js"
+  plugins:
+    expect: {}
+  ensure:
+    p95: 500
+    maxErrorRate: 1
+  phases:
+    - duration: 1
+      arrivalCount: 50
+      name: "Spike: 50 concurrent users hitting the webhook API"
+  defaults:
+    headers:
+      content-type: "application/json"
+
+before:
+  flow:
+    - log: "Prereqs: node scripts/seed-webhook-load-test-users.js && node scripts/mock-webhook-receiver.js (or `npm run test:load:webhooks` to do both automatically)"
+
+scenarios:
+  - name: "Register a webhook, trigger a delivery, list, then remove it"
+    beforeScenario: "assignTestUser"
+    flow:
+      - post:
+          url: "/webhooks"
+          json:
+            username: "{{ username }}"
+            signature: "{{ signature }}"
+            signerAddress: "{{ signerAddress }}"
+            url: "{{ webhookUrl }}"
+            events: ["payment.received"]
+          expect:
+            - statusCode: 201
+          capture:
+            - json: "$.webhook.id"
+              as: "webhookId"
+
+      - post:
+          url: "/webhooks/{{ webhookId }}/test"
+          json:
+            username: "{{ username }}"
+            signature: "{{ signature }}"
+            signerAddress: "{{ signerAddress }}"
+          expect:
+            - statusCode: 200
+
+      - get:
+          url: "/webhooks"
+          json:
+            username: "{{ username }}"
+            signature: "{{ signature }}"
+            signerAddress: "{{ signerAddress }}"
+          expect:
+            - statusCode: 200
+
+      - delete:
+          url: "/webhooks/{{ webhookId }}"
+          json:
+            username: "{{ username }}"
+            signature: "{{ signature }}"
+            signerAddress: "{{ signerAddress }}"
+          expect:
+            - statusCode: 200

stellar-payment-platform/package.json

@@ -10,6 +10,7 @@
     "test:memory": "node --expose-gc node_modules/.bin/jest --forceExit --testPathPattern=memory --globals '{\"gc\":true}'",
     "test:integration": "TEST_API_URL=http://localhost:5001 jest --forceExit --testPathPattern=integration",
     "test:load": "artillery run artillery.yml",
+    "test:load:webhooks": "node scripts/run-webhook-load-test.js",
     "listener": "node horizonListener.js",
     "db:seed": "node scripts/seed.js",
     "prisma:generate": "prisma generate",

stellar-payment-platform/scripts/mock-webhook-receiver.js

@@ -0,0 +1,20 @@
+// Stands in for a merchant's webhook endpoint during load testing so
+// deliveries triggered by artillery-webhooks.yml don't leave the machine.
+const http = require('http');
+
+const PORT = Number(process.env.MOCK_RECEIVER_PORT || 5099);
+
+const server = http.createServer((req, res) => {
+  if (req.method !== 'POST') {
+    res.writeHead(405).end();
+    return;
+  }
+  req.resume();
+  req.on('end', () => {
+    res.writeHead(200, { 'Content-Type': 'application/json' }).end('{"ok":true}');
+  });
+});
+
+server.listen(PORT, () => {
+  console.log(`[mock-webhook-receiver] listening on :${PORT}`);
+});

stellar-payment-platform/scripts/run-webhook-load-test.js

@@ -0,0 +1,30 @@
+// Cross-platform entry point for `npm run test:load:webhooks`: seeds test
+// users, starts the mock receiver, runs Artillery, then cleans up.
+const path = require('path');
+const { spawn } = require('child_process');
+
+const ROOT = path.join(__dirname, '..');
+
+const run = (command, args) =>
+  new Promise((resolve, reject) => {
+    const child = spawn(command, args, { cwd: ROOT, stdio: 'inherit', shell: true });
+    child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`${command} exited with code ${code}`))));
+  });
+
+const main = async () => {
+  await run('node', ['scripts/seed-webhook-load-test-users.js']);
+
+  const receiver = spawn('node', ['scripts/mock-webhook-receiver.js'], { cwd: ROOT, stdio: 'inherit', shell: true });
+  await new Promise((resolve) => setTimeout(resolve, 500));
+
+  try {
+    await run('npx', ['artillery', 'run', 'artillery-webhooks.yml']);
+  } finally {
+    receiver.kill();
+  }
+};
+
+main().catch((err) => {
+  console.error(err.message);
+  process.exitCode = 1;
+});

stellar-payment-platform/scripts/seed-webhook-load-test-users.js

@@ -0,0 +1,44 @@
+// Provisions Stellar-keypair-backed test users for the webhook load test
+// (artillery-webhooks.yml). Each user's secret key is written to a local
+// fixture so the Artillery processor can sign requests the same way a real
+// Freighter wallet would (see verifyFreighterSignedMessage in webhookRoutes.js).
+require('dotenv').config();
+
+const fs = require('fs');
+const path = require('path');
+const { Keypair } = require('@stellar/stellar-sdk');
+const { prisma } = require('../prismaClient');
+const { logger } = require('../src/logger');
+
+const USER_COUNT = Number(process.env.LOAD_TEST_USER_COUNT || 50);
+const FEDERATION_DOMAIN = process.env.FEDERATION_DOMAIN || 'localhost';
+const OUTPUT_PATH = path.join(__dirname, '.load-test-fixtures', 'webhook-users.json');
+
+const seedWebhookLoadTestUsers = async () => {
+  const users = [];
+
+  for (let i = 0; i < USER_COUNT; i++) {
+    const keypair = Keypair.random();
+    const username = `webhook-load-${i}*${FEDERATION_DOMAIN}`;
+    const address = keypair.publicKey();
+
+    await prisma.user.upsert({
+      where: { username },
+      update: { address },
+      create: { username, address },
+    });
+
+    users.push({ username, address, secret: keypair.secret() });
+  }
+
+  fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
+  fs.writeFileSync(OUTPUT_PATH, JSON.stringify(users, null, 2));
+  logger.info(`[seed-webhook-load-test-users] Wrote ${users.length} users to ${OUTPUT_PATH}`);
+};
+
+seedWebhookLoadTestUsers()
+  .catch((err) => {
+    logger.error('[seed-webhook-load-test-users] Failed:', err.message);
+    process.exitCode = 1;
+  })
+  .finally(() => prisma.$disconnect());

stellar-payment-platform/scripts/webhook-load-processor.js

@@ -0,0 +1,42 @@
+// Artillery processor for artillery-webhooks.yml. Signs each virtual user's
+// requests the way a Freighter wallet would (see verifyFreighterSignedMessage
+// in src/routes/v1/webhookRoutes.js), so the load test exercises the real
+// signature-verification path rather than a stubbed-out auth check.
+const crypto = require('crypto');
+const fs = require('fs');
+const path = require('path');
+const { Keypair } = require('@stellar/stellar-sdk');
+
+const FIXTURE_PATH = path.join(__dirname, '.load-test-fixtures', 'webhook-users.json');
+const SIGNED_MESSAGE_PREFIX = Buffer.from('Stellar Signed Message:\n', 'utf8');
+const MOCK_RECEIVER_URL = process.env.MOCK_RECEIVER_URL || 'http://localhost:5099';
+
+let users;
+try {
+  users = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8'));
+} catch {
+  throw new Error(
+    `Missing load-test fixtures at ${FIXTURE_PATH}. Run "node scripts/seed-webhook-load-test-users.js" first.`,
+  );
+}
+
+// Every webhook route authenticates with the same "webhook:<username>"
+// message (see authenticateWebhookCall), so one signature per VU covers
+// register, test-delivery, list, and delete.
+function assignTestUser(context, events, done) {
+  const user = users[Math.floor(Math.random() * users.length)];
+  const message = `webhook:${user.username}`;
+  const hash = crypto
+    .createHash('sha256')
+    .update(Buffer.concat([SIGNED_MESSAGE_PREFIX, Buffer.from(message, 'utf8')]))
+    .digest();
+
+  context.vars.username = user.username;
+  context.vars.signerAddress = user.address;
+  context.vars.signature = Keypair.fromSecret(user.secret).sign(hash).toString('base64');
+  context.vars.webhookUrl = `${MOCK_RECEIVER_URL}/sink/${context.vars.$uuid}`;
+
+  return done();
+}
+
+module.exports = { assignTestUser };

stellar-payment-platform/src/routes/v1/adminRoutes.js

@@ -254,6 +254,42 @@ module.exports = (redisClient) => {
     }),
   );
 
+  // ── GET /admin/webhooks/health ───────────────────────────────────────────
+  // Aggregates webhook delivery health so ops can spot broken merchant
+  // integrations: total/healthy/failing counts, a rolling 24h success rate,
+  // and the URLs that have been failing for more than 24h.
+  router.get('/admin/webhooks/health', adminAuth, asyncHandler(async (req, res) => {
+    const prisma = getPrisma();
+    const username = typeof req.query.username === 'string' ? req.query.username.trim() : '';
+    const where = username ? { username } : {};
+    const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
+
+    const [total, failing, activeLast24h, failingLast24h, failingOver24h] = await Promise.all([
+      prisma.webhook.count({ where }),
+      prisma.webhook.count({ where: { ...where, failingSince: { not: null } } }),
+      prisma.webhook.count({ where: { ...where, lastSentAt: { gte: dayAgo } } }),
+      prisma.webhook.count({ where: { ...where, lastSentAt: { gte: dayAgo }, failingSince: { not: null } } }),
+      prisma.webhook.findMany({
+        where: { ...where, failingSince: { lte: dayAgo } },
+        select: { id: true, username: true, url: true, failingSince: true },
+        orderBy: { failingSince: 'asc' },
+      }),
+    ]);
+
+    return res.status(200).json({
+      success: true,
+      summary: {
+        total,
+        healthy: total - failing,
+        failing,
+        successRate24h: activeLast24h
+          ? Number((((activeLast24h - failingLast24h) / activeLast24h) * 100).toFixed(2))
+          : null,
+      },
+      failingOver24h,
+    });
+  }));
+
   return router;
 };
 

stellar-payment-platform/src/routes/v1/index.js

@@ -2,7 +2,6 @@ const express = require('express');
 
 const userRoutes = require('./userRoutes');
 const receiptRoutes = require('./receiptRoutes');
-const contractRoutes = require('./contractRoutes');
 const webhookRoutes = require('./webhookRoutes');
 const statsRoutes = require('./statsRoutes');
 const historyRoutes = require('./historyRoutes');

stellar-payment-platform/src/webhookWorker.js

@@ -283,6 +283,8 @@ module.exports = {
   dispatchPaymentWebhooks,
   scheduleWebhookRetryJob,
   sendWebhook,
+  markWebhookSuccess,
+  markWebhookFailure,
   computeSignature,
   WEBHOOK_TIMEOUT_MS,
 };