diff --git a/README.md b/README.md index 69111c32..ccea1c25 100644 --- a/README.md +++ b/README.md @@ -391,6 +391,18 @@ Retrieves recent immutable audit trail records for mutating admin actions (`POST Mutating admin requests are intercepted by `auditLogMiddleware` and recorded asynchronously upon response completion. Sensitive keys (`password`, `secret`, `apiKey`, `token`, `signature`, `privateKey`, `seed`) are deeply redacted before persistence. +### `GET /admin/webhooks/health` +Aggregates webhook delivery health so operators can spot broken merchant integrations. +- **Query Parameters:** + - `username` (optional) – Scope the aggregates to one merchant's webhooks. +- **Headers:** `x-api-key` (required) – must match `ADMIN_API_KEY`. +- **Returns:** JSON object with `success: true`, a `summary` (`total`, `healthy`, `failing`, `successRate24h`), and `failingOver24h` — the webhooks that have been failing continuously for more than 24 hours. +- **Status Codes:** + - `200 OK`: Health snapshot retrieved successfully. + - `401 Unauthorized`: Missing or invalid API key. + +A webhook is "failing" while its `failingSince` timestamp is set (cleared on the next successful delivery). `successRate24h` is the share of webhooks with a delivery attempt in the last 24h that are currently healthy; it is `null` when nothing has been active in that window. + ### `GET /metrics` Prometheus scrape endpoint, served in the Prometheus text format. Exempt from the diff --git a/stellar-payment-platform/.gitignore b/stellar-payment-platform/.gitignore index 07008640..a15546d1 100644 --- a/stellar-payment-platform/.gitignore +++ b/stellar-payment-platform/.gitignore @@ -12,3 +12,6 @@ data/*.db-wal # Rotating log files (#294) logs/ + +# Generated webhook load-test keypairs (see scripts/seed-webhook-load-test-users.js) +scripts/.load-test-fixtures/ diff --git a/stellar-payment-platform/artillery-webhooks.yml b/stellar-payment-platform/artillery-webhooks.yml new file mode 100644 index 00000000..65ad8ffa --- /dev/null +++ b/stellar-payment-platform/artillery-webhooks.yml @@ -0,0 +1,64 @@ +config: + target: "{{ $processEnvironment.TARGET_URL || 'http://localhost:5000' }}" + processor: "./scripts/webhook-load-processor.js" + plugins: + expect: {} + ensure: + p95: 500 + maxErrorRate: 1 + phases: + - duration: 1 + arrivalCount: 50 + name: "Spike: 50 concurrent users hitting the webhook API" + defaults: + headers: + content-type: "application/json" + +before: + flow: + - log: "Prereqs: node scripts/seed-webhook-load-test-users.js && node scripts/mock-webhook-receiver.js (or `npm run test:load:webhooks` to do both automatically)" + +scenarios: + - name: "Register a webhook, trigger a delivery, list, then remove it" + beforeScenario: "assignTestUser" + flow: + - post: + url: "/webhooks" + json: + username: "{{ username }}" + signature: "{{ signature }}" + signerAddress: "{{ signerAddress }}" + url: "{{ webhookUrl }}" + events: ["payment.received"] + expect: + - statusCode: 201 + capture: + - json: "$.webhook.id" + as: "webhookId" + + - post: + url: "/webhooks/{{ webhookId }}/test" + json: + username: "{{ username }}" + signature: "{{ signature }}" + signerAddress: "{{ signerAddress }}" + expect: + - statusCode: 200 + + - get: + url: "/webhooks" + json: + username: "{{ username }}" + signature: "{{ signature }}" + signerAddress: "{{ signerAddress }}" + expect: + - statusCode: 200 + + - delete: + url: "/webhooks/{{ webhookId }}" + json: + username: "{{ username }}" + signature: "{{ signature }}" + signerAddress: "{{ signerAddress }}" + expect: + - statusCode: 200 diff --git a/stellar-payment-platform/package.json b/stellar-payment-platform/package.json index c545d7d2..658ddec4 100644 --- a/stellar-payment-platform/package.json +++ b/stellar-payment-platform/package.json @@ -10,6 +10,7 @@ "test:memory": "node --expose-gc node_modules/.bin/jest --forceExit --testPathPattern=memory --globals '{\"gc\":true}'", "test:integration": "TEST_API_URL=http://localhost:5001 jest --forceExit --testPathPattern=integration", "test:load": "artillery run artillery.yml", + "test:load:webhooks": "node scripts/run-webhook-load-test.js", "listener": "node horizonListener.js", "db:seed": "node scripts/seed.js", "prisma:generate": "prisma generate", diff --git a/stellar-payment-platform/scripts/mock-webhook-receiver.js b/stellar-payment-platform/scripts/mock-webhook-receiver.js new file mode 100644 index 00000000..b60ffddc --- /dev/null +++ b/stellar-payment-platform/scripts/mock-webhook-receiver.js @@ -0,0 +1,20 @@ +// Stands in for a merchant's webhook endpoint during load testing so +// deliveries triggered by artillery-webhooks.yml don't leave the machine. +const http = require('http'); + +const PORT = Number(process.env.MOCK_RECEIVER_PORT || 5099); + +const server = http.createServer((req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + req.resume(); + req.on('end', () => { + res.writeHead(200, { 'Content-Type': 'application/json' }).end('{"ok":true}'); + }); +}); + +server.listen(PORT, () => { + console.log(`[mock-webhook-receiver] listening on :${PORT}`); +}); diff --git a/stellar-payment-platform/scripts/run-webhook-load-test.js b/stellar-payment-platform/scripts/run-webhook-load-test.js new file mode 100644 index 00000000..35695660 --- /dev/null +++ b/stellar-payment-platform/scripts/run-webhook-load-test.js @@ -0,0 +1,30 @@ +// Cross-platform entry point for `npm run test:load:webhooks`: seeds test +// users, starts the mock receiver, runs Artillery, then cleans up. +const path = require('path'); +const { spawn } = require('child_process'); + +const ROOT = path.join(__dirname, '..'); + +const run = (command, args) => + new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd: ROOT, stdio: 'inherit', shell: true }); + child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`${command} exited with code ${code}`)))); + }); + +const main = async () => { + await run('node', ['scripts/seed-webhook-load-test-users.js']); + + const receiver = spawn('node', ['scripts/mock-webhook-receiver.js'], { cwd: ROOT, stdio: 'inherit', shell: true }); + await new Promise((resolve) => setTimeout(resolve, 500)); + + try { + await run('npx', ['artillery', 'run', 'artillery-webhooks.yml']); + } finally { + receiver.kill(); + } +}; + +main().catch((err) => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/stellar-payment-platform/scripts/seed-webhook-load-test-users.js b/stellar-payment-platform/scripts/seed-webhook-load-test-users.js new file mode 100644 index 00000000..9bcb3a2f --- /dev/null +++ b/stellar-payment-platform/scripts/seed-webhook-load-test-users.js @@ -0,0 +1,44 @@ +// Provisions Stellar-keypair-backed test users for the webhook load test +// (artillery-webhooks.yml). Each user's secret key is written to a local +// fixture so the Artillery processor can sign requests the same way a real +// Freighter wallet would (see verifyFreighterSignedMessage in webhookRoutes.js). +require('dotenv').config(); + +const fs = require('fs'); +const path = require('path'); +const { Keypair } = require('@stellar/stellar-sdk'); +const { prisma } = require('../prismaClient'); +const { logger } = require('../src/logger'); + +const USER_COUNT = Number(process.env.LOAD_TEST_USER_COUNT || 50); +const FEDERATION_DOMAIN = process.env.FEDERATION_DOMAIN || 'localhost'; +const OUTPUT_PATH = path.join(__dirname, '.load-test-fixtures', 'webhook-users.json'); + +const seedWebhookLoadTestUsers = async () => { + const users = []; + + for (let i = 0; i < USER_COUNT; i++) { + const keypair = Keypair.random(); + const username = `webhook-load-${i}*${FEDERATION_DOMAIN}`; + const address = keypair.publicKey(); + + await prisma.user.upsert({ + where: { username }, + update: { address }, + create: { username, address }, + }); + + users.push({ username, address, secret: keypair.secret() }); + } + + fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true }); + fs.writeFileSync(OUTPUT_PATH, JSON.stringify(users, null, 2)); + logger.info(`[seed-webhook-load-test-users] Wrote ${users.length} users to ${OUTPUT_PATH}`); +}; + +seedWebhookLoadTestUsers() + .catch((err) => { + logger.error('[seed-webhook-load-test-users] Failed:', err.message); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect()); diff --git a/stellar-payment-platform/scripts/webhook-load-processor.js b/stellar-payment-platform/scripts/webhook-load-processor.js new file mode 100644 index 00000000..78951b7e --- /dev/null +++ b/stellar-payment-platform/scripts/webhook-load-processor.js @@ -0,0 +1,42 @@ +// Artillery processor for artillery-webhooks.yml. Signs each virtual user's +// requests the way a Freighter wallet would (see verifyFreighterSignedMessage +// in src/routes/v1/webhookRoutes.js), so the load test exercises the real +// signature-verification path rather than a stubbed-out auth check. +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); +const { Keypair } = require('@stellar/stellar-sdk'); + +const FIXTURE_PATH = path.join(__dirname, '.load-test-fixtures', 'webhook-users.json'); +const SIGNED_MESSAGE_PREFIX = Buffer.from('Stellar Signed Message:\n', 'utf8'); +const MOCK_RECEIVER_URL = process.env.MOCK_RECEIVER_URL || 'http://localhost:5099'; + +let users; +try { + users = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')); +} catch { + throw new Error( + `Missing load-test fixtures at ${FIXTURE_PATH}. Run "node scripts/seed-webhook-load-test-users.js" first.`, + ); +} + +// Every webhook route authenticates with the same "webhook:" +// message (see authenticateWebhookCall), so one signature per VU covers +// register, test-delivery, list, and delete. +function assignTestUser(context, events, done) { + const user = users[Math.floor(Math.random() * users.length)]; + const message = `webhook:${user.username}`; + const hash = crypto + .createHash('sha256') + .update(Buffer.concat([SIGNED_MESSAGE_PREFIX, Buffer.from(message, 'utf8')])) + .digest(); + + context.vars.username = user.username; + context.vars.signerAddress = user.address; + context.vars.signature = Keypair.fromSecret(user.secret).sign(hash).toString('base64'); + context.vars.webhookUrl = `${MOCK_RECEIVER_URL}/sink/${context.vars.$uuid}`; + + return done(); +} + +module.exports = { assignTestUser }; diff --git a/stellar-payment-platform/src/routes/v1/adminRoutes.js b/stellar-payment-platform/src/routes/v1/adminRoutes.js index 53dd9f6d..d4eb88d5 100644 --- a/stellar-payment-platform/src/routes/v1/adminRoutes.js +++ b/stellar-payment-platform/src/routes/v1/adminRoutes.js @@ -254,6 +254,42 @@ module.exports = (redisClient) => { }), ); + // ── GET /admin/webhooks/health ─────────────────────────────────────────── + // Aggregates webhook delivery health so ops can spot broken merchant + // integrations: total/healthy/failing counts, a rolling 24h success rate, + // and the URLs that have been failing for more than 24h. + router.get('/admin/webhooks/health', adminAuth, asyncHandler(async (req, res) => { + const prisma = getPrisma(); + const username = typeof req.query.username === 'string' ? req.query.username.trim() : ''; + const where = username ? { username } : {}; + const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); + + const [total, failing, activeLast24h, failingLast24h, failingOver24h] = await Promise.all([ + prisma.webhook.count({ where }), + prisma.webhook.count({ where: { ...where, failingSince: { not: null } } }), + prisma.webhook.count({ where: { ...where, lastSentAt: { gte: dayAgo } } }), + prisma.webhook.count({ where: { ...where, lastSentAt: { gte: dayAgo }, failingSince: { not: null } } }), + prisma.webhook.findMany({ + where: { ...where, failingSince: { lte: dayAgo } }, + select: { id: true, username: true, url: true, failingSince: true }, + orderBy: { failingSince: 'asc' }, + }), + ]); + + return res.status(200).json({ + success: true, + summary: { + total, + healthy: total - failing, + failing, + successRate24h: activeLast24h + ? Number((((activeLast24h - failingLast24h) / activeLast24h) * 100).toFixed(2)) + : null, + }, + failingOver24h, + }); + })); + return router; }; diff --git a/stellar-payment-platform/src/routes/v1/index.js b/stellar-payment-platform/src/routes/v1/index.js index b55a65f0..1c9c6f44 100644 --- a/stellar-payment-platform/src/routes/v1/index.js +++ b/stellar-payment-platform/src/routes/v1/index.js @@ -2,7 +2,6 @@ const express = require('express'); const userRoutes = require('./userRoutes'); const receiptRoutes = require('./receiptRoutes'); -const contractRoutes = require('./contractRoutes'); const webhookRoutes = require('./webhookRoutes'); const statsRoutes = require('./statsRoutes'); const historyRoutes = require('./historyRoutes'); diff --git a/stellar-payment-platform/src/webhookWorker.js b/stellar-payment-platform/src/webhookWorker.js index 27ae0bff..3d5c390e 100644 --- a/stellar-payment-platform/src/webhookWorker.js +++ b/stellar-payment-platform/src/webhookWorker.js @@ -283,6 +283,8 @@ module.exports = { dispatchPaymentWebhooks, scheduleWebhookRetryJob, sendWebhook, + markWebhookSuccess, + markWebhookFailure, computeSignature, WEBHOOK_TIMEOUT_MS, }; diff --git a/stellar-payment-platform/tests/admin-webhook-health.test.js b/stellar-payment-platform/tests/admin-webhook-health.test.js new file mode 100644 index 00000000..4ac7f88b --- /dev/null +++ b/stellar-payment-platform/tests/admin-webhook-health.test.js @@ -0,0 +1,150 @@ +'use strict'; + +/** + * tests/admin-webhook-health.test.js + * + * Tests for GET /admin/webhooks/health (issue: webhook delivery observability). + */ + +const request = require('supertest'); + +// ── mocks ──────────────────────────────────────────────────────────────────── + +jest.mock('redis', () => ({ createClient: jest.fn(() => null) })); +jest.mock('../src/cleanup-cron', () => ({ scheduleCleanupJob: jest.fn() })); +jest.mock('../src/soft-delete-purge-cron', () => ({ scheduleSoftDeletePurgeJob: jest.fn() })); + +const mockWebhookCount = jest.fn(); +const mockWebhookFindMany = jest.fn(); + +jest.mock('../prismaClient', () => ({ + prisma: { + user: { + findUnique: jest.fn(), + findFirst: jest.fn(), + findMany: jest.fn(), + count: jest.fn(), + create: jest.fn(), + update: jest.fn(), + }, + payment: { findMany: jest.fn().mockResolvedValue([]) }, + webhook: { + count: mockWebhookCount, + findMany: mockWebhookFindMany, + }, + auditLog: { create: jest.fn().mockResolvedValue({}) }, + $transaction: jest.fn().mockResolvedValue([0, []]), + $queryRaw: jest.fn().mockResolvedValue([]), + $metrics: { json: jest.fn().mockResolvedValue({ counters: [], gauges: [], histograms: [] }) }, + }, + isPrismaConnectionError: () => false, +})); + +jest.mock('@stellar/stellar-sdk', () => ({ + Horizon: { Server: jest.fn().mockImplementation(() => ({ payments: jest.fn() })) }, + StrKey: { isValidEd25519PublicKey: jest.fn((v) => typeof v === 'string' && v.startsWith('G')) }, + Keypair: { fromPublicKey: jest.fn(() => ({ verify: jest.fn(() => true) })) }, +})); +jest.mock('pdfkit', () => jest.fn()); + +process.env.NODE_ENV = 'test'; +process.env.ADMIN_API_KEY = 'test-admin-key'; + +// Load app after mocks are in place. +const { app } = require('../server'); + +describe('GET /admin/webhooks/health', () => { + beforeEach(() => { + mockWebhookCount.mockReset(); + mockWebhookFindMany.mockReset(); + }); + + it('returns 401 without API key', async () => { + const res = await request(app).get('/api/v1/admin/webhooks/health'); + expect(res.status).toBe(401); + }); + + it('returns 401 with wrong API key', async () => { + const res = await request(app) + .get('/api/v1/admin/webhooks/health') + .set('x-api-key', 'wrong-key'); + expect(res.status).toBe(401); + }); + + it('aggregates totals, success rate, and stale failures', async () => { + // total, failing, activeLast24h, failingLast24h + mockWebhookCount + .mockResolvedValueOnce(10) + .mockResolvedValueOnce(3) + .mockResolvedValueOnce(4) + .mockResolvedValueOnce(1); + + const staleFailure = { + id: 'wh-1', + username: 'alice', + url: 'https://broken.example.com/hook', + failingSince: new Date('2026-01-01T00:00:00Z'), + }; + mockWebhookFindMany.mockResolvedValueOnce([staleFailure]); + + const res = await request(app) + .get('/api/v1/admin/webhooks/health') + .set('x-api-key', 'test-admin-key'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + summary: { + total: 10, + healthy: 7, + failing: 3, + successRate24h: 75, + }, + failingOver24h: [ + { + id: 'wh-1', + username: 'alice', + url: 'https://broken.example.com/hook', + failingSince: '2026-01-01T00:00:00.000Z', + }, + ], + }); + }); + + it('returns null successRate24h when nothing has been active', async () => { + mockWebhookCount + .mockResolvedValueOnce(2) + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(0); + mockWebhookFindMany.mockResolvedValueOnce([]); + + const res = await request(app) + .get('/api/v1/admin/webhooks/health') + .set('x-api-key', 'test-admin-key'); + + expect(res.status).toBe(200); + expect(res.body.summary.successRate24h).toBeNull(); + }); + + it('scopes all queries by username when provided', async () => { + mockWebhookCount + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(0); + mockWebhookFindMany.mockResolvedValueOnce([]); + + const res = await request(app) + .get('/api/v1/admin/webhooks/health?username=alice') + .set('x-api-key', 'test-admin-key'); + + expect(res.status).toBe(200); + expect(mockWebhookCount).toHaveBeenCalledWith({ where: { username: 'alice' } }); + expect(mockWebhookFindMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ username: 'alice' }), + }), + ); + }); +});