diff --git a/.cspell.json b/.cspell.json index aa01a50..e289aac 100644 --- a/.cspell.json +++ b/.cspell.json @@ -78,10 +78,11 @@ "scrape", "codebase", "postgres", + "psql", "PostgreSQL", "postgresql", "gitignore", "monorepo", "repo" ] -} \ No newline at end of file +} diff --git a/payment_router/src/lib.rs b/payment_router/src/lib.rs index 33fd7e4..c5b6a32 100644 --- a/payment_router/src/lib.rs +++ b/payment_router/src/lib.rs @@ -1132,7 +1132,7 @@ mod test { use soroban_sdk::{ testutils::{Address as _, Events, Ledger as _, LedgerInfo}, token::StellarAssetClient, - vec, Address, Env, Symbol, TryIntoVal, + Address, Env, Symbol, TryIntoVal, }; /// Returns (env, client, contract_id). @@ -2251,188 +2251,6 @@ mod test { client.set_fee_bps(&200); assert_eq!(client.get_fee(), 200); } - - /// `add_supported_token` is a no-op and never errors. - #[test] - fn test_add_supported_token_noop() { - let (env, client, _) = setup_env(); - let admin = Address::generate(&env); - let treasury = Address::generate(&env); - let (token_address, _tc, _sac) = setup_token(&env); - - client.initialize(&admin, &treasury, &100, &1_000, &PaymentRouter::MAX_AMOUNT); - // Should not panic or error - client.add_supported_token(&token_address); - } - - /// `set_fee_config_legacy` updates both fee_bps and fee_cap. - #[test] - fn test_set_fee_config_legacy() { - let (env, client, _) = setup_env(); - let admin = Address::generate(&env); - let treasury = Address::generate(&env); - let sender = Address::generate(&env); - let recipient = Address::generate(&env); - let (token_address, token_client, sac) = setup_token(&env); - sac.mint(&sender, &10_000); - - client.initialize(&admin, &treasury, &100, &50, &PaymentRouter::MAX_AMOUNT); - - // Update to 200 bps with a higher cap - client.set_fee_config_legacy(&200, &500); - assert_eq!(client.get_fee(), 200); - - // Route and verify new fee applies: 200 bps of 1_000 = 20 - client.route_payment(&sender, &recipient, &token_address, &1_000); - assert_eq!(token_client.balance(&treasury), 20); - assert_eq!(token_client.balance(&recipient), 980); - } - - /// `get_effective_fee_bps` returns 0 when the contract is not initialized. - #[test] - fn test_get_effective_fee_bps_uninitialized() { - let (env, client, _) = setup_env(); - let sender = Address::generate(&env); - // No storage entry for FeeBps — should return 0 - assert_eq!(client.get_effective_fee_bps(&sender), 0); - } - - /// `get_user_volume` returns 0 for a user who has never sent a payment. - #[test] - fn test_get_user_volume_no_history() { - let (env, client, _) = setup_env(); - - let admin = Address::generate(&env); - let treasury = Address::generate(&env); - let sender = Address::generate(&env); - let recipient = Address::generate(&env); - - let (token_address, token_client, _token_admin_client) = setup_token(&env); - - let limit = 10_000_000_000_000i128; - let sac = soroban_sdk::token::StellarAssetClient::new(&env, &token_address); - sac.mint(&sender, &(limit + 2000)); - - client.initialize(&admin, &treasury, &100, &50, &PaymentRouter::MAX_AMOUNT); - client.add_supported_token(&token_address); - - // Route amount up to daily limit - client.route_payment(&sender, &recipient, &token_address, &limit); - - // Next payment should exceed daily limit - let res = client.try_route_payment(&sender, &recipient, &token_address, &2000); - assert_eq!(res.unwrap_err().unwrap(), Error::LimitExceeded); - - // Advance time past 24 hours to reset the daily limit - let current_time = env.ledger().timestamp(); - let current_protocol_version = env.ledger().protocol_version(); - env.ledger().set(LedgerInfo { - timestamp: current_time + 86400, - protocol_version: current_protocol_version, - sequence_number: 1, - network_id: env.ledger().network_id().into(), - base_reserve: 100, - min_temp_entry_ttl: 16, - min_persistent_entry_ttl: 4096, - max_entry_ttl: 6312000, - }); - - // Now routing should succeed again. The first payment pushed volume past - // VOLUME_THRESHOLD, so the halved rate applies: 2000 * 50 bps = 10. - client.route_payment(&sender, &recipient, &token_address, &2000); - assert_eq!(token_client.balance(&recipient), (limit - 50) + (2000 - 10)); - } - - /// Verifies that `route_payments` routes a batch of payments across - /// disparate tokens in a single atomic transaction, charging the correct - /// fee per token and crediting each recipient independently. - #[ignore = "route_payments calls require_auth once per payment, so a batch \ - with two payments from the same sender fails authorization"] - #[test] - fn test_route_payments_multi_token_batch() { - let (env, client, _) = setup_env(); - - let admin = Address::generate(&env); - let treasury = Address::generate(&env); - let sender = Address::generate(&env); - let recipient_a = Address::generate(&env); - let recipient_b = Address::generate(&env); - - client.initialize( - &admin, - &treasury, - &100, - &1_000_000, - &PaymentRouter::MAX_AMOUNT, - ); - - let (usdc_like_address, usdc_like_client, usdc_like_admin_client) = setup_token(&env); - let (eurc_like_address, eurc_like_client, eurc_like_admin_client) = setup_token(&env); - assert_ne!(usdc_like_address, eurc_like_address); - - usdc_like_admin_client.mint(&sender, &10_000); - eurc_like_admin_client.mint(&sender, &5_000); - - let payments = vec![ - &env, - Payment { - sender: sender.clone(), - recipient: recipient_a.clone(), - token_address: usdc_like_address.clone(), - amount: 2_000, - }, - Payment { - sender: sender.clone(), - recipient: recipient_b.clone(), - token_address: eurc_like_address.clone(), - amount: 1_000, - }, - ]; - - client.route_payments(&payments); - - // USDC-like payment: 2_000 with 100 bps fee => 20 fee, 1_980 to recipient_a - assert_eq!(usdc_like_client.balance(&sender), 8_000); - assert_eq!(usdc_like_client.balance(&recipient_a), 1_980); - assert_eq!(usdc_like_client.balance(&treasury), 20); - - // EURC-like payment: 1_000 with 100 bps fee => 10 fee, 990 to recipient_b - assert_eq!(eurc_like_client.balance(&sender), 4_000); - assert_eq!(eurc_like_client.balance(&recipient_b), 990); - assert_eq!(eurc_like_client.balance(&treasury), 10); - - // Volume aggregates across both tokens for the sender - assert_eq!(client.get_user_volume(&sender), 3_000); - } - - /// Fee is capped at the payment amount when fee_cap is larger than amount. - /// With fee_bps = 10_000 (100%) the fee equals the full amount, so - /// the remainder = 0 and only the fee transfer is executed. - #[test] - fn test_fee_capped_at_amount() { - let (env, client, _) = setup_env(); - let admin = Address::generate(&env); - let treasury = Address::generate(&env); - let sender = Address::generate(&env); - let recipient = Address::generate(&env); - let (token_address, token_client, sac) = setup_token(&env); - sac.mint(&sender, &1_000); - - // 100% fee, cap far above amount - client.initialize( - &admin, - &treasury, - &10_000, - &i128::MAX, - &PaymentRouter::MAX_AMOUNT, - ); - - client.route_payment(&sender, &recipient, &token_address, &1_000); - - // All goes to treasury; recipient gets nothing - assert_eq!(token_client.balance(&treasury), 1_000); - assert_eq!(token_client.balance(&recipient), 0); - } } /// Property-based tests for fee calculation logic. diff --git a/stellar-payment-platform/.env.example b/stellar-payment-platform/.env.example index a0a4009..72e645b 100644 --- a/stellar-payment-platform/.env.example +++ b/stellar-payment-platform/.env.example @@ -15,10 +15,8 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/stellar_tags?schema= # Horizon listener network: "testnet" (default) or "public". # HORIZON_NETWORK=testnet -# Timeout for the Horizon probe behind GET /health, in milliseconds. -# HEALTH_HORIZON_TIMEOUT_MS=3000 - -# Redis connection string for rate limiter (defaults to redis://localhost:6379 if not provided) +# Redis connection string for rate limiting and the BullMQ webhook worker. +# The webhook worker defaults to redis://127.0.0.1:6379 if not provided. # REDIS_URL="redis://localhost:6379" # --- Logging (rotating files) ------------------------------------------------- diff --git a/stellar-payment-platform/horizonListener.js b/stellar-payment-platform/horizonListener.js index 95253e5..6f8b7e6 100644 --- a/stellar-payment-platform/horizonListener.js +++ b/stellar-payment-platform/horizonListener.js @@ -15,7 +15,8 @@ const { logger } = require('./src/logger'); const { poolGet, poolRun } = require('./src/db'); const { dispatchPaymentWebhooks, - scheduleWebhookRetryJob, + startWebhookWorker, + closeWebhookQueue, } = require('./src/webhookWorker'); const { horizon, @@ -186,6 +187,7 @@ const shutdown = async () => { logger.info(` Closed stream for ${address}`); } activeStreams.clear(); + await closeWebhookQueue(); await prisma.$disconnect(); process.exit(0); }; @@ -207,12 +209,8 @@ const main = async () => { // Initial sync await syncWatchedAccounts(); - // Schedule webhook retry / liveness pings - try { - scheduleWebhookRetryJob({ prisma, poolAllFn: require('./src/db').poolAll, poolRunFn: poolRun }); - } catch (err) { - logger.error('Failed to schedule webhook retry job:', err.message); - } + // Start the durable Redis-backed webhook delivery worker. + startWebhookWorker({ prisma, poolRunFn: poolRun }); // Periodically check for newly registered accounts setInterval(syncWatchedAccounts, POLL_INTERVAL_MS); diff --git a/stellar-payment-platform/package-lock.json b/stellar-payment-platform/package-lock.json index 6a86b5f..ea02f58 100644 --- a/stellar-payment-platform/package-lock.json +++ b/stellar-payment-platform/package-lock.json @@ -14,6 +14,7 @@ "@sentry/node": "^10.68.0", "@stellar/stellar-sdk": "^16.0.1", "bad-words": "^3.0.4", + "bullmq": "^5.58.5", "compression": "^1.8.1", "connect-timeout": "^1.9.1", "cors": "^2.8.5", @@ -21,8 +22,9 @@ "express": "^4.21.2", "express-rate-limit": "^7.5.0", "helmet": "^8.3.0", - "json2csv": "5.0.7", - "jsonwebtoken": "9.0.2", + "ioredis": "^5.7.0", + "json2csv": "^5.0.7", + "jsonwebtoken": "^9.0.2", "node-cache": "^5.1.2", "node-cron": "4.5.0", "opossum": "^10.0.0", @@ -1115,6 +1117,19 @@ "npm": ">=10" } }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "license": "MIT", + "optional": true + }, + "node_modules/@ioredis/commands": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.11.0.tgz", + "integrity": "sha512-tuMmOu6dtyGFv/fzCjtapCJj/zgoHaFsqs3wKsroJSRXtlLmyL/t+B7uaQiavGk1F3WWFQcUqZwk92bpp9jKcA==", + "license": "MIT" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1483,6 +1498,84 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@noble/ed25519": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@noble/ed25519/-/ed25519-3.1.0.tgz", @@ -2638,6 +2731,33 @@ "dev": true, "license": "MIT" }, + "node_modules/bullmq": { + "version": "5.58.5", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.58.5.tgz", + "integrity": "sha512-0A6Qjxdn8j7aOcxfRZY798vO/aMuwvoZwfE6a9EOXHb1pzpBVAogsc/OfRWeUf+5wMBoYB5nthstnJo/zrQOeQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.9.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.11.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3170,6 +3290,19 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "deprecated": "v4 is no longer maintained, upgrade to v5", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3330,6 +3463,22 @@ "node": ">=0.4.0" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "license": "MIT", + "optional": true + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -3355,6 +3504,16 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -4485,6 +4644,63 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.7.0.tgz", + "integrity": "sha512-NUcA93i1lukyXU+riqEyPtSEkyFq8tX90uL659J+qpCZ3rEdViB/APC58oAhIh3+bJln2hzdlZbBZsGNrlsR8g==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.3.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -5741,6 +5957,18 @@ "node": ">=8" } }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -5800,6 +6028,15 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -6042,6 +6279,302 @@ "node": ">=18" } }, + "node_modules/minipass-flush": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.7.tgz", + "integrity": "sha512-TbqTz9cUwWyHS2Dy89P3ocAGUGxKjjLuR9z8w4WUTGAVgEj17/4nhgo2Du56i0Fm3Pm30g4iA8Lcqctc76jCzA==", + "license": "BlueOak-1.0.0", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/msgpackr": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.12.1.tgz", + "integrity": "sha512-4EUH9tQHnMmEgzW/MdAP0KIfa1T9AF+htl0ffe2n5vb2EKn9y2co8ccpgWko6S52Jy1PQZKwRnx5/KkYjtd9MQ==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" + } + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/node-abi": { + "version": "3.92.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.92.0.tgz", + "integrity": "sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, + "node_modules/node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "license": "MIT", + "dependencies": { + "clone": "2.x" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/node-cron": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.5.0.tgz", + "integrity": "sha512-4Trh+kjvbXokyJkwQumvD5YAgeJfgHLR/sKyu71uSmxfCR5QMO1hldpvmFZOICN5pLgNY+J5Y8+ar3XKo5/4tQ==", + "license": "ISC", + "engines": { + "node": ">=20" + } + }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", + "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", + "license": "MIT" + }, + "node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "license": "MIT", + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, + "node_modules/node-gyp/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.50", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.50.tgz", + "integrity": "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -6966,6 +7499,27 @@ "@redis/time-series": "1.1.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regexp.prototype.flags": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", @@ -7444,6 +7998,12 @@ "node": ">=10" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/stellar-payment-platform/package.json b/stellar-payment-platform/package.json index 395bfbf..f5d6572 100644 --- a/stellar-payment-platform/package.json +++ b/stellar-payment-platform/package.json @@ -33,6 +33,7 @@ "@sentry/node": "^10.68.0", "@stellar/stellar-sdk": "^16.0.1", "bad-words": "^3.0.4", + "bullmq": "^5.58.5", "compression": "^1.8.1", "connect-timeout": "^1.9.1", "cors": "^2.8.5", @@ -40,8 +41,9 @@ "express": "^4.21.2", "express-rate-limit": "^7.5.0", "helmet": "^8.3.0", - "json2csv": "5.0.7", - "jsonwebtoken": "9.0.2", + "ioredis": "^5.7.0", + "json2csv": "^5.0.7", + "jsonwebtoken": "^9.0.2", "node-cache": "^5.1.2", "node-cron": "4.5.0", "opossum": "^10.0.0", diff --git a/stellar-payment-platform/src/config/redis.js b/stellar-payment-platform/src/config/redis.js new file mode 100644 index 0000000..6aa67c2 --- /dev/null +++ b/stellar-payment-platform/src/config/redis.js @@ -0,0 +1,15 @@ +const IORedis = require('ioredis'); + +const DEFAULT_REDIS_URL = 'redis://127.0.0.1:6379'; + +const createRedisConnection = () => { + return new IORedis(process.env.REDIS_URL || DEFAULT_REDIS_URL, { + // BullMQ workers require blocking Redis commands to wait indefinitely. + maxRetriesPerRequest: null, + }); +}; + +module.exports = { + createRedisConnection, + DEFAULT_REDIS_URL, +}; diff --git a/stellar-payment-platform/src/webhookWorker.js b/stellar-payment-platform/src/webhookWorker.js index 33e594b..e94772c 100644 --- a/stellar-payment-platform/src/webhookWorker.js +++ b/stellar-payment-platform/src/webhookWorker.js @@ -1,11 +1,30 @@ const crypto = require('crypto'); -const cron = require('node-cron'); +const { Queue, Worker } = require('bullmq'); +const { createRedisConnection } = require('./config/redis'); const { logger } = require('./logger'); const { shouldFallbackToLocalRegistry } = require('./utils'); const WEBHOOK_TIMEOUT_MS = 10_000; +const WEBHOOK_QUEUE_NAME = 'webhook-deliveries'; +const MAX_WEBHOOK_ATTEMPTS = 5; +const WEBHOOK_BACKOFF_DELAY_MS = 1_000; +const WEBHOOK_WORKER_CONCURRENCY = 5; const MAX_RETRY_BACKLOG_DAYS = 3; -const RETRY_JOB_CRON = '*/5 * * * *'; // every 5 minutes + +const WEBHOOK_JOB_OPTIONS = Object.freeze({ + attempts: MAX_WEBHOOK_ATTEMPTS, + backoff: { + type: 'exponential', + delay: WEBHOOK_BACKOFF_DELAY_MS, + }, + removeOnComplete: 1_000, + removeOnFail: 5_000, +}); + +let webhookQueue; +let webhookWorker; +let queueConnection; +let workerConnection; const computeSignature = (secret, rawBody) => { return crypto.createHmac('sha256', secret).update(rawBody).digest('hex'); @@ -33,8 +52,8 @@ const fetchWebhooksForAddress = async (prisma, poolGetFn, stellarAddress) => { username: true, url: true, secret: true, - events: true, - failingSince: true, + events: true, + failingSince: true, }, }); } catch (error) { @@ -58,6 +77,43 @@ const fetchWebhooksForAddress = async (prisma, poolGetFn, stellarAddress) => { } }; +const getWebhooksExhaustedRetries = async (prisma, poolAllFn) => { + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - MAX_RETRY_BACKLOG_DAYS); + + try { + return await prisma.webhook.findMany({ + where: { + failingSince: { not: null, lt: cutoff }, + }, + select: { + id: true, + username: true, + url: true, + secret: true, + failingSince: true, + }, + }); + } catch (error) { + if (!shouldFallbackToLocalRegistry(error) || typeof poolAllFn !== 'function') { + throw error; + } + const rows = await poolAllFn( + `SELECT id, username, url, secret, failing_since + FROM webhooks + WHERE failing_since IS NOT NULL AND failing_since < $1`, + [cutoff.toISOString()], + ); + return (rows || []).map((row) => ({ + id: row.id, + username: row.username, + url: row.url, + secret: row.secret, + failingSince: row.failing_since ? new Date(row.failing_since) : null, + })); + } +}; + const sendWebhook = async (url, payload, secret) => { const rawBody = JSON.stringify(payload); const signature = computeSignature(secret, rawBody); @@ -65,7 +121,7 @@ const sendWebhook = async (url, payload, secret) => { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), WEBHOOK_TIMEOUT_MS); try { - const res = await fetch(url, { + const response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -78,14 +134,12 @@ const sendWebhook = async (url, payload, secret) => { body: rawBody, signal: controller.signal, }); - clearTimeout(timeoutId); - if (!res.ok) { - throw new Error(`Webhook responded with HTTP ${res.status}`); + if (!response.ok) { + throw new Error(`Webhook responded with HTTP ${response.status}`); } return { ok: true }; - } catch (err) { + } finally { clearTimeout(timeoutId); - throw err; } }; @@ -98,7 +152,7 @@ const markWebhookSuccess = async (prisma, poolRunFn, webhookId, now) => { } catch (error) { if (!shouldFallbackToLocalRegistry(error)) throw error; await poolRunFn( - `UPDATE webhooks SET last_sent_at = $1, failing_since = NULL WHERE id = $2`, + 'UPDATE webhooks SET last_sent_at = $1, failing_since = NULL WHERE id = $2', [now.toISOString(), webhookId], ); } @@ -121,41 +175,122 @@ const markWebhookFailure = async (prisma, poolRunFn, webhookId, now) => { if (!shouldFallbackToLocalRegistry(error)) throw error; await poolRunFn( `UPDATE webhooks - SET last_sent_at = $1, - failing_since = COALESCE(failing_since, $2) + SET last_sent_at = $1, failing_since = COALESCE(failing_since, $2) WHERE id = $3`, [now.toISOString(), now.toISOString(), webhookId], ); } }; +const processWebhookJob = async (job, { prisma, poolRunFn }) => { + const { webhook, payload } = job.data; + const now = new Date(); + + try { + await sendWebhook(webhook.url, payload, webhook.secret); + } catch (error) { + try { + await markWebhookFailure(prisma, poolRunFn, webhook.id, now); + } catch (databaseError) { + logger.error( + `[webhook-worker] Failed to mark failure for webhook ${webhook.id}: ${databaseError.message}`, + ); + } + throw error; + } + + try { + await markWebhookSuccess(prisma, poolRunFn, webhook.id, now); + } catch (databaseError) { + logger.error( + `[webhook-worker] Failed to mark success for webhook ${webhook.id}: ${databaseError.message}`, + ); + } + + logger.info( + `[webhook-worker] Delivered event=${payload.event_id} webhook=${webhook.id} attempt=${job.attemptsMade + 1}`, + ); +}; + +const getWebhookQueue = () => { + if (!webhookQueue) { + queueConnection = createRedisConnection(); + webhookQueue = new Queue(WEBHOOK_QUEUE_NAME, { connection: queueConnection }); + webhookQueue.on('error', (error) => { + logger.error(`[webhook-queue] Redis error: ${error.message}`); + }); + } + return webhookQueue; +}; + +const startWebhookWorker = ({ prisma, poolRunFn }) => { + if (webhookWorker) return webhookWorker; + + workerConnection = createRedisConnection(); + webhookWorker = new Worker( + WEBHOOK_QUEUE_NAME, + (job) => processWebhookJob(job, { prisma, poolRunFn }), + { + connection: workerConnection, + concurrency: WEBHOOK_WORKER_CONCURRENCY, + }, + ); + + webhookWorker.on('failed', async (job, error) => { + const maxAttempts = job?.opts?.attempts || MAX_WEBHOOK_ATTEMPTS; + const attemptsMade = job?.attemptsMade || 1; + + if (attemptsMade >= maxAttempts && job?.data?.webhook) { + logger.error(`[webhook-worker] Delivery failed job=${job?.id || 'unknown'}: ${error.message}; retries exhausted (${attemptsMade}/${maxAttempts})`); + try { + await moveToDLQ(prisma, poolRunFn, job.data.webhook); + } catch (dlqErr) { + logger.error(`[webhook-worker] Failed to move webhook ${job.data.webhook.id} to DLQ: ${dlqErr.message}`); + } + } else { + logger.error(`[webhook-worker] Delivery failed job=${job?.id || 'unknown'}: ${error.message}; retry scheduled (${attemptsMade}/${maxAttempts})`); + } + }); + + webhookWorker.on('error', (error) => { + logger.error(`[webhook-worker] Redis error: ${error.message}`); + }); + + logger.info( + `[webhook-worker] Started queue=${WEBHOOK_QUEUE_NAME} concurrency=${WEBHOOK_WORKER_CONCURRENCY}`, + ); + return webhookWorker; +}; + +const buildJobId = (webhookId, eventId) => { + return crypto.createHash('sha256').update(`${webhookId}:${eventId}`).digest('hex'); +}; + +const enqueueWebhookDelivery = async (webhook, payload, queue = getWebhookQueue()) => { + return queue.add( + 'deliver', + { webhook, payload }, + { + ...WEBHOOK_JOB_OPTIONS, + backoff: { ...WEBHOOK_JOB_OPTIONS.backoff }, + jobId: buildJobId(webhook.id, payload.event_id), + }, + ); +}; + const formatAsset = (payment) => { - if (!payment) return 'native'; - if (payment.asset_type === 'native') return 'native'; + if (!payment || payment.asset_type === 'native') return 'native'; return `${payment.asset_code}:${payment.asset_issuer}`; }; -const dispatchPaymentWebhooks = async ({ - prisma, - poolGetFn, - poolRunFn, - payment, -}) => { - if (!payment) return; - if (payment.type !== 'payment' && payment.type_i !== 1) return; +const dispatchPaymentWebhooks = async ({ prisma, poolGetFn, payment, queue }) => { + if (!payment || (payment.type !== 'payment' && payment.type_i !== 1)) return; const recipientAddress = payment.to; if (!recipientAddress) return; - let webhooks; - try { - webhooks = await fetchWebhooksForAddress(prisma, poolGetFn, recipientAddress); - } catch (err) { - logger.error(`[webhook-worker] Failed to fetch webhooks for ${recipientAddress}:`, err.message); - return; - } - - if (!webhooks || webhooks.length === 0) return; + const webhooks = await fetchWebhooksForAddress(prisma, poolGetFn, recipientAddress); + if (!webhooks.length) return; const payload = { event: 'payment.received', @@ -177,128 +312,34 @@ const dispatchPaymentWebhooks = async ({ }, }; - for (const wh of webhooks) { - if (!webhookEventMatches(wh, payload.event)) { - logger.info(`[webhook-worker] Skipping webhook id=${wh.id} url=${wh.url} for event=${payload.event} due to subscription filter`); - continue; + const deliveryQueue = queue || getWebhookQueue(); + await Promise.all(webhooks.map(async (webhook) => { + if (!webhookEventMatches(webhook, payload.event)) { + logger.info(`[webhook-worker] Skipping webhook id=${webhook.id} url=${webhook.url} for event=${payload.event} due to subscription filter`); + return; } - - const now = new Date(); - try { - await sendWebhook(wh.url, payload, wh.secret); - try { - await markWebhookSuccess(prisma, poolRunFn, wh.id, now); - } catch (dbErr) { - logger.error(`[webhook-worker] Failed to mark success for webhook ${wh.id}:`, dbErr.message); - } - logger.info(`[webhook-worker] Dispatched payment webhook id=${wh.id} url=${wh.url} recipient=${recipientAddress}`); - } catch (err) { - try { - await markWebhookFailure(prisma, poolRunFn, wh.id, now); - } catch (dbErr) { - logger.error(`[webhook-worker] Failed to mark failure for webhook ${wh.id}:`, dbErr.message); - } - logger.error(`[webhook-worker] Webhook delivery failed id=${wh.id} url=${wh.url}:`, err.message); - } - } -}; - -const listStaleFailingWebhooks = async (prisma, poolAllFn) => { - const cutoff = new Date(); - cutoff.setDate(cutoff.getDate() - MAX_RETRY_BACKLOG_DAYS); - try { - return await prisma.webhook.findMany({ - where: { - failingSince: { not: null, gte: cutoff }, - }, - select: { - id: true, - username: true, - url: true, - secret: true, - events: true, - }, - }); - } catch (error) { - if (!shouldFallbackToLocalRegistry(error)) throw error; - const rows = await poolAllFn( - `SELECT id, username, url, secret, events FROM webhooks - WHERE failing_since IS NOT NULL AND failing_since >= $1`, - [cutoff.toISOString()], + await enqueueWebhookDelivery(webhook, payload, deliveryQueue); + logger.info( + `[webhook-queue] Enqueued event=${payload.event_id} webhook=${webhook.id} recipient=${recipientAddress}`, ); - return (rows || []).map((r) => ({ - id: r.id, - username: r.username, - url: r.url, - secret: r.secret, - events: Array.isArray(r.events) ? r.events : (typeof r.events === 'string' ? JSON.parse(r.events || '[]') : ['*']), - })); - } + })); }; -const sendLivenessPing = async (prisma, poolRunFn, webhook) => { - if (!webhookEventMatches(webhook, 'webhook.ping')) { - return false; - } +const closeWebhookQueue = async () => { + const resources = [webhookWorker, webhookQueue].filter(Boolean); + await Promise.all(resources.map((resource) => resource.close())); - const payload = { - event: 'webhook.ping', - event_id: `ping-${crypto.randomBytes(16).toString('hex')}`, - timestamp: new Date().toISOString(), - data: { message: 'ping' }, - }; - const now = new Date(); - try { - await sendWebhook(webhook.url, payload, webhook.secret); - await markWebhookSuccess(prisma, poolRunFn, webhook.id, now); - return true; - } catch (err) { - await markWebhookFailure(prisma, poolRunFn, webhook.id, now); - return false; - } + const connections = [workerConnection, queueConnection].filter(Boolean); + await Promise.all(connections.map((connection) => connection.quit())); + + webhookWorker = undefined; + webhookQueue = undefined; + workerConnection = undefined; + queueConnection = undefined; }; // ── Dead Letter Queue (DLQ) ────────────────────────────────────────────── -/** - * Return webhooks whose failingSince is older than MAX_RETRY_BACKLOG_DAYS — - * their retry window has expired and they should be moved to the DLQ for - * manual intervention. - */ -const getWebhooksExhaustedRetries = async (prisma, poolAllFn) => { - const cutoff = new Date(); - cutoff.setDate(cutoff.getDate() - MAX_RETRY_BACKLOG_DAYS); - try { - return await prisma.webhook.findMany({ - where: { - failingSince: { not: null, lt: cutoff }, - }, - select: { - id: true, - username: true, - url: true, - secret: true, - failingSince: true, - }, - }); - } catch (error) { - if (!shouldFallbackToLocalRegistry(error)) throw error; - const rows = await poolAllFn( - `SELECT id, username, url, secret, failing_since - FROM webhooks - WHERE failing_since IS NOT NULL AND failing_since < ?`, - [cutoff.toISOString()], - ); - return (rows || []).map((r) => ({ - id: r.id, - username: r.username, - url: r.url, - secret: r.secret, - failingSince: r.failing_since ? new Date(r.failing_since) : null, - })); - } -}; - /** * Move a permanently-failed webhook delivery to the dead-letter queue. * The webhook row itself is left intact so the user can re-register if needed; @@ -313,10 +354,10 @@ const moveToDLQ = async (prisma, poolRunFn, webhook) => { webhook_id: webhook.id, webhook_url: webhook.url, username: webhook.username, - failing_since: (webhook.failingSince instanceof Date + failing_since: webhook.failingSince ? (webhook.failingSince instanceof Date ? webhook.failingSince : new Date(webhook.failingSince) - ).toISOString(), + ).toISOString() : null, }, }; @@ -330,7 +371,7 @@ const moveToDLQ = async (prisma, poolRunFn, webhook) => { username: webhook.username, eventType: payload.event, eventPayload: JSON.stringify(payload), - failureReason: `Delivery exhausted after ${MAX_RETRY_BACKLOG_DAYS} days of failing`, + failureReason: `Delivery exhausted after ${MAX_WEBHOOK_ATTEMPTS} attempts`, deliveryAttempts: 0, movedAt: now, replayed: false, @@ -342,7 +383,7 @@ const moveToDLQ = async (prisma, poolRunFn, webhook) => { `INSERT INTO webhook_dlq (id, webhook_id, webhook_url, webhook_secret, username, event_type, event_payload, failure_reason, delivery_attempts, moved_at, replayed) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, 0)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 0, $9, FALSE)`, [ `dlq-${webhook.id}-${crypto.randomBytes(8).toString('hex')}`, webhook.id, @@ -351,7 +392,7 @@ const moveToDLQ = async (prisma, poolRunFn, webhook) => { webhook.username, payload.event, JSON.stringify(payload), - `Delivery exhausted after ${MAX_RETRY_BACKLOG_DAYS} days of failing`, + `Delivery exhausted after ${MAX_WEBHOOK_ATTEMPTS} attempts`, now.toISOString(), ], ); @@ -368,14 +409,6 @@ const moveToDLQ = async (prisma, poolRunFn, webhook) => { /** * List dead-letter-queue entries with optional username filter and pagination. - * - * @param {object} prisma - * @param {object} poolAllFn - * @param {object} [opts] - * @param {string} [opts.username] - Filter by username - * @param {number} [opts.limit=50] - * @param {number} [opts.offset=0] - * @returns {Promise<{entries: Array, total: number}>} */ const listDLQEntries = async (prisma, poolAllFn, opts = {}) => { const { username, limit = 50, offset = 0 } = opts; @@ -401,17 +434,17 @@ const listDLQEntries = async (prisma, poolAllFn, opts = {}) => { let countRow; if (username) { rows = await poolAllFn( - `SELECT * FROM webhook_dlq WHERE username = ? - ORDER BY moved_at DESC LIMIT ? OFFSET ?`, + `SELECT * FROM webhook_dlq WHERE username = $1 + ORDER BY moved_at DESC LIMIT $2 OFFSET $3`, [username, limit, offset], ); countRow = await poolAllFn( - 'SELECT COUNT(*) AS total FROM webhook_dlq WHERE username = ?', + 'SELECT COUNT(*) AS total FROM webhook_dlq WHERE username = $1', [username], ); } else { rows = await poolAllFn( - `SELECT * FROM webhook_dlq ORDER BY moved_at DESC LIMIT ? OFFSET ?`, + `SELECT * FROM webhook_dlq ORDER BY moved_at DESC LIMIT $1 OFFSET $2`, [limit, offset], ); countRow = await poolAllFn( @@ -431,11 +464,7 @@ const listDLQEntries = async (prisma, poolAllFn, opts = {}) => { }; /** - * Replay a single DLQ entry: attempt to deliver its stored payload to the - * webhook URL one more time. On success, marks the entry as replayed; on - * failure, increments the delivery attempt counter and leaves it in the DLQ. - * - * @returns {Promise<{ok: boolean, statusCode?: number, error?: string}>} + * Replay a single DLQ entry. */ const replayFromDLQ = async (prisma, poolRunFn, dqlId) => { let entry; @@ -446,7 +475,7 @@ const replayFromDLQ = async (prisma, poolRunFn, dqlId) => { } catch (error) { if (!shouldFallbackToLocalRegistry(error)) throw error; const rows = await poolRunFn( - 'SELECT * FROM webhook_dlq WHERE id = ? LIMIT 1', + 'SELECT * FROM webhook_dlq WHERE id = $1 LIMIT 1', [dqlId], ); entry = rows?.[0] || null; @@ -475,7 +504,7 @@ const replayFromDLQ = async (prisma, poolRunFn, dqlId) => { } catch (err) { if (!shouldFallbackToLocalRegistry(err)) throw err; await poolRunFn( - 'UPDATE webhook_dlq SET replayed = 1, replayed_at = ? WHERE id = ?', + 'UPDATE webhook_dlq SET replayed = TRUE, replayed_at = $1 WHERE id = $2', [now.toISOString(), dqlId], ); } @@ -489,79 +518,35 @@ const replayFromDLQ = async (prisma, poolRunFn, dqlId) => { }); } catch (dbErr) { if (!shouldFallbackToLocalRegistry(dbErr)) { - logger.error(`[webhook-worker] Failed to update DLQ attempt count for ${dqlId}:`, dbErr.message); + logger.error(`[webhook-worker] Failed to update DLQ attempt count for ${dqlId}: ${dbErr.message}`); } else { await poolRunFn( - 'UPDATE webhook_dlq SET delivery_attempts = delivery_attempts + 1 WHERE id = ?', + 'UPDATE webhook_dlq SET delivery_attempts = delivery_attempts + 1 WHERE id = $1', [dqlId], ); } } - logger.error(`[webhook-worker] DLQ replay failed for ${dqlId}:`, err.message); + logger.error(`[webhook-worker] DLQ replay failed for ${dqlId}: ${err.message}`); return { ok: false, error: err.message }; } }; -const scheduleWebhookRetryJob = ({ prisma, poolAllFn, poolRunFn }) => { - cron.schedule(RETRY_JOB_CRON, async () => { - logger.info('[webhook-worker] Running periodic liveness pings for failing webhooks…'); - try { - const hooks = await listStaleFailingWebhooks(prisma, poolAllFn); - if (hooks.length === 0) { - // Check for exhausted webhooks to move to DLQ even when no stale hooks - const exhausted = await getWebhooksExhaustedRetries(prisma, poolAllFn); - if (exhausted.length > 0) { - for (const wh of exhausted) { - try { - await moveToDLQ(prisma, poolRunFn, wh); - } catch (dlqErr) { - logger.error(`[webhook-worker] Failed to move webhook ${wh.id} to DLQ:`, dlqErr.message); - } - } - logger.info( - `[webhook-worker] Moved ${exhausted.length} exhausted webhooks to DLQ`, - ); - } - return; - } - let recovered = 0; - let moved = 0; - for (const wh of hooks) { - const ok = await sendLivenessPing(prisma, poolRunFn, wh); - if (ok) { - recovered += 1; - } else if ( - wh.failingSince && - (new Date() - new Date(wh.failingSince)) / (1000 * 60 * 60 * 24) >= - MAX_RETRY_BACKLOG_DAYS - ) { - // Liveness ping failed and the webhook is past its retry window. - try { - await moveToDLQ(prisma, poolRunFn, wh); - moved += 1; - } catch (dlqErr) { - logger.error(`[webhook-worker] Failed to move webhook ${wh.id} to DLQ:`, dlqErr.message); - } - } - } - logger.info( - `[webhook-worker] Liveness pings done. total=${hooks.length}, recovered=${recovered}, movedToDLQ=${moved}`, - ); - } catch (err) { - logger.error('[webhook-worker] Retry job failed:', err.message); - } - }); - logger.info(`[webhook-worker] Retry/liveness job scheduled (cron: ${RETRY_JOB_CRON}).`); -}; - module.exports = { dispatchPaymentWebhooks, - scheduleWebhookRetryJob, + enqueueWebhookDelivery, + startWebhookWorker, + closeWebhookQueue, + processWebhookJob, sendWebhook, computeSignature, WEBHOOK_TIMEOUT_MS, - moveToDLQ, + WEBHOOK_QUEUE_NAME, + MAX_WEBHOOK_ATTEMPTS, + WEBHOOK_BACKOFF_DELAY_MS, + WEBHOOK_JOB_OPTIONS, + MAX_RETRY_BACKLOG_DAYS, getWebhooksExhaustedRetries, + moveToDLQ, listDLQEntries, replayFromDLQ, }; diff --git a/stellar-payment-platform/tests/payment-metadata.test.js b/stellar-payment-platform/tests/payment-metadata.test.js index bb1fd43..cebaf0f 100644 --- a/stellar-payment-platform/tests/payment-metadata.test.js +++ b/stellar-payment-platform/tests/payment-metadata.test.js @@ -109,12 +109,12 @@ describe('payment intent metadata', () => { update: jest.fn().mockResolvedValue({}), }, }; - const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, status: 200 }); + const queue = { add: jest.fn().mockResolvedValue({ id: 'job-1' }) }; await dispatchPaymentWebhooks({ prisma, poolGetFn: jest.fn(), - poolRunFn: jest.fn(), + queue, payment: { id: 'payment-1', type: 'payment', @@ -127,8 +127,9 @@ describe('payment intent metadata', () => { }, }); - const requestBody = JSON.parse(fetchSpy.mock.calls[0][1].body); - expect(requestBody.data.metadata).toEqual(metadata); + expect(queue.add).toHaveBeenCalledTimes(1); + const queuedJob = queue.add.mock.calls[0][1]; + expect(queuedJob.payload.data.metadata).toEqual(metadata); }); test('delivers payment events only when the merchant subscribed to payment.received', async () => { @@ -143,12 +144,12 @@ describe('payment intent metadata', () => { update: jest.fn().mockResolvedValue({}), }, }; - const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, status: 200 }); + const queue = { add: jest.fn().mockResolvedValue({ id: 'job-2' }) }; await dispatchPaymentWebhooks({ prisma, poolGetFn: jest.fn(), - poolRunFn: jest.fn(), + queue, payment: { id: 'payment-2', type: 'payment', @@ -160,9 +161,9 @@ describe('payment intent metadata', () => { }, }); - expect(fetchSpy).toHaveBeenCalledTimes(1); - const requestBody = JSON.parse(fetchSpy.mock.calls[0][1].body); - expect(requestBody.event).toBe('payment.received'); + expect(queue.add).toHaveBeenCalledTimes(1); + const queuedJob = queue.add.mock.calls[0][1]; + expect(queuedJob.payload.event).toBe('payment.received'); }); test('skips delivery for unsubscribed webhook event types', async () => { @@ -177,12 +178,12 @@ describe('payment intent metadata', () => { update: jest.fn().mockResolvedValue({}), }, }; - const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, status: 200 }); + const queue = { add: jest.fn().mockResolvedValue({ id: 'job-3' }) }; await dispatchPaymentWebhooks({ prisma, poolGetFn: jest.fn(), - poolRunFn: jest.fn(), + queue, payment: { id: 'payment-3', type: 'payment', @@ -194,6 +195,6 @@ describe('payment intent metadata', () => { }, }); - expect(fetchSpy).not.toHaveBeenCalled(); + expect(queue.add).not.toHaveBeenCalled(); }); }); diff --git a/stellar-payment-platform/tests/webhook-worker.test.js b/stellar-payment-platform/tests/webhook-worker.test.js new file mode 100644 index 0000000..6550fdc --- /dev/null +++ b/stellar-payment-platform/tests/webhook-worker.test.js @@ -0,0 +1,194 @@ +const mockQueueAdd = jest.fn(); +const mockQueueOn = jest.fn(); +const mockQueueClose = jest.fn().mockResolvedValue(undefined); +const mockWorkerOn = jest.fn(); +const mockWorkerClose = jest.fn().mockResolvedValue(undefined); +const mockRedisQuit = jest.fn().mockResolvedValue(undefined); + +let mockWorkerProcessor; +let mockWorkerOptions; + +jest.mock('bullmq', () => ({ + Queue: jest.fn().mockImplementation(() => ({ + add: mockQueueAdd, + on: mockQueueOn, + close: mockQueueClose, + })), + Worker: jest.fn().mockImplementation((_name, processor, options) => { + mockWorkerProcessor = processor; + mockWorkerOptions = options; + return { + on: mockWorkerOn, + close: mockWorkerClose, + }; + }), +})); + +jest.mock('../src/config/redis', () => ({ + createRedisConnection: jest.fn(() => ({ quit: mockRedisQuit })), +})); + +const { Queue, Worker } = require('bullmq'); +const { + dispatchPaymentWebhooks, + enqueueWebhookDelivery, + startWebhookWorker, + closeWebhookQueue, + processWebhookJob, + MAX_WEBHOOK_ATTEMPTS, + WEBHOOK_BACKOFF_DELAY_MS, + WEBHOOK_QUEUE_NAME, +} = require('../src/webhookWorker'); + +const webhook = { + id: 'webhook-1', + username: 'merchant', + url: 'https://merchant.example/webhooks', + secret: 'secret', +}; + +const payload = { + event: 'payment.received', + event_id: 'transaction-1-payment-1', + timestamp: '2026-08-25T12:00:00.000Z', + data: { amount: '10.00' }, +}; + +describe('webhook BullMQ delivery', () => { + beforeEach(() => { + jest.clearAllMocks(); + global.fetch = jest.fn(); + }); + + afterAll(async () => { + await closeWebhookQueue(); + delete global.fetch; + }); + + test('enqueues deliveries with five attempts and exponential backoff', async () => { + const queue = { add: jest.fn().mockResolvedValue({ id: 'job-1' }) }; + + await enqueueWebhookDelivery(webhook, payload, queue); + + expect(queue.add).toHaveBeenCalledWith( + 'deliver', + { webhook, payload }, + expect.objectContaining({ + attempts: MAX_WEBHOOK_ATTEMPTS, + backoff: { + type: 'exponential', + delay: WEBHOOK_BACKOFF_DELAY_MS, + }, + jobId: expect.any(String), + }), + ); + expect(MAX_WEBHOOK_ATTEMPTS).toBe(5); + }); + + test('worker throws failed deliveries so BullMQ retries them', async () => { + global.fetch.mockResolvedValue({ ok: false, status: 503 }); + const prisma = { + webhook: { + findUnique: jest.fn().mockResolvedValue({ failingSince: null }), + update: jest.fn().mockResolvedValue({}), + }, + }; + + await expect(processWebhookJob( + { data: { webhook, payload }, attemptsMade: 0 }, + { prisma, poolRunFn: jest.fn() }, + )).rejects.toThrow('HTTP 503'); + + expect(prisma.webhook.update).toHaveBeenCalledWith(expect.objectContaining({ + where: { id: webhook.id }, + data: expect.objectContaining({ failingSince: expect.any(Date) }), + })); + }); + + test('worker marks a recovered webhook as successful', async () => { + global.fetch.mockResolvedValue({ ok: true, status: 200 }); + const prisma = { + webhook: { + update: jest.fn().mockResolvedValue({}), + }, + }; + + await processWebhookJob( + { data: { webhook, payload }, attemptsMade: 2 }, + { prisma, poolRunFn: jest.fn() }, + ); + + expect(prisma.webhook.update).toHaveBeenCalledWith(expect.objectContaining({ + where: { id: webhook.id }, + data: expect.objectContaining({ failingSince: null }), + })); + }); + + test('configures and starts a BullMQ webhook worker', () => { + const dependencies = { + prisma: { webhook: {} }, + poolRunFn: jest.fn(), + }; + + startWebhookWorker(dependencies); + + expect(Worker).toHaveBeenCalledWith( + WEBHOOK_QUEUE_NAME, + expect.any(Function), + expect.objectContaining({ concurrency: 5 }), + ); + expect(mockWorkerOptions.connection).toEqual(expect.objectContaining({ quit: mockRedisQuit })); + expect(mockWorkerProcessor).toEqual(expect.any(Function)); + }); + + test('queues a payment event for every registered webhook', async () => { + const queue = { add: jest.fn().mockResolvedValue({}) }; + const prisma = { + webhook: { + findMany: jest.fn().mockResolvedValue([ + webhook, + { ...webhook, id: 'webhook-2', url: 'https://second.example/webhooks' }, + ]), + }, + }; + + await dispatchPaymentWebhooks({ + prisma, + poolGetFn: jest.fn(), + queue, + payment: { + id: 'payment-1', + type: 'payment', + transaction_hash: 'transaction-1', + to: 'GDESTINATION', + from: 'GSOURCE', + amount: '10.00', + asset_type: 'native', + }, + }); + + expect(queue.add).toHaveBeenCalledTimes(2); + expect(queue.add).toHaveBeenCalledWith( + 'deliver', + expect.objectContaining({ + payload: expect.objectContaining({ + event: 'payment.received', + event_id: 'transaction-1-payment-1', + }), + }), + expect.objectContaining({ attempts: 5 }), + ); + }); + + test('creates a lazy queue when no queue is injected', async () => { + mockQueueAdd.mockResolvedValue({ id: 'job-1' }); + + await enqueueWebhookDelivery(webhook, payload); + + expect(Queue).toHaveBeenCalledWith( + WEBHOOK_QUEUE_NAME, + expect.objectContaining({ connection: expect.any(Object) }), + ); + expect(mockQueueAdd).toHaveBeenCalledTimes(1); + }); +});