diff --git a/listener/src/index.ts b/listener/src/index.ts index 2f21d1f3..852256eb 100644 --- a/listener/src/index.ts +++ b/listener/src/index.ts @@ -27,6 +27,10 @@ function loadConfig(): Config { eventsApiPort: parseInt(process.env.EVENTS_API_PORT || '8787'), eventsApiCorsOrigin: process.env.EVENTS_API_CORS_ORIGIN || 'http://localhost:5173', discord, + retryQueue: { + baseDelayMs: parseInt(process.env.RETRY_BASE_DELAY_MS || '5000'), + maxRetries: parseInt(process.env.RETRY_MAX_RETRIES || '5'), + }, }; } diff --git a/listener/src/services/discord-notification.ts b/listener/src/services/discord-notification.ts index 73d4e6de..2719584b 100644 --- a/listener/src/services/discord-notification.ts +++ b/listener/src/services/discord-notification.ts @@ -75,9 +75,6 @@ export class DiscordNotificationService { } this.deduplicator.markSent(fingerprint); - logger.info('Discord notification sent successfully', { - eventId: event.id, - contractAddress: contractConfig.address, logger.info('Discord notification delivered', { ...logContext, durationMs, diff --git a/listener/src/services/event-subscriber.test.ts b/listener/src/services/event-subscriber.test.ts index 9ee184a1..8e887983 100644 --- a/listener/src/services/event-subscriber.test.ts +++ b/listener/src/services/event-subscriber.test.ts @@ -535,7 +535,7 @@ describe('EventSubscriber', () => { await (subscriber as any).checkForEvents(); expect(mockLogger.warn).toHaveBeenCalledWith( - 'Failed to send Discord notification, event will still be processed', + 'Discord notification failed, adding to retry queue', expect.objectContaining({ eventId: 'event-1' }) ); }); diff --git a/listener/src/services/event-subscriber.ts b/listener/src/services/event-subscriber.ts index c2b06879..036a3778 100644 --- a/listener/src/services/event-subscriber.ts +++ b/listener/src/services/event-subscriber.ts @@ -9,6 +9,7 @@ import { validateEventPayload, } from '../utils/event-utils'; import { DiscordNotificationService } from './discord-notification'; +import { NotificationRetryQueue } from './notification-retry-queue'; export class EventSubscriber { private config: Config; @@ -17,23 +18,31 @@ export class EventSubscriber { private reconnectAttempts: number = 0; private lastCursors: Map = new Map(); private discordService: DiscordNotificationService | null = null; + private retryQueue: NotificationRetryQueue | null = null; constructor(config: Config) { this.config = config; this.server = new StellarSDK.rpc.Server(config.stellarRpcUrl); if (config.discord) { this.discordService = new DiscordNotificationService(config.discord); + this.retryQueue = new NotificationRetryQueue( + (event, contractConfig, requestId) => + this.discordService!.sendEventNotification(event, contractConfig, requestId), + config.retryQueue + ); } } async start(): Promise { this.isRunning = true; logger.info('Starting event subscriber service'); + this.retryQueue?.start(); this.poll(); } async stop(): Promise { this.isRunning = false; + this.retryQueue?.stop(); logger.info('Stopping event subscriber service'); } @@ -196,11 +205,12 @@ export class EventSubscriber { contractConfig, requestId ); - if (!success) { - logger.warn('Failed to send Discord notification, event will still be processed', { + if (!success && this.retryQueue) { + logger.warn('Discord notification failed, adding to retry queue', { requestId, eventId: event.id, }); + this.retryQueue.enqueue(event, contractConfig, requestId); } } diff --git a/listener/src/services/notification-retry-queue.test.ts b/listener/src/services/notification-retry-queue.test.ts new file mode 100644 index 00000000..d495bcac --- /dev/null +++ b/listener/src/services/notification-retry-queue.test.ts @@ -0,0 +1,279 @@ +import { xdr } from '@stellar/stellar-sdk'; +import * as StellarSDK from '@stellar/stellar-sdk'; +import { NotificationRetryQueue, NotificationFn } from './notification-retry-queue'; + +jest.mock('../utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, +})); + +function createMockEvent( + overrides: Partial = {} +): StellarSDK.rpc.Api.EventResponse { + return { + id: 'event-123', + type: 'contract', + ledger: 1000, + ledgerClosedAt: '2026-01-01T00:00:00Z', + transactionIndex: 1, + operationIndex: 0, + inSuccessfulContractCall: true, + txHash: 'abc123', + topic: [xdr.ScVal.scvSymbol('test_event')], + value: xdr.ScVal.scvString('test value'), + ...overrides, + }; +} + +const mockContractConfig = { address: 'CA123', events: ['test_event'] }; + +describe('NotificationRetryQueue', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + describe('enqueue', () => { + it('adds an item to the queue', () => { + const notificationFn: NotificationFn = jest.fn(); + const queue = new NotificationRetryQueue(notificationFn, { baseDelayMs: 1000 }); + + queue.enqueue(createMockEvent(), mockContractConfig); + + expect(queue.size()).toBe(1); + }); + + it('logs when an item is queued', () => { + const logger = jest.requireMock('../utils/logger').default; + const notificationFn: NotificationFn = jest.fn(); + const queue = new NotificationRetryQueue(notificationFn, { baseDelayMs: 1000 }); + + queue.enqueue(createMockEvent({ id: 'evt-q' }), mockContractConfig, 'req-1'); + + expect(logger.info).toHaveBeenCalledWith( + 'Notification queued for retry', + expect.objectContaining({ eventId: 'evt-q', requestId: 'req-1' }) + ); + }); + }); + + describe('processQueue', () => { + it('retries a notification after the base delay', async () => { + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(true); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 1000, + processIntervalMs: 100, + }); + queue.start(); + + queue.enqueue(createMockEvent(), mockContractConfig); + + // Before delay expires — should not have retried yet + jest.advanceTimersByTime(500); + await Promise.resolve(); + expect(notificationFn).not.toHaveBeenCalled(); + + // After delay expires — should retry + jest.advanceTimersByTime(600); + await Promise.resolve(); + await Promise.resolve(); + expect(notificationFn).toHaveBeenCalledTimes(1); + + queue.stop(); + }); + + it('removes the item from the queue on success', async () => { + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(true); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 100, + processIntervalMs: 50, + }); + queue.start(); + + queue.enqueue(createMockEvent(), mockContractConfig); + + jest.advanceTimersByTime(200); + await Promise.resolve(); + await Promise.resolve(); + + expect(queue.size()).toBe(0); + queue.stop(); + }); + + it('logs success on a successful retry', async () => { + const logger = jest.requireMock('../utils/logger').default; + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(true); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 100, + processIntervalMs: 50, + }); + queue.start(); + + queue.enqueue(createMockEvent({ id: 'evt-ok' }), mockContractConfig, 'req-ok'); + jest.advanceTimersByTime(200); + await Promise.resolve(); + await Promise.resolve(); + + expect(logger.info).toHaveBeenCalledWith( + 'Retry succeeded', + expect.objectContaining({ eventId: 'evt-ok', attempt: 1 }) + ); + queue.stop(); + }); + }); + + describe('exponential backoff', () => { + it('doubles the delay on each successive failure', async () => { + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(false); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 1000, + maxRetries: 5, + processIntervalMs: 100, + }); + queue.start(); + + queue.enqueue(createMockEvent(), mockContractConfig); + + // Trigger attempt 1 (after 1000 ms base delay) + jest.advanceTimersByTime(1100); + await Promise.resolve(); + await Promise.resolve(); + expect(notificationFn).toHaveBeenCalledTimes(1); + + // Trigger attempt 2 (after 2000 ms from attempt 1) + jest.advanceTimersByTime(2100); + await Promise.resolve(); + await Promise.resolve(); + expect(notificationFn).toHaveBeenCalledTimes(2); + + queue.stop(); + }); + + it('logs a warning with the next retry delay on failure', async () => { + const logger = jest.requireMock('../utils/logger').default; + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(false); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 1000, + maxRetries: 3, + processIntervalMs: 100, + }); + queue.start(); + + queue.enqueue(createMockEvent({ id: 'evt-backoff' }), mockContractConfig); + + jest.advanceTimersByTime(1100); + await Promise.resolve(); + await Promise.resolve(); + + expect(logger.warn).toHaveBeenCalledWith( + 'Retry failed, scheduling next attempt', + expect.objectContaining({ eventId: 'evt-backoff', attempt: 1, delayMs: 2000 }) + ); + queue.stop(); + }); + }); + + describe('max retries', () => { + it('stops retrying after maxRetries attempts', async () => { + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(false); + const maxRetries = 3; + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 100, + maxRetries, + processIntervalMs: 50, + }); + queue.start(); + queue.enqueue(createMockEvent(), mockContractConfig); + + const flush = async () => { + for (let i = 0; i < 5; i++) await Promise.resolve(); + }; + + // attempt 1 fires at t=100ms (base delay) + jest.advanceTimersByTime(100); + await flush(); + expect(notificationFn).toHaveBeenCalledTimes(1); + + // attempt 2 fires at t=300ms (100 + 100*2^1 = 300) + jest.advanceTimersByTime(200); + await flush(); + expect(notificationFn).toHaveBeenCalledTimes(2); + + // attempt 3 fires at t=700ms (300 + 100*2^2 = 700) + jest.advanceTimersByTime(400); + await flush(); + expect(notificationFn).toHaveBeenCalledTimes(maxRetries); + expect(queue.size()).toBe(0); + + queue.stop(); + }); + + it('logs an error when the notification permanently fails', async () => { + const logger = jest.requireMock('../utils/logger').default; + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(false); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 100, + maxRetries: 1, + processIntervalMs: 50, + }); + queue.start(); + + queue.enqueue(createMockEvent({ id: 'evt-dead' }), mockContractConfig, 'req-dead'); + + jest.advanceTimersByTime(200); + await Promise.resolve(); + await Promise.resolve(); + + expect(logger.error).toHaveBeenCalledWith( + 'Notification permanently failed after max retries', + expect.objectContaining({ eventId: 'evt-dead', totalAttempts: 1 }) + ); + queue.stop(); + }); + }); + + describe('start / stop', () => { + it('does not process items when stopped', async () => { + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(true); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 100, + processIntervalMs: 50, + }); + + queue.enqueue(createMockEvent(), mockContractConfig); + // Never call queue.start() + + jest.advanceTimersByTime(1000); + await Promise.resolve(); + + expect(notificationFn).not.toHaveBeenCalled(); + }); + + it('calling start twice does not double-process items', async () => { + const notificationFn: NotificationFn = jest.fn().mockResolvedValue(true); + const queue = new NotificationRetryQueue(notificationFn, { + baseDelayMs: 100, + processIntervalMs: 50, + }); + queue.start(); + queue.start(); // second call should be a no-op + + queue.enqueue(createMockEvent(), mockContractConfig); + + jest.advanceTimersByTime(200); + await Promise.resolve(); + await Promise.resolve(); + + expect(notificationFn).toHaveBeenCalledTimes(1); + queue.stop(); + }); + }); +}); diff --git a/listener/src/services/notification-retry-queue.ts b/listener/src/services/notification-retry-queue.ts new file mode 100644 index 00000000..f6c8c379 --- /dev/null +++ b/listener/src/services/notification-retry-queue.ts @@ -0,0 +1,147 @@ +import * as StellarSDK from '@stellar/stellar-sdk'; +import { ContractConfig } from '../types'; +import logger from '../utils/logger'; + +export interface RetryQueueOptions { + baseDelayMs?: number; + maxRetries?: number; + processIntervalMs?: number; +} + +interface RetryItem { + event: StellarSDK.rpc.Api.EventResponse; + contractConfig: ContractConfig; + retryCount: number; + nextRetryAt: number; + requestId?: string; +} + +const DEFAULTS = { + baseDelayMs: 5_000, + maxRetries: 5, + processIntervalMs: 5_000, +}; + +export type NotificationFn = ( + event: StellarSDK.rpc.Api.EventResponse, + contractConfig: ContractConfig, + requestId?: string +) => Promise; + +export class NotificationRetryQueue { + private queue: RetryItem[] = []; + private readonly baseDelayMs: number; + private readonly maxRetries: number; + private readonly processIntervalMs: number; + private timer: ReturnType | null = null; + private readonly notificationFn: NotificationFn; + + constructor(notificationFn: NotificationFn, options?: RetryQueueOptions) { + this.notificationFn = notificationFn; + this.baseDelayMs = options?.baseDelayMs ?? DEFAULTS.baseDelayMs; + this.maxRetries = options?.maxRetries ?? DEFAULTS.maxRetries; + this.processIntervalMs = options?.processIntervalMs ?? DEFAULTS.processIntervalMs; + } + + enqueue( + event: StellarSDK.rpc.Api.EventResponse, + contractConfig: ContractConfig, + requestId?: string + ): void { + const delayMs = this.calculateDelay(0); + const nextRetryAt = Date.now() + delayMs; + + logger.info('Notification queued for retry', { + requestId, + eventId: event.id, + contractAddress: contractConfig.address, + delayMs, + nextRetryAt: new Date(nextRetryAt).toISOString(), + maxRetries: this.maxRetries, + }); + + this.queue.push({ event, contractConfig, retryCount: 0, nextRetryAt, requestId }); + } + + start(): void { + if (this.timer !== null) return; + this.timer = setInterval(() => { + this.processQueue().catch((err) => + logger.error('Unexpected error in retry queue processor', { error: err }) + ); + }, this.processIntervalMs); + } + + stop(): void { + if (this.timer !== null) { + clearInterval(this.timer); + this.timer = null; + } + } + + size(): number { + return this.queue.length; + } + + private async processQueue(): Promise { + const now = Date.now(); + const due = this.queue.filter((item) => item.nextRetryAt <= now); + this.queue = this.queue.filter((item) => item.nextRetryAt > now); + + for (const item of due) { + await this.retryItem(item); + } + } + + private async retryItem(item: RetryItem): Promise { + const attempt = item.retryCount + 1; + + logger.info('Retrying failed notification', { + requestId: item.requestId, + eventId: item.event.id, + contractAddress: item.contractConfig.address, + attempt, + maxRetries: this.maxRetries, + }); + + const success = await this.notificationFn(item.event, item.contractConfig, item.requestId); + + if (success) { + logger.info('Retry succeeded', { + requestId: item.requestId, + eventId: item.event.id, + contractAddress: item.contractConfig.address, + attempt, + }); + return; + } + + if (attempt >= this.maxRetries) { + logger.error('Notification permanently failed after max retries', { + requestId: item.requestId, + eventId: item.event.id, + contractAddress: item.contractConfig.address, + totalAttempts: attempt, + }); + return; + } + + const delayMs = this.calculateDelay(attempt); + const nextRetryAt = Date.now() + delayMs; + + logger.warn('Retry failed, scheduling next attempt', { + requestId: item.requestId, + eventId: item.event.id, + contractAddress: item.contractConfig.address, + attempt, + delayMs, + nextRetryAt: new Date(nextRetryAt).toISOString(), + }); + + this.queue.push({ ...item, retryCount: attempt, nextRetryAt }); + } + + private calculateDelay(retryCount: number): number { + return this.baseDelayMs * Math.pow(2, retryCount); + } +} diff --git a/listener/src/types/index.ts b/listener/src/types/index.ts index 659a4c69..107ae76b 100644 --- a/listener/src/types/index.ts +++ b/listener/src/types/index.ts @@ -8,6 +8,11 @@ export interface DiscordConfig { webhookId: string; } +export interface RetryQueueConfig { + baseDelayMs?: number; + maxRetries?: number; +} + export interface Config { stellarNetwork: string; stellarRpcUrl: string; @@ -18,4 +23,5 @@ export interface Config { eventsApiPort: number; eventsApiCorsOrigin: string; discord?: DiscordConfig; + retryQueue?: RetryQueueConfig; }