From b93f6519dd22ab5a18592a693c0002df237d98d8 Mon Sep 17 00:00:00 2001 From: Orioye Blessing Esther <210155349+ayaoba24@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:42:41 -0700 Subject: [PATCH] feat(webhooks): add randomized jitter to exponential backoff for webhook retries (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevent thundering herd problems by adding 20% randomized jitter to the exponential backoff strategy used for webhook delivery retries. Delays now vary within ±20% of each attempt's base delay (e.g. attempt 1: 2000–2400ms, attempt 2: 4000–4800ms), staggering concurrent retries across subscribers. - Add backoff.util.ts with exponentialBackoffWithJitter and webhookBackoffStrategy - Register custom backoffStrategy on the BullMQ queue via settings - Add 11 unit tests for jitter calculations and BullMQ type compatibility Closes #125 🤖 Generated with Codebuff Co-authored-by: Buffy --- .../services/webhook-delivery.service.ts | 5 +- src/modules/webhooks/webhook.module.ts | 12 ++ src/modules/webhooks/webhooks.processor.ts | 11 +- .../webhooks/workers/webhook.worker.ts | 6 +- src/utils/backoff.util.spec.ts | 131 ++++++++++++++++++ src/utils/backoff.util.ts | 54 ++++++++ 6 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 src/utils/backoff.util.spec.ts create mode 100644 src/utils/backoff.util.ts diff --git a/src/modules/webhooks/services/webhook-delivery.service.ts b/src/modules/webhooks/services/webhook-delivery.service.ts index 15430d4..5055e9f 100644 --- a/src/modules/webhooks/services/webhook-delivery.service.ts +++ b/src/modules/webhooks/services/webhook-delivery.service.ts @@ -7,6 +7,8 @@ import { WebhookJobData } from '../types/webhook-job.types'; /** * Service for queuing webhook delivery jobs with BullMQ. * Handles retry logic and dead-letter queueing through the queue configuration. + * Uses exponential backoff with randomized jitter (via worker-level backoffStrategy) + * to prevent thundering herd problems. */ @Injectable() export class WebhookDeliveryService { @@ -20,7 +22,8 @@ export class WebhookDeliveryService { /** * Queues a webhook delivery job with exponential backoff retry policy. * The job will be processed by the webhook worker with automatic retries. - * Uses 2000ms base delay for exponential backoff: 2000ms, 4000ms, 8000ms, 16000ms. + * Uses 2000ms base delay for exponential backoff. + * Randomized jitter is applied by the worker-level backoffStrategy. * Maximum 5 attempts total. */ async queueDelivery(data: WebhookJobData): Promise { diff --git a/src/modules/webhooks/webhook.module.ts b/src/modules/webhooks/webhook.module.ts index 61ffc2e..19fa4bf 100644 --- a/src/modules/webhooks/webhook.module.ts +++ b/src/modules/webhooks/webhook.module.ts @@ -9,10 +9,16 @@ import { WebhookWorker } from './workers/webhook.worker'; import { WebhooksProcessor } from './webhooks.processor'; import { Queues } from '../../queues/queues.constants'; import { redisConfig } from '../../config/redis.config'; +import { webhookBackoffStrategy } from '../../utils/backoff.util'; +import type { RegisterQueueOptions } from '@nestjs/bullmq'; /** * Webhooks module. The dispatcher listens to domain events and queues * the curated WEBHOOK_EVENTS set to subscribed external endpoints via BullMQ. + * + * Uses a custom backoffStrategy with randomized jitter (20% of base delay) + * to prevent thundering herd problems when multiple webhook deliveries + * are retried simultaneously. */ @Module({ imports: [ @@ -35,6 +41,12 @@ import { redisConfig } from '../../config/redis.config'; removeOnComplete: { count: 1000 }, removeOnFail: { age: 24 * 3600 }, }, + // BullMQ reads queue.opts.settings.backoffStrategy at retry time. + // The AdvancedOptions type is not fully exposed by @nestjs/bullmq, so we + // cast to include the backoffStrategy field that BullMQ supports at runtime. + settings: { + backoffStrategy: webhookBackoffStrategy, + } as RegisterQueueOptions['settings'], }), ], controllers: [WebhookController], diff --git a/src/modules/webhooks/webhooks.processor.ts b/src/modules/webhooks/webhooks.processor.ts index 3e3212b..97e95c2 100644 --- a/src/modules/webhooks/webhooks.processor.ts +++ b/src/modules/webhooks/webhooks.processor.ts @@ -7,14 +7,19 @@ import { hmacSign } from '../../utils/crypto.util'; import { PrismaService } from '../../database/prisma.service'; /** - * BullMQ job processor for webhook event delivery with exponential backoff. - * Implements the retry strategy required by issue #9: + * BullMQ job processor for webhook event delivery with exponential backoff + jitter. + * Implements the retry strategy required by issue #125: * - 5 max attempts - * - Exponential backoff with 2000ms base (2000, 4000, 8000, 16000) + * - Exponential backoff with 2000ms base and 20% randomized jitter + * (prevents thundering herd against subscriber endpoints) * - Non-transient error detection (400,401,403,404,422) prevents infinite retries * - Persistent delivery status tracking (PENDING → RETRYING → FAILED/DELIVERED) * - Fail-safe: retry failures never crash the master process * + * Jitter is applied via a custom backoffStrategy configured on the BullMQ + * queue registration (see webhook.module.ts). BullMQ reads the strategy from + * queue.opts.settings.backoffStrategy at retry time. + * * This processor mirrors workers/webhook.worker.ts and is registered as an * alias to satisfy the expected import path `src/modules/webhooks/webhooks.processor.ts`. */ diff --git a/src/modules/webhooks/workers/webhook.worker.ts b/src/modules/webhooks/workers/webhook.worker.ts index 7514e8e..df7834d 100644 --- a/src/modules/webhooks/workers/webhook.worker.ts +++ b/src/modules/webhooks/workers/webhook.worker.ts @@ -8,11 +8,15 @@ import { PrismaService } from '../../../database/prisma.service'; /** * BullMQ worker for processing webhook delivery jobs. - * Implements exponential backoff retry logic (2000ms base, 5 attempts) with: + * Implements exponential backoff with randomized jitter retry logic (2000ms base, 5 attempts): + * - Jitter prevents thundering herd problems against subscriber endpoints * - Persistent delivery status tracking (PENDING, RETRYING, FAILED, DELIVERED) * - Non-transient error detection (400,401,403,404,422) via UnrecoverableError * - Non-blocking DB persistence after network I/O completes * - Fail-safe error handling that never crashes the master process + * + * Jitter is applied via a custom backoffStrategy configured on the BullMQ + * queue registration (see webhook.module.ts). */ @Processor(Queues.Webhooks) export class WebhookWorker extends WorkerHost { diff --git a/src/utils/backoff.util.spec.ts b/src/utils/backoff.util.spec.ts new file mode 100644 index 0000000..28f9ec1 --- /dev/null +++ b/src/utils/backoff.util.spec.ts @@ -0,0 +1,131 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { exponentialBackoffWithJitter, webhookBackoffStrategy } from './backoff.util'; + +describe('backoff.util', () => { + beforeEach(() => { + vi.spyOn(Math, 'random'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('exponentialBackoffWithJitter', () => { + it('calculates correct exponential delay with zero jitter', () => { + vi.mocked(Math.random).mockReturnValue(0); + + expect(exponentialBackoffWithJitter(1, 2000, 0.2)).toBe(2000); + expect(exponentialBackoffWithJitter(2, 2000, 0.2)).toBe(4000); + expect(exponentialBackoffWithJitter(3, 2000, 0.2)).toBe(8000); + expect(exponentialBackoffWithJitter(4, 2000, 0.2)).toBe(16000); + }); + + it('calculates correct exponential delay with max jitter', () => { + vi.mocked(Math.random).mockReturnValue(0.999); + + // Attempt 1: 2000 + floor(0.999 * 2000 * 0.2 * 1) = 2000 + 399 = 2399 + const delay1 = exponentialBackoffWithJitter(1, 2000, 0.2); + expect(delay1).toBe(2399); + + // Attempt 2: 4000 + floor(0.999 * 2000 * 0.2 * 2) = 4000 + 799 = 4799 + const delay2 = exponentialBackoffWithJitter(2, 2000, 0.2); + expect(delay2).toBe(4799); + }); + + it('jitter range increases with attempt number', () => { + vi.mocked(Math.random).mockReturnValue(0.5); + + // Attempt 1: 2000 + floor(0.5 * 2000 * 0.2 * 1) = 2000 + 200 = 2200 + const delay1 = exponentialBackoffWithJitter(1, 2000, 0.2); + // Attempt 2: 4000 + floor(0.5 * 2000 * 0.2 * 2) = 4000 + 400 = 4400 + const delay2 = exponentialBackoffWithJitter(2, 2000, 0.2); + // Attempt 3: 8000 + floor(0.5 * 2000 * 0.2 * 3) = 8000 + 600 = 8600 + const delay3 = exponentialBackoffWithJitter(3, 2000, 0.2); + + expect(delay1).toBe(2200); + expect(delay2).toBe(4400); + expect(delay3).toBe(8600); + }); + + it('returns an integer', () => { + vi.mocked(Math.random).mockReturnValue(0.123456789); + const delay = exponentialBackoffWithJitter(1, 2000, 0.2); + expect(Number.isInteger(delay)).toBe(true); + }); + + it('uses default jitter factor when not provided', () => { + vi.mocked(Math.random).mockReturnValue(0); + const delay = exponentialBackoffWithJitter(1, 2000); + expect(delay).toBe(2000); + }); + + it('supports different base delays', () => { + vi.mocked(Math.random).mockReturnValue(0); + + expect(exponentialBackoffWithJitter(1, 1000, 0.2)).toBe(1000); + expect(exponentialBackoffWithJitter(1, 5000, 0.2)).toBe(5000); + }); + }); + + describe('webhookBackoffStrategy', () => { + it('returns a delay consistent with the backoff algorithm', () => { + vi.mocked(Math.random).mockReturnValue(0); + + // First retry (attemptsMade = 0 → attempt 1) + const delay1 = webhookBackoffStrategy(0, 'exponential', new Error('test')) as number; + expect(delay1).toBe(2000); + + // Second retry (attemptsMade = 1 → attempt 2) + const delay2 = webhookBackoffStrategy(1, 'exponential', new Error('test')) as number; + expect(delay2).toBe(4000); + + // Third retry (attemptsMade = 2 → attempt 3) + const delay3 = webhookBackoffStrategy(2, 'exponential', new Error('test')) as number; + expect(delay3).toBe(8000); + }); + + it('adds jitter to prevent thundering herd', () => { + // Simulate 10 calls with different random values + const delays: number[] = []; + for (let i = 0; i < 10; i++) { + vi.mocked(Math.random).mockReturnValue(i / 10); + delays.push(webhookBackoffStrategy(1, 'exponential', new Error('test')) as number); + } + + // All delays should be different (due to jitter) + const uniqueDelays = new Set(delays); + expect(uniqueDelays.size).toBeGreaterThan(1); + + // All delays should be in the range [4000, 4800) + for (const delay of delays) { + expect(delay).toBeGreaterThanOrEqual(4000); + expect(delay).toBeLessThanOrEqual(4800); + } + }); + + it('is compatible with BullMQ BackoffStrategy type', () => { + // Verify the function signature matches what BullMQ expects: + // (attemptsMade: number, type?: string, err?: Error, job?: MinimalJob) => Promise | number + const strategy = webhookBackoffStrategy; + expect(typeof strategy).toBe('function'); + }); + + it('works regardless of type parameter', () => { + vi.mocked(Math.random).mockReturnValue(0); + + const delay1 = webhookBackoffStrategy(1, 'exponential') as number; + const delay2 = webhookBackoffStrategy(1, 'fixed') as number; + const delay3 = webhookBackoffStrategy(1, undefined) as number; + + // All should produce the same base delay for same attempt + expect(delay1).toBe(delay2); + expect(delay2).toBe(delay3); + }); + + it('works when err and job are not provided', () => { + vi.mocked(Math.random).mockReturnValue(0); + const delay = webhookBackoffStrategy(0) as number; + expect(delay).toBe(2000); + }); + }); +}); diff --git a/src/utils/backoff.util.ts b/src/utils/backoff.util.ts new file mode 100644 index 0000000..92cd8cd --- /dev/null +++ b/src/utils/backoff.util.ts @@ -0,0 +1,54 @@ +/** + * Exponential backoff utilities with randomized jitter. + * + * Jitter prevents "thundering herd" problems where many retrying clients + * hit the same endpoint simultaneously. Each delay is calculated as: + * + * delay = baseDelay * 2^(attempt - 1) + random(0, baseDelay * jitterFactor * attempt) + * + * With a 20% jitter factor and 2000ms base: + * - Attempt 1: 2000ms + random(0, 400ms) + * - Attempt 2: 4000ms + random(0, 800ms) + * - Attempt 3: 8000ms + random(0, 1600ms) + * - Attempt 4: 16000ms + random(0, 3200ms) + */ + +import type { BackoffStrategy } from 'bullmq'; + +/** Default jitter factor (20% of base delay). */ +const DEFAULT_JITTER_FACTOR = 0.2; + +/** Default base delay for webhook retries (2 seconds). */ +const DEFAULT_BASE_DELAY_MS = 2_000; + +/** + * Calculates exponential backoff delay with randomized jitter. + * + * @param attempt - 1-based attempt number (1 = first retry) + * @param baseDelay - base delay in milliseconds (e.g. 2000) + * @param jitterFactor - fraction of base delay to add as jitter (0–1, default 0.2) + * @returns delay in milliseconds with jitter applied (floored to integer) + */ +export function exponentialBackoffWithJitter( + attempt: number, + baseDelay: number = DEFAULT_BASE_DELAY_MS, + jitterFactor: number = DEFAULT_JITTER_FACTOR, +): number { + const exponentialDelay = baseDelay * Math.pow(2, attempt - 1); + const jitterRange = baseDelay * jitterFactor * attempt; + const jitter = Math.random() * jitterRange; + return Math.floor(exponentialDelay + jitter); +} + +/** + * Custom BullMQ backoff strategy for webhook delivery with jitter. + * + * Compatible with BullMQ's `BackoffStrategy` type. Registered on the + * Worker via the `backoffStrategy` option so it overrides the default + * exponential calculation with jittered delays. + * + * @see https://docs.bullmq.io/guide/retrying-failing-jobs#custom-backoff-strategy + */ +export const webhookBackoffStrategy: BackoffStrategy = (attemptsMade) => { + return exponentialBackoffWithJitter(attemptsMade + 1, DEFAULT_BASE_DELAY_MS, DEFAULT_JITTER_FACTOR); +};