diff --git a/src/Dafda.Tests/Builders/ConsumerBuilder.cs b/src/Dafda.Tests/Builders/ConsumerBuilder.cs index bbbc094..bedc6fe 100644 --- a/src/Dafda.Tests/Builders/ConsumerBuilder.cs +++ b/src/Dafda.Tests/Builders/ConsumerBuilder.cs @@ -18,6 +18,7 @@ internal class ConsumerBuilder private IDeadLetterQueue _deadLetterQueue = NullDeadLetterQueue.Instance; private int _maxRetries; private Func _deadLetterQueueBypass; + private Func _retryBackoff; public ConsumerBuilder WithUnitOfWork(IHandlerUnitOfWork unitOfWork) { @@ -78,6 +79,12 @@ public ConsumerBuilder WithDeadLetterQueueBypass(Func deadLette return this; } + public ConsumerBuilder WithRetryBackoff(Func retryBackoff) + { + _retryBackoff = retryBackoff; + return this; + } + public Consumer Build() => new Consumer( _registry, @@ -89,5 +96,6 @@ public Consumer Build() => _enableAutoCommit, _deadLetterQueue, _maxRetries, - _deadLetterQueueBypass); + _deadLetterQueueBypass, + _retryBackoff); } \ No newline at end of file diff --git a/src/Dafda.Tests/Configuration/TestDeadLetterQueueOptions.cs b/src/Dafda.Tests/Configuration/TestDeadLetterQueueOptions.cs index 242957b..ab74c71 100644 --- a/src/Dafda.Tests/Configuration/TestDeadLetterQueueOptions.cs +++ b/src/Dafda.Tests/Configuration/TestDeadLetterQueueOptions.cs @@ -66,4 +66,163 @@ public void bypass_predicate_is_not_affected_by_predicates_registered_afterwards Assert.False(predicate(new FormatException())); Assert.True(sut.BypassPredicate(new FormatException())); } + + [Fact] + public void has_no_retry_backoff_by_default() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Null(sut.RetryBackoff); + } + + [Fact] + public void fixed_retry_backoff_returns_the_same_delay_for_every_attempt() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithRetryBackoff(TimeSpan.FromSeconds(3)); + + Assert.Equal(TimeSpan.FromSeconds(3), sut.RetryBackoff(1)); + Assert.Equal(TimeSpan.FromSeconds(3), sut.RetryBackoff(2)); + Assert.Equal(TimeSpan.FromSeconds(3), sut.RetryBackoff(7)); + } + + [Fact] + public void exponential_retry_backoff_doubles_by_default() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithExponentialRetryBackoff(TimeSpan.FromSeconds(1)); + + Assert.Equal(TimeSpan.FromSeconds(1), sut.RetryBackoff(1)); + Assert.Equal(TimeSpan.FromSeconds(2), sut.RetryBackoff(2)); + Assert.Equal(TimeSpan.FromSeconds(4), sut.RetryBackoff(3)); + Assert.Equal(TimeSpan.FromSeconds(8), sut.RetryBackoff(4)); + } + + [Fact] + public void exponential_retry_backoff_uses_the_supplied_factor() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithExponentialRetryBackoff(TimeSpan.FromSeconds(1), factor: 3); + + Assert.Equal(TimeSpan.FromSeconds(1), sut.RetryBackoff(1)); + Assert.Equal(TimeSpan.FromSeconds(3), sut.RetryBackoff(2)); + Assert.Equal(TimeSpan.FromSeconds(9), sut.RetryBackoff(3)); + } + + [Fact] + public void exponential_retry_backoff_is_capped_by_max_delay() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithExponentialRetryBackoff(TimeSpan.FromSeconds(1), maxDelay: TimeSpan.FromSeconds(5)); + + Assert.Equal(TimeSpan.FromSeconds(1), sut.RetryBackoff(1)); + Assert.Equal(TimeSpan.FromSeconds(2), sut.RetryBackoff(2)); + Assert.Equal(TimeSpan.FromSeconds(4), sut.RetryBackoff(3)); + Assert.Equal(TimeSpan.FromSeconds(5), sut.RetryBackoff(4)); + Assert.Equal(TimeSpan.FromSeconds(5), sut.RetryBackoff(100)); + } + + [Fact] + public void throws_when_fixed_retry_backoff_delay_is_negative() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws(() => sut.WithRetryBackoff(TimeSpan.FromSeconds(-1))); + } + + [Fact] + public void throws_when_exponential_retry_backoff_initial_delay_is_negative() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws(() => sut.WithExponentialRetryBackoff(TimeSpan.FromSeconds(-1))); + } + + [Theory] + [InlineData(0d)] + [InlineData(-1d)] + [InlineData(double.NaN)] + public void throws_when_exponential_retry_backoff_factor_is_not_positive(double factor) + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws(() => sut.WithExponentialRetryBackoff(TimeSpan.FromSeconds(1), factor)); + } + + [Fact] + public void throws_when_exponential_retry_backoff_max_delay_is_negative() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws( + () => sut.WithExponentialRetryBackoff(TimeSpan.FromSeconds(1), maxDelay: TimeSpan.FromSeconds(-1))); + } + + [Fact] + public void throws_when_fixed_retry_backoff_delay_exceeds_the_supported_maximum() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws( + () => sut.WithRetryBackoff(DeadLetterQueueOptions.MaxSupportedRetryDelay + TimeSpan.FromMilliseconds(1))); + } + + [Fact] + public void throws_when_exponential_retry_backoff_initial_delay_exceeds_the_supported_maximum() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws( + () => sut.WithExponentialRetryBackoff(DeadLetterQueueOptions.MaxSupportedRetryDelay + TimeSpan.FromMilliseconds(1))); + } + + [Fact] + public void throws_when_exponential_retry_backoff_max_delay_exceeds_the_supported_maximum() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws( + () => sut.WithExponentialRetryBackoff( + TimeSpan.FromSeconds(1), + maxDelay: DeadLetterQueueOptions.MaxSupportedRetryDelay + TimeSpan.FromMilliseconds(1))); + } + + [Fact] + public void exponential_retry_backoff_clamps_overflow_to_the_supported_maximum() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithExponentialRetryBackoff(TimeSpan.FromSeconds(1)); + + Assert.Equal(DeadLetterQueueOptions.MaxSupportedRetryDelay, sut.RetryBackoff(64)); + Assert.Equal(DeadLetterQueueOptions.MaxSupportedRetryDelay, sut.RetryBackoff(int.MaxValue)); + } + + [Fact] + public void exponential_retry_backoff_prefers_an_explicit_max_delay_over_the_supported_maximum() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithExponentialRetryBackoff(TimeSpan.FromSeconds(1), maxDelay: TimeSpan.FromSeconds(30)); + + Assert.Equal(TimeSpan.FromSeconds(30), sut.RetryBackoff(64)); + Assert.Equal(TimeSpan.FromSeconds(30), sut.RetryBackoff(int.MaxValue)); + } + + [Fact] + public void exponential_retry_backoff_returns_zero_for_a_zero_initial_delay() + { + var sut = new DeadLetterQueueOptions("dlq") + .WithExponentialRetryBackoff(TimeSpan.Zero, factor: 1000); + + Assert.Equal(TimeSpan.Zero, sut.RetryBackoff(1)); + Assert.Equal(TimeSpan.Zero, sut.RetryBackoff(64)); + Assert.Equal(TimeSpan.Zero, sut.RetryBackoff(int.MaxValue)); + } + + [Fact] + public void throws_when_max_retries_is_negative() + { + var sut = new DeadLetterQueueOptions("dlq"); + + Assert.Throws(() => sut.WithMaxRetries(-1)); + } } diff --git a/src/Dafda.Tests/Consuming/TestConsumer.cs b/src/Dafda.Tests/Consuming/TestConsumer.cs index cdfb21a..3e2fe8f 100644 --- a/src/Dafda.Tests/Consuming/TestConsumer.cs +++ b/src/Dafda.Tests/Consuming/TestConsumer.cs @@ -518,12 +518,115 @@ public async Task dead_letters_exceptions_that_do_not_match_the_bypass() Assert.Equal(1, deadLetterQueueSpy.SendCount); } + [Fact] + public async Task does_not_delay_between_retries_when_no_backoff_is_configured() + { + var handlerInvocations = 0; + var handler = new MessageHandlerSpy(() => + { + handlerInvocations++; + throw new InvalidOperationException("boom"); + }); + + var deadLetterQueueSpy = new DeadLetterQueueSpy(); + + var sut = BuildConsumerWithHandler( + handler, + deadLetterQueue: deadLetterQueueSpy, + maxRetries: 2, + retryBackoff: null); + + await sut.ConsumeSingle(CancellationToken.None); + + Assert.Equal(3, handlerInvocations); + Assert.Equal(1, deadLetterQueueSpy.SendCount); + } + + [Fact] + public async Task applies_retry_backoff_for_each_retry_attempt_and_dead_letters_after_max_retries() + { + var handlerInvocations = 0; + var handler = new MessageHandlerSpy(() => + { + handlerInvocations++; + throw new InvalidOperationException("boom"); + }); + + var deadLetterQueueSpy = new DeadLetterQueueSpy(); + var backoffAttempts = new List(); + + var sut = BuildConsumerWithHandler( + handler, + deadLetterQueue: deadLetterQueueSpy, + maxRetries: 3, + retryBackoff: attempt => + { + backoffAttempts.Add(attempt); + return TimeSpan.FromMilliseconds(1); + }); + + await sut.ConsumeSingle(CancellationToken.None); + + Assert.Equal(new[] { 1, 2, 3 }, backoffAttempts); + Assert.Equal(4, handlerInvocations); + Assert.Equal(1, deadLetterQueueSpy.SendCount); + } + + [Fact] + public async Task does_not_apply_retry_backoff_when_handler_succeeds() + { + var handler = new MessageHandlerSpy(() => { }); + + var deadLetterQueueSpy = new DeadLetterQueueSpy(); + var backoffInvocations = 0; + + var sut = BuildConsumerWithHandler( + handler, + deadLetterQueue: deadLetterQueueSpy, + maxRetries: 3, + retryBackoff: _ => + { + backoffInvocations++; + return TimeSpan.Zero; + }); + + await sut.ConsumeSingle(CancellationToken.None); + + Assert.Equal(0, backoffInvocations); + Assert.Equal(0, deadLetterQueueSpy.SendCount); + } + + [Fact] + public async Task does_not_dead_letter_when_cancelled_during_retry_backoff() + { + using var cts = new CancellationTokenSource(); + + var handler = new MessageHandlerSpy(() => throw new InvalidOperationException("boom")); + + var deadLetterQueueSpy = new DeadLetterQueueSpy(); + + var sut = BuildConsumerWithHandler( + handler, + deadLetterQueue: deadLetterQueueSpy, + maxRetries: 3, + retryBackoff: _ => + { + cts.Cancel(); + return TimeSpan.FromMinutes(5); + }); + + await Assert.ThrowsAnyAsync(() => sut.ConsumeSingle(cts.Token)); + + Assert.Equal(0, deadLetterQueueSpy.SendCount); + } + private static Consumer BuildConsumerWithHandler( IMessageHandler handler, Func onCommit = null, IDeadLetterQueue deadLetterQueue = null, int maxRetries = 0, - Func deadLetterQueueBypass = null) + Func deadLetterQueueBypass = null, + Func retryBackoff = null) { var registration = new MessageRegistrationBuilder() .WithHandlerInstanceType(handler.GetType()) @@ -546,7 +649,8 @@ private static Consumer BuildConsumerWithHandler( .WithUnitOfWork(new UnitOfWorkStub(handler)) .WithMessageHandlerRegistry(registry) .WithMaxRetries(maxRetries) - .WithDeadLetterQueueBypass(deadLetterQueueBypass); + .WithDeadLetterQueueBypass(deadLetterQueueBypass) + .WithRetryBackoff(retryBackoff); if (deadLetterQueue != null) { diff --git a/src/Dafda/Configuration/ConsumerConfiguration.cs b/src/Dafda/Configuration/ConsumerConfiguration.cs index b7ac7bc..6ed72be 100644 --- a/src/Dafda/Configuration/ConsumerConfiguration.cs +++ b/src/Dafda/Configuration/ConsumerConfiguration.cs @@ -14,7 +14,8 @@ internal class ConsumerConfiguration( IConsumerErrorHandler consumerErrorHandler, Func deadLetterQueueFactory, int maxRetries, - Func deadLetterQueueBypass) + Func deadLetterQueueBypass, + Func retryBackoff = null) : ConsumerConfigurationBase(configuration, factories.UnitOfWorkFactory, consumerErrorHandler) { public ConsumerConfigurationFactories Factories { get; } = factories; @@ -23,4 +24,5 @@ internal class ConsumerConfiguration( public Func DeadLetterQueueFactory { get; } = deadLetterQueueFactory; public int MaxRetries { get; } = maxRetries; public Func DeadLetterQueueBypass { get; } = deadLetterQueueBypass; + public Func RetryBackoff { get; } = retryBackoff; } \ No newline at end of file diff --git a/src/Dafda/Configuration/ConsumerConfigurationBuilder.cs b/src/Dafda/Configuration/ConsumerConfigurationBuilder.cs index a10ab1b..25a5d95 100644 --- a/src/Dafda/Configuration/ConsumerConfigurationBuilder.cs +++ b/src/Dafda/Configuration/ConsumerConfigurationBuilder.cs @@ -208,6 +208,7 @@ internal ConsumerConfiguration Build() var deadLetterQueueFactory = BuildDeadLetterQueueFactory(configurations); var maxRetries = _deadLetterQueueOptions?.MaxRetries ?? 0; var deadLetterQueueBypass = _deadLetterQueueOptions?.BypassPredicate; + var retryBackoff = _deadLetterQueueOptions?.RetryBackoff; return new ConsumerConfiguration( configuration: configurations, @@ -217,7 +218,8 @@ internal ConsumerConfiguration Build() consumerErrorHandler: _consumerErrorHandler, deadLetterQueueFactory: deadLetterQueueFactory, maxRetries: maxRetries, - deadLetterQueueBypass: deadLetterQueueBypass); + deadLetterQueueBypass: deadLetterQueueBypass, + retryBackoff: retryBackoff); } private Func BuildDeadLetterQueueFactory(IDictionary configurations) diff --git a/src/Dafda/Configuration/ConsumerServiceCollectionExtensions.cs b/src/Dafda/Configuration/ConsumerServiceCollectionExtensions.cs index c41a8a9..f700ad3 100644 --- a/src/Dafda/Configuration/ConsumerServiceCollectionExtensions.cs +++ b/src/Dafda/Configuration/ConsumerServiceCollectionExtensions.cs @@ -47,7 +47,8 @@ public static void AddConsumer(this IServiceCollection services, Action + /// Maps a retry attempt number (the first retry is attempt 1) to the delay + /// awaited before that attempt is made. Returns null when no backoff has been + /// configured, in which case retries happen without any delay. + /// + internal Func RetryBackoff { get; private set; } + + /// + /// The largest delay that can be awaited between retry attempts, limited by the + /// underlying platform timer. + /// + internal static readonly TimeSpan MaxSupportedRetryDelay = TimeSpan.FromMilliseconds(int.MaxValue); + + /// + /// Wait a fixed before each retry attempt. + /// + /// + /// The delay awaited before every retry attempt. Must be zero or greater, and cannot + /// exceed . + /// + public DeadLetterQueueOptions WithRetryBackoff(TimeSpan delay) + { + EnsureDelayIsSupported(delay, "The retry backoff delay for a dead letter queue"); + + RetryBackoff = _ => delay; + return this; + } + + /// + /// Wait an exponentially increasing delay before each retry attempt. The delay before + /// retry attempt n (the first retry being attempt 1) is + /// multiplied by raised to the + /// power of n - 1, optionally capped by . + /// + /// + /// The delay awaited before the first retry attempt. Must be zero or greater, and cannot + /// exceed . + /// + /// The multiplier applied for each subsequent attempt. Must be greater than zero. + /// + /// An optional upper bound for the computed delay. Must be zero or greater and cannot exceed + /// when supplied. When omitted, the computed delay is + /// still clamped to . + /// + public DeadLetterQueueOptions WithExponentialRetryBackoff(TimeSpan initialDelay, double factor = 2.0, TimeSpan? maxDelay = null) + { + EnsureDelayIsSupported(initialDelay, "The retry backoff delay for a dead letter queue"); + + if (double.IsNaN(factor) || factor <= 0) + { + throw new InvalidConfigurationException("The retry backoff factor for a dead letter queue must be greater than zero."); + } + + if (maxDelay.HasValue) + { + EnsureDelayIsSupported(maxDelay.Value, "The maximum retry backoff delay for a dead letter queue"); + } + + RetryBackoff = attempt => CalculateExponentialDelay(initialDelay, factor, maxDelay, attempt); + return this; + } + + private static void EnsureDelayIsSupported(TimeSpan delay, string subject) + { + if (delay < TimeSpan.Zero) + { + throw new InvalidConfigurationException($"{subject} cannot be negative."); + } + + if (delay > MaxSupportedRetryDelay) + { + throw new InvalidConfigurationException($"{subject} cannot exceed {MaxSupportedRetryDelay}."); + } + } + + private static TimeSpan CalculateExponentialDelay(TimeSpan initialDelay, double factor, TimeSpan? maxDelay, int attempt) + { + var exponent = attempt < 1 ? 0 : attempt - 1; + var ticks = initialDelay.Ticks * Math.Pow(factor, exponent); + + if (double.IsNaN(ticks) || ticks <= 0) + { + return TimeSpan.Zero; + } + + var delay = ticks >= MaxSupportedRetryDelay.Ticks + ? MaxSupportedRetryDelay + : TimeSpan.FromTicks((long)ticks); + + if (maxDelay.HasValue && delay > maxDelay.Value) + { + delay = maxDelay.Value; + } + + return delay > MaxSupportedRetryDelay ? MaxSupportedRetryDelay : delay; + } + /// /// A predicate matching exceptions that should bypass the dead letter queue. /// When an exception matches, it is rethrown instead of being retried or diff --git a/src/Dafda/Consuming/Consumer.cs b/src/Dafda/Consuming/Consumer.cs index 166453b..ff8acc5 100644 --- a/src/Dafda/Consuming/Consumer.cs +++ b/src/Dafda/Consuming/Consumer.cs @@ -17,7 +17,8 @@ internal class Consumer( bool isAutoCommitEnabled = false, IDeadLetterQueue deadLetterQueue = null, int maxRetries = 0, - Func deadLetterQueueBypass = null) + Func deadLetterQueueBypass = null, + Func retryBackoff = null) : IConsumer, IDisposable { private readonly LocalMessageDispatcher _localMessageDispatcher = new( @@ -73,8 +74,16 @@ private async Task Dispatch(MessageResult messageResult, CancellationToken cance } catch (Exception exception) when (deadLetterQueueEnabled && !cancellationToken.IsCancellationRequested && !ShouldBypassDeadLetterQueue(exception)) { - if (attempt++ < maxRetries) + if (attempt < maxRetries) { + attempt++; + + var delay = GetRetryDelay(attempt); + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, cancellationToken); + } + continue; } @@ -84,6 +93,11 @@ private async Task Dispatch(MessageResult messageResult, CancellationToken cance } } + private TimeSpan GetRetryDelay(int attempt) + { + return retryBackoff == null ? TimeSpan.Zero : retryBackoff(attempt); + } + private bool ShouldBypassDeadLetterQueue(Exception exception) { return deadLetterQueueBypass != null && deadLetterQueueBypass(exception);