From cbbbb1571e9c3172ec57db3d0af49ff8166c582b Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Wed, 29 Jul 2026 15:55:28 +0530 Subject: [PATCH 01/12] Add jitter: Full/Equal modes plus decorrelated backoff Retrying on a fixed schedule means every client that failed together retries together, so a dependency coming back up gets a synchronized wave. Jitter is the fix. Two shapes ship here. `Jitter::Full` / `Jitter::Equal` with the `Jittered` wrapper and the `Backoff::jittered(mode)` combinator randomize any strategy's delays. Opt-in; retry's default stays deterministic. `DecorrelatedBackoff` is the AWS formula, `min(cap, rand(base, prev*3))`. It can't be a `Jitter` mode because the randomness is in the recurrence: each range is set by the delay actually drawn last time, so there's no deterministic sequence underneath to wrap. Validated config, `max_retries` give-up, and the same `BackoffConfigError` as the exponential strategy. Neither randomized type is `Clone`. Copying the RNG state would make every copy replay identical delays, which puts concurrent requests on one host into lockstep, which is the herd again. Hold the config, which is `Clone`, and build a fresh strategy from it. Both are seedable (`with_seed`) so jittered retries stay reproducible. Tests are mutation-checked: a no-op `Equal` mode, a dropped `mode` argument, a fixed seed in place of entropy, the raw draw fed back instead of the capped one, a smaller multiplier, and the cap applied before the draw all fail the suite. ADR004 records the `fastrand` dependency (measured against the cost of the `tracing` tree we already take), why decorrelated is a strategy rather than a mode, and why the randomized types aren't `Clone`. --- CHANGELOG.md | 20 ++ Cargo.lock | 7 + Cargo.toml | 1 + docs/adr/ADR004.md | 68 ++++++ src/backoff.rs | 539 ++++++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 5 +- 6 files changed, 630 insertions(+), 10 deletions(-) create mode 100644 docs/adr/ADR004.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 908a7f8..1392733 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- Jitter for backoff: the `Jitter` mode (`Full` / `Equal`) and a `Backoff::jittered(..)` combinator + that wraps any strategy, so a fleet of clients doesn't retry in lockstep. Opt-in; the RNG is + seedable for reproducible tests. +- `DecorrelatedBackoff`, built from a validated `DecorrelatedBackoffConfig`: each delay is drawn + from `base ..= prev * 3`, capped at `max_delay` (the AWS "Exponential Backoff and Jitter" + formula). It's a strategy rather than a `Jitter` mode because the randomness lives in the + recurrence, so there's no deterministic sequence for `Jittered` to wrap. Seedable via + `DecorrelatedBackoff::with_seed`. + +The randomized strategies (`DecorrelatedBackoff`, `Jittered`) are deliberately not `Clone`. A copy +would carry the RNG state and replay the same delays, which is the lockstep jitter exists to +prevent; clone the config and build a fresh strategy instead. + +Jitter needs a random number generator, so this adds `fastrand` as a required dependency (1198 +lines, no transitive dependencies of its own). It is not behind a feature flag; ADR004 records the +measurements and the reasoning. + ## [0.2.0] - 2026-07-26 Supersedes the yanked 0.1.1: that release removed public items in a patch, which was a breaking diff --git a/Cargo.lock b/Cargo.lock index 13bef64..32ab9b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,10 +2,17 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "mettle" version = "0.2.0" dependencies = [ + "fastrand", "pin-project-lite", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 5248d36..b331d4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ rustdoc-args = ["--cfg", "docsrs"] tokio = { version = "1", features = ["time"], optional = true } pin-project-lite = { version = "0.2", optional = true } tracing = { version = "0.1", default-features = false, features = ["std"] } +fastrand = "2" [features] default = ["async", "blocking"] diff --git a/docs/adr/ADR004.md b/docs/adr/ADR004.md new file mode 100644 index 0000000..53ba109 --- /dev/null +++ b/docs/adr/ADR004.md @@ -0,0 +1,68 @@ +# ADR004: Jitter, a fourth dependency, and why randomized strategies aren't `Clone` + +**Status:** Accepted + +## Context + +Retrying with a fixed backoff means every client that failed at the same moment retries at the +same moment. The dependency that just fell over gets a synchronized wave of traffic the instant +it comes back, which is the thundering herd. Jitter is the fix, and it's the first thing in +mettle that needs randomness, so it forces three decisions we hadn't had to make yet. + +## Decisions + +**1. `fastrand` is a required dependency, not a feature.** +Jitter needs a random number generator. `fastrand` gives us three things (`Rng::new()`, +`Rng::with_seed(u64)`, and `rng.u64(range)`) and nothing else comes with it. + +ADR003 moved Tokio behind a feature so sync-only users didn't pay for a runtime they never +touched, so the obvious move was to do the same here: `fastrand` optional, behind a `jitter` +feature. We measured instead of guessing. `fastrand` is 1198 lines with zero transitive +dependencies, and rebuilding it costs about the same as rebuilding the whole `tracing` tree +(`tracing` plus `tracing-core` plus `once_cell`), which ADR003 already accepts unconditionally. +Gating it would have put `#[cfg]` on roughly a dozen public items including a trait method, +doubled the feature matrix CI has to cover, and given anyone on a minimal build a missing-type +error instead of a working `Jitter`. That's a lot of friction to save less than what a dependency +we already took costs. + +The weight class is what matters, not the count. Tokio is a runtime; `fastrand` is a PRNG. We'd +make the same call again for something this size and revisit it for anything larger. + +We also considered inlining a small PRNG to keep the dependency count at two. We didn't, because +seeding from entropy portably is the part that's actually easy to get wrong, and getting it wrong +means a fleet seeded identically, which is the exact failure jitter exists to prevent. + +**2. Decorrelated jitter is its own `Backoff`, not a `Jitter` mode.** +`Jitter::Full` and `Jitter::Equal` are per-delay functions: hand them a delay, get a randomized +delay. `Jittered` wraps any strategy and applies one. Decorrelated jitter doesn't fit that +shape. Its next delay is drawn from a range set by the delay it actually drew last time +(`next = min(cap, rand(base, prev * 3))`), so the randomness is inside the recurrence and there's +no deterministic sequence underneath to wrap. It ships as `DecorrelatedBackoff`. + +`Jitter` is `#[non_exhaustive]`, so a future maintainer could try to add a `Decorrelated` variant +and then discover it's unimplementable. The enum's docs say why it isn't there. + +**3. Randomized strategies are not `Clone`.** +`ExponentialBackoff` is `Clone` and cloning it is the right thing to do: you get the same +deterministic sequence, which is what you asked for. Cloning `Jittered` or `DecorrelatedBackoff` +copies the RNG state, so every copy replays identical delays. A user who builds one and clones it +per request puts every concurrent request on that host into lockstep, and nothing warns them. +That's the herd again, one host at a time. + +We considered a manual `Clone` that reseeds, and rejected it: a `Clone` that doesn't clone breaks +the trait's contract and would silently break seeded tests. So the randomized types just aren't +`Clone`. Keep the config, which is `Clone`, and build a fresh strategy from it. + +This direction is also the reversible one. Adding `Clone` later is a minor version; taking it away +is a breaking change. + +## Consequences + +Jitter is available to everyone with no feature flag to discover, at the cost of one small +dependency. The `Backoff` trait now has strategies that behave differently under `Clone`, which +the trait docs have to explain rather than leaving to be discovered. Anyone reusing a randomized +policy across calls has to hold the config rather than the strategy, which is one extra line and +the only shape that's actually correct. + +Every randomized path is seedable (`Jittered::with_seed`, `DecorrelatedBackoff::with_seed`), so +none of this costs us deterministic tests. diff --git a/src/backoff.rs b/src/backoff.rs index 81018dd..8eeebb6 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -1,31 +1,55 @@ //! Backoff strategies: whether to retry, and how long to wait before each attempt. //! //! A [`Backoff`] is a stateful sequence of delays. `retry` takes it by value and drives that -//! owned instance; reuse one policy across calls by passing a fresh (or cloned) value. +//! owned instance; reuse one policy across calls by passing a fresh value. use std::num::NonZeroU32; use std::time::Duration; -// Defaults for `ExponentialBackoffConfig::default()`. +// Defaults shared by both config `Default` impls; `DEFAULT_FACTOR` is exponential-only. const DEFAULT_FACTOR: u32 = 2; const DEFAULT_BASE: Duration = Duration::from_millis(100); const DEFAULT_MAX_RETRIES: u32 = 3; const DEFAULT_MAX_DELAY: Duration = Duration::from_secs(30); +/// The multiplier in decorrelated jitter's `rand(base, prev * 3)`. Part of the published +/// algorithm, not a tuning knob. +const DECORRELATED_MULTIPLIER: u32 = 3; + /// A stateful sequence of retry delays. Pure: no I/O, no sleeping, no clock reads. /// /// A backoff is consumed as it runs, since each [`next_delay`](Backoff::next_delay) advances it, /// so `retry` takes it **by value** and drives that owned instance. A freshly constructed value -/// must represent an un-started sequence; to reuse one policy across calls, pass a fresh (or -/// cloned) value each time. +/// must represent an un-started sequence, so reuse one policy across calls by passing a fresh +/// value each time. A deterministic strategy such as [`ExponentialBackoff`] can be cloned +/// instead; the randomized ones are deliberately not [`Clone`], since a copy would replay the +/// same delays. /// /// Open on purpose: add a custom strategy by implementing it. pub trait Backoff { /// Delay before the next retry, or `None` to give up (e.g. retries exhausted). fn next_delay(&mut self) -> Option; + + /// Wrap this strategy so each delay is randomized by `mode` (see [`Jitter`]), seeding the RNG + /// from entropy. Composes with any strategy; reach for it to keep a fleet of clients from + /// retrying in lockstep: + /// + /// ``` + /// use mettle::{Backoff, ExponentialBackoff, Jitter}; + /// + /// let mut backoff = ExponentialBackoff::default().jittered(Jitter::Full); + /// let delay = backoff.next_delay(); // somewhere in 0 ..= 100ms + /// # let _ = delay; + /// ``` + fn jittered(self, mode: Jitter) -> Jittered + where + Self: Sized, + { + Jittered::new(self, mode) + } } -/// Why an [`ExponentialBackoff`] configuration was rejected. +/// Why a backoff configuration was rejected. Not every strategy can produce every variant. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum BackoffConfigError { @@ -33,7 +57,7 @@ pub enum BackoffConfigError { ZeroBase, /// `factor` was zero, so delays would collapse to zero (a busy-loop). ZeroFactor, - /// `max_delay` was smaller than `base`, which would cap the first delay below `base`. + /// `max_delay` was smaller than `base`, which would cap delays below `base`. MaxDelayBelowBase, } @@ -89,7 +113,8 @@ impl Default for ExponentialBackoffConfig { /// `retry` consumes it by value, so pass a fresh (or cloned) one to run the same policy again. /// /// Delays are deterministic: no jitter is applied, so a given config always yields the same -/// sequence. Jitter is planned; until then, wrap this in a custom [`Backoff`] if you need it. +/// sequence. Add randomness by wrapping it with [`Backoff::jittered`], for example +/// `ExponentialBackoff::default().jittered(Jitter::Full)`. #[derive(Debug, Clone)] pub struct ExponentialBackoff { factor: NonZeroU32, @@ -154,6 +179,225 @@ impl Backoff for ExponentialBackoff { } } +/// How to randomize a backoff's delays, so a fleet of clients doesn't retry in lockstep +/// (a thundering herd). Applied by wrapping any strategy; see [`Backoff::jittered`]. +/// +/// There is deliberately no `Decorrelated` variant. That formula feeds each drawn delay into the +/// next range, and a mode applied to one delay at a time has nowhere to keep it, so it ships as +/// its own strategy: [`DecorrelatedBackoff`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Jitter { + /// Each delay becomes a uniform random value in `0 ..= delay`. Maximum spread. + Full, + /// Each delay becomes a uniform random value in `delay/2 ..= delay`, keeping a floor under + /// every wait. + Equal, +} + +impl Jitter { + fn apply(self, delay: Duration, rng: &mut fastrand::Rng) -> Duration { + match self { + Self::Full => rand_duration(rng, Duration::ZERO, delay), + Self::Equal => { + let half = delay / 2; + half + rand_duration(rng, Duration::ZERO, delay - half) + } + } + } +} + +/// A uniform random `Duration` in `lo ..= hi`, or `lo` if `hi <= lo`. Computed in `u64` +/// nanoseconds; real delays sit well below that bound, and larger inputs saturate to it. +fn rand_duration(rng: &mut fastrand::Rng, lo: Duration, hi: Duration) -> Duration { + let lo = lo.as_nanos().min(u64::MAX as u128) as u64; + let hi = hi.as_nanos().min(u64::MAX as u128) as u64; + if hi <= lo { + return Duration::from_nanos(lo); + } + Duration::from_nanos(lo + rng.u64(0..=(hi - lo))) +} + +/// Any [`Backoff`] wrapped to randomize each delay by a [`Jitter`] mode. +/// +/// The inner strategy stays deterministic; only this layer is random. Its RNG is seedable with +/// [`with_seed`](Jittered::with_seed) so jittered retries stay reproducible in tests. Usually +/// built with [`Backoff::jittered`] rather than named directly. +/// +/// Not [`Clone`] on purpose: a copy would carry the RNG state and replay the same delays, which +/// is the lockstep jitter exists to prevent. Wrap a fresh inner strategy instead. +#[derive(Debug)] +pub struct Jittered { + inner: B, + mode: Jitter, + rng: fastrand::Rng, +} + +impl Jittered { + /// Wrap `inner`, seeding the RNG from entropy. + pub fn new(inner: B, mode: Jitter) -> Self { + Self { + inner, + mode, + rng: fastrand::Rng::new(), + } + } + + /// Wrap `inner` with a fixed `seed`, for reproducible tests. + pub fn with_seed(inner: B, mode: Jitter, seed: u64) -> Self { + Self { + inner, + mode, + rng: fastrand::Rng::with_seed(seed), + } + } +} + +impl Backoff for Jittered { + fn next_delay(&mut self) -> Option { + let delay = self.inner.next_delay()?; + Some(self.mode.apply(delay, &mut self.rng)) + } +} + +/// Parameters for [`DecorrelatedBackoff::new`]. Fill only what differs from [`Default`]: +/// +/// ``` +/// # use mettle::DecorrelatedBackoffConfig; +/// let _ = DecorrelatedBackoffConfig { max_retries: 8, ..Default::default() }; +/// ``` +#[derive(Debug, Clone)] +pub struct DecorrelatedBackoffConfig { + /// Lower bound on every delay, and where the first draw starts from (must be non-zero). + pub base: Duration, + /// Number of retries; `0` means one attempt, no retries + /// (total attempts = `max_retries + 1`). + pub max_retries: u32, + /// Upper bound on any single delay (must be >= `base`). + pub max_delay: Duration, +} + +impl Default for DecorrelatedBackoffConfig { + /// Sensible defaults: 100 ms base, 3 retries, 30 s cap. + fn default() -> Self { + Self { + base: DEFAULT_BASE, + max_retries: DEFAULT_MAX_RETRIES, + max_delay: DEFAULT_MAX_DELAY, + } + } +} + +/// Decorrelated jitter: each delay is drawn uniformly from `base ..= prev * 3`, capped at +/// `max_delay`, for at most `max_retries` retries. The formula is the one from AWS's +/// "Exponential Backoff and Jitter". +/// +/// ``` +/// use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig}; +/// +/// let mut backoff = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; +/// let delay = backoff.next_delay(); // somewhere in 100ms ..= 300ms +/// # let _ = delay; +/// # Ok::<_, mettle::BackoffConfigError>(()) +/// ``` +/// +/// This is a strategy rather than a [`Jitter`] mode because the randomness lives in the +/// recurrence: each range is set by the delay that was actually drawn last time, so there is no +/// deterministic sequence underneath for [`Jittered`] to wrap. +/// +/// One thing differs from [`ExponentialBackoff`]: the first delay is already random, somewhere in +/// `base ..= base * 3`, rather than exactly `base`. Against a jittered exponential, the difference +/// is the floor. Every delay here is at least `base`, where [`Jitter::Full`] can return anything +/// down to zero. Don't stack the two by calling [`jittered`](Backoff::jittered) on this: the +/// randomness is already in the recurrence, and wrapping it throws the `base` floor away. +/// +/// Build it with [`new`](DecorrelatedBackoff::new), or [`with_seed`](DecorrelatedBackoff::with_seed) +/// to make the delays reproducible in tests. Not [`Clone`] on purpose: a copy would carry the RNG +/// state and replay the same delays, which is the lockstep this strategy exists to prevent. Keep +/// the [`DecorrelatedBackoffConfig`] around and build a fresh one per call instead. +#[derive(Debug)] +pub struct DecorrelatedBackoff { + base: Duration, + max_delay: Duration, + prev: Duration, // the last delay handed out; starts at `base` + retries_left: u32, + rng: fastrand::Rng, +} + +impl DecorrelatedBackoff { + /// Validate a [`DecorrelatedBackoffConfig`] into a ready-to-run backoff, seeding the RNG from + /// entropy. + /// + /// # Errors + /// Returns [`BackoffConfigError`] if the config would produce a degenerate + /// (e.g. zero-delay) sequence. + pub fn new(config: DecorrelatedBackoffConfig) -> Result { + Self::build(config, fastrand::Rng::new()) + } + + /// As [`new`](DecorrelatedBackoff::new), but with a fixed `seed`, for reproducible tests. A + /// seed replays the same delays within a build; the exact values are not part of this crate's + /// API contract. + /// + /// # Errors + /// Returns [`BackoffConfigError`] if the config would produce a degenerate + /// (e.g. zero-delay) sequence. + pub fn with_seed( + config: DecorrelatedBackoffConfig, + seed: u64, + ) -> Result { + Self::build(config, fastrand::Rng::with_seed(seed)) + } + + fn build( + config: DecorrelatedBackoffConfig, + rng: fastrand::Rng, + ) -> Result { + let DecorrelatedBackoffConfig { + base, + max_retries, + max_delay, + } = config; + if base.is_zero() { + // Zero is absorbing here: prev = 0 makes every later range [0, 0] too. + return Err(BackoffConfigError::ZeroBase); + } + if max_delay < base { + return Err(BackoffConfigError::MaxDelayBelowBase); + } + Ok(Self { + base, + max_delay, + prev: base, + retries_left: max_retries, + rng, + }) + } +} + +impl Backoff for DecorrelatedBackoff { + fn next_delay(&mut self) -> Option { + if self.retries_left == 0 { + return None; // retries exhausted — give up + } + self.retries_left -= 1; + + // Draw from [base, prev * 3], then cap, which is the formula as published. + // `saturating_mul` so an enormous `prev` pins the top of the range at `Duration::MAX` + // instead of panicking. + let hi = self.prev.saturating_mul(DECORRELATED_MULTIPLIER); + let delay = rand_duration(&mut self.rng, self.base, hi) + .min(self.max_delay) + // `.min` alone can't drop below `base`, since `max_delay >= base` is validated. The + // `.max` is for a `base` past the u64-nanosecond range `rand_duration` works in, + // where the draw itself saturates low. Absurd as an input, but it keeps the + // "every delay is at least `base`" guarantee total rather than almost-total. + .max(self.base); + self.prev = delay; // the capped value seeds the next draw + Some(delay) + } +} + #[cfg(test)] mod tests { use super::*; @@ -206,8 +450,10 @@ mod tests { max_delay: Duration::MAX, }) .unwrap(); - let _ = b.next_delay(); // saturating_mul must not overflow-panic - let _ = b.next_delay(); + // `saturating_mul` must not overflow-panic, and must saturate rather than wrap to zero: + // a zero delay here would be the busy-loop the config validation exists to prevent. + assert!(b.next_delay().unwrap() > Duration::ZERO); + assert!(b.next_delay().unwrap() > Duration::ZERO); } #[test] @@ -236,4 +482,279 @@ mod tests { Err(BackoffConfigError::MaxDelayBelowBase) )); } + + // --- jitter --- + + fn exp6() -> ExponentialBackoff { + exp(1, 2, 100, 6) // underlying: 1, 2, 4, 8, 16, 32 (seconds) + } + + fn drain(mut b: impl Backoff) -> Vec { + std::iter::from_fn(move || b.next_delay()).collect() + } + + #[test] + fn full_jitter_stays_within_bounds() { + // Full jitter: every delay lies in [0, the underlying delay], and the sequence still ends + // exactly when the inner strategy is exhausted. + let plain = drain(exp6()); + let mut j = Jittered::with_seed(exp6(), Jitter::Full, 42); + for p in &plain { + let d = j.next_delay().unwrap(); + assert!( + d <= *p, + "full jitter exceeded the base delay: {d:?} > {p:?}" + ); + } + assert_eq!(j.next_delay(), None); + } + + #[test] + fn equal_jitter_keeps_a_floor() { + // Equal jitter: every delay lies in [d/2, d]. + let plain = drain(exp6()); + let mut j = Jittered::with_seed(exp6(), Jitter::Equal, 99); + for p in &plain { + let d = j.next_delay().unwrap(); + assert!( + d >= *p / 2 && d <= *p, + "equal jitter out of [d/2, d]: {d:?} for {p:?}" + ); + } + } + + #[test] + fn seed_makes_jitter_reproducible() { + // Same seed yields an identical sequence (so jittered retries stay testable); different + // seeds generally differ, which guards against a constant or broken RNG. Both modes, so + // neither can quietly become a no-op that still satisfies its bounds. + for mode in [Jitter::Full, Jitter::Equal] { + let seq = |seed| drain(Jittered::with_seed(exp6(), mode, seed)); + assert_eq!(seq(7), seq(7), "{mode:?} was not reproducible"); + assert_ne!(seq(1), seq(2), "{mode:?} ignored its seed"); + } + } + + #[test] + fn jitter_actually_moves_the_delay() { + // Bounds checks alone are satisfied by the identity function, so pin that each mode + // really randomizes: `Full` must reach below the halfway mark, `Equal` must land strictly + // inside (d/2, d) at least once. + let full = drain(Jittered::with_seed(exp6(), Jitter::Full, 5)); + let plain = drain(exp6()); + assert!( + full.iter().zip(&plain).any(|(d, p)| *d < *p / 2), + "full jitter never dropped below half the base delay" + ); + let equal = drain(Jittered::with_seed(exp6(), Jitter::Equal, 5)); + assert!( + equal + .iter() + .zip(&plain) + .any(|(d, p)| *d > *p / 2 && *d < *p), + "equal jitter never landed strictly inside (d/2, d)" + ); + } + + #[test] + fn jittered_combinator_passes_the_mode_through() { + // `Backoff::jittered` is the documented entry point, so drive it rather than only the + // `Jittered::` constructors. `Equal` keeps a floor that `Full` does not. + let mut b = exp6().jittered(Jitter::Equal); + let plain = drain(exp6()); + for p in &plain { + let d = b.next_delay().unwrap(); + assert!( + d >= *p / 2 && d <= *p, + "combinator lost the mode: {d:?} for {p:?}" + ); + } + assert_eq!(b.next_delay(), None); + } + + #[test] + fn unseeded_jitter_differs_between_instances() { + // The entropy-seeded path is what stops a fleet retrying in lockstep, and it's the reason + // these types aren't `Clone`. A regression to a fixed seed would pass every other test + // here. Two independent RNGs colliding across six delays is a 2^-64 event. + assert_ne!( + drain(exp6().jittered(Jitter::Full)), + drain(exp6().jittered(Jitter::Full)) + ); + } + + #[test] + fn jitter_handles_zero_and_extreme_delays() { + // A custom strategy can hand back zero or enormous delays; jitter must not panic. + struct Fixed(std::vec::IntoIter); + impl Backoff for Fixed { + fn next_delay(&mut self) -> Option { + self.0.next() + } + } + let inner = Fixed(vec![Duration::ZERO, Duration::MAX, secs(1)].into_iter()); + let mut j = Jittered::with_seed(inner, Jitter::Full, 1); + assert_eq!(j.next_delay(), Some(Duration::ZERO)); // rand(0..=0) + let _ = j.next_delay().unwrap(); // Duration::MAX saturates, no panic + assert!(j.next_delay().unwrap() <= secs(1)); + assert_eq!(j.next_delay(), None); + } + + // --- decorrelated jitter --- + + fn dec(base: u64, max_delay: u64, max_retries: u32, seed: u64) -> DecorrelatedBackoff { + DecorrelatedBackoff::with_seed( + DecorrelatedBackoffConfig { + base: secs(base), + max_retries, + max_delay: secs(max_delay), + }, + seed, + ) + .unwrap() + } + + #[test] + fn decorrelated_draws_each_delay_from_the_previous_one() { + // The recurrence, checked step by step across seeds: every delay sits in + // [base, min(cap, prev * 3)]. The `base` floor is what stops it collapsing toward zero, + // and it's the first non-zero `lo` anything passes to `rand_duration`. + let (base, cap) = (secs(1), secs(30)); + let mut ever_above_double = false; + for seed in 0..16 { + let mut prev = base; + let mut b = dec(1, 30, 40, seed); + while let Some(d) = b.next_delay() { + let hi = (prev * 3).min(cap); + assert!( + d >= base && d <= hi, + "{d:?} outside [{base:?}, {hi:?}] (seed {seed})" + ); + ever_above_double |= d > prev * 2 && d < cap; + prev = d; + } + } + // That bound is one-sided, so a smaller multiplier would satisfy it too. Only a range that + // really runs to `prev * 3` can land a delay past `prev * 2`. + assert!(ever_above_double, "no delay ever exceeded `prev * 2`"); + } + + #[test] + fn decorrelated_feeds_the_capped_delay_into_the_next_draw() { + // `prev` must be the delay handed out, not the raw draw. Feeding the uncapped draw back in + // lets `prev` grow without bound, so the sequence pins to the cap and stops coming down. + // Every per-delay bound still holds under that mutation, so what separates the two is how + // often the sequence recovers below the cap. + let cap = secs(2); + let (mut below, mut total) = (0u32, 0u32); + for seed in 0..32 { + for d in drain(dec(1, 2, 32, seed)) { + below += u32::from(d < cap); + total += 1; + } + } + // Feeding back the capped value holds this near 20%. Feeding back the raw draw collapses + // it to roughly 3%, since `prev` runs away after a handful of retries. + assert!( + below * 10 >= total, + "sequence stopped recovering below the cap: {below}/{total}" + ); + } + + #[test] + fn unseeded_decorrelated_differs_between_instances() { + // Same reasoning as the jitter twin: the entropy-seeded path is the whole reason this type + // isn't `Clone`, and a regression to a fixed seed would pass every other test here. + let cfg = || DecorrelatedBackoffConfig { + base: secs(1), + max_retries: 8, + max_delay: secs(60), + }; + assert_ne!( + drain(DecorrelatedBackoff::new(cfg()).unwrap()), + drain(DecorrelatedBackoff::new(cfg()).unwrap()) + ); + } + + #[test] + fn decorrelated_first_delay_is_already_random() { + // Unlike `ExponentialBackoff`, whose first delay is exactly `base`, `prev` starts at + // `base` so the very first delay is drawn from [base, base * 3]. + let firsts: Vec<_> = (0..16) + .map(|seed| dec(1, 1000, 1, seed).next_delay().unwrap()) + .collect(); + assert!(firsts.iter().all(|d| *d >= secs(1) && *d <= secs(3))); + assert!( + firsts.iter().any(|d| *d != secs(1)), + "first delay never moved off `base`" + ); + } + + #[test] + fn decorrelated_gives_up_after_max_retries() { + assert_eq!(drain(dec(1, 100, 5, 3)).len(), 5); + assert_eq!(dec(1, 100, 0, 3).next_delay(), None); // no retries — exactly one attempt + } + + #[test] + fn decorrelated_seed_is_reproducible() { + // Same seed, same sequence, so a jittered retry stays testable; different seeds differ, + // which guards against a constant or broken RNG. + let seq = |seed| drain(dec(1, 60, 8, seed)); + assert_eq!(seq(7), seq(7)); + assert_ne!(seq(1), seq(2)); + } + + #[test] + fn decorrelated_reaches_the_cap() { + // The cap applies after the draw, so `max_delay` is a value the sequence actually hits + // rather than an asymptote it approaches. + let hit = (0..32).any(|seed| drain(dec(1, 4, 24, seed)).contains(&secs(4))); + assert!(hit, "cap was never reached"); + } + + #[test] + fn decorrelated_base_equal_to_cap_is_constant() { + // Degenerate but legal: every draw is capped straight back to `base`, so the RNG can't + // move it. + assert_eq!(drain(dec(5, 5, 4, 12345)), vec![secs(5); 4]); + } + + #[test] + fn decorrelated_huge_base_does_not_panic() { + let mut b = DecorrelatedBackoff::with_seed( + DecorrelatedBackoffConfig { + base: Duration::from_secs(u64::MAX / 2), + max_retries: 3, + max_delay: Duration::MAX, + }, + 1, + ) + .unwrap(); + // `prev * 3` must saturate rather than overflow-panic. This `base` is also past the + // u64-nanosecond range `rand_duration` works in, where the draw saturates low, so it + // pins the `.max(base)` that keeps the "every delay is at least `base`" guarantee total. + let base = Duration::from_secs(u64::MAX / 2); + assert!(b.next_delay().unwrap() >= base); + assert!(b.next_delay().unwrap() >= base); + } + + #[test] + fn decorrelated_rejects_degenerate_configs() { + assert!(matches!( + DecorrelatedBackoff::new(DecorrelatedBackoffConfig { + base: secs(0), + ..Default::default() + }), + Err(BackoffConfigError::ZeroBase) + )); + assert!(matches!( + DecorrelatedBackoff::new(DecorrelatedBackoffConfig { + base: secs(10), + max_delay: secs(5), + ..Default::default() + }), + Err(BackoffConfigError::MaxDelayBelowBase) + )); + } } diff --git a/src/lib.rs b/src/lib.rs index e5c0c42..8a66660 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,7 +58,10 @@ mod shared; #[cfg(test)] mod test_support; -pub use backoff::{Backoff, BackoffConfigError, ExponentialBackoff, ExponentialBackoffConfig}; +pub use backoff::{ + Backoff, BackoffConfigError, DecorrelatedBackoff, DecorrelatedBackoffConfig, + ExponentialBackoff, ExponentialBackoffConfig, Jitter, Jittered, +}; #[cfg(feature = "async")] pub use clock::Clock; #[cfg(feature = "async")] From a46ea2f99a7b3ff78a04a3a4dd0c51035f9d0a95 Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Wed, 29 Jul 2026 16:01:10 +0530 Subject: [PATCH 02/12] Drive the randomized backoffs through the retry driver The backoff tests cover the delay sequences in isolation, but nothing exercised a strategy carrying its own RNG through the pin-projected state machine. Seeded, so the delays stay fixed. Blocking twin skipped: its driver is a plain loop, so it can't have the class of bug these cover (state surviving a move into a pinned future). --- src/retry.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/retry.rs b/src/retry.rs index 677ec62..7c0d622 100644 --- a/src/retry.rs +++ b/src/retry.rs @@ -685,4 +685,59 @@ mod tests { assert_eq!(out, Ok(42)); assert_eq!(events.get(), 2); // one event per retry } + + // --- randomized strategies driven through the real driver --- + + #[tokio::test] + async fn drives_a_decorrelated_backoff() { + // The backoff tests cover the delay sequence in isolation; this covers the wiring, that a + // strategy holding its own RNG survives being moved into the future and polled. Seeded, so + // the delays are fixed even though the strategy is randomized. + use crate::backoff::{DecorrelatedBackoff, DecorrelatedBackoffConfig}; + + let clock = MockClock::new(); + let out: Result = retry(|| async { Err("boom") }) + .backoff( + DecorrelatedBackoff::with_seed( + DecorrelatedBackoffConfig { + base: secs(1), + max_retries: 4, + max_delay: secs(20), + }, + 7, + ) + .unwrap(), + ) + .clock(clock.clone()) + .await; + + assert_eq!(out, Err("boom")); + let slept = clock.slept(); + assert_eq!(slept.len(), 4); // max_retries sleeps, then give up + assert!( + slept.iter().all(|d| *d >= secs(1) && *d <= secs(20)), + "delays escaped [base, max_delay]: {slept:?}" + ); + } + + #[tokio::test] + async fn drives_a_jittered_backoff() { + let clock = MockClock::new(); + let out: Result = retry(|| async { Err("boom") }) + .backoff(crate::backoff::Jittered::with_seed( + backoff(3), + crate::backoff::Jitter::Full, + 42, + )) + .clock(clock.clone()) + .await; + + assert_eq!(out, Err("boom")); + // Underlying exponential is 1s, 2s, 4s; full jitter can only shrink each one. + let slept = clock.slept(); + assert_eq!(slept.len(), 3); + for (d, cap) in slept.iter().zip([secs(1), secs(2), secs(4)]) { + assert!(*d <= cap, "jittered delay {d:?} exceeded {cap:?}"); + } + } } From c034aecb9a48d814fcbfa7360b36e606031c4d8b Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Wed, 29 Jul 2026 16:13:14 +0530 Subject: [PATCH 03/12] Document factor=1 as constant backoff instead of adding a type `ConstantBackoff` was on the 0.3.0 list, but `ExponentialBackoff` with `factor: 1` already produces exactly that: the delay never grows, so it stays at `base`. A separate type would be a permanent public commitment for something already expressible, so this is one doc line and a test pinning the behaviour instead. --- src/backoff.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/backoff.rs b/src/backoff.rs index 8eeebb6..bb86f76 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -81,7 +81,7 @@ impl std::error::Error for BackoffConfigError {} /// ``` #[derive(Debug, Clone)] pub struct ExponentialBackoffConfig { - /// Growth multiplier applied each retry (must be >= 1). + /// Growth multiplier applied each retry (must be >= 1). Set it to `1` for a constant delay. pub factor: u32, /// Delay before the first retry (must be non-zero). pub base: Duration, @@ -427,6 +427,13 @@ mod tests { assert_eq!(b.next_delay(), None); // 5 retries used up } + #[test] + fn factor_of_one_holds_the_delay_constant() { + // Documented on `ExponentialBackoffConfig::factor`, and the reason there's no separate + // `ConstantBackoff` type. + assert_eq!(drain(exp(7, 1, 100, 4)), vec![secs(7); 4]); + } + #[test] fn delay_is_capped_at_max() { let mut b = exp(10, 10, 30, 4); From 5e7e646d1bd09f9f691af97ee7d1307cdb200fc7 Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 11:05:49 +0530 Subject: [PATCH 04/12] Release 0.3.0 Jitter only. The RetryError work is breaking and ships separately, so this release is purely additive: existing code compiles unchanged and the default backoff stays deterministic. Bumps the version because cargo-semver-checks compares against the published 0.2.0, and adds a README section since jitter is user-facing. --- CHANGELOG.md | 4 ++++ Cargo.lock | 2 +- Cargo.toml | 4 ++-- README.md | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1392733..67096f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0] - 2026-08-09 + ### Added - Jitter for backoff: the `Jitter` mode (`Full` / `Equal`) and a `Backoff::jittered(..)` combinator that wraps any strategy, so a fleet of clients doesn't retry in lockstep. Opt-in; the RNG is @@ -16,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 formula). It's a strategy rather than a `Jitter` mode because the randomness lives in the recurrence, so there's no deterministic sequence for `Jittered` to wrap. Seedable via `DecorrelatedBackoff::with_seed`. +- Documented that `ExponentialBackoffConfig { factor: 1, .. }` gives a constant delay, so there is + no separate `ConstantBackoff` type to learn. The randomized strategies (`DecorrelatedBackoff`, `Jittered`) are deliberately not `Clone`. A copy would carry the RNG state and replay the same delays, which is the lockstep jitter exists to diff --git a/Cargo.lock b/Cargo.lock index 32ab9b3..81a7c27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "mettle" -version = "0.2.0" +version = "0.3.0" dependencies = [ "fastrand", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index b331d4b..5e74194 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "mettle" -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.85" -description = "A resilience toolkit for Rust: async and blocking retries with configurable backoff." +description = "A resilience toolkit for Rust: async and blocking retries with configurable backoff and jitter." readme = "README.md" license = "MIT OR Apache-2.0" keywords = ["retry", "backoff", "resilience", "async", "tokio"] diff --git a/README.md b/README.md index abff4b1..5cc4a9b 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,22 @@ let body = retry(|| async { fetch(&url).await }) No async runtime? The blocking twin is identical but ends in `.call()` instead of `.await`. +Retrying on a fixed schedule means every client that failed together retries together, so a service +that is coming back up gets a synchronized wave. Jitter spreads them out: + +```rust +use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff, Jitter}; + +// Randomize any strategy's delays... +let backoff = ExponentialBackoff::default().jittered(Jitter::Full); + +// ...or use decorrelated jitter, where each delay is drawn from the previous one. +let backoff = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; +``` + +Both seed from entropy by default and take a fixed seed (`with_seed`) when you want a test to +replay the same delays. + ## Tools Each tool comes with a runnable example. Start there: From 4851a2caa7db6b6f519a32f4ba900dc7416443a4 Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 11:26:19 +0530 Subject: [PATCH 05/12] Pin equal jitter's floor at a factor where the coincidence breaks Equal jitter's floor is delay/2. With the default factor of 2 that is also the previous ladder value, so the two readings give the same number and the existing tests -- which all use factor 2 -- cannot tell them apart. This is documentation rather than coverage, and the comment says so. Jitter::apply takes one Duration and keeps no history, so the previous-step reading is not implementable without a structural change; the test exists to stop that change happening by accident and to state the rule at a factor where the two numbers differ. --- src/backoff.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/backoff.rs b/src/backoff.rs index bb86f76..a10323a 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -530,6 +530,41 @@ mod tests { } } + #[test] + fn equal_jitter_floor_is_half_the_delay_not_the_previous_step() { + // Executable documentation rather than extra coverage. `Equal`'s floor is `delay / 2`, + // and with the default `factor: 2` that is also the previous ladder value, so the two + // readings are easy to confuse. A factor of 3 separates them: at the 9s step the previous + // delay is 3s but the floor is 4.5s. + // + // `Jitter::apply` takes one `Duration` and keeps no history, so the previous-step reading + // isn't implementable today and this can't fail without a structural change. It is here to + // stop that change being made by accident, and to state the rule at a factor where the + // coincidence doesn't hide it. + let ladder = drain(exp(1, 3, 10_000, 4)); // 1s, 3s, 9s, 27s + assert_eq!(ladder[2], secs(9)); + assert_eq!( + ladder[1], + secs(3), + "the previous step, which is NOT the floor" + ); + + for seed in 0..256 { + let mut j = Jittered::with_seed(exp(1, 3, 10_000, 4), Jitter::Equal, seed); + for (i, plain) in ladder.iter().enumerate() { + let d = j.next_delay().unwrap(); + assert!( + d >= *plain / 2, + "step {i} drew {d:?}, below half of {plain:?} (seed {seed})" + ); + assert!( + d <= *plain, + "step {i} drew {d:?}, above {plain:?} (seed {seed})" + ); + } + } + } + #[test] fn seed_makes_jitter_reproducible() { // Same seed yields an identical sequence (so jittered retries stay testable); different From 9a33c28acc5a97ce276cfa224ddfb80229c03aff Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 11:45:13 +0530 Subject: [PATCH 06/12] Cut Jitter::Equal, and the enum with it Equal jitter is dominated. AWS's own post says it "does slightly more work than Full Jitter, and takes much longer", and their SDK agrees: full jitter is the default for STANDARD and ADAPTIVE retry modes while equal survives only on the LEGACY throttling path. Simulating both against mettle's actual implementations under a contended resource reproduced it. Equal lost to full on total calls AND completion time in all five configurations tried, across client counts and base delays. An option nobody should pick is a trap rather than a choice, and "equal" reads as the safe middle option, which is exactly backwards. With one mode left the enum was ceremony, so `jittered()` now takes no argument and the `Jitter` type is gone. The cost is the extension point: `Jitter` was #[non_exhaustive] so a fourth mode would have been free to add, and now it would need a separate `jittered_with(mode)`. Accepted -- the literature defines three, we ship the one that measures best, and decorrelated cannot be a mode at all. Docs now say which to reach for: `.jittered()` composes with any strategy and spreads widest; `DecorrelatedBackoff` gives a floor, at the price of never retrying sooner than `base`. --- CHANGELOG.md | 11 ++- README.md | 11 ++- docs/adr/ADR004.md | 32 ++++++-- src/backoff.rs | 178 +++++++++++++-------------------------------- src/lib.rs | 2 +- src/retry.rs | 6 +- 6 files changed, 96 insertions(+), 144 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67096f8..184ecbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.3.0] - 2026-08-09 ### Added -- Jitter for backoff: the `Jitter` mode (`Full` / `Equal`) and a `Backoff::jittered(..)` combinator - that wraps any strategy, so a fleet of clients doesn't retry in lockstep. Opt-in; the RNG is - seedable for reproducible tests. +- Jitter for backoff: `Backoff::jittered()` wraps any strategy so each delay becomes a uniform + random value in `0 ..= delay` ("full jitter"), so a fleet of clients doesn't retry in lockstep. + Opt-in; the RNG is seedable for reproducible tests. + + There is no mode to pick. AWS's measurements had the alternative ("equal jitter", a floor at + `delay/2`) doing more work *and* finishing later than full jitter, and a simulation of mettle's + own implementations reproduced that in every configuration tried, so offering it would only + invite people to choose the worse one. For a floor under every wait, use `DecorrelatedBackoff`. - `DecorrelatedBackoff`, built from a validated `DecorrelatedBackoffConfig`: each delay is drawn from `base ..= prev * 3`, capped at `max_delay` (the AWS "Exponential Backoff and Jitter" formula). It's a strategy rather than a `Jitter` mode because the randomness lives in the diff --git a/README.md b/README.md index 5cc4a9b..77fb583 100644 --- a/README.md +++ b/README.md @@ -41,15 +41,20 @@ Retrying on a fixed schedule means every client that failed together retries tog that is coming back up gets a synchronized wave. Jitter spreads them out: ```rust -use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff, Jitter}; +use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff}; -// Randomize any strategy's delays... -let backoff = ExponentialBackoff::default().jittered(Jitter::Full); +// Randomize any strategy's delays into 0 ..= delay ("full jitter")... +let backoff = ExponentialBackoff::default().jittered(); // ...or use decorrelated jitter, where each delay is drawn from the previous one. let backoff = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; ``` +Which one: `.jittered()` works on any strategy, including one you wrote, and spreads delays as +widely as possible. `DecorrelatedBackoff` is its own strategy and never draws below its `base`, so +reach for it when you want a floor under every wait. The trade is that a floor also means never +retrying sooner than `base`, so a dependency that frees up early isn't picked up until then. + Both seed from entropy by default and take a fixed seed (`with_seed`) when you want a test to replay the same delays. diff --git a/docs/adr/ADR004.md b/docs/adr/ADR004.md index 53ba109..4ca616c 100644 --- a/docs/adr/ADR004.md +++ b/docs/adr/ADR004.md @@ -32,15 +32,35 @@ We also considered inlining a small PRNG to keep the dependency count at two. We seeding from entropy portably is the part that's actually easy to get wrong, and getting it wrong means a fleet seeded identically, which is the exact failure jitter exists to prevent. -**2. Decorrelated jitter is its own `Backoff`, not a `Jitter` mode.** -`Jitter::Full` and `Jitter::Equal` are per-delay functions: hand them a delay, get a randomized -delay. `Jittered` wraps any strategy and applies one. Decorrelated jitter doesn't fit that -shape. Its next delay is drawn from a range set by the delay it actually drew last time +**2. One kind of jitter, with no mode to choose.** +`Jittered` applies full jitter: every delay becomes a uniform random value in `0 ..= delay`. +There is no `Jitter` enum and no argument to `jittered()`. + +We shipped `Full` and `Equal` first, mirroring the AWS post that names them. Then we read the post's +own conclusion, which is that equal jitter "does slightly more work than Full Jitter, and takes much +longer", and simulated both against mettle's actual implementations under a contended resource. +Equal lost to full on both total calls and completion time in every configuration we tried, at +several client counts and base delays. AWS's own SDK agrees: full jitter is the default for its +STANDARD and ADAPTIVE retry modes, and equal survives only on the LEGACY throttling path. + +An option nobody should pick is not an option, it's a trap, and the name "equal" reads as the safe +middle choice, which is exactly backwards. So it's gone, and with one mode left the enum was pure +ceremony. + +The cost is the extension point. `Jitter` was `#[non_exhaustive]`, so adding a fourth mode later +would have been free; now it would need a separate `jittered_with(mode)` method. We accept that: +the literature defines three modes, we ship the one that measures best, and decorrelated cannot be +a mode at all (see below). + +**2b. Decorrelated jitter is its own `Backoff`, not a mode.** +A mode is a per-delay function: hand it a delay, get a randomized delay. Decorrelated doesn't fit +that shape. Its next delay is drawn from a range set by the delay it actually drew last time (`next = min(cap, rand(base, prev * 3))`), so the randomness is inside the recurrence and there's no deterministic sequence underneath to wrap. It ships as `DecorrelatedBackoff`. -`Jitter` is `#[non_exhaustive]`, so a future maintainer could try to add a `Decorrelated` variant -and then discover it's unimplementable. The enum's docs say why it isn't there. +It is also the answer for anyone who wants a floor, which is what `Equal` was reaching for and +getting wrong: decorrelated never draws below `base`. The trade is real and documented, since a +floor also means never retrying sooner than `base`. **3. Randomized strategies are not `Clone`.** `ExponentialBackoff` is `Clone` and cloning it is the right thing to do: you get the same diff --git a/src/backoff.rs b/src/backoff.rs index a10323a..8338453 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -30,22 +30,29 @@ pub trait Backoff { /// Delay before the next retry, or `None` to give up (e.g. retries exhausted). fn next_delay(&mut self) -> Option; - /// Wrap this strategy so each delay is randomized by `mode` (see [`Jitter`]), seeding the RNG - /// from entropy. Composes with any strategy; reach for it to keep a fleet of clients from - /// retrying in lockstep: + /// Wrap this strategy so every delay becomes a uniform random value in `0 ..= delay`, seeding + /// the RNG from entropy. + /// + /// This is "full jitter" from AWS's *Exponential Backoff and Jitter*. Composes with any + /// strategy, including one you wrote, which is the point: reach for it to stop a fleet of + /// clients retrying in lockstep. /// /// ``` - /// use mettle::{Backoff, ExponentialBackoff, Jitter}; + /// use mettle::{Backoff, ExponentialBackoff}; /// - /// let mut backoff = ExponentialBackoff::default().jittered(Jitter::Full); + /// let mut backoff = ExponentialBackoff::default().jittered(); /// let delay = backoff.next_delay(); // somewhere in 0 ..= 100ms /// # let _ = delay; /// ``` - fn jittered(self, mode: Jitter) -> Jittered + /// + /// Because the floor is zero, a retry can fire almost immediately. That is the mechanism, not + /// a flaw: it is what lets a freed-up dependency be picked up at once. If you need a floor + /// under every wait, use [`DecorrelatedBackoff`] instead, which never goes below its `base`. + fn jittered(self) -> Jittered where Self: Sized, { - Jittered::new(self, mode) + Jittered::new(self) } } @@ -114,7 +121,7 @@ impl Default for ExponentialBackoffConfig { /// /// Delays are deterministic: no jitter is applied, so a given config always yields the same /// sequence. Add randomness by wrapping it with [`Backoff::jittered`], for example -/// `ExponentialBackoff::default().jittered(Jitter::Full)`. +/// `ExponentialBackoff::default().jittered()`. #[derive(Debug, Clone)] pub struct ExponentialBackoff { factor: NonZeroU32, @@ -179,34 +186,6 @@ impl Backoff for ExponentialBackoff { } } -/// How to randomize a backoff's delays, so a fleet of clients doesn't retry in lockstep -/// (a thundering herd). Applied by wrapping any strategy; see [`Backoff::jittered`]. -/// -/// There is deliberately no `Decorrelated` variant. That formula feeds each drawn delay into the -/// next range, and a mode applied to one delay at a time has nowhere to keep it, so it ships as -/// its own strategy: [`DecorrelatedBackoff`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum Jitter { - /// Each delay becomes a uniform random value in `0 ..= delay`. Maximum spread. - Full, - /// Each delay becomes a uniform random value in `delay/2 ..= delay`, keeping a floor under - /// every wait. - Equal, -} - -impl Jitter { - fn apply(self, delay: Duration, rng: &mut fastrand::Rng) -> Duration { - match self { - Self::Full => rand_duration(rng, Duration::ZERO, delay), - Self::Equal => { - let half = delay / 2; - half + rand_duration(rng, Duration::ZERO, delay - half) - } - } - } -} - /// A uniform random `Duration` in `lo ..= hi`, or `lo` if `hi <= lo`. Computed in `u64` /// nanoseconds; real delays sit well below that bound, and larger inputs saturate to it. fn rand_duration(rng: &mut fastrand::Rng, lo: Duration, hi: Duration) -> Duration { @@ -218,7 +197,16 @@ fn rand_duration(rng: &mut fastrand::Rng, lo: Duration, hi: Duration) -> Duratio Duration::from_nanos(lo + rng.u64(0..=(hi - lo))) } -/// Any [`Backoff`] wrapped to randomize each delay by a [`Jitter`] mode. +/// Any [`Backoff`] wrapped so each delay becomes a uniform random value in `0 ..= delay`. +/// +/// This is "full jitter": the widest spread, so the density of clients attempting at any instant +/// is as low as it can be. There is no mode to choose. AWS's own measurements had the alternative +/// ("equal jitter", a floor at `delay/2`) doing more work *and* finishing later, so shipping it as +/// an option would only invite people to pick the worse one. +/// +/// Want a floor under every wait? That's [`DecorrelatedBackoff`], which never draws below its +/// `base`. Note the trade: a floor means never retrying sooner than `base`, so a dependency that +/// frees up early isn't picked up until then. /// /// The inner strategy stays deterministic; only this layer is random. Its RNG is seedable with /// [`with_seed`](Jittered::with_seed) so jittered retries stay reproducible in tests. Usually @@ -229,25 +217,22 @@ fn rand_duration(rng: &mut fastrand::Rng, lo: Duration, hi: Duration) -> Duratio #[derive(Debug)] pub struct Jittered { inner: B, - mode: Jitter, rng: fastrand::Rng, } impl Jittered { /// Wrap `inner`, seeding the RNG from entropy. - pub fn new(inner: B, mode: Jitter) -> Self { + pub fn new(inner: B) -> Self { Self { inner, - mode, rng: fastrand::Rng::new(), } } /// Wrap `inner` with a fixed `seed`, for reproducible tests. - pub fn with_seed(inner: B, mode: Jitter, seed: u64) -> Self { + pub fn with_seed(inner: B, seed: u64) -> Self { Self { inner, - mode, rng: fastrand::Rng::with_seed(seed), } } @@ -256,7 +241,7 @@ impl Jittered { impl Backoff for Jittered { fn next_delay(&mut self) -> Option { let delay = self.inner.next_delay()?; - Some(self.mode.apply(delay, &mut self.rng)) + Some(rand_duration(&mut self.rng, Duration::ZERO, delay)) } } @@ -301,13 +286,13 @@ impl Default for DecorrelatedBackoffConfig { /// # Ok::<_, mettle::BackoffConfigError>(()) /// ``` /// -/// This is a strategy rather than a [`Jitter`] mode because the randomness lives in the -/// recurrence: each range is set by the delay that was actually drawn last time, so there is no -/// deterministic sequence underneath for [`Jittered`] to wrap. +/// This is a strategy of its own rather than something [`Jittered`] could produce, because the +/// randomness lives in the recurrence: each range is set by the delay that was actually drawn last +/// time, so there is no deterministic sequence underneath to wrap. /// /// One thing differs from [`ExponentialBackoff`]: the first delay is already random, somewhere in /// `base ..= base * 3`, rather than exactly `base`. Against a jittered exponential, the difference -/// is the floor. Every delay here is at least `base`, where [`Jitter::Full`] can return anything +/// is the floor. Every delay here is at least `base`, where [`Backoff::jittered`] can return anything /// down to zero. Don't stack the two by calling [`jittered`](Backoff::jittered) on this: the /// randomness is already in the recurrence, and wrapping it throws the `base` floor away. /// @@ -505,7 +490,7 @@ mod tests { // Full jitter: every delay lies in [0, the underlying delay], and the sequence still ends // exactly when the inner strategy is exhausted. let plain = drain(exp6()); - let mut j = Jittered::with_seed(exp6(), Jitter::Full, 42); + let mut j = Jittered::with_seed(exp6(), 42); for p in &plain { let d = j.next_delay().unwrap(); assert!( @@ -516,99 +501,43 @@ mod tests { assert_eq!(j.next_delay(), None); } - #[test] - fn equal_jitter_keeps_a_floor() { - // Equal jitter: every delay lies in [d/2, d]. - let plain = drain(exp6()); - let mut j = Jittered::with_seed(exp6(), Jitter::Equal, 99); - for p in &plain { - let d = j.next_delay().unwrap(); - assert!( - d >= *p / 2 && d <= *p, - "equal jitter out of [d/2, d]: {d:?} for {p:?}" - ); - } - } - - #[test] - fn equal_jitter_floor_is_half_the_delay_not_the_previous_step() { - // Executable documentation rather than extra coverage. `Equal`'s floor is `delay / 2`, - // and with the default `factor: 2` that is also the previous ladder value, so the two - // readings are easy to confuse. A factor of 3 separates them: at the 9s step the previous - // delay is 3s but the floor is 4.5s. - // - // `Jitter::apply` takes one `Duration` and keeps no history, so the previous-step reading - // isn't implementable today and this can't fail without a structural change. It is here to - // stop that change being made by accident, and to state the rule at a factor where the - // coincidence doesn't hide it. - let ladder = drain(exp(1, 3, 10_000, 4)); // 1s, 3s, 9s, 27s - assert_eq!(ladder[2], secs(9)); - assert_eq!( - ladder[1], - secs(3), - "the previous step, which is NOT the floor" - ); - - for seed in 0..256 { - let mut j = Jittered::with_seed(exp(1, 3, 10_000, 4), Jitter::Equal, seed); - for (i, plain) in ladder.iter().enumerate() { - let d = j.next_delay().unwrap(); - assert!( - d >= *plain / 2, - "step {i} drew {d:?}, below half of {plain:?} (seed {seed})" - ); - assert!( - d <= *plain, - "step {i} drew {d:?}, above {plain:?} (seed {seed})" - ); - } - } - } - #[test] fn seed_makes_jitter_reproducible() { // Same seed yields an identical sequence (so jittered retries stay testable); different - // seeds generally differ, which guards against a constant or broken RNG. Both modes, so - // neither can quietly become a no-op that still satisfies its bounds. - for mode in [Jitter::Full, Jitter::Equal] { - let seq = |seed| drain(Jittered::with_seed(exp6(), mode, seed)); - assert_eq!(seq(7), seq(7), "{mode:?} was not reproducible"); - assert_ne!(seq(1), seq(2), "{mode:?} ignored its seed"); - } + // seeds generally differ, which guards against a constant or broken RNG. + let seq = |seed| drain(Jittered::with_seed(exp6(), seed)); + assert_eq!(seq(7), seq(7)); + assert_ne!(seq(1), seq(2)); } #[test] fn jitter_actually_moves_the_delay() { - // Bounds checks alone are satisfied by the identity function, so pin that each mode - // really randomizes: `Full` must reach below the halfway mark, `Equal` must land strictly - // inside (d/2, d) at least once. - let full = drain(Jittered::with_seed(exp6(), Jitter::Full, 5)); + // `d <= p` alone is satisfied by the identity function, so pin that jitter really + // randomizes: across the sequence it must land both below and above the halfway mark. + let jittered = drain(Jittered::with_seed(exp6(), 5)); let plain = drain(exp6()); assert!( - full.iter().zip(&plain).any(|(d, p)| *d < *p / 2), - "full jitter never dropped below half the base delay" + jittered.iter().zip(&plain).any(|(d, p)| *d < *p / 2), + "jitter never dropped below half the plain delay" ); - let equal = drain(Jittered::with_seed(exp6(), Jitter::Equal, 5)); assert!( - equal - .iter() - .zip(&plain) - .any(|(d, p)| *d > *p / 2 && *d < *p), - "equal jitter never landed strictly inside (d/2, d)" + jittered.iter().zip(&plain).any(|(d, p)| *d > *p / 2), + "jitter never rose above half the plain delay" ); } #[test] - fn jittered_combinator_passes_the_mode_through() { + fn jittered_combinator_wraps_the_inner_strategy() { // `Backoff::jittered` is the documented entry point, so drive it rather than only the - // `Jittered::` constructors. `Equal` keeps a floor that `Full` does not. - let mut b = exp6().jittered(Jitter::Equal); + // `Jittered::` constructors. It must bound each delay by the inner strategy's own value + // and end exactly when the inner one does. + let mut b = exp6().jittered(); let plain = drain(exp6()); for p in &plain { let d = b.next_delay().unwrap(); assert!( - d >= *p / 2 && d <= *p, - "combinator lost the mode: {d:?} for {p:?}" + d <= *p, + "combinator exceeded the inner delay: {d:?} for {p:?}" ); } assert_eq!(b.next_delay(), None); @@ -619,10 +548,7 @@ mod tests { // The entropy-seeded path is what stops a fleet retrying in lockstep, and it's the reason // these types aren't `Clone`. A regression to a fixed seed would pass every other test // here. Two independent RNGs colliding across six delays is a 2^-64 event. - assert_ne!( - drain(exp6().jittered(Jitter::Full)), - drain(exp6().jittered(Jitter::Full)) - ); + assert_ne!(drain(exp6().jittered()), drain(exp6().jittered())); } #[test] @@ -635,7 +561,7 @@ mod tests { } } let inner = Fixed(vec![Duration::ZERO, Duration::MAX, secs(1)].into_iter()); - let mut j = Jittered::with_seed(inner, Jitter::Full, 1); + let mut j = Jittered::with_seed(inner, 1); assert_eq!(j.next_delay(), Some(Duration::ZERO)); // rand(0..=0) let _ = j.next_delay().unwrap(); // Duration::MAX saturates, no panic assert!(j.next_delay().unwrap() <= secs(1)); diff --git a/src/lib.rs b/src/lib.rs index 8a66660..8216f14 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,7 @@ mod test_support; pub use backoff::{ Backoff, BackoffConfigError, DecorrelatedBackoff, DecorrelatedBackoffConfig, - ExponentialBackoff, ExponentialBackoffConfig, Jitter, Jittered, + ExponentialBackoff, ExponentialBackoffConfig, Jittered, }; #[cfg(feature = "async")] pub use clock::Clock; diff --git a/src/retry.rs b/src/retry.rs index 7c0d622..c19d458 100644 --- a/src/retry.rs +++ b/src/retry.rs @@ -724,11 +724,7 @@ mod tests { async fn drives_a_jittered_backoff() { let clock = MockClock::new(); let out: Result = retry(|| async { Err("boom") }) - .backoff(crate::backoff::Jittered::with_seed( - backoff(3), - crate::backoff::Jitter::Full, - 42, - )) + .backoff(crate::backoff::Jittered::with_seed(backoff(3), 42)) .clock(clock.clone()) .await; From 07f4f9c8074c141e53daea2c94599dfcbcb22288 Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 11:54:31 +0530 Subject: [PATCH 07/12] Make the deterministic path as ergonomic as the production one Two papercuts found by writing the API from a user's side rather than from inside the crate. `.jittered()` reads well but couldn't be seeded, and seeding meant `Jittered::with_seed(ExponentialBackoff::default(), 42)` -- so a test and the production config it was supposed to be testing looked nothing alike. `jittered_with_seed(seed)` closes that. And `Clock` wasn't implemented for references, so `.clock(&mock)` was a compile error (E0599) and every mock had to wrap its own state in Rc or Arc before it could be passed at all. That is friction on the exact path this crate advertises. Added for `&C` and `Arc` on both Clock traits; a mock is now a plain struct with a RefCell. Both additive. Verified from a downstream crate: a mock with no interior sharing now works through &clock, through Arc, and through the seeded combinator, all three producing identical delays. --- CHANGELOG.md | 7 +++++++ src/backoff.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ src/blocking/clock.rs | 22 ++++++++++++++++++++++ src/clock.rs | 29 +++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 184ecbc..8c41e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 formula). It's a strategy rather than a `Jitter` mode because the randomness lives in the recurrence, so there's no deterministic sequence for `Jittered` to wrap. Seedable via `DecorrelatedBackoff::with_seed`. +- `Backoff::jittered_with_seed(seed)`, so the ergonomic combinator can be seeded too. Previously + the readable form couldn't be made deterministic and the deterministic form meant naming + `Jittered` directly, which made a test and its production config look nothing alike. +- `Clock` is now implemented for `&C` and `Arc` (both the async and blocking traits). Writing a + mock clock and passing `.clock(&mock)` used to be a compile error, forcing every mock to wrap its + own state in `Rc`/`Arc` just to be usable. That was friction on the exact path this crate exists + to make easy. - Documented that `ExponentialBackoffConfig { factor: 1, .. }` gives a constant delay, so there is no separate `ConstantBackoff` type to learn. diff --git a/src/backoff.rs b/src/backoff.rs index 8338453..89a077b 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -54,6 +54,30 @@ pub trait Backoff { { Jittered::new(self) } + + /// As [`jittered`](Backoff::jittered), but with a fixed `seed`, so the delays repeat exactly. + /// + /// For tests. Turning on jitter otherwise costs you the ability to assert what your retry + /// waited, which is the one thing this crate is built to let you do: + /// + /// ``` + /// use mettle::{Backoff, ExponentialBackoff}; + /// + /// let delays = |seed| { + /// let mut b = ExponentialBackoff::default().jittered_with_seed(seed); + /// std::iter::from_fn(move || b.next_delay()).collect::>() + /// }; + /// assert_eq!(delays(42), delays(42)); + /// ``` + /// + /// Don't reach for this in production. A fleet that all seeds the same way retries in + /// lockstep, which is the thundering herd jitter exists to prevent. + fn jittered_with_seed(self, seed: u64) -> Jittered + where + Self: Sized, + { + Jittered::with_seed(self, seed) + } } /// Why a backoff configuration was rejected. Not every strategy can produce every variant. @@ -543,6 +567,25 @@ mod tests { assert_eq!(b.next_delay(), None); } + #[test] + fn jittered_with_seed_matches_the_named_constructor() { + // The combinator is the ergonomic path; it must not be a second-class one. Seeding + // through it has to give exactly what naming the type gives. + assert_eq!( + drain(exp6().jittered_with_seed(42)), + drain(Jittered::with_seed(exp6(), 42)) + ); + // And it must actually honour the seed rather than quietly reading entropy. + assert_eq!( + drain(exp6().jittered_with_seed(7)), + drain(exp6().jittered_with_seed(7)) + ); + assert_ne!( + drain(exp6().jittered_with_seed(1)), + drain(exp6().jittered_with_seed(2)) + ); + } + #[test] fn unseeded_jitter_differs_between_instances() { // The entropy-seeded path is what stops a fleet retrying in lockstep, and it's the reason diff --git a/src/blocking/clock.rs b/src/blocking/clock.rs index 18e07ae..16de7e4 100644 --- a/src/blocking/clock.rs +++ b/src/blocking/clock.rs @@ -17,6 +17,28 @@ pub trait Clock { fn sleep(&self, dur: Duration); } +// Same reasoning as the async twin: a test wants to keep its mock so it can advance time and read +// back what was slept, and without these `.clock(&mock)` doesn't compile. +impl Clock for &C { + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) { + (**self).sleep(dur); + } +} + +impl Clock for std::sync::Arc { + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) { + (**self).sleep(dur); + } +} + /// A [`Clock`] backed by [`std::thread::sleep`] and [`std::time::Instant`]. #[derive(Debug, Clone, Copy, Default)] pub struct StdClock; diff --git a/src/clock.rs b/src/clock.rs index 6ac9487..24f21ff 100644 --- a/src/clock.rs +++ b/src/clock.rs @@ -18,6 +18,35 @@ pub trait Clock { fn sleep(&self, dur: Duration) -> Self::Sleep; } +// A mock clock is normally something the test wants to keep hold of, so it can advance time and +// read back what was slept. Without these, `.clock(&mock)` fails to compile and the mock has to +// wrap its own state in `Rc`/`Arc` just to be usable, which is friction on the exact path this +// crate exists to make easy. Use `&C` when the retry doesn't outlive the clock, `Arc` when it +// does. +impl Clock for &C { + type Sleep = C::Sleep; + + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) -> Self::Sleep { + (**self).sleep(dur) + } +} + +impl Clock for std::sync::Arc { + type Sleep = C::Sleep; + + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) -> Self::Sleep { + (**self).sleep(dur) + } +} + /// A [`Clock`] backed by Tokio's timer, so `now` and `sleep` read the same clock /// (and both honor `tokio::time::pause`). #[derive(Debug, Clone, Copy, Default)] From 917fa7a76080ca65e255914bf360869c7bea677d Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 12:37:00 +0530 Subject: [PATCH 08/12] Warn about the one way 0.3.0 can break you The release is additive at the signature level, but adding `impl Clock for &C` collides with the same impl if a user already wrote it themselves -- most likely as a workaround for it being missing. Verified: a crate with both `impl Clock for MyClock` and `impl Clock for &MyClock` compiles against 0.2.0 and fails with E0119 against this branch. Blast radius is small and the fix is deleting a line that this release makes redundant, so the impls stay. But cargo-semver-checks does not flag added impls, so nothing else would have told anyone. It goes in the CHANGELOG. --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c41e8e..cc92b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mock clock and passing `.clock(&mock)` used to be a compile error, forcing every mock to wrap its own state in `Rc`/`Arc` just to be usable. That was friction on the exact path this crate exists to make easy. + + **One way this can break you.** If you already wrote `impl Clock for &YourClock` yourself, most + likely as a workaround for the above, that now collides with the impl this release adds and the + build fails with `E0119: conflicting implementations`. The fix is to delete your impl, which this + release makes redundant. Nothing else in 0.3.0 changes an existing signature, and + `cargo-semver-checks` does not flag added impls, so this is called out here rather than left to + be discovered. - Documented that `ExponentialBackoffConfig { factor: 1, .. }` gives a constant delay, so there is no separate `ConstantBackoff` type to learn. From 605426c99451a5d8a382461f9dd2359e89861a6a Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 13:46:35 +0530 Subject: [PATCH 09/12] Teach the docs that jitter exists Every gate was green and every one of these was still wrong, because nothing checks whether prose is current. The crate-level docs -- the docs.rs landing page, the first thing anyone reads -- never mentioned jitter at all. The ADR index never listed ADR004. And examples/retry.rs, which the README calls the place to start, showed none of the release's headline feature. Also corrected the 'available now' line, which still described 0.2.0. --- docs/adr/README.md | 1 + examples/retry.rs | 31 +++++++++++++++++++++++++++++-- src/lib.rs | 25 ++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/docs/adr/README.md b/docs/adr/README.md index 610d556..9708711 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -9,3 +9,4 @@ changes, edit its file and say what changed and why. | [001](ADR001.md) | Sans-IO core, hand-written async, one crate | Accepted | | [002](ADR002.md) | Public API, open traits, one validated `Backoff` | Accepted | | [003](ADR003.md) | Dependencies and Cargo features | Accepted | +| [004](ADR004.md) | Jitter, a fourth dependency, and `Clone` | Accepted | diff --git a/examples/retry.rs b/examples/retry.rs index c99e7e0..21a6adc 100644 --- a/examples/retry.rs +++ b/examples/retry.rs @@ -11,9 +11,15 @@ //! - `.max_elapsed(d)` gives up once the next wait would push total time past `d` //! - `.clock(clock)` supplies the time source (default: Tokio) //! +//! The default schedule is deterministic, which means a fleet of clients that failed together +//! retries together. `.jittered()` spreads them out; see step 4. +//! //! Run with: `cargo run --example retry` -use mettle::{ExponentialBackoff, ExponentialBackoffConfig, retry}; +use mettle::{ + Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff, + ExponentialBackoffConfig, retry, +}; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::time::Duration; @@ -24,7 +30,7 @@ enum FetchError { } #[tokio::main] -async fn main() { +async fn main() -> Result<(), mettle::BackoffConfigError> { // 1) Simplest form: pass the operation, take every default. let result = retry(flaky_fetch).await; println!("1. defaults: {result:?}"); // Ok("user data"), succeeds on attempt 3 @@ -42,6 +48,27 @@ async fn main() { .when(|e| matches!(e, FetchError::Timeout)) .await; println!("3. full: {result:?}"); // Ok("user data"), succeeds on attempt 3 + + // 4) Jitter, so a fleet doesn't retry in lockstep. `.jittered()` randomizes each delay into + // `0 ..= delay`; `DecorrelatedBackoff` instead draws each delay from the previous one and + // never goes below `base`. Both are opt-in: the default above stays deterministic. + let result = retry(flaky_fetch) + .backoff(fast_backoff().jittered()) + .when(|e| matches!(e, FetchError::Timeout)) + .await; + println!("4. jittered: {result:?}"); // same outcome, unpredictable delays + + let result = retry(flaky_fetch) + .backoff(DecorrelatedBackoff::new(DecorrelatedBackoffConfig { + base: Duration::from_millis(20), + max_retries: 5, + max_delay: Duration::from_secs(1), + })?) + .when(|e| matches!(e, FetchError::Timeout)) + .await; + println!("5. decorrel.: {result:?}"); // every delay at least 20ms + + Ok(()) } /// A flaky call: fails with `Timeout` twice, then succeeds. It resets after each success, so it diff --git a/src/lib.rs b/src/lib.rs index 8216f14..50ac609 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ //! don't hand-roll retry-and-backoff logic in every project. //! //! Available now: `retry()` (async) and `blocking::retry()` (sync), both with configurable -//! backoff. Timeout and circuit breaking are planned. +//! backoff and optional jitter. Timeout and circuit breaking are planned. //! //! # Quickstart //! @@ -27,6 +27,29 @@ //! the builder methods, then `.await`. No async runtime? The blocking twin is identical but ends //! in `.call()` instead of `.await`. //! +//! # Jitter +//! +//! A fixed schedule means every client that failed together retries together, so a service coming +//! back up gets a synchronized wave. Two ways to spread that out, both opt-in: +//! +//! ``` +//! use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff}; +//! +//! // Randomize any strategy's delays into `0 ..= delay` ("full jitter"). +//! let spread = ExponentialBackoff::default().jittered(); +//! +//! // Or draw each delay from the previous one, never below `base`. +//! let floored = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; +//! # let _ = (spread, floored); +//! # Ok::<_, mettle::BackoffConfigError>(()) +//! ``` +//! +//! [`jittered`](Backoff::jittered) wraps any strategy, including one you wrote. +//! [`DecorrelatedBackoff`] is its own strategy and keeps a floor under every wait, at the cost of +//! never retrying sooner than `base`. Both seed from entropy; use +//! [`jittered_with_seed`](Backoff::jittered_with_seed) or +//! [`DecorrelatedBackoff::with_seed`] when a test needs the delays to repeat. +//! //! # Observability //! //! Every retry emits a [`tracing`](https://docs.rs/tracing) event on target `mettle::retry` at From fec2e40a16a6fca6f976994034790a093cb3426c Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 16:40:28 +0530 Subject: [PATCH 10/12] Move the reasoning out of the CHANGELOG and into ADR004 A changelog answers 'what changed and what must I do'. Rationale is a different question with a different audience and a different lifetime: it belongs where someone goes looking for why, not where someone goes looking for whether to upgrade. The 0.3.0 entry had grown three paragraphs of argument about AWS's jitter measurements, why decorrelated is a strategy rather than a mode, and why the reference impls exist. That is ADR material. The entry is now the change list, the new dependency, and the one thing that can break a build, with a link out for the why. ADR004 gains the two decisions that had no home: why both a readable and a seeded jitter path exist, and why Clock and Now are implemented for references -- including the honest note that our own tests hid that gap for two releases by working around it on day one. --- CHANGELOG.md | 64 +++++++++++++++++++--------------------------- docs/adr/ADR004.md | 46 ++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc92b21..0dff7bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,44 +9,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.3.0] - 2026-08-09 +Adds jitter. Purely additive apart from one collision noted under Upgrading. + ### Added -- Jitter for backoff: `Backoff::jittered()` wraps any strategy so each delay becomes a uniform - random value in `0 ..= delay` ("full jitter"), so a fleet of clients doesn't retry in lockstep. - Opt-in; the RNG is seedable for reproducible tests. - - There is no mode to pick. AWS's measurements had the alternative ("equal jitter", a floor at - `delay/2`) doing more work *and* finishing later than full jitter, and a simulation of mettle's - own implementations reproduced that in every configuration tried, so offering it would only - invite people to choose the worse one. For a floor under every wait, use `DecorrelatedBackoff`. -- `DecorrelatedBackoff`, built from a validated `DecorrelatedBackoffConfig`: each delay is drawn - from `base ..= prev * 3`, capped at `max_delay` (the AWS "Exponential Backoff and Jitter" - formula). It's a strategy rather than a `Jitter` mode because the randomness lives in the - recurrence, so there's no deterministic sequence for `Jittered` to wrap. Seedable via - `DecorrelatedBackoff::with_seed`. -- `Backoff::jittered_with_seed(seed)`, so the ergonomic combinator can be seeded too. Previously - the readable form couldn't be made deterministic and the deterministic form meant naming - `Jittered` directly, which made a test and its production config look nothing alike. -- `Clock` is now implemented for `&C` and `Arc` (both the async and blocking traits). Writing a - mock clock and passing `.clock(&mock)` used to be a compile error, forcing every mock to wrap its - own state in `Rc`/`Arc` just to be usable. That was friction on the exact path this crate exists - to make easy. - - **One way this can break you.** If you already wrote `impl Clock for &YourClock` yourself, most - likely as a workaround for the above, that now collides with the impl this release adds and the - build fails with `E0119: conflicting implementations`. The fix is to delete your impl, which this - release makes redundant. Nothing else in 0.3.0 changes an existing signature, and - `cargo-semver-checks` does not flag added impls, so this is called out here rather than left to - be discovered. -- Documented that `ExponentialBackoffConfig { factor: 1, .. }` gives a constant delay, so there is - no separate `ConstantBackoff` type to learn. - -The randomized strategies (`DecorrelatedBackoff`, `Jittered`) are deliberately not `Clone`. A copy -would carry the RNG state and replay the same delays, which is the lockstep jitter exists to -prevent; clone the config and build a fresh strategy instead. - -Jitter needs a random number generator, so this adds `fastrand` as a required dependency (1198 -lines, no transitive dependencies of its own). It is not behind a feature flag; ADR004 records the -measurements and the reasoning. +- `Backoff::jittered()` and `Backoff::jittered_with_seed(seed)`. Wraps any strategy, including one + you wrote, so each delay becomes a uniform random value in `0 ..= delay` ("full jitter"). Opt-in: + the default schedule stays deterministic. The seeded form is for tests. +- `DecorrelatedBackoff`, from a validated `DecorrelatedBackoffConfig`. Draws each delay from + `base ..= prev * 3`, capped at `max_delay`, and never below `base`. Seedable with `with_seed`. +- `Clock` for `&C` and `Arc`, on both the async and blocking traits, so a mock clock can be + passed as `.clock(&mock)` and still be read afterwards. +- `ExponentialBackoffConfig { factor: 1, .. }` documented as the way to get a constant delay; there + is no separate `ConstantBackoff` type. + +### Changed +- New required dependency: `fastrand` (no transitive dependencies, not feature-gated). + +### Upgrading + +0.2.0 code compiles unchanged, with one exception. If you wrote `impl Clock for &YourClock` +yourself, it now collides with the impl this release adds and the build fails with +`E0119: conflicting implementations`. Delete yours; this release makes it redundant. +`cargo-semver-checks` does not flag added impls, so it would not have warned you. + +`Jittered` and `DecorrelatedBackoff` are deliberately not `Clone`, because a copy carries the RNG +state and replays the same delays. Keep the config and build a fresh strategy from it. + +Why any of this is shaped the way it is, including why there is no jitter *mode* to choose: +[ADR004](https://github.com/azeemshaik025/mettle/blob/main/docs/adr/ADR004.md). ## [0.2.0] - 2026-07-26 diff --git a/docs/adr/ADR004.md b/docs/adr/ADR004.md index 4ca616c..bc07980 100644 --- a/docs/adr/ADR004.md +++ b/docs/adr/ADR004.md @@ -1,4 +1,4 @@ -# ADR004: Jitter, a fourth dependency, and why randomized strategies aren't `Clone` +# ADR004: Jitter, and the decisions it forced **Status:** Accepted @@ -7,7 +7,9 @@ Retrying with a fixed backoff means every client that failed at the same moment retries at the same moment. The dependency that just fell over gets a synchronized wave of traffic the instant it comes back, which is the thundering herd. Jitter is the fix, and it's the first thing in -mettle that needs randomness, so it forces three decisions we hadn't had to make yet. +mettle that needs randomness, so it forced a set of decisions we hadn't had to make yet: what to +depend on, how many knobs to expose, what `Clone` means once a type carries RNG state, and what it +takes for a randomized policy to still be testable. ## Decisions @@ -76,6 +78,39 @@ the trait's contract and would silently break seeded tests. So the randomized ty This direction is also the reversible one. Adding `Clone` later is a minor version; taking it away is a breaking change. +**4. Both the readable path and the deterministic path exist, and they look alike.** +`jittered()` seeds from entropy, which is what production wants. `jittered_with_seed(seed)` fixes +the seed, which is what a test wants. + +The seeded constructor already existed as `Jittered::with_seed(inner, seed)`, so this looks +redundant. It isn't. Without the combinator, turning on jitter meant a test config +(`Jittered::with_seed(ExponentialBackoff::default(), 42)`) that looked nothing like the production +config it was supposed to be exercising. Deterministic testing is this crate's headline claim, so +the deterministic path being the awkward one was the wrong way round. + +Seeding in production is a mistake, since a fleet seeded identically retries in lockstep, so the +method's docs say so rather than leaving it implied by the name. + +**5. `Clock` and `Now` are implemented for `&C` and `Arc`.** +Both traits are taken by value: `retry(..).clock(c)` owns its clock, and a `CircuitBreaker` owns +its time source for its whole life. Without a reference impl, a test could hand over its mock and +then never read it back, which defeats the entire point of injecting time. + +We didn't notice for two releases because our own tests worked around it on day one: every mock in +this repo wraps its state in `Arc>` and is passed by `.clone()`, even in single-threaded +blocking tests where no sharing is needed. The workaround was uniform, prepaid, and therefore +invisible. It surfaced only when the API was written from outside the crate, in a separate crate +with a path dependency, doing what a reader of the README would do. + +The lesson is worth more than the fix: a test suite that lives inside the crate inherits its own +solutions and cannot tell you whether the front door opens. + +This is the one part of 0.3.0 that can break an existing user. Someone who hit the same papercut +and wrote `impl Clock for &TheirClock` themselves now collides with ours (E0119). We shipped it +anyway: the population is small, they get a compile error rather than silent misbehaviour, and the +fix is deleting a line this release makes redundant. `cargo-semver-checks` does not flag added +impls, so the CHANGELOG says it out loud. + ## Consequences Jitter is available to everyone with no feature flag to discover, at the cost of one small @@ -84,5 +119,8 @@ the trait docs have to explain rather than leaving to be discovered. Anyone reus policy across calls has to hold the config rather than the strategy, which is one extra line and the only shape that's actually correct. -Every randomized path is seedable (`Jittered::with_seed`, `DecorrelatedBackoff::with_seed`), so -none of this costs us deterministic tests. +Every randomized path is seedable, through the combinator as well as the constructor, so none of +this costs us deterministic tests. + +The reference impls make a mock clock an ordinary struct rather than something that needs interior +sharing, and they are the one thing here that can break an existing build. From f3ccd949628ddfeb0199ffe4a314161a356e2a5f Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 16:45:29 +0530 Subject: [PATCH 11/12] Split the testability decisions into ADR005 ADR004 was carrying two things that are not about jitter: why both a readable and a seeded jitter path exist, and why Clock and Now are implemented for references. Both came out of shipping jitter, but neither is a jitter decision -- they are about whether the deterministic path this crate advertises is actually usable. ADR005 also records the practice that found them, which is the part worth keeping: build a throwaway crate against a path dependency and write the code a first-time reader would write. Our own tests hid the reference-impl gap for two releases because they worked around it on day one, so a suite that lives inside the crate cannot tell you whether the front door opens. Numbering: retry-error and circuit-breaker will shift their ADRs to 006 and 007 when they rebase. --- CHANGELOG.md | 4 ++- docs/adr/ADR004.md | 49 +++++--------------------------- docs/adr/ADR005.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++ docs/adr/README.md | 1 + 4 files changed, 82 insertions(+), 43 deletions(-) create mode 100644 docs/adr/ADR005.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dff7bb..408bc02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,11 +31,13 @@ Adds jitter. Purely additive apart from one collision noted under Upgrading. yourself, it now collides with the impl this release adds and the build fails with `E0119: conflicting implementations`. Delete yours; this release makes it redundant. `cargo-semver-checks` does not flag added impls, so it would not have warned you. +[ADR005](https://github.com/azeemshaik025/mettle/blob/main/docs/adr/ADR005.md) explains why the +impls are there and why we shipped them anyway. `Jittered` and `DecorrelatedBackoff` are deliberately not `Clone`, because a copy carries the RNG state and replays the same delays. Keep the config and build a fresh strategy from it. -Why any of this is shaped the way it is, including why there is no jitter *mode* to choose: +Why the jitter side is shaped the way it is, including why there is no *mode* to choose: [ADR004](https://github.com/azeemshaik025/mettle/blob/main/docs/adr/ADR004.md). ## [0.2.0] - 2026-07-26 diff --git a/docs/adr/ADR004.md b/docs/adr/ADR004.md index bc07980..192222c 100644 --- a/docs/adr/ADR004.md +++ b/docs/adr/ADR004.md @@ -1,4 +1,4 @@ -# ADR004: Jitter, and the decisions it forced +# ADR004: Jitter, a fourth dependency, and why randomized strategies aren't `Clone` **Status:** Accepted @@ -7,9 +7,10 @@ Retrying with a fixed backoff means every client that failed at the same moment retries at the same moment. The dependency that just fell over gets a synchronized wave of traffic the instant it comes back, which is the thundering herd. Jitter is the fix, and it's the first thing in -mettle that needs randomness, so it forced a set of decisions we hadn't had to make yet: what to -depend on, how many knobs to expose, what `Clone` means once a type carries RNG state, and what it -takes for a randomized policy to still be testable. +mettle that needs randomness, so it forces three decisions we hadn't had to make yet: what to +depend on, how many knobs to expose, and what `Clone` means once a type carries RNG state. + +Shipping it also exposed two gaps in how testable the crate actually was. Those are ADR005. ## Decisions @@ -78,39 +79,6 @@ the trait's contract and would silently break seeded tests. So the randomized ty This direction is also the reversible one. Adding `Clone` later is a minor version; taking it away is a breaking change. -**4. Both the readable path and the deterministic path exist, and they look alike.** -`jittered()` seeds from entropy, which is what production wants. `jittered_with_seed(seed)` fixes -the seed, which is what a test wants. - -The seeded constructor already existed as `Jittered::with_seed(inner, seed)`, so this looks -redundant. It isn't. Without the combinator, turning on jitter meant a test config -(`Jittered::with_seed(ExponentialBackoff::default(), 42)`) that looked nothing like the production -config it was supposed to be exercising. Deterministic testing is this crate's headline claim, so -the deterministic path being the awkward one was the wrong way round. - -Seeding in production is a mistake, since a fleet seeded identically retries in lockstep, so the -method's docs say so rather than leaving it implied by the name. - -**5. `Clock` and `Now` are implemented for `&C` and `Arc`.** -Both traits are taken by value: `retry(..).clock(c)` owns its clock, and a `CircuitBreaker` owns -its time source for its whole life. Without a reference impl, a test could hand over its mock and -then never read it back, which defeats the entire point of injecting time. - -We didn't notice for two releases because our own tests worked around it on day one: every mock in -this repo wraps its state in `Arc>` and is passed by `.clone()`, even in single-threaded -blocking tests where no sharing is needed. The workaround was uniform, prepaid, and therefore -invisible. It surfaced only when the API was written from outside the crate, in a separate crate -with a path dependency, doing what a reader of the README would do. - -The lesson is worth more than the fix: a test suite that lives inside the crate inherits its own -solutions and cannot tell you whether the front door opens. - -This is the one part of 0.3.0 that can break an existing user. Someone who hit the same papercut -and wrote `impl Clock for &TheirClock` themselves now collides with ours (E0119). We shipped it -anyway: the population is small, they get a compile error rather than silent misbehaviour, and the -fix is deleting a line this release makes redundant. `cargo-semver-checks` does not flag added -impls, so the CHANGELOG says it out loud. - ## Consequences Jitter is available to everyone with no feature flag to discover, at the cost of one small @@ -119,8 +87,5 @@ the trait docs have to explain rather than leaving to be discovered. Anyone reus policy across calls has to hold the config rather than the strategy, which is one extra line and the only shape that's actually correct. -Every randomized path is seedable, through the combinator as well as the constructor, so none of -this costs us deterministic tests. - -The reference impls make a mock clock an ordinary struct rather than something that needs interior -sharing, and they are the one thing here that can break an existing build. +Every randomized path is seedable, so none of this costs us deterministic tests. Making that +seeding pleasant to reach is ADR005. diff --git a/docs/adr/ADR005.md b/docs/adr/ADR005.md new file mode 100644 index 0000000..6243b91 --- /dev/null +++ b/docs/adr/ADR005.md @@ -0,0 +1,71 @@ +# ADR005: Making the tested path as usable as the production path + +**Status:** Accepted + +## Context + +mettle's headline claim is that you can test a resilience policy exactly, with no sleeping and no +real clock. Shipping jitter put that claim under load for the first time, because a randomized +policy is only testable if you can pin the randomness *and* still read back what happened. + +Two things turned out to be wrong, and neither was a bug in the sense a test would catch. Both were +found the same way: by writing the API from outside the crate, in a separate crate with a path +dependency, doing what someone who had only read the README would do. That took about ten minutes +and found two problems that had been there since 0.1.0. + +## Decisions + +**1. The readable path and the deterministic path are the same shape.** +`jittered()` seeds from entropy, which is what production wants. `jittered_with_seed(seed)` fixes +the seed, which is what a test wants. + +The seeded constructor already existed as `Jittered::with_seed(inner, seed)`, so the combinator +looks redundant. It isn't. Without it, turning on jitter meant a production line reading +`ExponentialBackoff::default().jittered()` and a test line reading +`Jittered::with_seed(ExponentialBackoff::default(), 42)`. Those are different constructions, so a +test no longer visibly exercised the thing it was testing. For a crate that sells deterministic +testing, the deterministic path being the awkward one is the wrong way round. + +Seeding in production is a mistake, since a fleet seeded identically retries in lockstep, so the +method's docs say so outright rather than leaving it implied. + +**2. `Clock` and `Now` are implemented for `&C` and `Arc`.** +Both traits are taken by value. `retry(..).clock(c)` owns its clock, and a `CircuitBreaker` owns its +time source for its whole life. Without a reference impl a test can hand over its mock and then +never read it back, which defeats the entire point of injecting time. Passing `&mock` was a compile +error; the only way through was to give the mock interior sharing so it could be cloned. + +`&C` covers the case where the retry doesn't outlive the clock. `Arc` covers the case where it +does, which is the usual shape for a breaker and for anything spawned. + +**3. Before a release, use the crate from outside it.** +This is the decision that generalises, so it is written down as one. + +We missed the reference-impl gap for two releases because our own tests worked around it on day +one. Every mock in this repo wraps its state in `Arc>` and gets passed by `.clone()`, even +in single-threaded blocking tests where nothing is shared. Written once, copied everywhere, and from +the inside it looked like "how you write a mock clock here" rather than like a tax. Nobody notices a +papercut they have already bandaged. + +A test suite that lives inside the crate inherits its own solutions. It can be thorough and +mutation-checked and still tell you nothing about whether the front door opens. So the release +checklist includes building a throwaway crate against a path dependency and writing the code a +first-time reader would write. + +## Consequences + +A mock clock is now an ordinary struct with a `Cell` or a `RefCell`, not something that needs +interior sharing before it can be passed at all. `examples/breaker.rs` lost its `Rc>` for +exactly this reason, which is the clearest evidence the friction was real: our own example was +working around it. + +Decision 2 is the one part of 0.3.0 that can break an existing build. Someone who hit the same +papercut and wrote `impl Clock for &TheirClock` themselves now collides with ours, and the build +fails with `E0119: conflicting implementations`. We shipped it anyway: the affected population is +small, the failure is a compile error rather than silent misbehaviour, and the fix is deleting a +line this release makes redundant. `cargo-semver-checks` does not flag added impls, so nothing in CI +would have caught it and the CHANGELOG says it out loud instead. + +Decision 3 costs a few minutes per release and has no automation behind it. The cheap way to make it +structural later is a `tests/` directory, since integration tests compile against the crate's public +face and would have caught this without anyone thinking to look. diff --git a/docs/adr/README.md b/docs/adr/README.md index 9708711..9811e6b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,3 +10,4 @@ changes, edit its file and say what changed and why. | [002](ADR002.md) | Public API, open traits, one validated `Backoff` | Accepted | | [003](ADR003.md) | Dependencies and Cargo features | Accepted | | [004](ADR004.md) | Jitter, a fourth dependency, and `Clone` | Accepted | +| [005](ADR005.md) | Making the tested path as usable as the production path | Accepted | From 78390bc2377ee9994fd1e63897a543ab13e9e7a4 Mon Sep 17 00:00:00 2001 From: Azeem Shaik Date: Sun, 9 Aug 2026 17:02:58 +0530 Subject: [PATCH 12/12] Say why the RNG is concrete while the clock is a trait Fair question to get asked: the crate injects time through a trait, so why is randomness a bare fastrand::Rng? Because the two problems are not the same shape. You cannot seed the system clock, so the only way a test controls time is to replace the source, which is what the Clock trait is for. Randomness has a cheaper answer: seeding IS the determinism mechanism, and with_seed already gives full reproducibility without replacing anything. A trait would buy the ability to swap the algorithm, which nobody has asked for and which costs a second type parameter on two public types forever. The coupling is three functions (Rng::new, Rng::with_seed, rng.u64) and fastrand appears in no public signature, so the abstraction is not needed to keep the option of swapping it either. Also fixes an inconsistency the question surfaced: DecorrelatedBackoff's seeded constructor warned that exact values are not an API contract, and the two jitter seed paths -- the ones people reach for first -- did not. --- docs/adr/ADR004.md | 19 +++++++++++++++++++ src/backoff.rs | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/docs/adr/ADR004.md b/docs/adr/ADR004.md index 192222c..36dd0bc 100644 --- a/docs/adr/ADR004.md +++ b/docs/adr/ADR004.md @@ -35,6 +35,25 @@ We also considered inlining a small PRNG to keep the dependency count at two. We seeding from entropy portably is the part that's actually easy to get wrong, and getting it wrong means a fleet seeded identically, which is the exact failure jitter exists to prevent. +**1b. The RNG is a concrete `fastrand::Rng`, not a trait.** +`Clock` is injected through a trait, so it's fair to ask why randomness isn't. The two look +symmetrical and aren't. + +You cannot seed the system clock. The only way a test controls time is to replace the source +outright, which is why `Clock` has to be a trait. Randomness has a cheaper answer: seeding *is* the +determinism mechanism, and `with_seed` already gives complete reproducibility without replacing +anything. A trait would buy the ability to swap the *algorithm*, which is a different capability and +one nobody has asked for. Jitter does not need to be unpredictable to an adversary. + +The coupling is also small enough that the abstraction isn't needed to keep our options open. We use +exactly three things (`Rng::new`, `Rng::with_seed`, `rng.u64(range)`), `fastrand` appears in no +public signature, and swapping it would be a change inside one file. Against that, a trait means a +second type parameter on `Jittered` and `DecorrelatedBackoff` forever. + +What we do owe users is honesty about what a seed guarantees: the same delays within a build, not +across `fastrand` versions. Every seeded entry point says so, and this crate's own tests assert +properties rather than golden values for that reason. + **2. One kind of jitter, with no mode to choose.** `Jittered` applies full jitter: every delay becomes a uniform random value in `0 ..= delay`. There is no `Jitter` enum and no argument to `jittered()`. diff --git a/src/backoff.rs b/src/backoff.rs index 89a077b..d0f970e 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -72,6 +72,9 @@ pub trait Backoff { /// /// Don't reach for this in production. A fleet that all seeds the same way retries in /// lockstep, which is the thundering herd jitter exists to prevent. + /// + /// A seed replays the same delays within a build. The exact values are not part of this + /// crate's API contract, so assert on properties rather than on specific numbers. fn jittered_with_seed(self, seed: u64) -> Jittered where Self: Sized, @@ -254,6 +257,9 @@ impl Jittered { } /// Wrap `inner` with a fixed `seed`, for reproducible tests. + /// + /// A seed replays the same delays within a build. The exact values are not part of this + /// crate's API contract, so assert on properties rather than on specific numbers. pub fn with_seed(inner: B, seed: u64) -> Self { Self { inner,