From f0a1832477b878251854e607c572098388adce8a Mon Sep 17 00:00:00 2001
From: Jonas Heller <168646044+jheller1212@users.noreply.github.com>
Date: Wed, 24 Jun 2026 20:43:54 +0200
Subject: [PATCH] Harden retry backoff for many participants sharing one API
 key
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Workshops run ~30 participants on a single shared key. The old backoff
used fixed delays (5s, 15s) with only 2 retries, so a synchronized burst
(everyone clicks Start together) hit 429 and all retried on the same
beat — a thundering herd that re-triggered the limit. Add jitter to both
fallback and Retry-After waits so concurrent clients fan out, and raise
to 5 attempts (4 retries) to ride out transient spikes instead of erroring.
---
 src/lib/api/conversation.ts | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/lib/api/conversation.ts b/src/lib/api/conversation.ts
index 4aa3cc4..57808c8 100644
--- a/src/lib/api/conversation.ts
+++ b/src/lib/api/conversation.ts
@@ -5,12 +5,26 @@ import { APIError } from './types';
 /** HTTP status codes that are safe to retry (rate-limit / server overload). */
 const RETRYABLE_STATUSES = new Set([429, 503, 529]);
 
-/** Fallback delays (ms) for retry attempts 1 and 2 when no Retry-After header is present. */
-const FALLBACK_DELAYS_MS = [5_000, 15_000];
+/** Total attempts (1 initial + N retries). Tuned for many participants sharing
+ *  one API key in a workshop, where transient 429 bursts are expected. */
+const MAX_ATTEMPTS = 5;
+
+/** Base fallback delays (ms) per retry when no Retry-After header is present.
+ *  Actual waits are jittered (see jitteredDelay) so concurrent clients don't all
+ *  retry on the same beat and re-trigger the rate limit (thundering herd). */
+const FALLBACK_DELAYS_MS = [2_000, 6_000, 15_000, 30_000];
 
 /** Maximum number of ms we will honour from a Retry-After header. */
 const MAX_RETRY_AFTER_MS = 60_000;
 
+/** Spread the wait across 50–100% of the base (fallback) or add 0–2s on top of a
+ *  server-provided Retry-After, so 30 simultaneous retries fan out over time. */
+function jitteredDelay(baseMs: number, fromRetryAfter: boolean): number {
+  return fromRetryAfter
+    ? baseMs + Math.round(Math.random() * 2_000)
+    : Math.round(baseMs * (0.5 + Math.random() * 0.5));
+}
+
 /** Sleep that rejects immediately if the signal aborts, so Stop interrupts retry waits. */
 function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
   return new Promise((resolve, reject) => {
@@ -45,15 +59,16 @@ export async function generateResponse(
 
   let lastError: Error = new Error('Unknown error');
 
-  for (let attempt = 0; attempt < 3; attempt++) {
+  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
     if (attempt > 0) {
       // Respect Retry-After header if the previous error included one; otherwise use
       // fixed fallback delays that are long enough for typical overload scenarios.
-      let delayMs = FALLBACK_DELAYS_MS[attempt - 1] ?? 15_000;
-      if (lastError instanceof APIError && lastError.retryAfter != null) {
-        delayMs = Math.min(lastError.retryAfter * 1000, MAX_RETRY_AFTER_MS);
-      }
-      await abortableDelay(delayMs, signal);
+      // Either way the wait is jittered to de-synchronise concurrent clients.
+      const fromRetryAfter = lastError instanceof APIError && lastError.retryAfter != null;
+      const base = fromRetryAfter
+        ? Math.min((lastError as APIError).retryAfter! * 1000, MAX_RETRY_AFTER_MS)
+        : (FALLBACK_DELAYS_MS[attempt - 1] ?? 30_000);
+      await abortableDelay(jitteredDelay(base, fromRetryAfter), signal);
     }
 
     try {