Jittered retry backoff + more retries for shared-key workshops

src/lib/api/conversation.ts

@@ -5,12 +5,26 @@ import { APIError } from './types';
 /** HTTP status codes that are safe to retry (rate-limit / server overload). */
 const RETRYABLE_STATUSES = new Set([429, 503, 529]);
 
-/** Fallback delays (ms) for retry attempts 1 and 2 when no Retry-After header is present. */
-const FALLBACK_DELAYS_MS = [5_000, 15_000];
+/** Total attempts (1 initial + N retries). Tuned for many participants sharing
+ *  one API key in a workshop, where transient 429 bursts are expected. */
+const MAX_ATTEMPTS = 5;
+
+/** Base fallback delays (ms) per retry when no Retry-After header is present.
+ *  Actual waits are jittered (see jitteredDelay) so concurrent clients don't all
+ *  retry on the same beat and re-trigger the rate limit (thundering herd). */
+const FALLBACK_DELAYS_MS = [2_000, 6_000, 15_000, 30_000];
 
 /** Maximum number of ms we will honour from a Retry-After header. */
 const MAX_RETRY_AFTER_MS = 60_000;
 
+/** Spread the wait across 50–100% of the base (fallback) or add 0–2s on top of a
+ *  server-provided Retry-After, so 30 simultaneous retries fan out over time. */
+function jitteredDelay(baseMs: number, fromRetryAfter: boolean): number {
+  return fromRetryAfter
+    ? baseMs + Math.round(Math.random() * 2_000)
+    : Math.round(baseMs * (0.5 + Math.random() * 0.5));
+}
+
 /** Sleep that rejects immediately if the signal aborts, so Stop interrupts retry waits. */
 function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
   return new Promise((resolve, reject) => {
@@ -45,15 +59,16 @@ export async function generateResponse(
 
   let lastError: Error = new Error('Unknown error');
 
-  for (let attempt = 0; attempt < 3; attempt++) {
+  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
     if (attempt > 0) {
       // Respect Retry-After header if the previous error included one; otherwise use
       // fixed fallback delays that are long enough for typical overload scenarios.
-      let delayMs = FALLBACK_DELAYS_MS[attempt - 1] ?? 15_000;
-      if (lastError instanceof APIError && lastError.retryAfter != null) {
-        delayMs = Math.min(lastError.retryAfter * 1000, MAX_RETRY_AFTER_MS);
-      }
-      await abortableDelay(delayMs, signal);
+      // Either way the wait is jittered to de-synchronise concurrent clients.
+      const fromRetryAfter = lastError instanceof APIError && lastError.retryAfter != null;
+      const base = fromRetryAfter
+        ? Math.min((lastError as APIError).retryAfter! * 1000, MAX_RETRY_AFTER_MS)
+        : (FALLBACK_DELAYS_MS[attempt - 1] ?? 30_000);
+      await abortableDelay(jitteredDelay(base, fromRetryAfter), signal);
     }
 
     try {