Graceful capacity handling: request queue + visible waiting state

src/components/ChatPanel.tsx

@@ -11,6 +11,7 @@ import { Message } from '../types';
 interface ChatPanelProps {
   messages: Message[];
   isLoading: boolean;
+  waitStatus?: string | null;
   userInput: string;
   onUserInputChange: (value: string) => void;
   onSendMessage: () => void;
@@ -64,6 +65,7 @@ interface ChatPanelProps {
 export function ChatPanel({
   messages,
   isLoading,
+  waitStatus,
   userInput,
   onUserInputChange,
   onSendMessage,
@@ -340,6 +342,7 @@ export function ChatPanel({
                 <ConversationDisplay
                   messages={messages}
                   isLoading={isLoading}
+                  waitStatus={waitStatus}
                   botName1={botName1}
                   botName2={botName2}
                   bubbleColor1={bubbleColor1}

src/components/ConversationDisplay.tsx

@@ -15,6 +15,7 @@ const messageEntrance = {
 interface ConversationDisplayProps {
   messages: Message[];
   isLoading?: boolean;
+  waitStatus?: string | null;
   botName1: string;
   botName2: string;
   bubbleColor1: string;
@@ -48,6 +49,7 @@ function TokenBadge({ wordCount }: { wordCount: number }) {
 export function ConversationDisplay({
   messages,
   isLoading = false,
+  waitStatus = null,
   botName1,
   botName2,
   bubbleColor1,
@@ -173,7 +175,9 @@ export function ConversationDisplay({
             <span className="typing-dot w-2 h-2 bg-sky-400 rounded-full" style={{ animation: 'typing-bounce 1.2s ease-in-out infinite', animationDelay: '160ms' }} />
             <span className="typing-dot w-2 h-2 bg-sky-400 rounded-full" style={{ animation: 'typing-bounce 1.2s ease-in-out infinite', animationDelay: '320ms' }} />
           </div>
-          Generating…
+          {waitStatus
+            ? <span className="text-amber-500 dark:text-amber-400">{waitStatus}</span>
+            : 'Generating…'}
         </motion.div>
       )}
 

src/components/ResearchInterface.tsx

@@ -598,6 +598,7 @@ export function ResearchInterface({
             <ChatPanel
               messages={engine.messages}
               isLoading={engine.isLoading}
+              waitStatus={engine.waitStatus}
               userInput={settings.userInput}
               onUserInputChange={settings.setUserInput}
               onSendMessage={() => engine.handleSendMessage(settings.userInput, setCurrentView)}

src/hooks/useConversationEngine.ts

@@ -1,5 +1,6 @@
 import { useState, useRef, useEffect, useCallback } from 'react';
 import { generateResponse } from '../lib/api/conversation';
+import type { WaitStatus } from '../lib/api/conversation';
 import { supabase } from '../lib/supabase';
 import { trackEvent } from '../lib/analytics';
 import { APIError } from '../lib/api/types';
@@ -50,6 +51,15 @@ export function useConversationEngine(opts: ConversationEngineOptions) {
   const [interactionCount, setInteractionCount] = useState(0);
   const [repetitionCurrent, setRepetitionCurrent] = useState(0);
   const [stoppingTriggers, setStoppingTriggers] = useState<Record<string, string>>({});
+  // Human-readable "waiting for capacity" message shown while a turn is queued
+  // behind the per-browser request gate or sleeping through a rate-limit retry.
+  const [waitStatus, setWaitStatus] = useState<string | null>(null);
+
+  const describeWait = (w: WaitStatus): string | null => {
+    if (!w) return null;
+    if (w.kind === 'queued') return 'Many requests at once — your turn is queued…';
+    return `High demand — the API is rate-limiting. Retrying in ${Math.max(1, Math.round(w.retryInMs / 1000))}s…`;
+  };
 
   // Refs for latest values in async callbacks
   const autoInteractRef = useRef(opts.autoInteract);
@@ -152,8 +162,12 @@ export function useConversationEngine(opts: ConversationEngineOptions) {
 
       const ac = new AbortController();
       abortControllerRef.current = ac;
-      const response = await generateResponse(config, remappedMessages, ac.signal);
+      const response = await generateResponse(
+        config, remappedMessages, ac.signal,
+        (w) => setWaitStatus(describeWait(w)),
+      );
       abortControllerRef.current = null;
+      setWaitStatus(null);
 
       if (isStoppedRef.current) { setIsLoading(false); return; }
 
@@ -217,6 +231,7 @@ export function useConversationEngine(opts: ConversationEngineOptions) {
       // If the user pressed Stop after the response arrived, fall through without
       // finalizing — handleStop already reset the loading state.
     } catch (error) {
+      setWaitStatus(null);
       // Stop button aborts in-flight requests and retry waits — not an error to surface
       if (error instanceof Error && error.name === 'AbortError' && isStoppedRef.current) {
         setIsLoading(false);
@@ -459,6 +474,7 @@ export function useConversationEngine(opts: ConversationEngineOptions) {
       abortControllerRef.current.abort();
       abortControllerRef.current = null;
     }
+    setWaitStatus(null);
     setIsLoading(false);
   };
 
@@ -479,13 +495,15 @@ export function useConversationEngine(opts: ConversationEngineOptions) {
     setIsLoading(false);
     setErrors([]);
     setStoppingTriggers({});
+    setWaitStatus(null);
     initialChainRef.current = null;
     isStoppedRef.current = false;
   };
 
   return {
     messages, setMessages,
     isLoading,
+    waitStatus,
     errors, setErrors,
     interactionCount,
     repetitionCurrent,

src/lib/api/conversation.ts

@@ -1,6 +1,15 @@
 import { Message, ChatConfig } from '../../types';
 import { createProvider } from './factory';
 import { APIError } from './types';
+import { withRequestSlot } from './requestGate';
+
+/** Why a turn is currently waiting, surfaced to the UI so the user sees a
+ *  "waiting for capacity" state instead of a silent spinner or an error.
+ *  `null` means the request is actively in flight (clear any waiting state). */
+export type WaitStatus =
+  | { kind: 'queued' }
+  | { kind: 'rate-limited'; retryInMs: number; attempt: number }
+  | null;
 
 /** HTTP status codes that are safe to retry (rate-limit / server overload). */
 const RETRYABLE_STATUSES = new Set([429, 503, 529]);
@@ -47,7 +56,8 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
 export async function generateResponse(
   config: ChatConfig,
   messages: Message[],
-  signal?: AbortSignal
+  signal?: AbortSignal,
+  onWait?: (status: WaitStatus) => void,
 ): Promise<Message> {
   const startTime = Date.now();
   const provider = createProvider(config.model);
@@ -68,17 +78,25 @@ export async function generateResponse(
       const base = fromRetryAfter
         ? Math.min((lastError as APIError).retryAfter! * 1000, MAX_RETRY_AFTER_MS)
         : (FALLBACK_DELAYS_MS[attempt - 1] ?? 30_000);
-      await abortableDelay(jitteredDelay(base, fromRetryAfter), signal);
+      const delayMs = jitteredDelay(base, fromRetryAfter);
+      onWait?.({ kind: 'rate-limited', retryInMs: delayMs, attempt });
+      await abortableDelay(delayMs, signal);
     }
 
     try {
-      const result = await provider.makeRequest({
-        apiKey: config.apiKey,
-        orgId: config.orgId,
-        model: config.modelVersion,
-        temperature: config.temperature,
-        maxTokens: config.maxTokens
-      }, apiMessages, signal);
+      const result = await withRequestSlot(
+        () => {
+          onWait?.(null); // slot acquired — request is now actually in flight
+          return provider.makeRequest({
+            apiKey: config.apiKey,
+            orgId: config.orgId,
+            model: config.modelVersion,
+            temperature: config.temperature,
+            maxTokens: config.maxTokens
+          }, apiMessages, signal);
+        },
+        () => onWait?.({ kind: 'queued' }),
+      );
 
       const timeTaken = Date.now() - startTime;
       const wordCount = result.content.trim().split(/\s+/).filter(Boolean).length;

src/lib/api/requestGate.ts

@@ -0,0 +1,45 @@
+/**
+ * Per-browser concurrency gate for outbound provider requests.
+ *
+ * In a workshop a single participant may run several AI-to-AI conversations at
+ * once. Without a gate, each conversation turn fires immediately and a browser
+ * can fan out a burst of simultaneous requests against the shared API key. This
+ * caps the number of in-flight requests per browser and queues the rest FIFO,
+ * so excess turns wait for a free slot instead of all hitting the key at once.
+ *
+ * The gate wraps only the actual in-flight request — not retry backoff waits —
+ * so a slot is released while a turn is sleeping between retry attempts.
+ */
+
+const MAX_CONCURRENT = 4;
+
+let active = 0;
+const waiters: Array<() => void> = [];
+
+/** Number of requests currently waiting for a free slot. */
+export function queueDepth(): number {
+  return waiters.length;
+}
+
+/**
+ * Run `fn` once a concurrency slot is free. If the gate is saturated, `onQueued`
+ * is called and the request waits FIFO. The slot is always released, even if
+ * `fn` throws.
+ */
+export async function withRequestSlot<T>(
+  fn: () => Promise<T>,
+  onQueued?: () => void,
+): Promise<T> {
+  if (active >= MAX_CONCURRENT) {
+    onQueued?.();
+    await new Promise<void>((resolve) => waiters.push(resolve));
+  }
+  active++;
+  try {
+    return await fn();
+  } finally {
+    active--;
+    const next = waiters.shift();
+    if (next) next();
+  }
+}