From 05175408cb553210bc829e37d8cab612a1025d46 Mon Sep 17 00:00:00 2001 From: Jonas Heller <168646044+jheller1212@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:52:32 +0200 Subject: [PATCH] Graceful capacity handling: per-browser request queue + visible wait state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the shared workshop key is near its limit, turns now degrade gracefully instead of erroring: - New requestGate caps concurrent outbound provider calls per browser (4) and queues the rest FIFO, so one participant running several conversations doesn't burst the key. - generateResponse reports a WaitStatus (queued / rate-limited+retry) via an onWait callback. The engine surfaces it as waitStatus, shown in the chat loading indicator ('Many requests at once — your turn is queued…' / 'High demand — retrying in Ns…') in amber, so the user sees the app is waiting for capacity rather than a frozen spinner or an error. - Slot is held only during the in-flight request, released during retry backoff. Status cleared on response, stop, reset, and error. --- src/components/ChatPanel.tsx | 3 ++ src/components/ConversationDisplay.tsx | 6 +++- src/components/ResearchInterface.tsx | 1 + src/hooks/useConversationEngine.ts | 20 +++++++++++- src/lib/api/conversation.ts | 36 +++++++++++++++------ src/lib/api/requestGate.ts | 45 ++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 src/lib/api/requestGate.ts diff --git a/src/components/ChatPanel.tsx b/src/components/ChatPanel.tsx index 199aaa3..f4ef1c4 100644 --- a/src/components/ChatPanel.tsx +++ b/src/components/ChatPanel.tsx @@ -11,6 +11,7 @@ import { Message } from '../types'; interface ChatPanelProps { messages: Message[]; isLoading: boolean; + waitStatus?: string | null; userInput: string; onUserInputChange: (value: string) => void; onSendMessage: () => void; @@ -64,6 +65,7 @@ interface ChatPanelProps { export function ChatPanel({ messages, isLoading, + waitStatus, userInput, onUserInputChange, onSendMessage, @@ -340,6 +342,7 @@ export function ChatPanel({ - Generating… + {waitStatus + ? {waitStatus} + : 'Generating…'} )} diff --git a/src/components/ResearchInterface.tsx b/src/components/ResearchInterface.tsx index bc1e25d..a47ff42 100644 --- a/src/components/ResearchInterface.tsx +++ b/src/components/ResearchInterface.tsx @@ -598,6 +598,7 @@ export function ResearchInterface({ engine.handleSendMessage(settings.userInput, setCurrentView)} diff --git a/src/hooks/useConversationEngine.ts b/src/hooks/useConversationEngine.ts index a640d62..5944ac3 100644 --- a/src/hooks/useConversationEngine.ts +++ b/src/hooks/useConversationEngine.ts @@ -1,5 +1,6 @@ import { useState, useRef, useEffect, useCallback } from 'react'; import { generateResponse } from '../lib/api/conversation'; +import type { WaitStatus } from '../lib/api/conversation'; import { supabase } from '../lib/supabase'; import { trackEvent } from '../lib/analytics'; import { APIError } from '../lib/api/types'; @@ -50,6 +51,15 @@ export function useConversationEngine(opts: ConversationEngineOptions) { const [interactionCount, setInteractionCount] = useState(0); const [repetitionCurrent, setRepetitionCurrent] = useState(0); const [stoppingTriggers, setStoppingTriggers] = useState>({}); + // Human-readable "waiting for capacity" message shown while a turn is queued + // behind the per-browser request gate or sleeping through a rate-limit retry. + const [waitStatus, setWaitStatus] = useState(null); + + const describeWait = (w: WaitStatus): string | null => { + if (!w) return null; + if (w.kind === 'queued') return 'Many requests at once — your turn is queued…'; + return `High demand — the API is rate-limiting. Retrying in ${Math.max(1, Math.round(w.retryInMs / 1000))}s…`; + }; // Refs for latest values in async callbacks const autoInteractRef = useRef(opts.autoInteract); @@ -152,8 +162,12 @@ export function useConversationEngine(opts: ConversationEngineOptions) { const ac = new AbortController(); abortControllerRef.current = ac; - const response = await generateResponse(config, remappedMessages, ac.signal); + const response = await generateResponse( + config, remappedMessages, ac.signal, + (w) => setWaitStatus(describeWait(w)), + ); abortControllerRef.current = null; + setWaitStatus(null); if (isStoppedRef.current) { setIsLoading(false); return; } @@ -217,6 +231,7 @@ export function useConversationEngine(opts: ConversationEngineOptions) { // If the user pressed Stop after the response arrived, fall through without // finalizing — handleStop already reset the loading state. } catch (error) { + setWaitStatus(null); // Stop button aborts in-flight requests and retry waits — not an error to surface if (error instanceof Error && error.name === 'AbortError' && isStoppedRef.current) { setIsLoading(false); @@ -459,6 +474,7 @@ export function useConversationEngine(opts: ConversationEngineOptions) { abortControllerRef.current.abort(); abortControllerRef.current = null; } + setWaitStatus(null); setIsLoading(false); }; @@ -479,6 +495,7 @@ export function useConversationEngine(opts: ConversationEngineOptions) { setIsLoading(false); setErrors([]); setStoppingTriggers({}); + setWaitStatus(null); initialChainRef.current = null; isStoppedRef.current = false; }; @@ -486,6 +503,7 @@ export function useConversationEngine(opts: ConversationEngineOptions) { return { messages, setMessages, isLoading, + waitStatus, errors, setErrors, interactionCount, repetitionCurrent, diff --git a/src/lib/api/conversation.ts b/src/lib/api/conversation.ts index 57808c8..cbf72f7 100644 --- a/src/lib/api/conversation.ts +++ b/src/lib/api/conversation.ts @@ -1,6 +1,15 @@ import { Message, ChatConfig } from '../../types'; import { createProvider } from './factory'; import { APIError } from './types'; +import { withRequestSlot } from './requestGate'; + +/** Why a turn is currently waiting, surfaced to the UI so the user sees a + * "waiting for capacity" state instead of a silent spinner or an error. + * `null` means the request is actively in flight (clear any waiting state). */ +export type WaitStatus = + | { kind: 'queued' } + | { kind: 'rate-limited'; retryInMs: number; attempt: number } + | null; /** HTTP status codes that are safe to retry (rate-limit / server overload). */ const RETRYABLE_STATUSES = new Set([429, 503, 529]); @@ -47,7 +56,8 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise { export async function generateResponse( config: ChatConfig, messages: Message[], - signal?: AbortSignal + signal?: AbortSignal, + onWait?: (status: WaitStatus) => void, ): Promise { const startTime = Date.now(); const provider = createProvider(config.model); @@ -68,17 +78,25 @@ export async function generateResponse( const base = fromRetryAfter ? Math.min((lastError as APIError).retryAfter! * 1000, MAX_RETRY_AFTER_MS) : (FALLBACK_DELAYS_MS[attempt - 1] ?? 30_000); - await abortableDelay(jitteredDelay(base, fromRetryAfter), signal); + const delayMs = jitteredDelay(base, fromRetryAfter); + onWait?.({ kind: 'rate-limited', retryInMs: delayMs, attempt }); + await abortableDelay(delayMs, signal); } try { - const result = await provider.makeRequest({ - apiKey: config.apiKey, - orgId: config.orgId, - model: config.modelVersion, - temperature: config.temperature, - maxTokens: config.maxTokens - }, apiMessages, signal); + const result = await withRequestSlot( + () => { + onWait?.(null); // slot acquired — request is now actually in flight + return provider.makeRequest({ + apiKey: config.apiKey, + orgId: config.orgId, + model: config.modelVersion, + temperature: config.temperature, + maxTokens: config.maxTokens + }, apiMessages, signal); + }, + () => onWait?.({ kind: 'queued' }), + ); const timeTaken = Date.now() - startTime; const wordCount = result.content.trim().split(/\s+/).filter(Boolean).length; diff --git a/src/lib/api/requestGate.ts b/src/lib/api/requestGate.ts new file mode 100644 index 0000000..37e5377 --- /dev/null +++ b/src/lib/api/requestGate.ts @@ -0,0 +1,45 @@ +/** + * Per-browser concurrency gate for outbound provider requests. + * + * In a workshop a single participant may run several AI-to-AI conversations at + * once. Without a gate, each conversation turn fires immediately and a browser + * can fan out a burst of simultaneous requests against the shared API key. This + * caps the number of in-flight requests per browser and queues the rest FIFO, + * so excess turns wait for a free slot instead of all hitting the key at once. + * + * The gate wraps only the actual in-flight request — not retry backoff waits — + * so a slot is released while a turn is sleeping between retry attempts. + */ + +const MAX_CONCURRENT = 4; + +let active = 0; +const waiters: Array<() => void> = []; + +/** Number of requests currently waiting for a free slot. */ +export function queueDepth(): number { + return waiters.length; +} + +/** + * Run `fn` once a concurrency slot is free. If the gate is saturated, `onQueued` + * is called and the request waits FIFO. The slot is always released, even if + * `fn` throws. + */ +export async function withRequestSlot( + fn: () => Promise, + onQueued?: () => void, +): Promise { + if (active >= MAX_CONCURRENT) { + onQueued?.(); + await new Promise((resolve) => waiters.push(resolve)); + } + active++; + try { + return await fn(); + } finally { + active--; + const next = waiters.shift(); + if (next) next(); + } +}