diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 3043ddee..31b74d0f 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -189,20 +189,8 @@ export function createApp(opts: AppOptions = {}): { app.use("/api/agents", agentsRouter); app.get("/openapi.json", (_req: Request, res: Response) => { - res.json(getOpenapiJson()); + res.json(openapiSpec); }); - app.get("/openapi.yaml", (_req: Request, res: Response) => { - res.setHeader("Content-Type", "text/yaml; charset=utf-8"); - res.send(getOpenapiYaml()); - }); - app.get("/docs/swagger.json", (_req: Request, res: Response) => { - res.json(getOpenapiJson()); - }); - app.get("/docs/swagger.yaml", (_req: Request, res: Response) => { - res.setHeader("Content-Type", "text/yaml; charset=utf-8"); - res.send(getOpenapiYaml()); - }); - app.use("/docs", swaggerUi.serve, swaggerUi.setup(openapiSpec, swaggerUiOptions)); // ── Task routes ──────────────────────────────────────────────────────────── // Authenticated task creation uses the tighter authed limiter. diff --git a/backend/src/api/routes/agents.ts b/backend/src/api/routes/agents.ts index 1f8d987a..d5401485 100644 --- a/backend/src/api/routes/agents.ts +++ b/backend/src/api/routes/agents.ts @@ -19,6 +19,18 @@ export interface AgentsRouterOptions { db?: AgentDb; } +const STELLAR_PUBLIC_KEY_REGEX = /^G[A-Z2-7]{55}$/; + +const RegisterAgentSchema = z.object({ + agentId: z.string(), + capabilities: z.array(z.string()), + pricingXLM: z.number().positive("Price must be positive"), + endpoint: z.string().url(), + stellarPublicKey: z + .string() + .regex(STELLAR_PUBLIC_KEY_REGEX, "Invalid Stellar public key format"), +}); + const DEFAULT_HEALTH_TIMEOUT_MS = 3_000; // Mirrors the RegisterAgentRequest schema documented in api/docs.ts. @@ -41,8 +53,7 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { * @openapi * /api/agents: * get: - * summary: List registered AI agents - * description: Retrieves registered agents matching optional capability, minimum reputation, and maximum price filters. + * summary: List registered agents * operationId: listAgents * tags: [Agents] * security: [] @@ -51,47 +62,27 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { * name: capability * schema: { type: string } * description: Filter agents that support this capability - * example: "research" * - in: query * name: minReputation * schema: { type: number } - * description: Minimum reputation score threshold - * example: 80.0 * - in: query * name: maxPriceXLM * schema: { type: number } - * description: Maximum price per task execution in XLM - * example: 1.5 * responses: * 200: - * description: Array of matching registered agents - * headers: - * X-RateLimit-Limit: - * $ref: '#/components/headers/X-RateLimit-Limit' - * X-RateLimit-Remaining: - * $ref: '#/components/headers/X-RateLimit-Remaining' - * X-RateLimit-Reset: - * $ref: '#/components/headers/X-RateLimit-Reset' + * description: List of agents * content: * application/json: * schema: * type: array * items: * $ref: '#/components/schemas/Agent' - * example: - * - id: "agent_crypto_analyst_01" - * capabilities: ["research", "report"] - * pricingXLM: 0.25 - * endpoint: "https://agent-crypto.example.com/api" - * stellarPublicKey: "GABZXN7PIRZGNMHGA728XZVOG2GUFIDLAZ6AF2I2MD2OCYTAF2K1K4XYZ" - * reputationScore: 98.5 - * lastSeenAt: "2026-08-25T17:20:00.000Z" * 500: * description: Internal server error * content: * application/json: * schema: - * $ref: '#/components/schemas/InternalServerError' + * $ref: '#/components/schemas/Error' */ // GET /api/agents — supports cursor pagination when ?cursor or ?limit present router.get("/", (req: Request, res: Response, next: NextFunction): void => { @@ -147,6 +138,7 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { } catch { res.status(500).json({ error: "Internal Server Error" }); } + res.json(agent); }); router.get("/:id/health", async (req: Request, res: Response, next: NextFunction): Promise => { @@ -157,9 +149,9 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { return; } - const startedAt = Date.now(); - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), healthTimeoutMs); + const startedAt = Date.now(); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), healthTimeoutMs); try { const response = await fetch(agent.endpoint, { @@ -246,6 +238,22 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { } catch (error) { next(error); } + + const db = getDb(); + const agent = { + id: data.agentId, + capabilities: data.capabilities, + pricingXLM: data.pricingXLM, + endpoint: data.endpoint, + stellarPublicKey: data.stellarPublicKey, + reputationScore: 0, + lastSeenAt: new Date().toISOString(), + status: 'online' as const + }; + + db.upsert(agent); + + res.status(201).json(agent); }); router.post("/:id/heartbeat", heartbeatRateLimitMiddleware, (req: Request, res: Response, next: NextFunction): void => { @@ -266,6 +274,13 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { } catch (error) { next(error); } + + db.upsert({ ...agent, lastSeenAt: new Date().toISOString(), status: 'online' }); + const updated = db.findById(req.params.id); + res.status(200).json({ + status: "ok", + lastSeenAt: updated?.lastSeenAt ?? new Date().toISOString(), + }); }); router.delete("/:id", (req: Request, res: Response, next: NextFunction): void => { @@ -299,6 +314,9 @@ export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { } catch (error) { next(error); } + + db.delete(req.params.id); + res.json({ message: "Agent deleted successfully" }); }); return router; diff --git a/backend/src/services/venice/cache.ts b/backend/src/services/venice/cache.ts index e1184f59..c3bd9ad3 100644 --- a/backend/src/services/venice/cache.ts +++ b/backend/src/services/venice/cache.ts @@ -120,6 +120,45 @@ export class VeniceResponseCache { return null; } + /** + * Graceful degradation: return the most recent cached entry for the prompt + * even if it is stale/expired. Used when all providers fail so the task + * can proceed without failing. Returns null if nothing is cached at all. + */ + getStale(prompt: string, agentType: string, modelVersion: string): string | null { + // Prefer fresh hit first (already tried via get), then fall back to stale + const exact = this.store.get(buildCacheKey(prompt, agentType, modelVersion)); + if (exact) { + this.recordHit(); + return exact.content; + } + + // Fuzzy stale search — ignore expiry, pick highest similarity then most recent + const norm = normalizePrompt(prompt); + let best: CachedEntry | null = null; + let bestScore = 0; + let bestTime = 0; + for (const entry of this.store.values()) { + if (entry.agentType !== agentType) continue; + if (entry.modelVersion !== modelVersion) continue; + const score = similarity(norm, entry.prompt); + if (score >= this.options.similarityThreshold) { + if (score > bestScore || (score === bestScore && entry.createdAt > bestTime)) { + bestScore = score; + bestTime = entry.createdAt; + best = entry; + } + } + } + + if (best) { + this.recordHit(); + return best.content; + } + + return null; + } + set(prompt: string, agentType: string, modelVersion: string, content: string): void { const now = Date.now(); const key = buildCacheKey(prompt, agentType, modelVersion); diff --git a/backend/src/services/venice/client.ts b/backend/src/services/venice/client.ts index 2638e618..5e621d23 100644 --- a/backend/src/services/venice/client.ts +++ b/backend/src/services/venice/client.ts @@ -12,6 +12,7 @@ import type { VeniceClientConfig, VeniceClientLike, VeniceMessage, + VeniceProviderConfig, } from './types.js'; interface CacheEnvConfig { @@ -19,6 +20,8 @@ interface CacheEnvConfig { VENICE_CACHE_TTL_MS: number; VENICE_CACHE_CODING_TTL_MS: number; VENICE_CACHE_SIMILARITY_THRESHOLD: number; + VENICE_REQUEST_TIMEOUT_MS: number; + VENICE_PROVIDER_MAX_RETRIES: number; } const CONFIG_FALLBACK: CacheEnvConfig = { @@ -26,6 +29,8 @@ const CONFIG_FALLBACK: CacheEnvConfig = { VENICE_CACHE_TTL_MS: 24 * 60 * 60 * 1000, VENICE_CACHE_CODING_TTL_MS: 60 * 60 * 1000, VENICE_CACHE_SIMILARITY_THRESHOLD: 0.8, + VENICE_REQUEST_TIMEOUT_MS: 10_000, + VENICE_PROVIDER_MAX_RETRIES: 3, }; const log = createLogger({ module: 'VeniceClient' }); @@ -40,38 +45,99 @@ const MODEL_MAP: Record = { const DEFAULT_MAX_TOKENS = 2048; const HARD_TOKEN_CAP = 8192; -const RETRY_DELAYS_MS = [200, 400, 800]; -const RETRYABLE_STATUS_CODES = new Set([429, 503]); +const RETRY_DELAYS_MS = [200, 400, 800, 1600]; +const RETRYABLE_STATUS_CODES = new Set([429, 503, 500, 502, 504]); const NON_RETRYABLE_STATUS_CODES = new Set([400, 401, 422]); const DEFAULT_CHAT_MODEL = 'llama-3.3-70b'; export class VeniceClient implements VeniceClientLike { - private readonly apiKey: string; - private readonly baseUrl: string; + private readonly providers: VeniceProviderConfig[]; private readonly breaker: CircuitBreaker; private readonly cache: VeniceResponseCache; private readonly deduplicator: RequestDeduplicator; private readonly modelVersion: string; + private readonly timeoutMs: number; + private readonly maxRetries: number; + private readonly enableCacheFallback: boolean; + + // Backward compat: expose primary for existing callers + private get apiKey(): string { + return this.providers[0]?.apiKey ?? ''; + } + private get baseUrl(): string { + return this.providers[0]?.baseUrl ?? 'https://api.venice.ai/api/v1'; + } constructor(config: VeniceClientConfig) { this.apiKey = config.apiKey; this.baseUrl = config.baseUrl ?? getConfig().VENICE_BASE_URL; this.breaker = config.circuitBreaker ?? new CircuitBreaker(); - const env = this.resolveConfig(); - this.modelVersion = config.modelVersion ?? env.VENICE_MODEL_VERSION; + const env = this.resolveConfig() as any; + this.modelVersion = config.modelVersion ?? env.VENICE_MODEL_VERSION ?? CONFIG_FALLBACK.VENICE_MODEL_VERSION; + this.timeoutMs = config.timeoutMs ?? env.VENICE_REQUEST_TIMEOUT_MS ?? CONFIG_FALLBACK.VENICE_REQUEST_TIMEOUT_MS; + this.maxRetries = config.maxRetries ?? env.VENICE_PROVIDER_MAX_RETRIES ?? CONFIG_FALLBACK.VENICE_PROVIDER_MAX_RETRIES; + this.enableCacheFallback = config.enableCacheFallback ?? true; + + // Build ordered provider chain: explicit providers wins, otherwise build from config + env fallbacks + if (config.providers && config.providers.length > 0) { + this.providers = config.providers.map((p) => ({ + apiKey: p.apiKey, + baseUrl: p.baseUrl ?? 'https://api.venice.ai/api/v1', + name: p.name, + })); + } else { + this.providers = this.buildProvidersFromEnv(config); + } + const cacheConfig = config.cacheConfig ?? {}; this.cache = config.cache ?? new VeniceResponseCache({ - defaultTtlMs: cacheConfig.defaultTtlMs ?? env.VENICE_CACHE_TTL_MS, - codingTtlMs: cacheConfig.codingTtlMs ?? env.VENICE_CACHE_CODING_TTL_MS, + defaultTtlMs: cacheConfig.defaultTtlMs ?? env.VENICE_CACHE_TTL_MS ?? CONFIG_FALLBACK.VENICE_CACHE_TTL_MS, + codingTtlMs: cacheConfig.codingTtlMs ?? env.VENICE_CACHE_CODING_TTL_MS ?? CONFIG_FALLBACK.VENICE_CACHE_CODING_TTL_MS, similarityThreshold: - cacheConfig.similarityThreshold ?? env.VENICE_CACHE_SIMILARITY_THRESHOLD, + cacheConfig.similarityThreshold ?? env.VENICE_CACHE_SIMILARITY_THRESHOLD ?? CONFIG_FALLBACK.VENICE_CACHE_SIMILARITY_THRESHOLD, }); this.deduplicator = config.deduplicator ?? new RequestDeduplicator(); } + private buildProvidersFromEnv(config: VeniceClientConfig): VeniceProviderConfig[] { + const primary: VeniceProviderConfig = { + apiKey: config.apiKey, + baseUrl: config.baseUrl ?? 'https://api.venice.ai/api/v1', + name: 'primary', + }; + const providers: VeniceProviderConfig[] = [primary]; + + // Try to read fallback env vars via getConfig (if available) + try { + const cfg: any = getConfig(); + const fallbackKeys: string = cfg.VENICE_FALLBACK_API_KEYS ?? ''; + const fallbackUrls: string = cfg.VENICE_FALLBACK_BASE_URLS ?? ''; + if (fallbackKeys) { + const keys = fallbackKeys + .split(',') + .map((k: string) => k.trim()) + .filter(Boolean); + const urls = fallbackUrls + ? fallbackUrls.split(',').map((u: string) => u.trim()).filter(Boolean) + : []; + keys.forEach((key: string, idx: number) => { + providers.push({ + apiKey: key, + baseUrl: urls[idx] ?? urls[0] ?? primary.baseUrl ?? 'https://api.venice.ai/api/v1', + name: `fallback-${idx + 1}`, + }); + }); + } + } catch { + // No config available (e.g. in tests) — just use primary + } + + return providers; + } + private resolveConfig(): CacheEnvConfig { try { return getConfig() as unknown as CacheEnvConfig; @@ -92,6 +158,11 @@ export class VeniceClient implements VeniceClientLike { return this.breaker.getFailureCount(); } + /** Expose provider chain for observability / tests. */ + getProviders(): VeniceProviderConfig[] { + return [...this.providers]; + } + /** Current cache hit rate (0..1) for monitoring. */ getCacheHitRate(): number { return this.cache.getHitRate(); @@ -151,7 +222,19 @@ export class VeniceClient implements VeniceClientLike { throw new TokenBudgetExceededError(maxTokens, HARD_TOKEN_CAP); } - this.breaker.assertClosed(); + // Circuit breaker check — but allow stale cache fallback even when open + try { + this.breaker.assertClosed(); + } catch (e) { + if (this.enableCacheFallback && !options?.force) { + const stale = this.cache.getStale(promptForLogging, agentType, this.modelVersion); + if (stale !== null) { + log.warn({ agentType, model, circuitState: this.breaker.getState() }, 'venice circuit open — serving stale cache'); + return stale; + } + } + throw e; + } const force = options?.force === true; const cacheKey = buildCacheKey(promptForLogging, agentType, this.modelVersion); @@ -170,9 +253,23 @@ export class VeniceClient implements VeniceClientLike { const runFetch = (): Promise => this.runVeniceFetch({ messages, model, options, promptForLogging, agentType }); - const result = force - ? await runFetch() - : await this.deduplicator.dedup(cacheKey, runFetch); + let result: string; + try { + result = force ? await runFetch() : await this.deduplicator.dedup(cacheKey, runFetch); + } catch (err) { + // Graceful degradation: if all providers failed and we have stale cache, return it + if (this.enableCacheFallback && !force) { + const stale = this.cache.getStale(promptForLogging, agentType, this.modelVersion); + if (stale !== null) { + log.warn( + { agentType, model, error: err instanceof Error ? err.message : String(err) }, + 'venice all providers failed — serving stale cache (graceful degradation)', + ); + return stale; + } + } + throw err; + } if (!force) { this.cache.set(promptForLogging, agentType, this.modelVersion, result); @@ -204,25 +301,57 @@ export class VeniceClient implements VeniceClientLike { max_tokens: options?.maxTokens ?? DEFAULT_MAX_TOKENS, }); - try { - const response = await this.fetchWithRetry(body, () => { retries++; }); - const data: unknown = await response.json(); - const content = (data as any)?.choices?.[0]?.message?.content; - if (typeof content !== 'string') { - throw new Error('Venice response missing expected content field'); - } + let lastError: Error | undefined; - this.breaker.recordSuccess(); - this.logRequest(requestId, agentType, model, promptForLogging, Date.now() - start, 'ok', retries); - return content; - } catch (err) { - if (err instanceof CircuitOpenError || err instanceof TokenBudgetExceededError) { - throw err; + // Try providers in order (fallback chain) + for (let pIndex = 0; pIndex < this.providers.length; pIndex++) { + const provider = this.providers[pIndex]!; + const isLastProvider = pIndex === this.providers.length - 1; + + try { + const response = await this.fetchWithRetryForProvider( + body, + provider, + () => { retries++; }, + ); + const data: unknown = await response.json(); + const content = (data as any)?.choices?.[0]?.message?.content; + if (typeof content !== 'string') { + throw new Error('Venice response missing expected content field'); + } + + this.breaker.recordSuccess(); + this.logRequest(requestId, agentType, model, promptForLogging, Date.now() - start, 'ok', retries, provider.name); + return content; + } catch (err) { + if (err instanceof CircuitOpenError || err instanceof TokenBudgetExceededError) { + throw err; + } + lastError = err instanceof Error ? err : new Error(String(err)); + // Non-retryable 400/422 on last provider should not failover further — but we still try next if available + const isNonRetryable = lastError.message.includes('non-retryable'); + // For 401, trying next provider with different key may succeed, so we do failover + if (pIndex < this.providers.length - 1) { + const nextProvider = this.providers[pIndex + 1]!.name ?? `fallback-${pIndex + 1}`; + log.warn( + { agentType, model, failedProvider: provider.name, nextProvider, error: lastError.message, retries }, + 'venice provider failed — failing over to next provider', + ); + // small backoff before failover to next provider + await this.sleep(100); + continue; + } + // Last provider failed — record failure for circuit breaker + this.breaker.recordFailure(); + this.logRequest(requestId, agentType, model, promptForLogging, Date.now() - start, 'error', retries, provider.name); + // If we have stale cache fallback enabled, the caller (createCompletion) will handle it + throw lastError; } - this.breaker.recordFailure(); - this.logRequest(requestId, agentType, model, promptForLogging, Date.now() - start, 'error', retries); - throw err; } + + // Should not reach here, but fallback + this.breaker.recordFailure(); + throw lastError ?? new Error('Venice AI is unreachable (all providers failed)'); } async stream( @@ -252,95 +381,141 @@ export class VeniceClient implements VeniceClientLike { }); let accumulated = ''; - try { - const response = await this.fetchWithRetry(body, () => { retries++; }); + let lastError: Error | undefined; - if (!response.body) { - throw new Error('Venice stream response has no body'); - } + for (let pIndex = 0; pIndex < this.providers.length; pIndex++) { + const provider = this.providers[pIndex]!; + try { + const response = await this.fetchWithRetryForProvider(body, provider, () => { retries++; }); - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let done = false; - - while (!done) { - const result = await reader.read(); - done = result.done; - if (result.value) { - const text = decoder.decode(result.value, { stream: !done }); - const lines = text.split('\n'); - for (const line of lines) { - if (!line.startsWith('data: ')) continue; - const payload = line.slice(6).trim(); - if (payload === '[DONE]') continue; - try { - const parsed = JSON.parse(payload); - const delta = parsed?.choices?.[0]?.delta?.content; - if (typeof delta === 'string' && delta.length > 0) { - accumulated += delta; - onChunk(delta); + if (!response.body) { + throw new Error('Venice stream response has no body'); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let done = false; + + while (!done) { + const result = await reader.read(); + done = result.done; + if (result.value) { + const text = decoder.decode(result.value, { stream: !done }); + const lines = text.split('\n'); + for (const line of lines) { + if (!line.startsWith('data: ')) continue; + const payload = line.slice(6).trim(); + if (payload === '[DONE]') continue; + try { + const parsed = JSON.parse(payload); + const delta = parsed?.choices?.[0]?.delta?.content; + if (typeof delta === 'string' && delta.length > 0) { + accumulated += delta; + onChunk(delta); + } + } catch { + // skip malformed SSE chunks } - } catch { - // skip malformed SSE chunks } } } - } - this.breaker.recordSuccess(); - this.logRequest(requestId, agentType, model, prompt, Date.now() - start, 'ok', retries); - } catch (err) { - if (err instanceof CircuitOpenError || err instanceof TokenBudgetExceededError) { - throw err; + this.breaker.recordSuccess(); + this.logRequest(requestId, agentType, model, prompt, Date.now() - start, 'ok', retries, provider.name); + return; + } catch (err) { + if (err instanceof CircuitOpenError || err instanceof TokenBudgetExceededError) { + throw err; + } + lastError = err instanceof Error ? err : new Error(String(err)); + if (pIndex < this.providers.length - 1) { + log.warn({ agentType, model, failedProvider: provider.name, error: lastError.message }, 'venice stream provider failed — failover'); + await this.sleep(100); + continue; + } + this.breaker.recordFailure(); + this.logRequest(requestId, agentType, model, prompt, Date.now() - start, 'error', retries, provider.name); + throw new Error( + `Venice stream error after ${accumulated.length} characters accumulated: ${lastError.message}` + ); } - this.breaker.recordFailure(); - this.logRequest(requestId, agentType, model, prompt, Date.now() - start, 'error', retries); - throw new Error( - `Venice stream error after ${accumulated.length} characters accumulated` - ); } + + throw lastError ?? new Error('Venice stream failed (all providers)'); } - private async fetchWithRetry( + /** + * Per-provider fetch with retries, exponential backoff and per-call timeout. + */ + private async fetchWithRetryForProvider( body: string, + provider: VeniceProviderConfig, onRetry: () => void ): Promise { let lastError: Error | undefined; + const maxAttempts = Math.min(this.maxRetries, RETRY_DELAYS_MS.length) + 1; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + let timeoutId: ReturnType | undefined; + const controller = new AbortController(); + if (this.timeoutMs > 0) { + timeoutId = setTimeout(() => controller.abort(), this.timeoutMs); + } - for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) { try { - const response = await fetch(`${this.baseUrl}/chat/completions`, { + const response = await fetch(`${provider.baseUrl ?? this.baseUrl}/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${this.apiKey}`, + 'Authorization': `Bearer ${provider.apiKey}`, }, body, + signal: controller.signal, }); + if (timeoutId) clearTimeout(timeoutId); + if (response.ok) { return response; } - if (NON_RETRYABLE_STATUS_CODES.has(response.status)) { + if (NON_RETRYABLE_STATUS_CODES.has(response.status) && response.status !== 401) { + // 401 may succeed on fallback with different key, so we treat it as retriable for failover throw new Error(`Venice returned non-retryable status: ${response.status}`); } - if (RETRYABLE_STATUS_CODES.has(response.status) && attempt < RETRY_DELAYS_MS.length) { + // 401 is special: allow failover to next provider, not retry same provider + if (response.status === 401) { + throw new Error(`Venice returned non-retryable status: ${response.status}`); + } + + if (RETRYABLE_STATUS_CODES.has(response.status) && attempt < maxAttempts - 1) { onRetry(); - await this.sleep(RETRY_DELAYS_MS[attempt]!); + await this.sleep(this.backoffDelay(attempt)); continue; } throw new Error(`Venice returned status: ${response.status}`); } catch (err) { + if (timeoutId) clearTimeout(timeoutId); + // AbortError from timeout + if (err instanceof Error && err.name === 'AbortError') { + lastError = new Error(`Venice request timed out after ${this.timeoutMs}ms`); + if (attempt < maxAttempts - 1) { + onRetry(); + await this.sleep(this.backoffDelay(attempt)); + continue; + } + throw lastError; + } if (err instanceof Error && err.message.startsWith('Venice returned')) { + // For non-retryable, don't retry same provider — throw to allow failover to next provider throw err; } lastError = err instanceof Error ? err : new Error(String(err)); - if (attempt < RETRY_DELAYS_MS.length) { + if (attempt < maxAttempts - 1) { onRetry(); - await this.sleep(RETRY_DELAYS_MS[attempt]!); + await this.sleep(this.backoffDelay(attempt)); continue; } } @@ -349,6 +524,23 @@ export class VeniceClient implements VeniceClientLike { throw lastError ?? new Error('Venice AI is unreachable'); } + private backoffDelay(attempt: number): number { + const base = RETRY_DELAYS_MS[attempt] ?? 800; + // Add jitter ±20% to avoid thundering herd + const jitter = base * 0.2 * (Math.random() * 2 - 1); + return Math.max(50, Math.round(base + jitter)); + } + + // Legacy fetchWithRetry kept for backward compat (delegates to primary provider) + private async fetchWithRetry( + body: string, + onRetry: () => void + ): Promise { + const primary = this.providers[0]; + if (!primary) throw new Error('No Venice providers configured'); + return this.fetchWithRetryForProvider(body, primary, onRetry); + } + private sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } @@ -360,7 +552,8 @@ export class VeniceClient implements VeniceClientLike { prompt: string, durationMs: number, status: 'ok' | 'error', - retries: number + retries: number, + providerName?: string ): void { const promptTokenEstimate = Math.ceil(prompt.length / 4); log.info({ @@ -371,6 +564,7 @@ export class VeniceClient implements VeniceClientLike { durationMs, status, retries, + provider: providerName ?? 'primary', circuitState: this.breaker.getState(), promptPreview: prompt.slice(0, 200), }, 'venice request'); diff --git a/backend/src/services/venice/types.ts b/backend/src/services/venice/types.ts index fe436789..b1fab55f 100644 --- a/backend/src/services/venice/types.ts +++ b/backend/src/services/venice/types.ts @@ -30,10 +30,24 @@ export interface VeniceChatOptions extends CompleteOptions { model?: string; } +export interface VeniceProviderConfig { + apiKey: string; + baseUrl?: string; + name?: string; +} + export interface VeniceClientConfig { apiKey: string; baseUrl?: string; circuitBreaker?: CircuitBreaker; + /** Ordered fallback providers; first is primary. When supplied, overrides apiKey/baseUrl. */ + providers?: VeniceProviderConfig[]; + /** Per-call timeout in ms. Default: 10_000. */ + timeoutMs?: number; + /** Retries per provider with exponential backoff. Default: 3. */ + maxRetries?: number; + /** When true, stale cache is returned if all providers fail. Default: true. */ + enableCacheFallback?: boolean; /** Model version used as part of the cache key; changing it invalidates entries. */ modelVersion?: string; /** Cache behaviour; built-in defaults are used when omitted. */