From 56502b60c542f1b07dc46ca3d90dc1f6d163eb69 Mon Sep 17 00:00:00 2001 From: louisrodriguez1990-crypto Date: Mon, 11 May 2026 03:40:16 -0400 Subject: [PATCH] feat(ai): add SmartRouter for unified multi-provider LLM routing Implements SmartRouter class that automatically detects the correct LLM provider from model name prefix, routes requests, and falls back through a configurable chain when a provider fails. Supports OpenAI, Gemini, Llama, and Cohere with token usage tracking and messages array input. Closes #286 Co-Authored-By: Claude Sonnet 4.6 --- JS/edgechains/arakoodev/src/ai/src/index.ts | 2 + .../ai/src/lib/smart-router/smartRouter.ts | 277 ++++++++++++++++++ .../src/ai/src/tests/smart-router.test.ts | 220 ++++++++++++++ 3 files changed, 499 insertions(+) create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/smart-router/smartRouter.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/tests/smart-router.test.ts diff --git a/JS/edgechains/arakoodev/src/ai/src/index.ts b/JS/edgechains/arakoodev/src/ai/src/index.ts index 2c98f37d..0390ff65 100644 --- a/JS/edgechains/arakoodev/src/ai/src/index.ts +++ b/JS/edgechains/arakoodev/src/ai/src/index.ts @@ -3,3 +3,5 @@ export { GeminiAI } from "./lib/gemini/gemini.js"; export { LlamaAI } from "./lib/llama/llama.js"; export { RetellAI } from "./lib/retell-ai/retell.js"; export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js"; +export { SmartRouter } from "./lib/smart-router/smartRouter.js"; +export type { SmartRouterChatResult } from "./lib/smart-router/smartRouter.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/smart-router/smartRouter.ts b/JS/edgechains/arakoodev/src/ai/src/lib/smart-router/smartRouter.ts new file mode 100644 index 00000000..40cc6b3c --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/smart-router/smartRouter.ts @@ -0,0 +1,277 @@ +import axios from "axios"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type ProviderName = "openai" | "gemini" | "llama" | "cohere"; + +interface MessageOption { + role: "user" | "assistant" | "system"; + content: string; +} + +interface SmartRouterOptions { + openaiApiKey?: string; + geminiApiKey?: string; + llamaApiKey?: string; + cohereApiKey?: string; + /** Override routing priority. Defaults to openai → gemini → llama → cohere */ + fallbackChain?: ProviderName[]; +} + +interface SmartRouterChatOptions { + model: string; + prompt?: string; + messages?: MessageOption[]; + max_tokens?: number; + temperature?: number; + stream?: boolean; +} + +export interface SmartRouterChatResult { + content: string; + provider: ProviderName; + model: string; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; +} + +// --------------------------------------------------------------------------- +// Provider detection +// --------------------------------------------------------------------------- + +const PROVIDER_PATTERNS: Record = { + openai: /^(gpt-|o1-|o3-|chatgpt-|text-embedding-|dall-e-)/i, + gemini: /^(gemini-|palm-|bison-)/i, + llama: /^(llama-|meta-llama\/|mixtral-|mistral-|qwen-|deepseek-)/i, + cohere: /^(command-|cohere-)/i, +}; + +function detectProvider(model: string): ProviderName | null { + for (const [provider, pattern] of Object.entries(PROVIDER_PATTERNS) as [ProviderName, RegExp][]) { + if (pattern.test(model)) return provider; + } + return null; +} + +// --------------------------------------------------------------------------- +// Per-provider chat implementations +// --------------------------------------------------------------------------- + +async function chatOpenAI( + model: string, + messages: MessageOption[], + options: SmartRouterChatOptions, + apiKey: string +): Promise { + const response = await axios.post( + "https://api.openai.com/v1/chat/completions", + { + model, + messages, + max_tokens: options.max_tokens ?? 256, + temperature: options.temperature ?? 0.7, + stream: false, + }, + { headers: { Authorization: `Bearer ${apiKey}`, "content-type": "application/json" } } + ); + const choice = response.data.choices[0]; + return { + content: choice.message.content, + provider: "openai", + model, + usage: response.data.usage + ? { + prompt_tokens: response.data.usage.prompt_tokens, + completion_tokens: response.data.usage.completion_tokens, + total_tokens: response.data.usage.total_tokens, + } + : undefined, + }; +} + +async function chatGemini( + model: string, + messages: MessageOption[], + options: SmartRouterChatOptions, + apiKey: string +): Promise { + const geminiModel = model.startsWith("gemini-") ? model : "gemini-pro"; + const url = `https://generativelanguage.googleapis.com/v1/models/${geminiModel}:generateContent?key=${apiKey}`; + const contents = messages.map((m) => ({ role: m.role === "assistant" ? "model" : "user", parts: [{ text: m.content }] })); + const response = await axios.post( + url, + { contents, generationConfig: { maxOutputTokens: options.max_tokens ?? 256, temperature: options.temperature ?? 0.7 } }, + { headers: { "content-type": "application/json" } } + ); + const candidate = response.data.candidates[0]; + const usage = response.data.usageMetadata; + return { + content: candidate.content.parts[0].text, + provider: "gemini", + model: geminiModel, + usage: usage + ? { + prompt_tokens: usage.promptTokenCount ?? 0, + completion_tokens: usage.candidatesTokenCount ?? 0, + total_tokens: usage.totalTokenCount ?? 0, + } + : undefined, + }; +} + +async function chatLlama( + model: string, + messages: MessageOption[], + options: SmartRouterChatOptions, + apiKey: string +): Promise { + const response = await axios.post( + "https://api.llama-api.com/chat/completions", + { + model: model || "llama-13b-chat", + messages, + max_tokens: options.max_tokens ?? 256, + temperature: options.temperature ?? 0.7, + }, + { headers: { Authorization: `Bearer ${apiKey}`, "content-type": "application/json" } } + ); + const choice = response.data.choices[0]; + return { + content: choice.message.content, + provider: "llama", + model, + usage: response.data.usage + ? { + prompt_tokens: response.data.usage.prompt_tokens, + completion_tokens: response.data.usage.completion_tokens, + total_tokens: response.data.usage.total_tokens, + } + : undefined, + }; +} + +async function chatCohere( + model: string, + messages: MessageOption[], + options: SmartRouterChatOptions, + apiKey: string +): Promise { + const lastUser = [...messages].reverse().find((m) => m.role === "user"); + const chatHistory = messages + .slice(0, -1) + .filter((m) => m.role !== "system") + .map((m) => ({ role: m.role === "assistant" ? "CHATBOT" : "USER", message: m.content })); + + const response = await axios.post( + "https://api.cohere.ai/v1/chat", + { + model: model || "command-r", + message: lastUser?.content ?? "", + chat_history: chatHistory, + max_tokens: options.max_tokens ?? 256, + temperature: options.temperature ?? 0.7, + }, + { headers: { Authorization: `Bearer ${apiKey}`, "content-type": "application/json" } } + ); + const meta = response.data.meta?.tokens; + return { + content: response.data.text, + provider: "cohere", + model: model || "command-r", + usage: meta + ? { + prompt_tokens: meta.input_tokens ?? 0, + completion_tokens: meta.output_tokens ?? 0, + total_tokens: (meta.input_tokens ?? 0) + (meta.output_tokens ?? 0), + } + : undefined, + }; +} + +// --------------------------------------------------------------------------- +// SmartRouter +// --------------------------------------------------------------------------- + +export class SmartRouter { + private keys: Record; + private fallbackChain: ProviderName[]; + + constructor(options: SmartRouterOptions = {}) { + this.keys = { + openai: options.openaiApiKey ?? process.env.OPENAI_API_KEY ?? "", + gemini: options.geminiApiKey ?? process.env.GEMINI_API_KEY ?? "", + llama: options.llamaApiKey ?? process.env.LLAMA_API_KEY ?? "", + cohere: options.cohereApiKey ?? process.env.COHERE_API_KEY ?? "", + }; + this.fallbackChain = options.fallbackChain ?? ["openai", "gemini", "llama", "cohere"]; + } + + detectProvider(model: string): ProviderName | null { + return detectProvider(model); + } + + isModelSupported(model: string): boolean { + return detectProvider(model) !== null; + } + + listProviders(): ProviderName[] { + return ["openai", "gemini", "llama", "cohere"]; + } + + private buildMessages(options: SmartRouterChatOptions): MessageOption[] { + if (options.messages) return options.messages; + if (options.prompt) return [{ role: "user", content: options.prompt }]; + throw new Error("SmartRouter.chat requires either `prompt` or `messages`"); + } + + private async callProvider( + provider: ProviderName, + model: string, + messages: MessageOption[], + options: SmartRouterChatOptions + ): Promise { + const key = this.keys[provider]; + if (!key) throw new Error(`No API key configured for provider: ${provider}`); + + switch (provider) { + case "openai": + return chatOpenAI(model, messages, options, key); + case "gemini": + return chatGemini(model, messages, options, key); + case "llama": + return chatLlama(model, messages, options, key); + case "cohere": + return chatCohere(model, messages, options, key); + } + } + + async chat(options: SmartRouterChatOptions): Promise { + const messages = this.buildMessages(options); + const primary = detectProvider(options.model); + + // Build attempt list: primary provider first, then fallback chain order + const order: ProviderName[] = primary + ? [primary, ...this.fallbackChain.filter((p) => p !== primary)] + : [...this.fallbackChain]; + + const errors: string[] = []; + for (const provider of order) { + if (!this.keys[provider]) continue; + try { + return await this.callProvider(provider, options.model, messages, options); + } catch (err: any) { + const msg = err?.response?.data?.error?.message ?? err?.message ?? String(err); + errors.push(`${provider}: ${msg}`); + } + } + + throw new Error( + `SmartRouter: all providers failed.\n${errors.join("\n")}` + ); + } +} diff --git a/JS/edgechains/arakoodev/src/ai/src/tests/smart-router.test.ts b/JS/edgechains/arakoodev/src/ai/src/tests/smart-router.test.ts new file mode 100644 index 00000000..483af7c4 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/tests/smart-router.test.ts @@ -0,0 +1,220 @@ +import axios from "axios"; +import { SmartRouter } from "../lib/smart-router/smartRouter"; + +jest.mock("axios"); +const mockedAxios = axios as jest.Mocked; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeOpenAIReply(content: string) { + return { + data: { + choices: [{ message: { role: "assistant", content } }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }, + }; +} + +function makeGeminiReply(content: string) { + return { + data: { + candidates: [{ content: { parts: [{ text: content }], role: "model" } }], + usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 20, totalTokenCount: 30 }, + }, + }; +} + +function makeLlamaReply(content: string) { + return { + data: { + choices: [{ message: { role: "assistant", content } }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }, + }; +} + +function makeCohereReply(content: string) { + return { + data: { + text: content, + meta: { tokens: { input_tokens: 10, output_tokens: 20 } }, + }, + }; +} + +// --------------------------------------------------------------------------- +// detectProvider +// --------------------------------------------------------------------------- + +describe("SmartRouter.detectProvider", () => { + const router = new SmartRouter(); + + it("detects openai from gpt-* prefix", () => { + expect(router.detectProvider("gpt-4o")).toBe("openai"); + expect(router.detectProvider("gpt-3.5-turbo")).toBe("openai"); + expect(router.detectProvider("o1-mini")).toBe("openai"); + expect(router.detectProvider("chatgpt-4o-latest")).toBe("openai"); + }); + + it("detects gemini from gemini-* prefix", () => { + expect(router.detectProvider("gemini-pro")).toBe("gemini"); + expect(router.detectProvider("gemini-1.5-flash")).toBe("gemini"); + expect(router.detectProvider("palm-2")).toBe("gemini"); + }); + + it("detects llama from llama-* and meta-llama/ prefix", () => { + expect(router.detectProvider("llama-3-70b")).toBe("llama"); + expect(router.detectProvider("meta-llama/Llama-3-8b")).toBe("llama"); + expect(router.detectProvider("mixtral-8x7b")).toBe("llama"); + expect(router.detectProvider("mistral-7b")).toBe("llama"); + }); + + it("detects cohere from command-* prefix", () => { + expect(router.detectProvider("command-r")).toBe("cohere"); + expect(router.detectProvider("command-r-plus")).toBe("cohere"); + }); + + it("returns null for unknown model", () => { + expect(router.detectProvider("unknown-model-xyz")).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// isModelSupported / listProviders +// --------------------------------------------------------------------------- + +describe("SmartRouter utilities", () => { + const router = new SmartRouter(); + + it("isModelSupported returns true for known prefixes", () => { + expect(router.isModelSupported("gpt-4o")).toBe(true); + expect(router.isModelSupported("command-r")).toBe(true); + }); + + it("isModelSupported returns false for unknown models", () => { + expect(router.isModelSupported("claude-3-opus")).toBe(false); + }); + + it("listProviders returns all four", () => { + expect(router.listProviders()).toEqual(["openai", "gemini", "llama", "cohere"]); + }); +}); + +// --------------------------------------------------------------------------- +// chat — primary routing +// --------------------------------------------------------------------------- + +describe("SmartRouter.chat — primary routing", () => { + afterEach(() => jest.clearAllMocks()); + + it("routes gpt-4o to OpenAI and returns content + usage", async () => { + mockedAxios.post.mockResolvedValueOnce(makeOpenAIReply("Hello from OpenAI")); + const router = new SmartRouter({ openaiApiKey: "sk-test" }); + const result = await router.chat({ model: "gpt-4o", prompt: "hi" }); + expect(result.provider).toBe("openai"); + expect(result.content).toBe("Hello from OpenAI"); + expect(result.usage?.total_tokens).toBe(30); + }); + + it("routes gemini-pro to Gemini", async () => { + mockedAxios.post.mockResolvedValueOnce(makeGeminiReply("Hello from Gemini")); + const router = new SmartRouter({ geminiApiKey: "gm-test" }); + const result = await router.chat({ model: "gemini-pro", prompt: "hi" }); + expect(result.provider).toBe("gemini"); + expect(result.content).toBe("Hello from Gemini"); + }); + + it("routes llama-3 to Llama", async () => { + mockedAxios.post.mockResolvedValueOnce(makeLlamaReply("Hello from Llama")); + const router = new SmartRouter({ llamaApiKey: "ll-test" }); + const result = await router.chat({ model: "llama-3-70b", prompt: "hi" }); + expect(result.provider).toBe("llama"); + expect(result.content).toBe("Hello from Llama"); + }); + + it("routes command-r to Cohere", async () => { + mockedAxios.post.mockResolvedValueOnce(makeCohereReply("Hello from Cohere")); + const router = new SmartRouter({ cohereApiKey: "co-test" }); + const result = await router.chat({ model: "command-r", prompt: "hi" }); + expect(result.provider).toBe("cohere"); + expect(result.content).toBe("Hello from Cohere"); + }); +}); + +// --------------------------------------------------------------------------- +// chat — fallback chain +// --------------------------------------------------------------------------- + +describe("SmartRouter.chat — fallback chain", () => { + afterEach(() => jest.clearAllMocks()); + + it("falls back to Gemini when OpenAI fails", async () => { + mockedAxios.post + .mockRejectedValueOnce(new Error("OpenAI 429")) + .mockResolvedValueOnce(makeGeminiReply("Fallback from Gemini")); + + const router = new SmartRouter({ openaiApiKey: "sk-test", geminiApiKey: "gm-test" }); + const result = await router.chat({ model: "gpt-4o", prompt: "hi" }); + expect(result.provider).toBe("gemini"); + expect(result.content).toBe("Fallback from Gemini"); + }); + + it("throws when all providers fail", async () => { + mockedAxios.post.mockRejectedValue(new Error("network error")); + const router = new SmartRouter({ openaiApiKey: "sk-test", geminiApiKey: "gm-test" }); + await expect(router.chat({ model: "gpt-4o", prompt: "hi" })).rejects.toThrow("SmartRouter: all providers failed"); + }); + + it("skips providers with no API key configured", async () => { + mockedAxios.post.mockResolvedValueOnce(makeGeminiReply("Gemini only")); + // Only gemini key provided — openai should be skipped + const router = new SmartRouter({ geminiApiKey: "gm-test" }); + const result = await router.chat({ model: "gpt-4o", prompt: "hi" }); + expect(result.provider).toBe("gemini"); + expect(mockedAxios.post).toHaveBeenCalledTimes(1); + }); + + it("respects custom fallback chain order", async () => { + mockedAxios.post + .mockRejectedValueOnce(new Error("Cohere fail")) + .mockResolvedValueOnce(makeOpenAIReply("OpenAI after cohere fail")); + + const router = new SmartRouter({ + openaiApiKey: "sk-test", + cohereApiKey: "co-test", + fallbackChain: ["cohere", "openai"], + }); + const result = await router.chat({ model: "gpt-4o", prompt: "hi" }); + expect(result.provider).toBe("openai"); + }); +}); + +// --------------------------------------------------------------------------- +// chat — message array support +// --------------------------------------------------------------------------- + +describe("SmartRouter.chat — messages array", () => { + afterEach(() => jest.clearAllMocks()); + + it("accepts messages array instead of prompt", async () => { + mockedAxios.post.mockResolvedValueOnce(makeOpenAIReply("ok")); + const router = new SmartRouter({ openaiApiKey: "sk-test" }); + const result = await router.chat({ + model: "gpt-4o", + messages: [ + { role: "system", content: "You are helpful." }, + { role: "user", content: "Hello" }, + ], + }); + expect(result.content).toBe("ok"); + }); + + it("throws if neither prompt nor messages provided", async () => { + const router = new SmartRouter({ openaiApiKey: "sk-test" }); + await expect(router.chat({ model: "gpt-4o" })).rejects.toThrow( + "requires either `prompt` or `messages`" + ); + }); +});