diff --git a/JS/edgechains/arakoodev/src/ai/src/index.ts b/JS/edgechains/arakoodev/src/ai/src/index.ts index 2c98f37d..8267ad89 100644 --- a/JS/edgechains/arakoodev/src/ai/src/index.ts +++ b/JS/edgechains/arakoodev/src/ai/src/index.ts @@ -3,3 +3,5 @@ export { GeminiAI } from "./lib/gemini/gemini.js"; export { LlamaAI } from "./lib/llama/llama.js"; export { RetellAI } from "./lib/retell-ai/retell.js"; export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js"; +export { SmartRouter } from "./router/router.js"; +export * from "./router/types.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/router/router.ts b/JS/edgechains/arakoodev/src/ai/src/router/router.ts new file mode 100644 index 00000000..a4c0f821 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/router/router.ts @@ -0,0 +1,170 @@ +import { + UnifiedChatOptions, + UnifiedChatResponse, + RouterOptions, + ProviderConfig, + UnifiedMessage, +} from "./types"; +import { OpenAI } from "../lib/openai/openai"; +import { GeminiAI } from "../lib/gemini/gemini"; +import { LlamaAI } from "../lib/llama/llama"; + +export class SmartRouter { + private options: RouterOptions; + + constructor(options: RouterOptions) { + this.options = { + max_retries: 2, + ...options, + }; + } + + async chat(chatOptions: UnifiedChatOptions): Promise { + if (this.options.routing_strategy === "fallback") { + return this.executeWithFallback(chatOptions); + } else { + return this.executeWithLoadBalance(chatOptions); + } + } + + private async executeWithFallback(chatOptions: UnifiedChatOptions): Promise { + let lastError: any; + for (const providerConfig of this.options.providers) { + try { + return await this.callProvider(providerConfig, chatOptions); + } catch (error) { + console.warn( + `Provider ${providerConfig.provider} (${providerConfig.model}) failed: ${error.message}. Trying next...` + ); + lastError = error; + continue; + } + } + throw new Error(`All providers failed. Last error: ${lastError?.message}`); + } + + private async executeWithLoadBalance(chatOptions: UnifiedChatOptions): Promise { + // Weighted random selection + const totalWeight = this.options.providers.reduce((sum, p) => sum + (p.weight || 1), 0); + let random = Math.random() * totalWeight; + + let selectedProvider = this.options.providers[this.options.providers.length - 1]; + for (const provider of this.options.providers) { + random -= provider.weight || 1; + if (random <= 0) { + selectedProvider = provider; + break; + } + } + + return this.callProvider(selectedProvider, chatOptions); + } + + private async callProvider( + config: ProviderConfig, + options: UnifiedChatOptions + ): Promise { + switch (config.provider) { + case "openai": + return this.callOpenAI(config, options); + case "gemini": + return this.callGemini(config, options); + case "llama": + return this.callLlama(config, options); + default: + throw new Error(`Provider ${config.provider} not implemented in router yet.`); + } + } + + private async callOpenAI( + config: ProviderConfig, + options: UnifiedChatOptions + ): Promise { + const client = new OpenAI({ apiKey: config.apiKey, orgId: config.orgId }); + const result = await client.chat({ + model: config.model as any, + messages: options.messages, + max_tokens: options.max_tokens, + temperature: options.temperature, + frequency_penalty: options.frequency_penalty, + }); + + return { + id: `router-oa-${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: config.model, + choices: [ + { + index: 0, + message: result as any, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }, + }; + } + + private async callGemini( + config: ProviderConfig, + options: UnifiedChatOptions + ): Promise { + const client = new GeminiAI({ apiKey: config.apiKey }); + const result = await client.chat({ + prompt: this.messagesToPrompt(options.messages), + temperature: options.temperature, + max_output_tokens: options.max_tokens, + }); + + return { + id: `router-gemini-${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: config.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: result.candidates[0].content.parts[0].text, + }, + finish_reason: result.candidates[0].finishReason || "stop", + }, + ], + usage: { + prompt_tokens: result.usageMetadata?.promptTokenCount || 0, + completion_tokens: result.usageMetadata?.candidatesTokenCount || 0, + total_tokens: result.usageMetadata?.totalTokenCount || 0, + }, + }; + } + + private async callLlama( + config: ProviderConfig, + options: UnifiedChatOptions + ): Promise { + const client = new LlamaAI({ apiKey: config.apiKey || "" }); + const result = await client.chat({ + model: config.model, + messages: options.messages, + max_tokens: options.max_tokens, + temperature: options.temperature, + }); + + // Llama API is OpenAI-compatible + return result as UnifiedChatResponse; + } + + private messagesToPrompt(messages: UnifiedMessage[]): string { + return messages + .map((m) => { + const roleName = m.role === "system" ? "Instruction" : m.role; + return `${roleName}: ${m.content}`; + }) + .join("\n\n"); + } +} diff --git a/JS/edgechains/arakoodev/src/ai/src/router/types.ts b/JS/edgechains/arakoodev/src/ai/src/router/types.ts new file mode 100644 index 00000000..fb74309f --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/router/types.ts @@ -0,0 +1,49 @@ +import { role } from "../types"; + +export interface UnifiedMessage { + role: role; + content: string; + name?: string; +} + +export interface UnifiedChatOptions { + model: string; + messages: UnifiedMessage[]; + max_tokens?: number; + temperature?: number; + stream?: boolean; + frequency_penalty?: number; + [key: string]: any; // Allow for provider-specific options +} + +export interface UnifiedChatResponse { + id: string; + object: string; + created: number; + model: string; + choices: { + index: number; + message: UnifiedMessage; + finish_reason: string; + }[]; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; +} + +export interface ProviderConfig { + provider: "openai" | "gemini" | "anthropic" | "llama" | "custom"; + apiKey?: string; + model: string; + weight?: number; // For load balancing + baseUrl?: string; + orgId?: string; +} + +export interface RouterOptions { + routing_strategy: "fallback" | "load-balance"; + providers: ProviderConfig[]; + max_retries?: number; +} diff --git a/JS/edgechains/examples/smart-router/index.ts b/JS/edgechains/examples/smart-router/index.ts new file mode 100644 index 00000000..f0157371 --- /dev/null +++ b/JS/edgechains/examples/smart-router/index.ts @@ -0,0 +1,36 @@ +import { SmartRouter } from "@arakoodev/edgechains.js/ai"; + +async function main() { + const router = new SmartRouter({ + routing_strategy: "fallback", + providers: [ + { + provider: "openai", + model: "gpt-3.5-turbo", + apiKey: process.env.OPENAI_API_KEY, + }, + { + provider: "gemini", + model: "gemini-pro", + apiKey: process.env.GEMINI_API_KEY, + }, + ], + }); + + try { + const response = await router.chat({ + model: "auto", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello, who are you?" }, + ], + }); + + console.log("Response:", response.choices[0].message.content); + console.log("Usage:", response.usage); + } catch (error) { + console.error("Error:", error.message); + } +} + +main();