diff --git a/src/core/llm/providers/AIModelProviderManager.ts b/src/core/llm/providers/AIModelProviderManager.ts index c3e7aa1b1e..364f5414f6 100644 --- a/src/core/llm/providers/AIModelProviderManager.ts +++ b/src/core/llm/providers/AIModelProviderManager.ts @@ -30,6 +30,7 @@ import { XAIProvider, XAIProviderConfig } from './implementations/XAIProvider'; import { GeminiProvider, GeminiProviderConfig } from './implementations/GeminiProvider'; import { ClaudeCodeProvider, ClaudeCodeProviderConfig } from './implementations/ClaudeCodeProvider'; import { GeminiCLIProvider, GeminiCLIProviderConfig } from './implementations/GeminiCLIProvider'; +import { LiteLLMProvider, LiteLLMProviderConfig } from './implementations/LiteLLMProvider'; import { GMIError, GMIErrorCode, createGMIErrorFromError } from '../../utils/errors.js'; // Corrected import path /** @@ -153,6 +154,9 @@ export class AIModelProviderManager { case 'gemini-cli': providerInstance = new GeminiCLIProvider(); break; + case 'litellm': + providerInstance = new LiteLLMProvider(); + break; default: console.warn(`AIModelProviderManager: Unknown provider ID '${providerEntry.providerId}'. Skipping.`); continue; diff --git a/src/core/llm/providers/__tests__/LiteLLMProvider.test.ts b/src/core/llm/providers/__tests__/LiteLLMProvider.test.ts new file mode 100644 index 0000000000..ced56a748a --- /dev/null +++ b/src/core/llm/providers/__tests__/LiteLLMProvider.test.ts @@ -0,0 +1,225 @@ +/** + * @fileoverview Tests for the LiteLLMProvider. + * + * LiteLLMProvider delegates to OpenAIProvider with a custom baseURL, + * same pattern as GroqProvider. Tests verify initialization, delegation, + * dynamic model discovery, and error handling. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { LiteLLMProvider } from '../implementations/LiteLLMProvider.js'; +import { OpenAIProvider } from '../implementations/OpenAIProvider.js'; + +// Mock OpenAIProvider to avoid real HTTP calls +vi.mock('../implementations/OpenAIProvider.js', () => { + const MockOpenAIProvider = vi.fn().mockImplementation(() => ({ + initialize: vi.fn().mockResolvedValue(undefined), + generateCompletion: vi.fn().mockResolvedValue({ + id: 'chatcmpl-test', + object: 'chat.completion', + created: 0, + modelId: 'anthropic/claude-sonnet-4-6', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'OK' }, + finishReason: 'stop', + }, + ], + usage: { promptTokens: 13, completionTokens: 4, totalTokens: 17 }, + }), + generateCompletionStream: vi.fn().mockImplementation(async function* () { + yield { + id: 'chatcmpl-test', + object: 'chat.completion.chunk', + created: 0, + modelId: 'anthropic/claude-sonnet-4-6', + choices: [ + { + index: 0, + message: { role: 'assistant', content: '' }, + finishReason: 'stop', + }, + ], + responseTextDelta: 'OK', + isFinal: true, + usage: { promptTokens: 13, completionTokens: 4, totalTokens: 17 }, + }; + }), + generateEmbeddings: vi.fn().mockResolvedValue({ + object: 'list', + data: [{ object: 'embedding', embedding: [0.1, 0.2, 0.3], index: 0 }], + model: 'text-embedding-3-small', + usage: { prompt_tokens: 5, total_tokens: 5 }, + }), + checkHealth: vi + .fn() + .mockResolvedValue({ isHealthy: true, details: 'OK' }), + shutdown: vi.fn().mockResolvedValue(undefined), + })); + return { OpenAIProvider: MockOpenAIProvider }; +}); + +describe('LiteLLMProvider', () => { + let provider: LiteLLMProvider; + + beforeEach(() => { + vi.clearAllMocks(); + provider = new LiteLLMProvider(); + }); + + describe('initialization', () => { + it('throws when API key is missing', async () => { + await expect( + provider.initialize({ apiKey: '' }), + ).rejects.toThrow('API key is required'); + }); + + it('initializes with valid config', async () => { + await provider.initialize({ + apiKey: 'sk-test', + baseURL: 'http://localhost:4000/v1', + }); + + expect(provider.isInitialized).toBe(true); + expect(provider.providerId).toBe('litellm'); + expect(provider.defaultModelId).toBe('gpt-4o-mini'); + }); + + it('uses custom default model', async () => { + await provider.initialize({ + apiKey: 'sk-test', + defaultModelId: 'anthropic/claude-sonnet-4-6', + }); + + expect(provider.defaultModelId).toBe('anthropic/claude-sonnet-4-6'); + }); + + it('delegates to OpenAIProvider with proxy baseURL', async () => { + await provider.initialize({ + apiKey: 'sk-test', + baseURL: 'http://proxy:4000/v1', + }); + + const mockInstance = (OpenAIProvider as unknown as ReturnType).mock.results[0].value; + expect(mockInstance.initialize).toHaveBeenCalledWith( + expect.objectContaining({ + apiKey: 'sk-test', + baseURL: 'http://proxy:4000/v1', + }), + ); + }); + }); + + describe('generateCompletion', () => { + it('delegates to OpenAIProvider', async () => { + await provider.initialize({ apiKey: 'sk-test' }); + + const res = await provider.generateCompletion( + 'anthropic/claude-sonnet-4-6', + [{ role: 'user', content: 'Say OK' }], + { temperature: 0 }, + ); + + expect(res.choices[0].message.content).toBe('OK'); + expect(res.choices[0].finishReason).toBe('stop'); + expect(res.usage?.totalTokens).toBe(17); + }); + }); + + describe('generateCompletionStream', () => { + it('yields streamed chunks from delegate', async () => { + await provider.initialize({ apiKey: 'sk-test' }); + + const chunks: any[] = []; + for await (const chunk of provider.generateCompletionStream( + 'anthropic/claude-sonnet-4-6', + [{ role: 'user', content: 'Say OK' }], + {}, + )) { + chunks.push(chunk); + } + + expect(chunks.length).toBeGreaterThan(0); + expect(chunks[chunks.length - 1].isFinal).toBe(true); + }); + }); + + describe('generateEmbeddings', () => { + it('delegates to OpenAIProvider', async () => { + await provider.initialize({ apiKey: 'sk-test' }); + + const res = await provider.generateEmbeddings( + 'text-embedding-3-small', + ['hello'], + ); + + expect(res.data[0].embedding).toEqual([0.1, 0.2, 0.3]); + }); + }); + + describe('listAvailableModels', () => { + it('fetches models from proxy /v1/models', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + data: [ + { id: 'gpt-4o-mini' }, + { id: 'anthropic/claude-sonnet-4-6' }, + ], + }), + }); + vi.stubGlobal('fetch', mockFetch); + + await provider.initialize({ apiKey: 'sk-test' }); + const models = await provider.listAvailableModels(); + + expect(models).toHaveLength(2); + expect(models[0].modelId).toBe('gpt-4o-mini'); + expect(models[1].modelId).toBe('anthropic/claude-sonnet-4-6'); + + vi.unstubAllGlobals(); + }); + + it('returns empty array on fetch error', async () => { + const mockFetch = vi.fn().mockResolvedValue({ ok: false }); + vi.stubGlobal('fetch', mockFetch); + + await provider.initialize({ apiKey: 'sk-test' }); + const models = await provider.listAvailableModels(); + + expect(models).toEqual([]); + + vi.unstubAllGlobals(); + }); + + it('returns empty array on network failure', async () => { + const mockFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED')); + vi.stubGlobal('fetch', mockFetch); + + await provider.initialize({ apiKey: 'sk-test' }); + const models = await provider.listAvailableModels(); + + expect(models).toEqual([]); + + vi.unstubAllGlobals(); + }); + }); + + describe('checkHealth', () => { + it('delegates to OpenAIProvider', async () => { + await provider.initialize({ apiKey: 'sk-test' }); + + const health = await provider.checkHealth(); + expect(health.isHealthy).toBe(true); + }); + }); + + describe('shutdown', () => { + it('delegates and resets state', async () => { + await provider.initialize({ apiKey: 'sk-test' }); + await provider.shutdown(); + + expect(provider.isInitialized).toBe(false); + }); + }); +}); diff --git a/src/core/llm/providers/implementations/LiteLLMProvider.ts b/src/core/llm/providers/implementations/LiteLLMProvider.ts new file mode 100644 index 0000000000..7829aaa0f3 --- /dev/null +++ b/src/core/llm/providers/implementations/LiteLLMProvider.ts @@ -0,0 +1,209 @@ +// File: backend/agentos/core/llm/providers/implementations/LiteLLMProvider.ts + +/** + * @fileoverview Implements the IProvider interface for LiteLLM, a unified AI + * gateway that provides access to 100+ LLM providers (Anthropic, OpenAI, + * Google, Bedrock, Azure, Ollama, etc.) through an OpenAI-compatible proxy. + * + * Like {@link GroqProvider}, this is a thin wrapper around {@link OpenAIProvider} + * since the LiteLLM proxy exposes a fully OpenAI-compatible API. The wrapper + * exists so AIModelProviderManager can identify LiteLLM-specific configuration + * (provider ID, dynamic model catalog via `/v1/models`, etc.). + * + * @module backend/agentos/core/llm/providers/implementations/LiteLLMProvider + * @implements {IProvider} + */ + +import { + IProvider, + ChatMessage, + ModelCompletionOptions, + ModelCompletionResponse, + ModelInfo, + ProviderEmbeddingOptions, + ProviderEmbeddingResponse, +} from '../IProvider'; +import { OpenAIProvider } from './OpenAIProvider'; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +/** + * Configuration for the LiteLLMProvider. + * + * @example + * const config: LiteLLMProviderConfig = { + * apiKey: process.env.LITELLM_API_KEY!, + * baseURL: 'http://localhost:4000/v1', + * defaultModelId: 'anthropic/claude-sonnet-4-6', + * }; + */ +export interface LiteLLMProviderConfig { + /** LiteLLM proxy master key or virtual key. */ + apiKey: string; + /** + * Base URL of the LiteLLM proxy. + * @default "http://localhost:4000/v1" + */ + baseURL?: string; + /** + * Default model to use when none is specified. + * Uses LiteLLM's provider-prefixed format (e.g. "anthropic/claude-sonnet-4-6"). + * @default "gpt-4o-mini" + */ + defaultModelId?: string; + /** Request timeout in milliseconds. @default 60000 */ + requestTimeout?: number; +} + +// --------------------------------------------------------------------------- +// Provider implementation +// --------------------------------------------------------------------------- + +/** + * @class LiteLLMProvider + * @implements {IProvider} + * + * Thin wrapper around {@link OpenAIProvider} that targets a self-hosted + * LiteLLM proxy. All request/response handling is delegated to the + * underlying OpenAI provider since LiteLLM exposes a fully + * OpenAI-compatible API. + * + * Models are dynamic and depend on the proxy configuration. Use + * {@link listAvailableModels} to discover what the proxy serves. + * + * @example + * const litellm = new LiteLLMProvider(); + * await litellm.initialize({ + * apiKey: process.env.LITELLM_API_KEY!, + * baseURL: 'http://localhost:4000/v1', + * }); + * const res = await litellm.generateCompletion( + * 'anthropic/claude-sonnet-4-6', + * messages, + * {} + * ); + */ +export class LiteLLMProvider implements IProvider { + /** @inheritdoc */ + public readonly providerId: string = 'litellm'; + /** @inheritdoc */ + public isInitialized: boolean = false; + /** @inheritdoc */ + public defaultModelId?: string; + + private delegate = new OpenAIProvider(); + private proxyBaseURL: string = 'http://localhost:4000/v1'; + private proxyApiKey: string = ''; + + constructor() {} + + /** + * Initializes the provider by configuring the underlying OpenAI delegate + * with the LiteLLM proxy URL and API key. + */ + public async initialize(config: LiteLLMProviderConfig): Promise { + if (!config.apiKey) { + throw new Error( + 'API key is required for LiteLLMProvider. Set LITELLM_API_KEY.', + ); + } + + this.defaultModelId = config.defaultModelId ?? 'gpt-4o-mini'; + this.proxyBaseURL = config.baseURL ?? 'http://localhost:4000/v1'; + this.proxyApiKey = config.apiKey; + + await this.delegate.initialize({ + apiKey: config.apiKey, + baseURL: this.proxyBaseURL, + defaultModelId: this.defaultModelId, + requestTimeout: config.requestTimeout ?? 60000, + }); + + this.isInitialized = true; + console.log( + `LiteLLMProvider initialized. Proxy: ${this.proxyBaseURL}, default model: ${this.defaultModelId}.`, + ); + } + + /** @inheritdoc */ + public async generateCompletion( + modelId: string, + messages: ChatMessage[], + options: ModelCompletionOptions, + ): Promise { + return this.delegate.generateCompletion(modelId, messages, options); + } + + /** @inheritdoc */ + public async *generateCompletionStream( + modelId: string, + messages: ChatMessage[], + options: ModelCompletionOptions, + ): AsyncGenerator { + yield* this.delegate.generateCompletionStream(modelId, messages, options); + } + + /** @inheritdoc */ + public async generateEmbeddings( + modelId: string, + texts: string[], + options?: ProviderEmbeddingOptions, + ): Promise { + return this.delegate.generateEmbeddings(modelId, texts, options); + } + + /** + * Queries the LiteLLM proxy's `/v1/models` endpoint to discover + * available models dynamically. + */ + public async listAvailableModels( + filter?: { capability?: string }, + ): Promise { + try { + const url = this.proxyBaseURL.replace(/\/+$/, '') + '/models'; + const response = await fetch(url, { + headers: { Authorization: `Bearer ${this.proxyApiKey}` }, + }); + if (!response.ok) { + return []; + } + const data = (await response.json()) as { + data?: Array<{ id: string; owned_by?: string }>; + }; + const models: ModelInfo[] = (data.data ?? []).map((m) => ({ + modelId: m.id, + providerId: 'litellm', + displayName: m.id, + capabilities: ['chat'] as string[], + supportsStreaming: true, + status: 'active' as const, + })); + if (filter?.capability) { + return models.filter((m) => m.capabilities.includes(filter.capability!)); + } + return models; + } catch { + return []; + } + } + + /** @inheritdoc */ + public async getModelInfo(modelId: string): Promise { + const models = await this.listAvailableModels(); + return models.find((m) => m.modelId === modelId); + } + + /** @inheritdoc */ + public async checkHealth(): Promise<{ isHealthy: boolean; details?: unknown }> { + return this.delegate.checkHealth(); + } + + /** @inheritdoc */ + public async shutdown(): Promise { + await this.delegate.shutdown(); + this.isInitialized = false; + console.log('LiteLLMProvider shutdown complete.'); + } +}