diff --git a/Memory/src/config/index.ts b/Memory/src/config/index.ts index becf2f5d5..d3bc5df1d 100644 --- a/Memory/src/config/index.ts +++ b/Memory/src/config/index.ts @@ -15,6 +15,8 @@ export type LlmProviderName = | "" | "local_only" | "openai_compatible" + | "sglang" + | "vllm" | "gemini" | "anthropic" | "bedrock" @@ -749,6 +751,10 @@ function unavailableLlm(defaults: LlmConfig): Record { function memoryLlmProvider(provider: string): LlmProviderName { switch (provider) { + case "sglang": + return "sglang"; + case "vllm": + return "vllm"; case "anthropic": return "anthropic"; case "google": @@ -1137,6 +1143,8 @@ function llmProvider(value: unknown, fallback: LlmProviderName): LlmProviderName provider === "" || provider === "local_only" || provider === "openai_compatible" || + provider === "sglang" || + provider === "vllm" || provider === "gemini" || provider === "anthropic" || provider === "bedrock" || diff --git a/Memory/src/model/llm.ts b/Memory/src/model/llm.ts index 1492f6f95..8d7157163 100644 --- a/Memory/src/model/llm.ts +++ b/Memory/src/model/llm.ts @@ -270,6 +270,8 @@ class HttpLlmClient implements LlmClient { private completeOnce(messages: LlmMessage[], options: Required> & LlmCompletionOptions): Promise { switch (this.config.provider) { case "openai_compatible": + case "sglang": + case "vllm": return this.completeOpenAiCompatible(messages, options); case "gemini": return this.completeGemini(messages, options); @@ -303,6 +305,11 @@ class HttpLlmClient implements LlmClient { const thinkingBudget = thinking.enabled && thinkingUsesEnableThinking(this.config.vendor ?? "", base, model) ? this.config.thinkingBudget : undefined; + const thinkingPayload = openAiCompatibleThinkingPayload( + this.config.provider, + thinking, + this.config.extraBody + ); const agentRegion = resolveMemoryAgentRegion(this.config.sourceProvider); const response = await postJsonWithRetry({ actualModelContext: this.config.actualModelContext, @@ -323,10 +330,10 @@ class HttpLlmClient implements LlmClient { ...(!omitTemperature ? { temperature: options.temperature ?? this.config.temperature } : {}), max_tokens: options.maxTokens ?? this.config.maxTokens, stream: false, - ...thinking.fields, + ...thinkingPayload.fields, ...(thinkingBudget !== undefined ? { thinking_budget: thinkingBudget } : {}), ...(options.jsonMode && !omitJsonMode ? { response_format: { type: "json_object" } } : {}), - ...(this.config.extraBody ?? {}) + ...thinkingPayload.extraBody } }); const choice = response.choices?.[0]; @@ -561,6 +568,43 @@ class HttpLlmClient implements LlmClient { } } +function openAiCompatibleThinkingPayload( + provider: LlmConfig["provider"], + thinking: ThinkingControl, + configuredExtraBody: Record | undefined +): { fields: Record; extraBody: Record } { + const extraBody = configuredExtraBody ?? {}; + const configuredChatTemplateKwargs = extraBody.chat_template_kwargs; + if ( + (provider !== "sglang" && provider !== "vllm") || + !("enable_thinking" in thinking.fields) || + (configuredChatTemplateKwargs !== undefined && + !isUnknownRecord(configuredChatTemplateKwargs)) + ) { + return { fields: thinking.fields, extraBody }; + } + const chatTemplateKwargs = configuredChatTemplateKwargs ?? {}; + + const fields = { ...thinking.fields }; + delete fields.enable_thinking; + return { + fields, + extraBody: { + ...extraBody, + chat_template_kwargs: { + ...chatTemplateKwargs, + ...(!Object.hasOwn(chatTemplateKwargs, "enable_thinking") + ? { enable_thinking: thinking.enabled } + : {}) + } + } + }; +} + +function isUnknownRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + function openAiCompatibleThinkingControl(input: { vendor: string; endpoint: string; diff --git a/Memory/tests/llm-thinking.test.ts b/Memory/tests/llm-thinking.test.ts index 0c0110f8d..c12a466ae 100644 --- a/Memory/tests/llm-thinking.test.ts +++ b/Memory/tests/llm-thinking.test.ts @@ -76,6 +76,65 @@ describe("memory LLM thinking configuration", () => { expect(requestBody(fetchMock)).not.toHaveProperty("thinking_budget"); }); + it.each(["sglang", "vllm"] as const)( + "routes Qwen thinking through chat_template_kwargs for the %s provider", + async (provider) => { + const fetchMock = openAiFetch(); + vi.stubGlobal("fetch", fetchMock); + const client = createLlmClient(llmConfig({ + provider, + vendor: "qwen", + endpoint: `https://${provider}.example/v1`, + model: "qwen3.8-27b", + enableThinking: true, + extraBody: { chat_template_kwargs: { tokenizer_option: "preserved" } } + })); + + await client.complete([{ role: "user", content: "filter" }], { + operation: "retrieval.filter", + thinkingMode: "disabled" + }); + expect(requestBody(fetchMock)).toMatchObject({ + chat_template_kwargs: { + enable_thinking: false, + tokenizer_option: "preserved" + } + }); + expect(requestBody(fetchMock)).not.toHaveProperty("enable_thinking"); + + fetchMock.mockClear(); + await client.complete([{ role: "user", content: "evolve" }], { + operation: "evolution.induction", + thinkingMode: "enabled" + }); + expect(requestBody(fetchMock)).toMatchObject({ + chat_template_kwargs: { enable_thinking: true } + }); + } + ); + + it("keeps chat_template_kwargs opt-in data isolated on the generic OpenAI-compatible provider", async () => { + const fetchMock = openAiFetch(); + vi.stubGlobal("fetch", fetchMock); + const client = createLlmClient(llmConfig({ + provider: "openai_compatible", + vendor: "qwen", + endpoint: "https://cloud-provider.example/v1", + model: "qwen3.8-27b", + extraBody: { chat_template_kwargs: { tokenizer_option: "preserved" } } + })); + + await client.complete([{ role: "user", content: "filter" }], { + operation: "retrieval.filter", + thinkingMode: "disabled" + }); + + expect(requestBody(fetchMock)).toMatchObject({ + enable_thinking: false, + chat_template_kwargs: { tokenizer_option: "preserved" } + }); + }); + it.each([ ["DeepSeek", "deepseek", "https://api.deepseek.com", "deepseek-v4-pro"], ["Zhipu", "zhipu", "https://open.bigmodel.cn/api/paas/v4", "glm-5.1"],