diff --git a/docs/website/content/docs/cli/http-server/index.mdx b/docs/website/content/docs/cli/http-server/index.mdx index f456d9bfd9..6ec2725ca9 100644 --- a/docs/website/content/docs/cli/http-server/index.mdx +++ b/docs/website/content/docs/cli/http-server/index.mdx @@ -516,6 +516,39 @@ curl http://localhost:11434/v1/chat/completions \ A `tools` request for a model loaded without `config.tools: true` returns `400 tools_not_enabled` rather than answering in prose. +**Forcing a tool call** with `tool_choice`: + +```bash +curl http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "my-llm", + "messages": [{"role": "user", "content": "What is the weather in London?"}], + "tools": [{ "type": "function", "function": { "name": "get_weather" } }], + "tool_choice": "required" + }' +``` + +`tool_choice` accepts `"auto"` (the default), `"none"`, `"required"`, or +`{ "type": "function", "function": { "name": "get_weather" } }` to force one +specific tool. On llama.cpp-backed models `required` and a named tool constrain +the sampler with the chat template's tool grammar, so the model calls a tool +instead of answering in prose. + +Both forms need a matching entry in `tools`: a demanding `tool_choice` with no +`tools`, or a name that isn't declared, returns `400 invalid_tool_choice`. A bare +tool name in place of the object form is rejected the same way — use the object +form to target one tool. + +`auto`, `none` and `required` are reserved: a tool carrying one of those names +cannot be targeted through `tool_choice` and the request returns +`400 invalid_tool_choice`. Rename the tool to target it. + +A tool call the model emits that fails to parse or validate is dropped, so the +response comes back with `finish_reason: "stop"` and no `tool_calls`. The server +log records what happened (`toolerrors=1 (PARSE_ERROR)`) — the OpenAI response +shape has no field for it. + #### Message content `messages[].content` accepts both the plain string form and the OpenAI **array-of-parts** form (`[{ "type": "text", "text": "…" }, …]`) that modern clients such as Cline and Open WebUI send. Parts of type `text` are concatenated into a single string; non-text parts (`image_url`, `input_audio`, `file`) are **silently dropped** — the chat surface is text-only and vision is out of scope. Both shapes below are valid: @@ -671,6 +704,8 @@ When generation is truncated because it hit `max_output_tokens` / `max_tokens`, The following Responses-API features are intentionally rejected with `400`: `conversation`, `background: true`, and built-in tools (`web_search`, `file_search`, `code_interpreter`). `function`-typed tools work when the model was loaded with `config.tools: true`; otherwise the request returns `400 tools_not_enabled`. +`tool_choice` is supported in the same shapes as chat completions, except that the object form is the Responses API's flattened `{ "type": "function", "name": "get_weather" }`. + #### `GET /v1/responses/:id` Retrieve a previously stored response by id. diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 2709a4ae84..b6bd5bcb89 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,54 @@ # Changelog +## [0.14.0] + +📦 **NPM:** https://www.npmjs.com/package/@qvac/cli/v/0.14.0 + +QVAC CLI 0.14.0 follows `@qvac/sdk` 0.20.0. `qvac serve` adds `DELETE /qvac/v1/kv_cache`, MiniMax-H3 fields on `/v1/videos`, the 0.8.x TTS load options on `/v1/audio/speech`, and `tool_choice` on `/v1/chat/completions` and `/v1/responses`. `qvac configure` no longer offers the removed diffusion CPU flags, and `qvac verify bundle` looks for TTS host packages next to `@qvac/tts-ggml` instead of under its `prebuilds/` directory. + +Install `@qvac/cli@0.14.0` with `@qvac/sdk@^0.20.0`. Publish this cut after SDK 0.20.0 is on npm. + +## Breaking Changes + +### Diffusion CPU flags in `qvac configure` + +`clip_on_cpu`, `vae_on_cpu`, and `control_net_cpu` are gone from the diffusion schema. Configure prompts for `params_backend`, `backend`, `max_vram`, and `stream_layers` instead. Existing configs that still set the old keys fail validation. + +**Before:** + +```json +{ + "clip_on_cpu": true, + "vae_on_cpu": true, + "control_net_cpu": true +} +``` + +**After:** + +```json +{ + "params_backend": "te=cpu,vae=cpu", + "backend": "controlnet=cpu" +} +``` + +CPU layer streaming: + +```json +{ + "params_backend": "diffusion=cpu", + "max_vram": -1, + "stream_layers": true +} +``` + +## Features + +`qvac serve` mounts `DELETE /qvac/v1/kv_cache` on the default QVAC surface to reclaim automatic KV caches. `/v1/videos` accepts MiniMax-H3 generation params. `/v1/audio/speech` passes through the TTS load options from `@qvac/tts-ggml` 0.8.x. `/v1/chat/completions` and `/v1/responses` accept `tool_choice` (`auto` / `none` / `required` / named tool) and log unparseable tool calls as `toolError` events. + +`qvac verify bundle` accepts `@qvac/tts-ggml` 0.9.x, where the native binary lives in a per-platform package (`@qvac/tts-ggml-darwin-arm64` and siblings) beside the meta package. + ## [0.13.1] 📦 **NPM:** https://www.npmjs.com/package/@qvac/cli/v/0.13.1 diff --git a/packages/cli/NOTICE b/packages/cli/NOTICE index 9ae69fd028..1ef2752971 100644 --- a/packages/cli/NOTICE +++ b/packages/cli/NOTICE @@ -31,29 +31,29 @@ JavaScript Dependencies @qvac/embed-llamacpp@0.37.0 https://github.com/tetherto/qvac @qvac/error@0.1.1 - @qvac/fabric@0.10.0 + @qvac/fabric@0.16.0 https://github.com/tetherto/qvac @qvac/infer-base@0.4.2 https://github.com/tetherto/qvac @qvac/infer-base@0.6.2 https://github.com/tetherto/qvac - @qvac/inference@0.19.0 + @qvac/inference@0.20.0 https://github.com/tetherto/qvac @qvac/langdetect-text@0.1.2 https://github.com/tetherto/qvac - @qvac/llm-llamacpp@0.49.1 + @qvac/llm-llamacpp@0.53.0 https://github.com/tetherto/qvac @qvac/logging@0.1.1 https://github.com/tetherto/qvac @qvac/ocr-ggml@0.21.0 https://github.com/tetherto/qvac - @qvac/rag@0.8.0 + @qvac/rag@0.8.1 https://github.com/tetherto/qvac @qvac/registry-client@0.6.1 https://github.com/tetherto/qvac @qvac/registry-schema@0.3.0 https://github.com/tetherto/qvac - @qvac/sdk@0.19.0 + @qvac/sdk@0.20.0 https://github.com/tetherto/qvac @qvac/translation-nmtcpp@0.13.0 https://github.com/tetherto/qvac diff --git a/packages/cli/README.md b/packages/cli/README.md index bae3c1b160..74da3b755c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -476,7 +476,7 @@ For tests that touch `qvac serve --openai`, `@qvac/ai-sdk-provider`, or agent-to [`test/AGENT_STACK_E2E.md`](./test/AGENT_STACK_E2E.md). It defines which layer owns SDK e2e, CLI contract tests, CLI in-process HTTP e2e, CLI spawned-binary e2e, provider integration, and plugin integration. -The CLI depends on the published `@qvac/sdk` (`^0.17.0`), which provides the +The CLI depends on the published `@qvac/sdk` (`^0.20.0`), which provides the `./commands` subpath that `bundle`/`verify` re-export and the server runtime the `serve` commands use. A normal `npm install` pulls it from the registry — no local SDK build is required. @@ -504,7 +504,7 @@ npm run dev:unlink ``` This runs `git checkout HEAD -- package.json` and re-installs, so the -committed `@qvac/sdk` dependency (`^0.17.0`) is restored regardless of what you +committed `@qvac/sdk` dependency (`^0.20.0`) is restored regardless of what you swapped in locally. `package-lock.json` is gitignored and is regenerated by the trailing `npm install`. diff --git a/packages/cli/changelog/0.14.0/CHANGELOG.md b/packages/cli/changelog/0.14.0/CHANGELOG.md new file mode 100644 index 0000000000..8b34079f0d --- /dev/null +++ b/packages/cli/changelog/0.14.0/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog v0.14.0 + +Release Date: 2026-09-17 + +## ✨ Features + +- Integrate diffusion layer streaming in the SDK. (see PR [#4389](https://github.com/tetherto/qvac/pull/4389)) - See [breaking changes](./breaking.md) + +## 🔌 API + +- Expose KV-cache reclaim over serve. (see PR [#4249](https://github.com/tetherto/qvac/pull/4249)) - See [API changes](./api.md) +- Integrate MiniMax-H3 video generation across inference and SDK. (see PR [#4351](https://github.com/tetherto/qvac/pull/4351)) - See [API changes](./api.md) +- Close the SDK gaps against @qvac/tts-ggml 0.8.x. (see PR [#4414](https://github.com/tetherto/qvac/pull/4414)) - See [API changes](./api.md) +- Update @qvac/tts-ggml to 0.9.1. (see PR [#4428](https://github.com/tetherto/qvac/pull/4428)) - See [API changes](./api.md) +- Accept tool_choice on the serve OpenAI routes. (see PR [#4524](https://github.com/tetherto/qvac/pull/4524)) - See [API changes](./api.md) + +## ⚙️ Infrastructure + +- Typecheck CLI push CI against the in-repo SDK on non-release branches. (see PR [#4470](https://github.com/tetherto/qvac/pull/4470)) diff --git a/packages/cli/changelog/0.14.0/CHANGELOG_LLM.md b/packages/cli/changelog/0.14.0/CHANGELOG_LLM.md new file mode 100644 index 0000000000..46e9ee67bc --- /dev/null +++ b/packages/cli/changelog/0.14.0/CHANGELOG_LLM.md @@ -0,0 +1,48 @@ +# QVAC CLI v0.14.0 Release Notes + +📦 **NPM:** https://www.npmjs.com/package/@qvac/cli/v/0.14.0 + +QVAC CLI 0.14.0 follows `@qvac/sdk` 0.20.0. `qvac serve` adds `DELETE /qvac/v1/kv_cache`, MiniMax-H3 fields on `/v1/videos`, the 0.8.x TTS load options on `/v1/audio/speech`, and `tool_choice` on `/v1/chat/completions` and `/v1/responses`. `qvac configure` no longer offers the removed diffusion CPU flags, and `qvac verify bundle` looks for TTS host packages next to `@qvac/tts-ggml` instead of under its `prebuilds/` directory. + +Install `@qvac/cli@0.14.0` with `@qvac/sdk@^0.20.0`. Publish this cut after SDK 0.20.0 is on npm. + +## Breaking Changes + +### Diffusion CPU flags in `qvac configure` + +`clip_on_cpu`, `vae_on_cpu`, and `control_net_cpu` are gone from the diffusion schema. Configure prompts for `params_backend`, `backend`, `max_vram`, and `stream_layers` instead. Existing configs that still set the old keys fail validation. + +**Before:** + +```json +{ + "clip_on_cpu": true, + "vae_on_cpu": true, + "control_net_cpu": true +} +``` + +**After:** + +```json +{ + "params_backend": "te=cpu,vae=cpu", + "backend": "controlnet=cpu" +} +``` + +CPU layer streaming: + +```json +{ + "params_backend": "diffusion=cpu", + "max_vram": -1, + "stream_layers": true +} +``` + +## Features + +`qvac serve` mounts `DELETE /qvac/v1/kv_cache` on the default QVAC surface to reclaim automatic KV caches. `/v1/videos` accepts MiniMax-H3 generation params. `/v1/audio/speech` passes through the TTS load options from `@qvac/tts-ggml` 0.8.x. `/v1/chat/completions` and `/v1/responses` accept `tool_choice` (`auto` / `none` / `required` / named tool) and log unparseable tool calls as `toolError` events. + +`qvac verify bundle` accepts `@qvac/tts-ggml` 0.9.x, where the native binary lives in a per-platform package (`@qvac/tts-ggml-darwin-arm64` and siblings) beside the meta package. diff --git a/packages/cli/changelog/0.14.0/api.md b/packages/cli/changelog/0.14.0/api.md new file mode 100644 index 0000000000..10650bd40d --- /dev/null +++ b/packages/cli/changelog/0.14.0/api.md @@ -0,0 +1,73 @@ +# 🔌 API Changes v0.14.0 + +## Expose KV-cache reclaim over serve + +PR: [#4249](https://github.com/tetherto/qvac/pull/4249) + +```bash +curl -X DELETE http://localhost:11434/qvac/v1/kv_cache +``` + +```json +{ "object": "kv_cache.reclaim", "deleted": true } +``` + +--- + +## Integrate MiniMax-H3 video generation across inference and SDK + +PR: [#4351](https://github.com/tetherto/qvac/pull/4351) + +`POST /v1/videos` accepts MiniMax-H3 fields on the OpenAI-shaped video job (H3 text-encoder / video VAE / audio VAE sources via `serve.models`). Poll `GET /v1/videos/{id}` then fetch bytes from `GET /v1/videos/{id}/content`. + +--- + +## Close the SDK gaps against @qvac/tts-ggml 0.8.x + +PR: [#4414](https://github.com/tetherto/qvac/pull/4414) + +`POST /v1/audio/speech` passes through the TTS load options that 0.8.x exposed (CosyVoice3, Chatterbox, Supertonic, Parler). + +--- + +## Update @qvac/tts-ggml to 0.9.1 + +PR: [#4428](https://github.com/tetherto/qvac/pull/4428) + +```bash +# tts-ggml 0.9.0 installs the host's binaries next to the meta package +node_modules/@qvac/tts-ggml/ # JavaScript only: addon: true, no prebuilds/ +node_modules/@qvac/tts-ggml-darwin-arm64/ # os/cpu filtered optionalDependency + addon/package.json # { "name": "@qvac/tts-ggml", "addon": true } + addon/prebuilds/darwin-arm64/qvac__tts-ggml.bare + +# Passes on this branch; on main it reports missing-prebuild for every host +qvac verify bundle --addons-source ./node_modules --host darwin-arm64 +``` + +```text +@qvac/tts-ggml@0.9.0 is missing a prebuild for linux-x64 +(expected …/@qvac/tts-ggml/prebuilds/linux-x64/*.bare). +No per-platform package @qvac/tts-ggml-linux-x64 is installed alongside it either. +``` + +--- + +## Accept tool_choice on the serve OpenAI routes + +PR: [#4524](https://github.com/tetherto/qvac/pull/4524) + +`POST /v1/chat/completions` and `POST /v1/responses` accept `tool_choice`: `"auto"` | `"none"` | `"required"` | a named-tool object. Chat uses `{ type: "function", function: { name } }`; Responses flattens to `{ type: "function", name }`. A demanding choice with no matching tool returns `400 invalid_tool_choice`. Parse failures are dropped from the OpenAI response and logged as `toolError` events. + +```bash +curl http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "my-llm", + "messages": [{"role": "user", "content": "What is the weather in London?"}], + "tools": [{ "type": "function", "function": { "name": "get_weather" } }], + "tool_choice": "required" + }' +``` + +--- diff --git a/packages/cli/changelog/0.14.0/breaking.md b/packages/cli/changelog/0.14.0/breaking.md new file mode 100644 index 0000000000..3a2ff5b9c9 --- /dev/null +++ b/packages/cli/changelog/0.14.0/breaking.md @@ -0,0 +1,36 @@ +# 💥 Breaking Changes v0.14.0 + +## Integrate diffusion layer streaming in the SDK + +PR: [#4389](https://github.com/tetherto/qvac/pull/4389) + +**BEFORE:** + +```typescript +const modelConfig = { + clip_on_cpu: true, + vae_on_cpu: true, + control_net_cpu: true +} +``` + +**AFTER:** + +```typescript +const modelConfig = { + params_backend: 'te=cpu,vae=cpu', + backend: 'controlnet=cpu' +} +``` + +To run the text encoder or VAE graph on CPU, add `te=cpu` or `vae=cpu` to `backend`. + +CPU layer streaming requires CPU diffusion parameter residency and graph cutting enabled by `max_vram`: + +```typescript +const modelConfig = { + params_backend: 'diffusion=cpu', + max_vram: -1, + stream_layers: true +} +``` diff --git a/packages/cli/package.json b/packages/cli/package.json index 4696fb6498..97ec785ef8 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@qvac/cli", - "version": "0.13.1", + "version": "0.14.0", "description": "Command-line interface for the QVAC ecosystem", "author": "Tether", "license": "Apache-2.0", @@ -59,7 +59,7 @@ "@fastify/swagger": "^9.0.0", "@fastify/swagger-ui": "^6.1.1", "@inquirer/prompts": "8.5.2", - "@qvac/sdk": "^0.19.0", + "@qvac/sdk": "^0.20.0", "close-with-grace": "^2.1.0", "commander": "^14.0.3", "fastify": "^5.0.0", diff --git a/packages/cli/src/serve/extensions/openai/adapters/completion-result.ts b/packages/cli/src/serve/extensions/openai/adapters/completion-result.ts index e7ca00aec7..ce48090af5 100644 --- a/packages/cli/src/serve/extensions/openai/adapters/completion-result.ts +++ b/packages/cli/src/serve/extensions/openai/adapters/completion-result.ts @@ -1,4 +1,4 @@ -import type { CompletionRun, CompletionStats, ToolCall } from '@qvac/sdk' +import type { CompletionRun, CompletionStats, ToolCall, ToolCallError } from '@qvac/sdk' import { HttpError } from '@/serve/lib/http-error' export type OpenAiFinishReason = 'stop' | 'length' | 'tool_calls' @@ -8,6 +8,13 @@ export interface DrainedCompletion { /** Concatenated `thinkingDelta` text; empty when the SDK captured no reasoning. */ thinking: string toolCalls: ToolCall[] + /** + * Tool-call regions the addon emitted but could not parse or validate. OpenAI + * has no field for these, so routes log them rather than returning them -- + * without that line an empty `stop` response looks like the model simply + * chose not to call a tool. + */ + toolErrors: ToolCallError[] stats: CompletionStats | undefined /** * Terminal reason from the SDK `completionDone` event (`eos` / `length` / @@ -46,6 +53,7 @@ export async function drainCompletion( let text = '' let thinking = '' const toolCalls: ToolCall[] = [] + const toolErrors: ToolCallError[] = [] let stats: CompletionStats | undefined let stopReason: string | undefined @@ -58,6 +66,8 @@ export async function drainCompletion( onThinking?.(event.text) } else if (event.type === 'toolCall') { toolCalls.push(event.call) + } else if (event.type === 'toolError') { + toolErrors.push(event.error) } else if (event.type === 'completionStats') { stats = event.stats } else if (event.type === 'completionDone') { @@ -78,7 +88,29 @@ export async function drainCompletion( const finishReason: OpenAiFinishReason = toolCalls.length > 0 ? 'tool_calls' : stopReason === 'length' ? 'length' : 'stop' - return { text, thinking, toolCalls, stats, stopReason, completionTokens, finishReason } + return { + text, + thinking, + toolCalls, + toolErrors, + stats, + stopReason, + completionTokens, + finishReason + } +} + +/** + * Render drained tool-call failures for the request log: a count plus each + * distinct error code, e.g. ` toolerrors=2 (PARSE_ERROR)`. Empty string when + * the run produced none, so it appends cleanly to an existing log line. + */ +export function formatToolErrors(toolErrors: ToolCallError[]): string { + if (toolErrors.length === 0) { + return '' + } + const codes = [...new Set(toolErrors.map((err) => err.code))].join(',') + return ` toolerrors=${toolErrors.length} (${codes})` } /** diff --git a/packages/cli/src/serve/extensions/openai/adapters/response-writers.ts b/packages/cli/src/serve/extensions/openai/adapters/response-writers.ts index e0522c05d8..837487e33d 100644 --- a/packages/cli/src/serve/extensions/openai/adapters/response-writers.ts +++ b/packages/cli/src/serve/extensions/openai/adapters/response-writers.ts @@ -1,7 +1,10 @@ import type { ServerResponse } from 'node:http' import type { CompletionRun, Tool } from '@qvac/sdk' import { sendSSE, endSSE } from '@/serve/lib/sse' -import { drainCompletion } from '@/serve/extensions/openai/adapters/completion-result' +import { + drainCompletion, + formatToolErrors +} from '@/serve/extensions/openai/adapters/completion-result' import { sdkToolCallsToOpenai } from '@/serve/extensions/openai/adapters/tool-calls' import type { GenerationParams, ResponseFormat } from '@/serve/extensions/openai/schemas/common' import { @@ -50,7 +53,8 @@ export async function writeBlockingResponse( p: ResponsesHandlerParams, result: CompletionRun ): Promise> { - const { text, toolCalls, stats, stopReason, completionTokens } = await drainCompletion(result) + const { text, toolCalls, toolErrors, stats, stopReason, completionTokens } = + await drainCompletion(result) const responseObject = buildResponseObject({ id: p.rid, @@ -82,7 +86,9 @@ export async function writeBlockingResponse( p.ctx.responsesStore.put(rec) } - p.ctx.logger.info(` responses done id=${p.rid} stored=${p.storeEnabled}`) + p.ctx.logger.info( + ` responses done id=${p.rid} stored=${p.storeEnabled}${formatToolErrors(toolErrors)}` + ) if (!res.headersSent) { const payload = JSON.stringify(responseObject) @@ -130,7 +136,7 @@ export async function writeStreamingResponse( response_id: p.rid }) - const { toolCalls, stats, stopReason, completionTokens } = await drainCompletion( + const { toolCalls, toolErrors, stats, stopReason, completionTokens } = await drainCompletion( result, (token) => { fullText += token @@ -263,6 +269,8 @@ export async function writeStreamingResponse( responseObject['status'] === 'incomplete' ? 'response.incomplete' : 'response.completed' sendSSE(res, { type: terminalType, response: responseObject }) endSSE(res, { sentinel: false }) - p.ctx.logger.info(` responses stream done id=${p.rid} stored=${p.storeEnabled}`) + p.ctx.logger.info( + ` responses stream done id=${p.rid} stored=${p.storeEnabled}${formatToolErrors(toolErrors)}` + ) return responseObject } diff --git a/packages/cli/src/serve/extensions/openai/routes/chat.ts b/packages/cli/src/serve/extensions/openai/routes/chat.ts index 3f977e5e1f..f1af8124e3 100644 --- a/packages/cli/src/serve/extensions/openai/routes/chat.ts +++ b/packages/cli/src/serve/extensions/openai/routes/chat.ts @@ -6,6 +6,7 @@ import { HttpError } from '@/serve/lib/http-error' import { initSSE, sendSSE, endSSE } from '@/serve/lib/sse' import { drainCompletion, + formatToolErrors, type OpenAiFinishReason } from '@/serve/extensions/openai/adapters/completion-result' import { requireModel } from '@/serve/core/plugins/require-model' @@ -22,9 +23,11 @@ import { import { resolveToolDialect } from '@/serve/lib/tool-dialect' import { InvalidResponseFormatError, + InvalidToolChoiceError, UnsupportedImageContentError } from '@/serve/extensions/openai/schemas/common' import { sdkToolCallsToOpenaiDeltas } from '@/serve/extensions/openai/adapters/tool-calls' +import { openaiState } from '@/serve/extensions/openai/state' import { buildUsage, chatCompletionChunk, @@ -59,6 +62,9 @@ async function prepare( if (err instanceof InvalidResponseFormatError) { throw new HttpError(400, 'invalid_response_format', err.message) } + if (err instanceof InvalidToolChoiceError) { + throw new HttpError(400, 'invalid_tool_choice', err.message) + } if (err instanceof UnsupportedImageContentError) { throw new HttpError(400, 'unsupported_image_content', err.message) } @@ -121,8 +127,8 @@ function formatStats(stats: CompletionStats | undefined): string { const descriptions = { completion: ` OpenAI-compatible chat completion. Accepts a chat-style \`messages\` array, -optional \`tools\` for function-calling, and an optional \`response_format\` -(\`text\` / \`json_object\` / \`json_schema\`). +optional \`tools\` for function-calling, an optional \`tool_choice\`, and an +optional \`response_format\` (\`text\` / \`json_object\` / \`json_schema\`). **Streaming**: pass \`stream: true\` to receive Server-Sent Events. The stream ends with \`data: [DONE]\\n\\n\` (OpenAI compatibility). @@ -132,6 +138,17 @@ ends with \`data: [DONE]\\n\\n\` (OpenAI compatibility). \`invalid_response_format\`. A \`tools\` request for a model loaded without \`config.tools: true\` is rejected with \`tools_not_enabled\`. +**\`tool_choice\`**: \`"auto"\` (default), \`"none"\`, \`"required"\`, or +\`{ type: 'function', function: { name } }\` to force one tool. \`required\` and +a named tool constrain generation with the chat template's tool grammar. Both +need a matching entry in \`tools\`; anything else — including a bare tool name +in place of the object form — is rejected with \`invalid_tool_choice\`. + +**Unparseable tool calls**: a tool call the model emits but that fails to parse +or validate is dropped, so the response carries \`finish_reason: "stop"\` and no +\`tool_calls\`. The server log records the count and error codes +(\`toolerrors=N (PARSE_ERROR)\`); OpenAI has no response field for them. + **Ignored params** (warned, not rejected): \`logit_bias\`, \`n\`, \`user\`, \`seed\`, \`logprobs\`, \`top_logprobs\`, \`frequency_penalty\`, \`presence_penalty\`, \`stop\`. @@ -195,7 +212,8 @@ async function runBlocking( ): Promise { const { history, tmpPaths } = await writeChatImages(p.history) try { - const result = completion({ + const completionFn = openaiState(req.server.qvac).completionOverride ?? completion + const result = completionFn({ modelId: p.sdkModelId, history, stream: false, @@ -211,11 +229,12 @@ async function runBlocking( }) req.bindCancel(result.requestId) - const { text, thinking, toolCalls, stats, completionTokens, finishReason } = + const { text, thinking, toolCalls, toolErrors, stats, completionTokens, finishReason } = await drainCompletion(result) req.server.qvac.logger.info( - ` completion done tokens=${completionTokens} finish=${finishReason}${formatStats(stats)}` + ` completion done tokens=${completionTokens} finish=${finishReason}` + + `${formatStats(stats)}${formatToolErrors(toolErrors)}` ) reply.send( @@ -245,7 +264,8 @@ async function runStreaming( ): Promise { const { history, tmpPaths } = await writeChatImages(p.history) try { - const result = completion({ + const completionFn = openaiState(req.server.qvac).completionOverride ?? completion + const result = completionFn({ modelId: p.sdkModelId, history, stream: true, @@ -275,7 +295,7 @@ async function runStreaming( sendSSE(raw, chunk({ role: 'assistant', content: '' }, null)) - const { toolCalls, stats, completionTokens, finishReason } = await drainCompletion( + const { toolCalls, toolErrors, stats, completionTokens, finishReason } = await drainCompletion( result, (token) => sendSSE(raw, chunk({ content: token }, null)), (token) => sendSSE(raw, chunk({ reasoning_content: token }, null)) @@ -283,7 +303,8 @@ async function runStreaming( const hasToolCalls = toolCalls.length > 0 req.server.qvac.logger.info( - ` streaming done tokens=${completionTokens} finish=${finishReason}${formatStats(stats)}` + ` streaming done tokens=${completionTokens} finish=${finishReason}` + + `${formatStats(stats)}${formatToolErrors(toolErrors)}` ) if (hasToolCalls) { diff --git a/packages/cli/src/serve/extensions/openai/routes/responses.ts b/packages/cli/src/serve/extensions/openai/routes/responses.ts index a7a4fcc072..747037be33 100644 --- a/packages/cli/src/serve/extensions/openai/routes/responses.ts +++ b/packages/cli/src/serve/extensions/openai/routes/responses.ts @@ -20,6 +20,7 @@ import { } from '@/serve/extensions/openai/schemas/responses' import { InvalidResponseFormatError, + InvalidToolChoiceError, type GenerationParams, type ResponseFormat } from '@/serve/extensions/openai/schemas/common' @@ -75,6 +76,11 @@ addressable via GET / DELETE / input_items. - \`tools[].type\` other than \`function\` (e.g. \`web_search\`, \`file_search\`, \`code_interpreter\`) → \`400 invalid_tool_type\` - structured output (\`json_object\`/\`json_schema\`) combined with non-empty \`tools\` → \`400 invalid_response_format\` - \`tools\` on a model loaded without \`config.tools: true\` → \`400 tools_not_enabled\` +- \`tool_choice\` demanding a tool with no matching \`tools\` entry → \`400 invalid_tool_choice\` + +**\`tool_choice\`**: \`"auto"\` (default), \`"none"\`, \`"required"\`, or +\`{ type: 'function', name }\` to force one tool. \`required\` and a named tool +constrain generation with the chat template's tool grammar. **Streaming** (\`stream: true\`) emits the OpenAI Responses SSE event sequence (\`response.created\` → \`response.output_text.delta\` … → \`response.completed\`) @@ -135,6 +141,9 @@ const plugin: FastifyPluginAsyncZod = async (app) => { if (err instanceof InvalidResponseFormatError) { throw new HttpError(400, 'invalid_response_format', err.message) } + if (err instanceof InvalidToolChoiceError) { + throw new HttpError(400, 'invalid_tool_choice', err.message) + } throw err } @@ -212,8 +221,10 @@ const plugin: FastifyPluginAsyncZod = async (app) => { previousResponseId: params.previousResponseId } + const completionFn = openaiState(req.server.qvac).completionOverride ?? completion + if (streaming) { - const result = completion({ + const result = completionFn({ modelId: params.sdkModelId, history: params.history, stream: true, @@ -227,7 +238,7 @@ const plugin: FastifyPluginAsyncZod = async (app) => { initSSE(reply, { [VOLATILE_HEADER]: RESPONSES_VOLATILE_STUB }) await writeStreamingResponse(reply.raw, writerParams, result) } else { - const result = completion({ + const result = completionFn({ modelId: params.sdkModelId, history: params.history, stream: false, diff --git a/packages/cli/src/serve/extensions/openai/schemas/chat.ts b/packages/cli/src/serve/extensions/openai/schemas/chat.ts index 54472d9396..5f0b23f4a8 100644 --- a/packages/cli/src/serve/extensions/openai/schemas/chat.ts +++ b/packages/cli/src/serve/extensions/openai/schemas/chat.ts @@ -9,8 +9,11 @@ import { responseFormat, toolDef, openaiToolsToSdk, + toolChoice, extractGenerationParams, extractResponseFormat, + extractToolChoice, + withToolChoice, UnsupportedImageContentError, type GenerationParams, type ResponseFormat, @@ -23,6 +26,7 @@ export const chatCompletionsBody = z messages: z.array(chatMessage), stream: z.boolean().optional(), tools: z.array(toolDef).optional(), + tool_choice: toolChoice.optional(), response_format: responseFormat.optional(), temperature: z.number().optional(), top_p: z.number().optional(), @@ -270,12 +274,13 @@ export interface SdkChatArgs { export function toSdkChatArgs(body: ChatCompletionsBody, dialect: ToolDialect): SdkChatArgs { const responseFmt = extractResponseFormat(body as Record) + const tools = openaiToolsToSdk(body.tools as Parameters[0]) return { history: openaiMessagesToHistory(body.messages as OpenAIMessage[], dialect), - tools: openaiToolsToSdk(body.tools as Parameters[0]), - generationParams: extractGenerationParams( - body as Record, - 'max_completion_tokens' + tools, + generationParams: withToolChoice( + extractGenerationParams(body as Record, 'max_completion_tokens'), + extractToolChoice(body as Record, tools) ), responseFormat: responseFmt, stream: Boolean(body.stream) diff --git a/packages/cli/src/serve/extensions/openai/schemas/common.ts b/packages/cli/src/serve/extensions/openai/schemas/common.ts index 3a3e69fb52..d4e9376d23 100644 --- a/packages/cli/src/serve/extensions/openai/schemas/common.ts +++ b/packages/cli/src/serve/extensions/openai/schemas/common.ts @@ -92,6 +92,8 @@ export interface GenerationParams { repeat_penalty?: number reasoning_budget?: -1 | 0 remove_thinking_from_context?: boolean + /** `auto` | `none` | `required` | a declared tool's name. */ + tool_choice?: string } export type ResponseFormat = @@ -176,6 +178,118 @@ export class UnsupportedImageContentError extends Error { } } +export class InvalidToolChoiceError extends Error { + constructor(message: string) { + super(message) + this.name = 'InvalidToolChoiceError' + } +} + +const TOOL_CHOICE_MODES = new Set(['auto', 'none', 'required']) + +/** + * OpenAI `tool_choice` accepts either a mode string or an object naming one + * function. Chat nests the name under `function`; Responses flattens it onto + * the object itself. Both collapse to the bare name the SDK takes. + */ +export const toolChoice = z.union([ + z.string(), + z + .object({ + type: z.string(), + function: z.object({ name: z.string() }).passthrough().optional(), + name: z.string().optional() + }) + .passthrough() +]) + +/** + * Translate OpenAI `tool_choice` into the SDK's string form, rejecting what + * the SDK would reject anyway so the caller gets a 400 instead of a 500 out + * of `completion()`. + */ +export function extractToolChoice( + body: Record, + tools: Tool[] | undefined +): string | undefined { + const raw = body['tool_choice'] + if (raw === undefined || raw === null) return undefined + + const choice = toolChoiceToSdk(raw) + + if (choice === 'none' || choice === 'auto') return choice + + if (!tools || tools.length === 0) { + throw new InvalidToolChoiceError( + `"tool_choice" ${JSON.stringify(choice)} requires at least one entry in "tools".` + ) + } + if (choice !== 'required' && !tools.some((tool) => tool.name === choice)) { + throw new InvalidToolChoiceError( + `"tool_choice" names ${JSON.stringify(choice)}, which is not one of the declared tools.` + ) + } + return choice +} + +/** + * Fold a resolved `tool_choice` into the generation params, which are + * `undefined` when the request set none of the other knobs. + */ +export function withToolChoice( + params: GenerationParams | undefined, + choice: string | undefined +): GenerationParams | undefined { + if (choice === undefined) return params + return { ...(params ?? {}), tool_choice: choice } +} + +function toolChoiceToSdk(raw: unknown): string { + if (typeof raw === 'string') { + if (TOOL_CHOICE_MODES.has(raw)) return raw + throw new InvalidToolChoiceError( + `"tool_choice" must be "auto", "none", "required", or an object naming a function ` + + `(got ${JSON.stringify(raw)}).` + ) + } + + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new InvalidToolChoiceError('"tool_choice" must be a string or an object.') + } + + const obj = raw as Record + if (obj['type'] !== 'function') { + throw new InvalidToolChoiceError( + `"tool_choice.type" must be "function" (got ${JSON.stringify(obj['type'])}).` + ) + } + + // Chat: { type, function: { name } }. Responses: { type, name }. + const nested = obj['function'] + const nestedName = + typeof nested === 'object' && nested !== null && !Array.isArray(nested) + ? (nested as Record)['name'] + : undefined + const name = typeof nestedName === 'string' ? nestedName : obj['name'] + + if (typeof name !== 'string' || name.length === 0) { + throw new InvalidToolChoiceError( + '"tool_choice" must carry a non-empty function name ("function.name" or "name").' + ) + } + // The SDK encodes mode and target in one string, so a tool actually named + // `auto`/`none`/`required` would read back as the mode and quietly invert the + // request -- targeting `none` would disable tool calling. The object form is + // unambiguous here and nowhere downstream, so the collision is caught here. + if (TOOL_CHOICE_MODES.has(name)) { + throw new InvalidToolChoiceError( + `"tool_choice" cannot target a tool named ${JSON.stringify(name)}: the name is reserved ` + + `for the "auto" / "none" / "required" modes. Rename the tool to target it.` + ) + } + return name +} + export function extractResponseFormat(body: Record): ResponseFormat | undefined { const raw = body['response_format'] if (raw === undefined || raw === null) return undefined diff --git a/packages/cli/src/serve/extensions/openai/schemas/responses.ts b/packages/cli/src/serve/extensions/openai/schemas/responses.ts index 79e0a0301d..556da47e3b 100644 --- a/packages/cli/src/serve/extensions/openai/schemas/responses.ts +++ b/packages/cli/src/serve/extensions/openai/schemas/responses.ts @@ -4,9 +4,12 @@ import type { Tool } from '@qvac/sdk' import { responseFormat, toolDef, + toolChoice, normalizeToolParameters, extractResponseFormat, extractGenerationParams, + extractToolChoice, + withToolChoice, type GenerationParams, type ResponseFormat } from '@/serve/extensions/openai/schemas/common' @@ -22,6 +25,7 @@ export const responsesBody = z conversation: z.unknown().optional(), background: z.boolean().optional(), tools: z.array(toolDef).optional(), + tool_choice: toolChoice.optional(), text: z.unknown().optional(), response_format: responseFormat.optional(), temperature: z.number().optional(), @@ -424,7 +428,10 @@ export function toSdkResponsesArgs(body: ResponsesBody): SdkResponsesArgs { return { history, tools, - generationParams: extractGenerationParams(body as Record, 'max_output_tokens'), + generationParams: withToolChoice( + extractGenerationParams(body as Record, 'max_output_tokens'), + extractToolChoice(body as Record, tools) + ), responseFormat: responseFmt, storeEnabled, previousResponseId, diff --git a/packages/cli/src/serve/extensions/openai/state.ts b/packages/cli/src/serve/extensions/openai/state.ts index d82388a2e6..ae78796026 100644 --- a/packages/cli/src/serve/extensions/openai/state.ts +++ b/packages/cli/src/serve/extensions/openai/state.ts @@ -25,6 +25,8 @@ export interface OpenAIState { transcribeOverride?: ( opts: Parameters[0] ) => Promise & { requestId: string } + /** Test seam — overrides `completion()` from `@qvac/sdk` when set. */ + completionOverride?: typeof sdk.completion /** Test seam — overrides `video()` from `@qvac/sdk` when set. */ videoOverride?: typeof sdk.video /** Test seam — overrides `cancel()` from `@qvac/sdk` when set. */ diff --git a/packages/cli/test/e2e/model/real-model.test.ts b/packages/cli/test/e2e/model/real-model.test.ts index 1b1e679c97..e5ca12608c 100644 --- a/packages/cli/test/e2e/model/real-model.test.ts +++ b/packages/cli/test/e2e/model/real-model.test.ts @@ -320,6 +320,43 @@ describe('chat completions (tools / structured output)', () => { assert.ok(['stop', 'tool_calls', 'length'].includes(body.choices[0].finish_reason)) }) + // What this case is for: a real run puts `tool_choice` through the SDK's + // strict generationParams schema and its tools refinement, which a stubbed + // `completion()` cannot reach. The 200 is the assertion that carries that. + // + // Whether the sampler then lands a parseable call is not pinned here. On this + // shared server the kv cache already holds turns rendered with thinking on, + // and this request turns it off; against a stale prefix the model spends the + // budget on repeated fragments and finishes on `length` (the reply comes back + // reporting more cached tokens than prompt tokens). Grammar behaviour is + // covered deterministically by the addon's own tool-calling integration test. + it('honours tool_choice required end to end', async () => { + const res = await post('/v1/chat/completions', { + model: E2E.llm, + messages: [{ role: 'user', content: 'Tell me the current conditions in Oslo.' }], + max_tokens: 128, + reasoning_budget: false, + tool_choice: 'required', + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get weather', + parameters: { type: 'object', properties: { city: { type: 'string' } } } + } + } + ] + }) + assert.equal(res.statusCode, 200, res.payload) + const body = res.json() as any + assert.ok(['stop', 'tool_calls', 'length'].includes(body.choices[0].finish_reason), res.payload) + const calls = body.choices[0].message.tool_calls + if (calls !== undefined) { + assert.equal(calls[0].function.name, 'get_weather') + } + }) + // A follow-up turn replays a prior assistant tool call as history. The server // re-renders it in the model's own dialect (resolved via getLoadedModelInfo); // rendering it in a foreign dialect made the model emit a malformed tool frame diff --git a/packages/cli/test/e2e/openai/tool-choice.test.ts b/packages/cli/test/e2e/openai/tool-choice.test.ts new file mode 100644 index 0000000000..f220830886 --- /dev/null +++ b/packages/cli/test/e2e/openai/tool-choice.test.ts @@ -0,0 +1,492 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import type { CompletionRun, ToolCall, ToolCallError } from '@qvac/sdk' +import { createServer } from '../helpers/server.js' +import { openaiState } from '@/serve/extensions/openai/state' +import { JSON_HEADERS, assertStatusAndError, collectSSE } from '../helpers/http.js' + +const CHAT_TOOLS = [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get weather', + parameters: { type: 'object', properties: { city: { type: 'string' } } } + } + } +] + +const RESPONSES_TOOLS = [ + { + type: 'function', + name: 'get_weather', + description: 'Get weather', + parameters: { type: 'object', properties: { city: { type: 'string' } } } + } +] + +// tools is on because the success cases below need it. Ordering already keeps +// the two gates apart: `toSdkChatArgs` throws before `assertToolsEnabled` runs, +// on both routes, so `invalid_tool_choice` wins either way. +const CONFIG = { + serve: { + models: { + 'test-llm': { + model: 'QWEN3_600M_INST_Q4', + preload: false, + config: { ctx_size: 2048, tools: true } + } + } + } +} + +function server(t: Parameters[0]) { + return createServer(t, { + config: CONFIG, + loadModelOverride: () => Promise.resolve('mock-model-id') + }) +} + +describe('serve: tool_choice rejections', () => { + it( + 'chat: a demanding tool_choice with no tools returns 400 invalid_tool_choice', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tool_choice: 'required' + } + }) + assertStatusAndError(res, 400, 'invalid_tool_choice') + } + ) + + it( + 'chat: a tool_choice naming an undeclared tool returns 400 invalid_tool_choice', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS, + tool_choice: { type: 'function', function: { name: 'get_stock' } } + } + }) + assertStatusAndError(res, 400, 'invalid_tool_choice') + } + ) + + // A bare name is not OpenAI's way to force a tool even when it matches a + // declared one -- the object form is. + it( + 'chat: a bare tool name in place of the object form returns 400 invalid_tool_choice', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS, + tool_choice: 'get_weather' + } + }) + assertStatusAndError(res, 400, 'invalid_tool_choice') + } + ) + + it( + 'responses: a demanding tool_choice with no tools returns 400 invalid_tool_choice', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + const res = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + input: 'What is the weather in Lagos?', + tool_choice: 'required' + } + }) + assertStatusAndError(res, 400, 'invalid_tool_choice') + } + ) + + it( + 'responses: an undeclared name in the flattened form returns 400 invalid_tool_choice', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + const res = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + input: 'What is the weather in Lagos?', + tools: RESPONSES_TOOLS, + tool_choice: { type: 'function', name: 'get_stock' } + } + }) + assertStatusAndError(res, 400, 'invalid_tool_choice') + } + ) +}) + +// Hands back a finished run so the assertions are about what serve passed the +// SDK and what it made of the answer -- no model is loaded. +function stubRun(opts: { text?: string; toolCalls?: ToolCall[]; toolErrors?: ToolCallError[] }) { + async function* events(): AsyncGenerator { + let seq = 0 + if (opts.text !== undefined) yield { type: 'contentDelta', seq: seq++, text: opts.text } + for (const call of opts.toolCalls ?? []) yield { type: 'toolCall', seq: seq++, call } + for (const error of opts.toolErrors ?? []) yield { type: 'toolError', seq: seq++, error } + yield { type: 'completionStats', seq: seq++, stats: { emittedTokens: 3 } } + yield { type: 'completionDone', seq: seq++, stopReason: 'eos' } + } + return { + requestId: 'tool-choice-request', + events: events(), + final: Promise.resolve(undefined), + text: Promise.resolve(opts.text ?? ''), + toolCalls: Promise.resolve(opts.toolCalls ?? []), + stats: Promise.resolve(undefined), + tokenStream: (async function* () {})(), + toolCallStream: (async function* () {})() + } as unknown as CompletionRun +} + +// `completion()` is overloaded, so the override's inferred parameter is the +// narrowest form. These are the fields the assertions read. +interface SeenRequest { + tools?: { name: string }[] + generationParams?: { tool_choice?: string } +} + +interface ChatChunk { + choices?: { + delta?: { tool_calls?: { function?: { name?: string } }[] } + finish_reason?: string | null + }[] +} + +const WEATHER_CALL: ToolCall = { + id: 'call_1', + name: 'get_weather', + arguments: { city: 'Lagos' } +} + +describe('serve: tool_choice success path', () => { + it( + 'chat: required reaches the SDK and the tool call comes back on the wire', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ toolCalls: [WEATHER_CALL] }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS, + tool_choice: 'required' + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'required') + assert.equal(seen[0]?.tools?.length, 1) + + const body = res.json() + assert.equal(body.choices[0].finish_reason, 'tool_calls') + assert.equal(body.choices[0].message.tool_calls[0].function.name, 'get_weather') + } + ) + + it( + 'chat: the object form arrives at the SDK as the bare tool name', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ toolCalls: [WEATHER_CALL] }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS, + tool_choice: { type: 'function', function: { name: 'get_weather' } } + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'get_weather') + } + ) + + it('chat: none reaches the SDK with no tools declared', { timeout: 15000 }, async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ text: 'It is warm.' }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tool_choice: 'none' + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'none') + assert.equal(res.json().choices[0].finish_reason, 'stop') + }) + + it('chat: omitting tool_choice leaves it off the SDK request', { timeout: 15000 }, async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ text: 'It is warm.' }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, undefined) + }) + + // The documented shape for a forced call the addon could not parse: still a + // 200, no tool_calls, and finish_reason 'stop'. + it( + 'chat: a run of only tool errors answers 200 with no tool calls', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + openaiState(app.qvac).completionOverride = () => + stubRun({ + toolErrors: [{ code: 'PARSE_ERROR', message: 'bad json', raw: '{' }] + }) + + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS, + tool_choice: 'required' + } + }) + + assert.equal(res.statusCode, 200, res.payload) + const choice = res.json().choices[0] + assert.equal(choice.finish_reason, 'stop') + assert.equal(choice.message.tool_calls, undefined) + } + ) + + it( + 'responses: the flattened object form arrives as the bare tool name', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ toolCalls: [WEATHER_CALL] }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + input: 'What is the weather in Lagos?', + tools: RESPONSES_TOOLS, + tool_choice: { type: 'function', name: 'get_weather' } + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'get_weather') + } + ) + + // The mode string and the object form take different branches through + // extractToolChoice, so the object-form case above does not cover this. + it( + 'responses: required reaches the SDK and the call renders as a function_call item', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ toolCalls: [WEATHER_CALL] }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + input: 'What is the weather in Lagos?', + tools: RESPONSES_TOOLS, + tool_choice: 'required' + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'required') + + const body = res.json<{ output: { type: string; name?: string }[] }>() + const call = body.output.find((item) => item.type === 'function_call') + assert.ok(call, `no function_call item in ${res.payload}`) + assert.equal(call.name, 'get_weather') + } + ) +}) + +// runStreaming and the streaming branch of the responses route each build their +// own argument object for completionFn, so the blocking cases above say nothing +// about them -- and streaming is how agent clients call these routes. +describe('serve: tool_choice on the streaming path', () => { + it( + 'chat: required reaches the SDK and the tool call arrives as deltas', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ toolCalls: [WEATHER_CALL] }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + messages: [{ role: 'user', content: 'What is the weather in Lagos?' }], + tools: CHAT_TOOLS, + tool_choice: 'required', + stream: true + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'required') + + const events = collectSSE(res.payload) + const chunks = events + .map((e) => e.data) + .filter((d): d is ChatChunk => d !== '[DONE]' && typeof d === 'object' && d !== null) + const withCalls = chunks.find((c) => c.choices?.[0]?.delta?.tool_calls !== undefined) + assert.ok(withCalls, `no tool_calls delta in ${res.payload}`) + assert.equal(withCalls.choices?.[0]?.delta?.tool_calls?.[0]?.function?.name, 'get_weather') + assert.ok( + chunks.some((c) => c.choices?.[0]?.finish_reason === 'tool_calls'), + `no tool_calls finish_reason in ${res.payload}` + ) + } + ) + + it( + 'responses: required reaches the SDK on the streaming branch', + { timeout: 15000 }, + async (t) => { + const app = await server(t) + await app.ready() + const seen: SeenRequest[] = [] + openaiState(app.qvac).completionOverride = (params) => { + seen.push(params as SeenRequest) + return stubRun({ toolCalls: [WEATHER_CALL] }) + } + + const res = await app.inject({ + method: 'POST', + url: '/v1/responses', + headers: JSON_HEADERS, + payload: { + model: 'test-llm', + input: 'What is the weather in Lagos?', + tools: RESPONSES_TOOLS, + tool_choice: 'required', + stream: true + } + }) + + assert.equal(res.statusCode, 200, res.payload) + assert.equal(seen[0]?.generationParams?.tool_choice, 'required') + + // Responses names its events in the JSON payload, not an SSE `event:` line. + const types = collectSSE(res.payload) + .map((e) => e.data) + .filter((d): d is { type: string } => typeof d === 'object' && d !== null) + .map((d) => d.type) + assert.ok(types.includes('response.completed'), `no response.completed in ${res.payload}`) + assert.ok( + types.includes('response.function_call_arguments.done'), + `tool call did not stream in ${res.payload}` + ) + } + ) +}) diff --git a/packages/cli/test/unit/extensions/openai/completion-result.test.ts b/packages/cli/test/unit/extensions/openai/completion-result.test.ts index adaf136659..180700b247 100644 --- a/packages/cli/test/unit/extensions/openai/completion-result.test.ts +++ b/packages/cli/test/unit/extensions/openai/completion-result.test.ts @@ -1,16 +1,18 @@ import { describe, it } from 'node:test' import assert from 'node:assert/strict' -import type { CompletionRun, CompletionStats, ToolCall } from '@qvac/sdk' +import type { CompletionRun, CompletionStats, ToolCall, ToolCallError } from '@qvac/sdk' import { InferenceCancelledError } from '@qvac/sdk' import { drainCompletion, - completionTokensFromStats + completionTokensFromStats, + formatToolErrors } from '@/serve/extensions/openai/adapters/completion-result' import { HttpError } from '@/serve/lib/http-error' function fakeRun(opts: { tokens?: string[] toolCalls?: ToolCall[] + toolErrors?: ToolCallError[] stats?: CompletionStats stopReason?: string final?: Promise @@ -19,6 +21,7 @@ function fakeRun(opts: { let seq = 0 for (const t of opts.tokens ?? []) yield { type: 'contentDelta', seq: seq++, text: t } for (const call of opts.toolCalls ?? []) yield { type: 'toolCall', seq: seq++, call } + for (const error of opts.toolErrors ?? []) yield { type: 'toolError', seq: seq++, error } if (opts.stats !== undefined) yield { type: 'completionStats', seq: seq++, stats: opts.stats } yield { type: 'completionDone', seq: seq++, stopReason: opts.stopReason ?? 'eos' } } @@ -137,3 +140,55 @@ describe('drainCompletion', () => { ) }) }) + +describe('drainCompletion tool errors', () => { + it('collects toolError events alongside successful calls', async () => { + const drained = await drainCompletion( + fakeRun({ + toolCalls: [{ id: 'call_1', name: 'get_weather', arguments: { city: 'Lugano' } }], + toolErrors: [{ code: 'PARSE_ERROR', message: 'bad json', raw: '{' }] + }) + ) + assert.deepEqual(drained.toolErrors, [{ code: 'PARSE_ERROR', message: 'bad json', raw: '{' }]) + assert.equal(drained.toolCalls.length, 1) + assert.equal(drained.finishReason, 'tool_calls') + }) + + // Every tool call failing leaves no calls and no content, so finish_reason is + // the ordinary 'stop' -- the log line is the only signal the model tried. + it('reports stop when every tool call failed to parse', async () => { + const drained = await drainCompletion( + fakeRun({ + toolErrors: [ + { code: 'PARSE_ERROR', message: 'bad json' }, + { code: 'VALIDATION_ERROR', message: 'city must be a string' } + ] + }) + ) + assert.equal(drained.toolErrors.length, 2) + assert.equal(drained.toolCalls.length, 0) + assert.equal(drained.finishReason, 'stop') + }) + + it('leaves toolErrors empty on a clean run', async () => { + const drained = await drainCompletion(fakeRun({ tokens: ['hi'] })) + assert.deepEqual(drained.toolErrors, []) + }) +}) + +describe('formatToolErrors', () => { + it('returns an empty string when there are none', () => { + assert.equal(formatToolErrors([]), '') + }) + + it('counts errors and lists each distinct code once', () => { + assert.equal( + formatToolErrors([ + { code: 'PARSE_ERROR', message: 'a' }, + { code: 'PARSE_ERROR', message: 'b' }, + { code: 'UNKNOWN_TOOL', message: 'c' } + ]), + ' toolerrors=3 (PARSE_ERROR,UNKNOWN_TOOL)' + ) + }) +}) diff --git a/packages/cli/test/unit/extensions/openai/schemas.test.ts b/packages/cli/test/unit/extensions/openai/schemas.test.ts index 78a0e6821a..17122b6048 100644 --- a/packages/cli/test/unit/extensions/openai/schemas.test.ts +++ b/packages/cli/test/unit/extensions/openai/schemas.test.ts @@ -7,11 +7,16 @@ import { extractResponseFormat, InvalidResponseFormatError, UnsupportedImageContentError, - extractGenerationParams + extractGenerationParams, + extractToolChoice, + withToolChoice, + InvalidToolChoiceError } from '@/serve/extensions/openai/schemas/common' import { openaiMessagesToHistory, writeChatImages, + chatCompletionsBody, + toSdkChatArgs, type OpenAIMessage } from '@/serve/extensions/openai/schemas/chat' import { @@ -1182,3 +1187,206 @@ describe('legacyPromptToHistory', () => { assert.deepEqual(legacyPromptToHistory('hello'), [{ role: 'user', content: 'hello' }]) }) }) + +describe('extractToolChoice', () => { + const emptyParams = { type: 'object' as const, properties: {} } + const tools = [ + { type: 'function' as const, name: 'get_weather', description: '', parameters: emptyParams }, + { type: 'function' as const, name: 'get_time', description: '', parameters: emptyParams } + ] + + it('returns undefined when the request sets no tool_choice', () => { + assert.equal(extractToolChoice({}, tools), undefined) + assert.equal(extractToolChoice({ tool_choice: null }, tools), undefined) + }) + + it('passes auto and none through without needing tools', () => { + assert.equal(extractToolChoice({ tool_choice: 'auto' }, undefined), 'auto') + assert.equal(extractToolChoice({ tool_choice: 'none' }, undefined), 'none') + }) + + it('accepts required when tools are declared', () => { + assert.equal(extractToolChoice({ tool_choice: 'required' }, tools), 'required') + }) + + it('flattens the chat object form to the bare tool name', () => { + assert.equal( + extractToolChoice( + { tool_choice: { type: 'function', function: { name: 'get_weather' } } }, + tools + ), + 'get_weather' + ) + }) + + it('flattens the responses object form to the bare tool name', () => { + assert.equal( + extractToolChoice({ tool_choice: { type: 'function', name: 'get_time' } }, tools), + 'get_time' + ) + }) + + it('rejects required with no tools', () => { + assert.throws( + () => extractToolChoice({ tool_choice: 'required' }, undefined), + InvalidToolChoiceError + ) + assert.throws(() => extractToolChoice({ tool_choice: 'required' }, []), InvalidToolChoiceError) + }) + + it('rejects a name that is not among the declared tools', () => { + assert.throws( + () => extractToolChoice({ tool_choice: { type: 'function', name: 'get_stock' } }, tools), + InvalidToolChoiceError + ) + }) + + it('rejects a bare string that is neither a mode nor an object form', () => { + assert.throws( + () => extractToolChoice({ tool_choice: 'get_weather' }, tools), + InvalidToolChoiceError + ) + assert.throws(() => extractToolChoice({ tool_choice: 'atuo' }, tools), InvalidToolChoiceError) + }) + + it('rejects a non-function tool_choice type', () => { + assert.throws( + () => extractToolChoice({ tool_choice: { type: 'custom', name: 'x' } }, tools), + InvalidToolChoiceError + ) + }) + + it('rejects an object form with no usable name', () => { + assert.throws( + () => extractToolChoice({ tool_choice: { type: 'function' } }, tools), + InvalidToolChoiceError + ) + assert.throws( + () => extractToolChoice({ tool_choice: { type: 'function', function: { name: '' } } }, tools), + InvalidToolChoiceError + ) + }) + + it('rejects an array', () => { + assert.throws(() => extractToolChoice({ tool_choice: [] }, tools), InvalidToolChoiceError) + }) + + // Collapsing to the bare name would read back as the mode and invert the + // request: targeting a tool called `none` would switch tool calling off. + it('rejects targeting a tool whose name collides with a mode', () => { + const reserved = ['auto', 'none', 'required'] + for (const name of reserved) { + const declared = [ + { type: 'function' as const, name, description: '', parameters: emptyParams } + ] + assert.throws( + () => + extractToolChoice({ tool_choice: { type: 'function', function: { name } } }, declared), + InvalidToolChoiceError, + `chat object form naming ${name}` + ) + assert.throws( + () => extractToolChoice({ tool_choice: { type: 'function', name } }, declared), + InvalidToolChoiceError, + `responses object form naming ${name}` + ) + } + }) + + it('still reads the bare mode strings as modes', () => { + const declared = [ + { type: 'function' as const, name: 'none', description: '', parameters: emptyParams } + ] + assert.equal(extractToolChoice({ tool_choice: 'none' }, declared), 'none') + assert.equal(extractToolChoice({ tool_choice: 'required' }, declared), 'required') + }) +}) + +describe('withToolChoice', () => { + it('leaves params untouched when there is no choice', () => { + assert.equal(withToolChoice(undefined, undefined), undefined) + assert.deepEqual(withToolChoice({ temp: 0.2 }, undefined), { temp: 0.2 }) + }) + + it('creates params when the request set only a tool_choice', () => { + assert.deepEqual(withToolChoice(undefined, 'required'), { tool_choice: 'required' }) + }) + + it('merges the choice into existing params', () => { + assert.deepEqual(withToolChoice({ temp: 0.2 }, 'get_weather'), { + temp: 0.2, + tool_choice: 'get_weather' + }) + }) +}) + +describe('chat tool_choice wiring', () => { + const weatherTool = { + type: 'function', + function: { + name: 'get_weather', + description: 'Get weather', + parameters: { type: 'object', properties: { city: { type: 'string' } } } + } + } + + function body(extra: Record): Record { + return { + model: 'my-llm', + messages: [{ role: 'user', content: 'Weather in Lugano?' }], + tools: [weatherTool], + ...extra + } + } + + it('accepts both tool_choice forms through the request schema', () => { + assert.equal(chatCompletionsBody.safeParse(body({ tool_choice: 'required' })).success, true) + assert.equal( + chatCompletionsBody.safeParse( + body({ tool_choice: { type: 'function', function: { name: 'get_weather' } } }) + ).success, + true + ) + }) + + it('threads a mode choice into generationParams', () => { + const args = toSdkChatArgs( + chatCompletionsBody.parse(body({ tool_choice: 'required' })), + 'hermes' + ) + assert.equal(args.generationParams?.tool_choice, 'required') + }) + + it('threads the object form through as the bare tool name', () => { + const args = toSdkChatArgs( + chatCompletionsBody.parse( + body({ tool_choice: { type: 'function', function: { name: 'get_weather' } } }) + ), + 'hermes' + ) + assert.equal(args.generationParams?.tool_choice, 'get_weather') + }) + + it('keeps tool_choice alongside the other generation params', () => { + const args = toSdkChatArgs( + chatCompletionsBody.parse(body({ tool_choice: 'required', temperature: 0.3 })), + 'hermes' + ) + assert.equal(args.generationParams?.tool_choice, 'required') + assert.equal(args.generationParams?.temp, 0.3) + }) + + it('leaves generationParams undefined when nothing was set', () => { + const args = toSdkChatArgs(chatCompletionsBody.parse(body({})), 'hermes') + assert.equal(args.generationParams, undefined) + }) + + it('rejects a demanding tool_choice with no tools', () => { + const parsed = chatCompletionsBody.parse({ + model: 'my-llm', + messages: [{ role: 'user', content: 'hi' }], + tool_choice: 'required' + }) + assert.throws(() => toSdkChatArgs(parsed, 'hermes'), InvalidToolChoiceError) + }) +})