diff --git a/CHANGELOG.md b/CHANGELOG.md index 8123ba4e..804e32fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Render structured typed tool results as JSON text for chat templates while preserving string results and source objects. + - Preserve Qwen XML tool argument types using declared schemas and reject invalid or undeclared calls without exposing executable tool deltas. - Settle pending LiteRT-LM requests when a worker stops, close their response ports, and report unverified native cleanup as an error. diff --git a/lib/src/core/template/template_render_context.dart b/lib/src/core/template/template_render_context.dart index b3760581..e61829fa 100644 --- a/lib/src/core/template/template_render_context.dart +++ b/lib/src/core/template/template_render_context.dart @@ -74,6 +74,8 @@ class TemplateRenderContext { ); /// Serializes [messages] into the JSON shape expected by a template handler. + /// Typed tool results become JSON text (or text parts for multimodal + /// templates); string results and the original typed messages are unchanged. static List> messagesForTemplate( List messages, { TemplateToolCallSerialization toolCallSerialization = @@ -86,6 +88,16 @@ class TemplateRenderContext { final rendered = multimodal ? message.toJsonMultimodal() : message.toJson(); + final toolResults = message.parts.whereType(); + if (toolResults.isNotEmpty) { + final result = toolResults.first.result; + final text = result is String ? result : jsonEncode(result); + rendered['content'] = multimodal + ? [ + {'type': 'text', 'text': text}, + ] + : text; + } if (rendered['tool_calls'] is List) { hasToolCalls = true; } diff --git a/test/e2e/template/llama_cpp_chat_tests_e2e_test.dart b/test/e2e/template/llama_cpp_chat_tests_e2e_test.dart index 6caf3857..791d3ba6 100644 --- a/test/e2e/template/llama_cpp_chat_tests_e2e_test.dart +++ b/test/e2e/template/llama_cpp_chat_tests_e2e_test.dart @@ -8,6 +8,7 @@ import 'dart:io'; import 'package:llamadart/llamadart.dart'; import 'package:test/test.dart'; +import '../../support/qwen35_tool_result_fixture.dart' as typed; import '../../support/qwen_tool_schema_fixture.dart'; void main() { @@ -117,6 +118,79 @@ void main() { } }, ); + test( + 'typed Qwen result history matches pinned upstream template rendering', + () async { + final build = + Platform.environment['LLAMA_CPP_CHAT_TEST_BUILD_DIR'] ?? + '${Directory.current.path}/.dart_tool/llama_cpp_chat_tests'; + final binary = File('$build/bin/test-chat-template'); + expect( + binary.existsSync(), + isTrue, + reason: 'Run the upstream selection to build test-chat-template.', + ); + final temp = Directory.systemTemp.createTempSync( + 'qwen-tool-result-parity-', + ); + addTearDown(() => temp.deleteSync(recursive: true)); + for (final thinking in [true, false]) { + // Independent upstream input oracle: the public typed payload is encoded + // as JSON text, without calling the production normalization helper. + final messages = + jsonDecode( + jsonEncode( + typed + .qwenResultHistory(stringControl: true) + .map((m) => m.toJson()) + .toList(), + ), + ) + as List; + ((messages[1]['tool_calls'] as List).single['function'] + as Map)['arguments'] = + typed.qwenResultPayload; + final input = File('${temp.path}/input.json') + ..writeAsStringSync( + jsonEncode({ + 'messages': messages, + 'tools': [typed.qwenResultTool.toJson()], + 'bos_token': '<|im_start|>', + 'eos_token': '<|im_end|>', + 'add_generation_prompt': true, + 'enable_thinking': thinking, + }), + ); + final output = File('${temp.path}/prompt.txt'); + final result = await Process.run(binary.path, [ + '--no-common', + '--json', + input.path, + '--output', + output.path, + File(typed.qwenResultTemplatePath).absolute.path, + ]); + expect( + result.exitCode, + 0, + reason: 'Upstream Qwen render failed: ${result.stderr}', + ); + expect(output.existsSync(), isTrue); + final rendered = typed.renderQwenResultHistory( + choice: ToolChoice.auto, + thinking: thinking, + ); + // Upstream tojson uses spaced separators; Dart emits compact JSON. + // Canonicalize only the tools declaration, preserving history and the + // generation/thinking suffix byte-for-byte. + expect( + _canonicalToolDeclarations(rendered.prompt), + _canonicalToolDeclarations(output.readAsStringSync()), + ); + expect(rendered.prompt, contains(jsonEncode(typed.qwenResultPayload))); + } + }, + ); } String _canonicalToolDeclarations(String prompt) => prompt.replaceFirstMapped( diff --git a/test/e2e/template/specialized_tool_grammar_validation_e2e_test.dart b/test/e2e/template/specialized_tool_grammar_validation_e2e_test.dart index 323a48a1..52fc4473 100644 --- a/test/e2e/template/specialized_tool_grammar_validation_e2e_test.dart +++ b/test/e2e/template/specialized_tool_grammar_validation_e2e_test.dart @@ -22,6 +22,7 @@ import 'package:llamadart/src/core/template/handlers/hunyuan_v3_handler.dart'; import 'package:llamadart/src/core/template/handlers/llama_cpp_specialized_handlers.dart'; import 'package:test/test.dart'; +import '../../support/qwen35_tool_result_fixture.dart' as typed; import '../../support/qwen_tool_schema_fixture.dart'; void main() { @@ -37,6 +38,35 @@ void main() { expect(File(validator).existsSync(), isTrue); }); + test('Qwen typed Map result history preserves compiled follow-up grammar', () { + for (final thinking in [true, false]) { + final rendered = typed.renderQwenResultHistory(thinking: thinking); + expect(rendered.prompt, contains(jsonEncode(typed.qwenResultPayload))); + expect(rendered.grammarLazy, isFalse); + // Qwen's existing grammar constrains the XML envelope and declared names; + // scalar typing is reconstructed by the output parser, tested separately. + // Its grammar root begins at the tool envelope, not the thinking prefix. + _expectGrammar( + validator, + rendered.grammar!, + valid: [typed.qwenResultEnvelope], + invalid: [ + typed.qwenResultEnvelope.replaceFirst( + '', + '', + ), + typed.qwenResultEnvelope.replaceFirst( + '', + '', + ), + typed.qwenResultEnvelope.replaceFirst('', ''), + typed.qwenResultEnvelope.replaceFirst('', ''), + 'No tool', + ], + ); + } + }); + test('Qwen typed result history preserves compiled follow-up grammar', () { for (final thinking in [true, false]) { final rendered = renderQwenResultHistory( diff --git a/test/fixtures/Qwen3_5-0_8B.md b/test/fixtures/Qwen3_5-0_8B.md new file mode 100644 index 00000000..09131fe9 --- /dev/null +++ b/test/fixtures/Qwen3_5-0_8B.md @@ -0,0 +1,9 @@ +# Qwen3.5 0.8B template fixture + +Exact `tokenizer.chat_template` extracted from `ggml-org/Qwen3.5-0.8B-GGUF`, revision `8fea620810c4afa23dd6443f999a48574c1611a3`, file `Qwen3.5-0.8B-Q4_0.gguf`. + +- Model SHA256: `57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf`. +- Template UTF-8 SHA256: `273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80`. +- Source: https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/tree/8fea620810c4afa23dd6443f999a48574c1611a3 + +Used by `test/integration/core/template/qwen35_tool_result_test.dart` to reproduce mapping-valued tool-result rejection and verify render-boundary JSON normalization. The template is unchanged; the fix belongs to Dart input serialization. diff --git a/test/integration/core/template/qwen35_tool_result_test.dart b/test/integration/core/template/qwen35_tool_result_test.dart new file mode 100644 index 00000000..17dd1089 --- /dev/null +++ b/test/integration/core/template/qwen35_tool_result_test.dart @@ -0,0 +1,157 @@ +@TestOn('vm') +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart/src/core/engine/chat_completion_stream_parser.dart'; +import 'package:test/test.dart'; + +import '../../../support/qwen35_tool_result_fixture.dart'; + +void main() { + test('exact Qwen3.5 template accepts structured typed tool results', () { + final source = File( + 'test/fixtures/templates/Qwen3_5-0_8B.jinja', + ).readAsStringSync(); + final payload = { + 'city': 'MontrĂ©al đŸ‘‹', + 'temperature_celsius': 17, + 'nested': [true, null], + 'escaped': '"quoted"\nline', + }; + final messages = [ + const LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'Call get_weather for MontrĂ©al.', + ), + const LlamaChatMessage.withContent( + role: LlamaChatRole.assistant, + content: [ + LlamaToolCallContent( + id: 'call_0', + name: 'get_weather', + arguments: {'city': 'MontrĂ©al'}, + rawJson: '{"city":"MontrĂ©al"}', + ), + ], + ), + LlamaChatMessage.withContent( + role: LlamaChatRole.tool, + content: [ + LlamaToolResultContent( + id: 'call_0', + name: 'get_weather', + result: payload, + ), + ], + ), + const LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'What is the temperature_celsius?', + ), + ]; + final output = ChatTemplateEngine.render( + templateSource: source, + messages: messages, + metadata: const {}, + toolChoice: ToolChoice.none, + enableThinking: false, + ); + expect( + output.prompt, + contains('\n${jsonEncode(payload)}\n'), + ); + expect(output.prompt, contains('')); + expect(output.prompt, contains('MontrĂ©al')); + expect(messages[2].toJson()['content'], same(payload)); + expect(output.prompt, isNot(contains('{city:'))); + }); + + test('typed result history preserves tool choices and thinking prefixes', () { + for (final choice in ToolChoice.values) { + for (final thinking in [true, false]) { + final actual = renderQwenResultHistory( + choice: choice, + thinking: thinking, + ); + final control = renderQwenResultHistory( + choice: choice, + thinking: thinking, + stringControl: true, + ); + expect(actual.prompt, control.prompt); + expect(actual.prompt, contains(jsonEncode(qwenResultPayload))); + expect(actual.grammar, control.grammar); + expect(actual.grammar, choice == ToolChoice.none ? isNull : isNotNull); + expect(actual.grammarLazy, choice == ToolChoice.auto); + expect(actual.thinkingForcedOpen, thinking); + expect(actual.preservedTokens, control.preservedTokens); + expect(actual.additionalStops, control.additionalStops); + } + } + }); + + // Keep strict reconstruction and rollback oracles for typed result histories. + for (final malformed in [false, true]) { + test( + 'typed result history ${malformed ? 'rolls back undeclared tools' : 'preserves schema-directed scalar and container types'}', + () async { + for (final choice in [ToolChoice.auto, ToolChoice.required]) { + for (final thinking in [true, false]) { + final rendered = renderQwenResultHistory( + choice: choice, + thinking: thinking, + ); + final body = malformed + ? qwenResultEnvelope.replaceFirst( + '', + '', + ) + : qwenResultEnvelope; + final prefix = thinking ? 'reason\n\n' : ''; + final chunks = await ChatCompletionStreamParser.parse( + tokenStream: Stream.fromIterable([prefix, ...body.split('')]), + templateResult: rendered, + parseToolCallsEnabled: true, + enableThinking: thinking, + modelName: 'qwen-typed-result-history', + completionId: 'typed-result-$choice-$thinking-$malformed', + tools: [qwenResultTool], + ).toList(); + final contentChunks = chunks + .map((c) => c.choices.single.delta.content ?? '') + .where((c) => c.isNotEmpty) + .toList(); + final reasoning = chunks + .map((c) => c.choices.single.delta.thinking ?? '') + .join(); + final calls = chunks + .expand( + (c) => + c.choices.single.delta.toolCalls ?? + const [], + ) + .toList(); + expect(reasoning.trim(), thinking ? 'reason' : ''); + if (malformed) { + expect(calls, isEmpty); + expect(contentChunks, [body]); + expect(chunks.last.choices.single.finishReason, 'stop'); + } else { + expect(contentChunks, isEmpty); + expect(calls, hasLength(1)); + expect(calls.single.function!.name, 'inspect'); + expect( + jsonDecode(calls.single.function!.arguments!), + qwenResultPayload, + ); + expect(chunks.last.choices.single.finishReason, 'tool_calls'); + } + } + } + }, + ); + } +} diff --git a/test/support/qwen35_tool_result_fixture.dart b/test/support/qwen35_tool_result_fixture.dart new file mode 100644 index 00000000..41933f6f --- /dev/null +++ b/test/support/qwen35_tool_result_fixture.dart @@ -0,0 +1,91 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart/llamadart.dart'; + +// Exact affected model template; provenance is in fixtures/Qwen3_5-0_8B.md. +const qwenResultTemplatePath = 'test/fixtures/templates/Qwen3_5-0_8B.jinja'; +const qwenResultPayload = { + 'code': '123', + 'options': {}, + 'items': [], + 'count': 7, + 'active': true, + 'empty': null, +}; +final qwenResultTool = ToolDefinition( + name: 'inspect', + description: 'Report the typed values from the previous tool result.', + parameters: [ + ToolParam.string('code', required: true), + ToolParam.object('options', properties: [], required: true), + ToolParam.array( + 'items', + itemType: ToolParam.string('item'), + required: true, + ), + ToolParam.integer('count', required: true), + ToolParam.boolean('active', required: true), + ToolParam.nullType('empty', required: true), + ], + handler: (_) async => qwenResultPayload, +); + +List qwenResultHistory({bool stringControl = false}) => [ + const LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'Inspect these values.', + ), + LlamaChatMessage.withContent( + role: LlamaChatRole.assistant, + content: [ + LlamaToolCallContent( + id: 'call_0', + name: 'inspect', + arguments: qwenResultPayload, + rawJson: jsonEncode(qwenResultPayload), + ), + ], + ), + LlamaChatMessage.withContent( + role: LlamaChatRole.tool, + content: [ + LlamaToolResultContent( + id: 'call_0', + name: 'inspect', + result: stringControl + ? jsonEncode(qwenResultPayload) + : qwenResultPayload, + ), + ], + ), + const LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'Use inspect again with the same values.', + ), +]; + +LlamaChatTemplateResult renderQwenResultHistory({ + ToolChoice choice = ToolChoice.required, + bool thinking = true, + bool stringControl = false, +}) => ChatTemplateEngine.render( + templateSource: File(qwenResultTemplatePath).readAsStringSync(), + messages: qwenResultHistory(stringControl: stringControl), + metadata: const {}, + tools: [qwenResultTool], + toolChoice: choice, + enableThinking: thinking, +); + +// Qwen XML envelope shape follows pinned llama.cpp test-chat.cpp Qwen3.5 +// emissions; numeric-looking strings and empty containers exercise schema types. +const qwenResultEnvelope = + '\n\n' + '\n123\n\n' + '\n{}\n\n' + '\n[]\n\n' + '\n7\n\n' + '\ntrue\n\n' + '\nnull\n\n' + '\n'; diff --git a/test/unit/core/template/template_render_context_test.dart b/test/unit/core/template/template_render_context_test.dart index 54bfd73b..25f225cc 100644 --- a/test/unit/core/template/template_render_context_test.dart +++ b/test/unit/core/template/template_render_context_test.dart @@ -9,6 +9,79 @@ import 'package:test/test.dart'; void main() { group('TemplateRenderContext', () { + for (final payload in [ + { + 'city': 'MontrĂ©al đŸ‘‹', + 'quote': '"line\\next\n', + 'nested': [17, true, null], + }, + [ + 1, + {'value': '한글'}, + ], + 17, + 1.5, + false, + null, + 'unchanged "text" đŸ‘‹', + ]) { + for (final multimodal in [false, true]) { + test( + 'serializes typed tool result ${payload.runtimeType}, multimodal=$multimodal', + () { + final part = LlamaToolResultContent( + id: 'call_0', + name: 'get_weather', + result: payload, + ); + final message = LlamaChatMessage.withContent( + role: LlamaChatRole.tool, + content: [part], + ); + final before = message.toJson(); + final rendered = TemplateRenderContext.messagesForTemplate([ + message, + ], multimodal: multimodal).single; + final expected = payload is String ? payload : jsonEncode(payload); + expect( + rendered['content'], + multimodal + ? [ + {'type': 'text', 'text': expected}, + ] + : expected, + ); + expect(rendered['tool_call_id'], 'call_0'); + expect(rendered['name'], 'get_weather'); + expect(message.toJson(), before); + expect(identical(part.result, payload), true); + }, + ); + } + } + test( + 'tool-result normalization does not stringify ordinary media parts', + () { + final message = LlamaChatMessage.withContent( + role: LlamaChatRole.user, + content: [ + const LlamaTextContent('look đŸ‘‹'), + LlamaImageContent(bytes: Uint8List.fromList([1, 2, 3])), + ], + ); + expect( + TemplateRenderContext.messagesForTemplate([message]).single, + message.toJson(), + ); + expect( + TemplateRenderContext.messagesForTemplate([ + message, + ], multimodal: true).single, + message.toJsonMultimodal(), + ); + }, + ); + test('serializes tool-call policy without mutating typed messages', () { final imageBytes = Uint8List.fromList([1, 2, 3, 4]); final messages = [ diff --git a/website/docs/changelog/recent-releases.md b/website/docs/changelog/recent-releases.md index 846fa6fd..a31f8dbb 100644 --- a/website/docs/changelog/recent-releases.md +++ b/website/docs/changelog/recent-releases.md @@ -9,6 +9,8 @@ For canonical full release notes, use: ## Unreleased +- Render structured typed tool results as JSON text for chat templates while preserving string results and source objects. + - Preserve Qwen XML tool argument types using declared schemas and reject invalid or undeclared calls without exposing executable tool deltas. - Settle pending LiteRT-LM requests when a worker stops, close their response ports, and report unverified native cleanup as an error. diff --git a/website/docs/guides/tool-calling.md b/website/docs/guides/tool-calling.md index 3581588b..7869bfc3 100644 --- a/website/docs/guides/tool-calling.md +++ b/website/docs/guides/tool-calling.md @@ -47,6 +47,12 @@ final stream = engine.create( 4. Append tool result message. 5. Call `engine.create(...)` again for final assistant response. +`LlamaToolResultContent.result` can contain JSON-compatible objects, arrays, +scalars, or null. The shared template renderer encodes these as JSON text; +string results remain unchanged. This conversion does not mutate the typed +result or change its public JSON representation. Multimodal templates receive +the encoded result as a text part. + Qwen XML tool calls are validated against the tools supplied to `engine.create`. Schema-declared strings such as `"123"` retain their type. Unknown functions, unknown or duplicate parameters, missing required values, and invalid value