diff --git a/CHANGELOG.md b/CHANGELOG.md index 4107fba5..29a50bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Suppress caller stop markers in native GGUF streams, including split markers and speculative decoding. + - Fix encoded audio bytes leaking into string chat-template prompts, restoring native Qwen3-ASR file/bytes transcription parity. * Explain unavailable native thinking-budget helpers with bounded, path-free loader diagnostics. diff --git a/doc/testing_matrix.md b/doc/testing_matrix.md index 427bbfec..6bf27246 100644 --- a/doc/testing_matrix.md +++ b/doc/testing_matrix.md @@ -406,3 +406,23 @@ When an agent creates or updates a PR: ancestry with `tool/git/safe_pr_head_update.dart` to prevent stale head rewinds. Consult `doc/pr_branch_writer_inventory.md` for writer scope and the remaining GitHub-managed governance boundary. + +### Native GGUF stop sequences + +`dart test -p vm test/integration/stop_sequences_test.dart` uses the small test +GGUF to cover ordinary and speculative stop handling, Unicode, token boundaries, +completion, cancellation, and subsequent generation. Ordinary tests constrain +output with grammar; speculative tests use deterministic unrestricted controls +because grammar sampling is unsupported there, and assert actual draft acceptance. + +Use a compliant chat model (Gemma 4 E2B or Qwen3.5 0.8B) for the unforced public +chat fixture; the control must emit `alpha cedar17 omega` and caller stop +`cedar17` must leave exactly `alpha `, under single-piece and default batching: + +```bash +dart run tool/testing/run_local_e2e.dart --scenario gguf-stop-sequences \ + --model-path /path/to/chat.gguf --backend cpu +``` + +Repeat with `--backend metal` on macOS when available. Record the source commit, +native runtime tag, printed model SHA-256, and native offload logs with results. diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index 0c17f2e4..3423f75f 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -23,6 +23,7 @@ import '../../core/template/media_placeholders.dart'; import '../../core/models/inference/model_params.dart'; import '../../core/template/chat_template_engine.dart'; import 'load_param_helpers.dart'; +import 'stop_sequence_buffer.dart'; import 'bindings.dart'; import 'llama_cpp_raw_bindings.dart' as raw_bindings; @@ -5399,7 +5400,7 @@ class LlamaCppService { ) async* { final cancelToken = Pointer.fromAddress(cancelTokenAddress); int currentPos = startPos; - final accumulatedBytes = []; + final stopBuffer = StopSequenceBuffer(stopSequences); final evalStopwatch = Stopwatch()..start(); var sampleMicros = 0; var evalMicros = 0; @@ -5429,17 +5430,10 @@ class LlamaCppService { if (n > 0) { final bytes = pieceBuf.asTypedList(n).toList(); - yield bytes; generatedTokens++; - - if (stopSequences.isNotEmpty) { - accumulatedBytes.addAll(bytes); - if (accumulatedBytes.length > 64) { - accumulatedBytes.removeRange(0, accumulatedBytes.length - 64); - } - final text = utf8.decode(accumulatedBytes, allowMalformed: true); - if (stopSequences.any((s) => text.endsWith(s))) break; - } + final visible = stopBuffer.add(bytes); + if (visible.isNotEmpty) yield visible; + if (stopBuffer.isStopped) break; } batch.n_tokens = 1; @@ -5456,6 +5450,9 @@ class LlamaCppService { if (decodeStatus != 0) break; } + final remaining = stopBuffer.finish(); + if (remaining.isNotEmpty) yield remaining; + evalStopwatch.stop(); ctx.lastPerfEvalMs = evalMicros / 1000.0; ctx.lastPerfSampleMs = sampleMicros / 1000.0; @@ -5491,7 +5488,7 @@ class LlamaCppService { int currentPos = startPos; int? pendingSampledToken; - final accumulatedBytes = []; + final stopBuffer = StopSequenceBuffer(stopSequences); final evalStopwatch = Stopwatch()..start(); var sampleMicros = 0; var evalMicros = 0; @@ -5536,17 +5533,9 @@ class LlamaCppService { if (n > 0) { final bytes = pieceBuf.asTypedList(n).toList(); - yield bytes; - if (stopSequences.isNotEmpty) { - accumulatedBytes.addAll(bytes); - if (accumulatedBytes.length > 64) { - accumulatedBytes.removeRange(0, accumulatedBytes.length - 64); - } - final text = utf8.decode(accumulatedBytes, allowMalformed: true); - if (stopSequences.any((s) => text.endsWith(s))) { - shouldStop = true; - } - } + final visible = stopBuffer.add(bytes); + if (visible.isNotEmpty) yield visible; + shouldStop = stopBuffer.isStopped; } if (shouldStop) { @@ -5850,20 +5839,9 @@ class LlamaCppService { if (n > 0) { final bytes = pieceBuf.asTypedList(n).toList(); - yield bytes; - if (stopSequences.isNotEmpty) { - accumulatedBytes.addAll(bytes); - if (accumulatedBytes.length > 64) { - accumulatedBytes.removeRange(0, accumulatedBytes.length - 64); - } - final text = utf8.decode( - accumulatedBytes, - allowMalformed: true, - ); - if (stopSequences.any((s) => text.endsWith(s))) { - shouldStop = true; - } - } + final visible = stopBuffer.add(bytes); + if (visible.isNotEmpty) yield visible; + shouldStop = stopBuffer.isStopped; } if (shouldStop || generatedTokens >= params.maxTokens) { @@ -5880,6 +5858,8 @@ class LlamaCppService { } } } + final remaining = stopBuffer.finish(); + if (remaining.isNotEmpty) yield remaining; } finally { malloc.free(draftPtr); malloc.free(idxPtr); diff --git a/lib/src/backends/llama_cpp/stop_sequence_buffer.dart b/lib/src/backends/llama_cpp/stop_sequence_buffer.dart new file mode 100644 index 00000000..dd136f92 --- /dev/null +++ b/lib/src/backends/llama_cpp/stop_sequence_buffer.dart @@ -0,0 +1,94 @@ +import 'dart:collection'; +import 'dart:convert'; + +/// Holds only bytes that could still complete a caller stop sequence. +/// +/// Matching UTF-8 bytes avoids decoding incomplete token pieces. A completed +/// marker and everything after it are suppressed, even within the same piece. +class StopSequenceBuffer { + /// Creates a per-generation buffer, ignoring empty and duplicate markers. + StopSequenceBuffer(List stops) + : _stops = stops + .where((stop) => stop.isNotEmpty) + .toSet() + .map((stop) => _StopPattern(utf8.encode(stop))) + .toList(growable: false); + + final List<_StopPattern> _stops; + final ListQueue _pending = ListQueue(); + + /// Whether a complete marker has been consumed. + bool get isStopped => _isStopped; + bool _isStopped = false; + + /// Consumes a token piece and returns bytes safe to expose immediately. + List add(List bytes) { + if (_isStopped) return const []; + if (_stops.isEmpty) return bytes; + final previousLength = _pending.length; + _pending.addAll(bytes); + var stopIndex = _pending.length; + var retainedLength = 0; + for (final stop in _stops) { + for (var i = 0; i < bytes.length; i++) { + if (stop.advance(bytes[i])) { + final start = previousLength + i + 1 - stop.bytes.length; + if (start < stopIndex) stopIndex = start; + } + } + if (stop.matched > retainedLength) retainedLength = stop.matched; + } + if (stopIndex < _pending.length) { + final visible = _pending.take(stopIndex).toList(); + _pending.clear(); + _isStopped = true; + return visible; + } + final safeLength = _pending.length - retainedLength; + return List.generate( + safeLength, + (_) => _pending.removeFirst(), + growable: false, + ); + } + + /// Releases an unfinished marker prefix when generation ends without a match. + List finish() { + final remaining = _pending.toList(); + _pending.clear(); + for (final stop in _stops) { + stop.matched = 0; + } + return remaining; + } +} + +// KMP prefix state processes only newly received bytes. Repeated prefixes in +// long caller markers must not cause rescans of the entire pending suffix on +// every token. Work is linear in incoming bytes per stop (amortized). +class _StopPattern { + _StopPattern(this.bytes) : fallback = List.filled(bytes.length, 0) { + var prefix = 0; + for (var i = 1; i < bytes.length; i++) { + while (prefix > 0 && bytes[i] != bytes[prefix]) { + prefix = fallback[prefix - 1]; + } + if (bytes[i] == bytes[prefix]) prefix++; + fallback[i] = prefix; + } + } + + final List bytes; + final List fallback; + int matched = 0; + + bool advance(int byte) { + while (matched > 0 && byte != bytes[matched]) { + matched = fallback[matched - 1]; + } + if (byte == bytes[matched]) matched++; + if (matched != bytes.length) return false; + matched = fallback[matched - 1]; + return true; + } +} diff --git a/lib/src/core/models/inference/generation_params.dart b/lib/src/core/models/inference/generation_params.dart index b9b8c953..58b4e290 100644 --- a/lib/src/core/models/inference/generation_params.dart +++ b/lib/src/core/models/inference/generation_params.dart @@ -620,7 +620,12 @@ class GenerationParams { /// If null, a seed based on the current time will be used. final int? seed; - /// List of strings that, if generated, will immediately stop the generation process. + /// Strings that end generation when matched, excluding the marker from output. + /// + /// Empty strings are ignored. Native GGUF matches across token boundaries and + /// inside token pieces; an unfinished prefix is emitted if generation ends + /// without a full match. Exact entries in [preservedTokens] remain available + /// to the native chat parser instead of acting as text stops. final List stopSequences; /// GBNF grammar string for structured output (e.g., "root ::= \"hello\" | \"world\""). diff --git a/test/e2e/backends/gguf_stop_sequences_e2e_test.dart b/test/e2e/backends/gguf_stop_sequences_e2e_test.dart new file mode 100644 index 00000000..1bfd3dcc --- /dev/null +++ b/test/e2e/backends/gguf_stop_sequences_e2e_test.dart @@ -0,0 +1,77 @@ +@TestOn('vm') +@Tags(['local-only', 'e2e']) +@Timeout(Duration(minutes: 5)) +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; +import 'package:test/test.dart'; + +void main() { + test('public GGUF chat suppresses caller marker and recovers', () async { + final model = Platform.environment['GGUF_STOP_MODEL']; + expect(model, isNotNull, reason: 'Set GGUF_STOP_MODEL to a chat GGUF'); + final backendName = Platform.environment['GGUF_STOP_BACKEND'] ?? 'cpu'; + final backend = GpuBackend.values.byName(backendName); + final engine = LlamaEngine(LlamaBackend()); + addTearDown(engine.dispose); + await engine.setNativeLogLevel(LlamaLogLevel.info); + final checksum = await sha256.bind(File(model!).openRead()).first; + print( + jsonEncode({ + 'model_sha256': '$checksum', + 'backend': backendName, + 'dart': Platform.version, + 'os': Platform.operatingSystemVersion, + }), + ); + await engine.loadModel( + model, + modelParams: ModelParams( + contextSize: 1024, + gpuLayers: backend == GpuBackend.cpu ? 0 : 99, + preferredBackend: backend, + numberOfThreads: 4, + numberOfThreadsBatch: 4, + ), + ); + Future run(List stops, int batching) async { + final text = StringBuffer(); + await for (final chunk in engine.create( + [ + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'Reply with exactly: alpha cedar17 omega', + ), + ], + enableThinking: false, + params: GenerationParams( + temp: 0, + seed: 1, + maxTokens: 32, + stopSequences: stops, + streamBatchTokenThreshold: batching, + streamBatchByteThreshold: batching == 1 ? 1 : 512, + ), + )) { + for (final choice in chunk.choices) { + text.write(choice.delta.content ?? ''); + } + } + return text.toString(); + } + + final control = await run([], 8); + print(jsonEncode({'control': control})); + expect(control, 'alpha cedar17 omega'); + for (final batching in [1, 8]) { + final stopped = await run(['cedar17'], batching); + print(jsonEncode({'batch_tokens': batching, 'stopped': stopped})); + expect(stopped, 'alpha '); + } + expect(await run([], 8), control); + }); +} diff --git a/test/integration/stop_sequences_test.dart b/test/integration/stop_sequences_test.dart new file mode 100644 index 00000000..9e22a707 --- /dev/null +++ b/test/integration/stop_sequences_test.dart @@ -0,0 +1,207 @@ +@TestOn('vm') +@Timeout(Duration(minutes: 5)) +library; + +import 'dart:convert'; + +import 'package:llamadart/llamadart.dart'; +import 'package:test/test.dart'; + +import '../test_helper.dart'; + +void main() { + late LlamaBackend backend; + late int model; + late int context; + + setUpAll(() async { + backend = LlamaBackend(); + final file = await TestHelper.getTestModel(); + const params = ModelParams( + contextSize: 512, + gpuLayers: 0, + preferredBackend: GpuBackend.cpu, + numberOfThreads: 2, + numberOfThreadsBatch: 2, + ); + model = await backend.modelLoad(file.path, params); + context = await backend.contextCreate(model, params); + }); + tearDownAll(() async { + await backend.contextFree(context); + await backend.modelFree(model); + await backend.dispose(); + }); + + group('ordinary GGUF stops', () { + GenerationParams params(String text, List stops) => + GenerationParams( + maxTokens: 128, + temp: 0, + seed: 1, + penalty: 1, + grammar: 'root ::= ${jsonEncode(text)}', + stopSequences: stops, + streamBatchTokenThreshold: 1, + streamBatchByteThreshold: 1, + ); + + Future run(String text, List stops) async { + final chunks = await backend + .generate(context, 'Once upon a time', params(text, stops)) + .toList(); + return utf8.decode(chunks.expand((chunk) => chunk).toList()); + } + + test('control, marker suppression, and next generation recovery', () async { + const text = 'alpha cedar17 omega'; + expect(await run(text, []), text); + expect(await run(text, ['cedar17']), 'alpha '); + expect(await run(text, []), text); + // The first sampled piece is subject to suppression too. + expect(await run(text, ['a']), isEmpty); + }); + + test('matches inside a token and ignores empty markers', () async { + const text = 'Once upon a time'; + final tokens = await backend.tokenize(model, text, addSpecial: false); + final pieces = []; + for (final token in tokens) { + pieces.add(await backend.detokenize(model, [token])); + } + final piece = pieces.firstWhere((piece) => piece.trim().length >= 3); + final stop = piece.substring(1, 2); + expect( + await run(text, ['', stop, stop]), + text.substring(0, text.indexOf(stop)), + ); + }); + + test('Unicode, overlap, and unfinished prefix at EOG', () async { + expect(await run('café 🦊終 omega', ['🦊終']), 'café '); + expect(await run('alpha cedar', ['cedar17']), 'alpha cedar'); + expect(await run('alpha abc omega', ['bc', 'abc']), 'alpha '); + expect(await run('alpha cedar17 omega', ['']), 'alpha cedar17 omega'); + }); + + test('preserved parser tokens are not consumed as caller stops', () async { + const text = 'alpha cedar17 omega'; + final chunks = await backend + .generate( + context, + 'Once upon a time', + params(text, ['cedar17']).copyWith(preservedTokens: ['cedar17']), + ) + .toList(); + expect(utf8.decode(chunks.expand((chunk) => chunk).toList()), text); + }); + + test('unfinished prefix at token limit is retained', () async { + final limited = params('alpha cedar17 omega', []).copyWith(maxTokens: 2); + Future generate(GenerationParams value) async => utf8.decode( + (await backend.generate(context, 'Once upon a time', value).toList()) + .expand((chunk) => chunk) + .toList(), + ); + final control = await generate(limited); + expect(control, isNotEmpty); + expect( + await generate(limited.copyWith(stopSequences: ['$control suffix'])), + control, + ); + }); + + test('cancel with pending marker then regenerate', () async { + final text = List.filled(60, 'alpha cedar17 omega ').join(); + var chunks = 0; + await for (final _ in backend.generate( + context, + 'Once upon a time', + params(text, ['cedar17-not-present']), + )) { + chunks++; + backend.cancelGeneration(); + } + expect(chunks, greaterThan(0)); + expect(await run('alpha cedar17 omega', ['cedar17']), 'alpha '); + }); + }); + + group('speculative GGUF stops without unsupported grammar sampling', () { + const prompt = + 'Once upon a time there was a little girl. ' + 'Once upon a time there was a little girl. ' + 'Once upon a time there was a'; + const params = GenerationParams( + maxTokens: 80, + temp: 0, + seed: 1, + penalty: 1, + streamBatchTokenThreshold: 1, + streamBatchByteThreshold: 1, + speculativeDecodingConfig: SpeculativeDecodingConfig.ngramSimple( + ngramSizeN: 1, + ngramSizeM: 4, + ngramMinHits: 1, + ), + ); + Future run(GenerationParams value) async => utf8.decode( + (await backend.generate(context, prompt, value).toList()) + .expand((chunk) => chunk) + .toList(), + ); + + test('sampled and accepted pieces suppress stops and recover', () async { + final control = await run(params); + final perf = await (backend as BackendPerformanceDiagnostics) + .getPerformanceContext(context); + expect(perf!.speculativeAcceptedDraftTokens, greaterThan(0)); + expect(control.length, greaterThan(40)); + // Derive markers from an unrestricted deterministic control rather than + // asking a tiny base model to follow chat instructions. Exact prefixes + // remain the oracle for every stopped request. + for (final start in [0, 5, 20, control.length ~/ 2]) { + final stop = control.substring(start, start + 5); + final stopped = await run(params.copyWith(stopSequences: [stop])); + expect(stopped, control.substring(0, control.indexOf(stop))); + } + expect(await run(params), control); + expect(await run(params.copyWith(stopSequences: [''])), control); + expect( + await run(params.copyWith(stopSequences: ['$control unfinished'])), + control, + ); + final marker = control.substring(10, 20); + expect( + await run( + params.copyWith(stopSequences: [marker], preservedTokens: [marker]), + ), + control, + ); + }); + + test('rejected speculative request releases generation state', () async { + final control = await run(params); + await expectLater( + run(params.copyWith(grammar: 'root ::= "alpha"')), + throwsA(isA()), + ); + expect(await run(params), control); + }); + + test('cancellation and subsequent generation recover', () async { + final control = await run(params); + var chunks = 0; + await for (final _ in backend.generate( + context, + prompt, + params.copyWith(stopSequences: ['not a matching stop']), + )) { + chunks++; + backend.cancelGeneration(); + } + expect(chunks, greaterThan(0)); + expect(await run(params), control); + }); + }); +} diff --git a/test/unit/backends/llama_cpp/llama_cpp_service_test.dart b/test/unit/backends/llama_cpp/llama_cpp_service_test.dart index 20cb18f6..f20422b3 100644 --- a/test/unit/backends/llama_cpp/llama_cpp_service_test.dart +++ b/test/unit/backends/llama_cpp/llama_cpp_service_test.dart @@ -17,6 +17,18 @@ import 'package:path/path.dart' as path; import 'package:test/test.dart'; void main() { + test('preserved template tokens remain excluded from native text stops', () { + final stops = _invokePrivateForTesting>( + LlamaCppService(), + '_effectiveStopSequences', + [ + ['', 'cedar17', 'suffix'], + [''], + ], + ); + expect(stops, ['cedar17', 'suffix']); + }); + group('reasoning-budget resolver diagnostics', () { Object? resolve( LlamaCppService service, diff --git a/test/unit/backends/llama_cpp/stop_sequence_buffer_test.dart b/test/unit/backends/llama_cpp/stop_sequence_buffer_test.dart new file mode 100644 index 00000000..724c50c8 --- /dev/null +++ b/test/unit/backends/llama_cpp/stop_sequence_buffer_test.dart @@ -0,0 +1,152 @@ +import 'dart:convert'; + +import 'package:llamadart/src/backends/llama_cpp/stop_sequence_buffer.dart'; +import 'package:test/test.dart'; + +void main() { + List collect(List> pieces, List stops) { + final buffer = StopSequenceBuffer(stops); + return [ + for (final piece in pieces) ...buffer.add(piece), + ...buffer.finish(), + ]; + } + + test('withholds only a possible marker prefix and releases mismatches', () { + final buffer = StopSequenceBuffer(['cedar17']); + expect(buffer.add(utf8.encode('alpha ce')), utf8.encode('alpha ')); + expect(buffer.add(utf8.encode('d')), isEmpty); + expect(buffer.add(utf8.encode('X')), utf8.encode('cedX')); + expect(buffer.finish(), isEmpty); + expect(buffer.isStopped, isFalse); + }); + + test( + 'suppresses complete marker and token tail and ignores later pieces', + () { + final buffer = StopSequenceBuffer(['cedar17']); + expect( + buffer.add(utf8.encode('alpha cedar17 omega')), + utf8.encode('alpha '), + ); + expect(buffer.isStopped, isTrue); + expect(buffer.add(utf8.encode('extra')), isEmpty); + expect(buffer.finish(), isEmpty); + }, + ); + + test('overlapping repeated patterns agree with a complete-text oracle', () { + const stops = ['aaa', 'aab', 'bab']; + for (var bits = 0; bits < 256; bits++) { + final text = List.generate( + 8, + (i) => bits & (1 << i) == 0 ? 'a' : 'b', + ).join(); + final matches = + stops.map(text.indexOf).where((index) => index >= 0).toList()..sort(); + final expected = matches.isEmpty + ? text + : text.substring(0, matches.first); + final bytes = utf8.encode(text); + for (var split = 0; split <= bytes.length; split++) { + expect( + collect([bytes.sublist(0, split), bytes.sublist(split)], stops), + utf8.encode(expected), + reason: '$text at $split', + ); + } + expect( + collect([ + for (final byte in bytes) [byte], + ], stops), + utf8.encode(expected), + ); + } + }); + + test('long repeated prefixes match without losing the preceding bytes', () { + final marker = '${'x' * 10000}y'; + final buffer = StopSequenceBuffer([marker]); + final visible = []; + for (var i = 0; i < 20000; i++) { + visible.addAll(buffer.add([120])); + } + visible.addAll(buffer.add([121, 122])); + expect(visible, List.filled(10000, 120)); + expect(buffer.isStopped, isTrue); + expect(buffer.finish(), isEmpty); + }); + + test('long irrelevant stops do not delay ordinary streaming', () { + final buffer = StopSequenceBuffer(['x' * 10000, 'cedar17']); + for (var i = 0; i < 100; i++) { + expect(buffer.add(utf8.encode('hello ')), utf8.encode('hello ')); + } + expect(buffer.finish(), isEmpty); + }); + + test('releases incomplete marker once at normal completion', () { + final buffer = StopSequenceBuffer(['cedar17']); + expect(buffer.add(utf8.encode('cedar1')), isEmpty); + expect(buffer.finish(), utf8.encode('cedar1')); + expect(buffer.finish(), isEmpty); + expect(buffer.isStopped, isFalse); + }); + + test('empty and duplicate stops do not terminate generation', () { + final bytes = utf8.encode('alpha 🦊 café'); + expect(collect([bytes], ['', '']), bytes); + expect(collect([bytes], ['', 'absent', 'absent']), bytes); + }); + + test('earliest complete overlapping marker wins in either list order', () { + for (final stops in [ + ['ab', 'abc', 'bc'], + ['bc', 'abc', 'ab'], + ['abc', 'ab', 'bc'], + ]) { + expect(collect([utf8.encode('xabc tail')], stops), utf8.encode('x')); + } + expect( + collect([utf8.encode('xaab tail')], ['aab', 'ab']), + utf8.encode('x'), + ); + expect(collect([utf8.encode('abc')], ['abcd', 'bc']), utf8.encode('a')); + }); + + test('preserves arbitrary non-marker bytes without lossy UTF-8 decoding', () { + const bytes = [0xff, 0xc3, 0, 0xa9]; + expect(collect([bytes], ['🦊']), bytes); + }); + + for (final fixture in [ + (text: 'alpha cedar17 omega', stop: 'cedar17', expected: 'alpha '), + (text: 'café 🦊終わり omega', stop: '🦊終わり', expected: 'café '), + (text: 'aaaaab tail', stop: 'aaab', expected: 'aa'), + (text: 'no marker ced', stop: 'cedar17', expected: 'no marker ced'), + (text: 'prefix ${'x' * 130} tail', stop: 'x' * 130, expected: 'prefix '), + ]) { + test('all two-way byte splits: ${fixture.stop}', () { + final bytes = utf8.encode(fixture.text); + for (var split = 0; split <= bytes.length; split++) { + expect( + collect( + [bytes.sublist(0, split), bytes.sublist(split)], + [fixture.stop], + ), + utf8.encode(fixture.expected), + reason: 'split=$split', + ); + } + expect( + collect( + [ + for (final byte in bytes) [byte], + ], + [fixture.stop], + ), + utf8.encode(fixture.expected), + ); + }); + } +} diff --git a/tool/testing/run_local_e2e.dart b/tool/testing/run_local_e2e.dart index babe5a51..861a1870 100644 --- a/tool/testing/run_local_e2e.dart +++ b/tool/testing/run_local_e2e.dart @@ -230,6 +230,31 @@ List buildLocalE2eScenarios({String? projectRoot}) { ), ], ), + LocalE2eScenario( + name: 'gguf-stop-sequences', + group: LocalE2eScenarioGroup.dartLocalOnly, + description: 'Verify public GGUF caller stop suppression and recovery.', + requiresDevice: false, + stepsBuilder: (context) => [ + LocalE2eCommandStep( + workingDirectory: context.projectRoot, + executable: 'dart', + arguments: const [ + 'test', + '-p', + 'vm', + '--run-skipped', + 'test/e2e/backends/gguf_stop_sequences_e2e_test.dart', + ], + environment: { + if (context.modelPath != null) + 'GGUF_STOP_MODEL': context.modelPath!, + 'GGUF_STOP_BACKEND': context.backend, + }, + description: 'GGUF public stop-sequence regression', + ), + ], + ), LocalE2eScenario( name: 'gguf-chat-features-smoke', group: LocalE2eScenarioGroup.dartLocalOnly, diff --git a/tool/testing/test_matrix.dart b/tool/testing/test_matrix.dart index 2851e293..7e83ab8a 100644 --- a/tool/testing/test_matrix.dart +++ b/tool/testing/test_matrix.dart @@ -34,6 +34,19 @@ class TestMatrixRow { /// The canonical contributor-facing validation matrix. const List testMatrixRows = [ + TestMatrixRow( + id: 'gguf-stop-sequences', + tier: 'targeted', + mode: 'local-only', + covers: 'public GGUF caller stop suppression, batching, and recovery', + command: + 'dart run tool/testing/run_local_e2e.dart --scenario ' + 'gguf-stop-sequences --model-path --backend cpu', + useWhen: + 'Native GGUF streaming or stop-sequence changes. Use a compliant ' + 'Gemma 4 or Qwen3.5 chat model; pair with the deterministic ordinary/' + 'speculative test/integration/stop_sequences_test.dart suite.', + ), TestMatrixRow( id: 'static-format-analyze', tier: 'essential', diff --git a/website/docs/changelog/recent-releases.md b/website/docs/changelog/recent-releases.md index 1a60bee0..e9bf8730 100644 --- a/website/docs/changelog/recent-releases.md +++ b/website/docs/changelog/recent-releases.md @@ -9,6 +9,8 @@ For canonical full release notes, use: ## Unreleased +- Suppress caller stop markers in native GGUF streams, including split markers and speculative decoding. + - Fix encoded audio bytes leaking into string chat-template prompts, restoring native Qwen3-ASR file/bytes transcription parity. - Explain unavailable native thinking-budget helpers with bounded, path-free loader diagnostics. diff --git a/website/docs/configuration/runtime-parameters.md b/website/docs/configuration/runtime-parameters.md index 704d2d25..88c7f71a 100644 --- a/website/docs/configuration/runtime-parameters.md +++ b/website/docs/configuration/runtime-parameters.md @@ -30,6 +30,13 @@ await engine.loadModel( ); ``` +Native GGUF `stopSequences` suppress the first completed marker and any text +following it, including markers split across tokens or embedded inside a token. +Empty stops are ignored. Unfinished marker prefixes are emitted when generation +ends without a match. Template tokens listed in `preservedTokens` remain +available to the chat parser; identical stop entries are excluded from native +text matching. This applies to ordinary and speculative generation. + Important fields: - `contextSize`: total context window.