fix: suppress caller stop markers in native GGUF streams

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Unreleased
 
+- Suppress caller stop markers in native GGUF streams, including split markers and speculative decoding.
+
 - Fix encoded audio bytes leaking into string chat-template prompts, restoring native Qwen3-ASR file/bytes transcription parity.
 
 * Explain unavailable native thinking-budget helpers with bounded, path-free loader diagnostics.

doc/testing_matrix.md

@@ -406,3 +406,23 @@ When an agent creates or updates a PR:
    ancestry with `tool/git/safe_pr_head_update.dart` to prevent stale head
    rewinds. Consult `doc/pr_branch_writer_inventory.md` for writer scope and the
    remaining GitHub-managed governance boundary.
+
+### Native GGUF stop sequences
+
+`dart test -p vm test/integration/stop_sequences_test.dart` uses the small test
+GGUF to cover ordinary and speculative stop handling, Unicode, token boundaries,
+completion, cancellation, and subsequent generation. Ordinary tests constrain
+output with grammar; speculative tests use deterministic unrestricted controls
+because grammar sampling is unsupported there, and assert actual draft acceptance.
+
+Use a compliant chat model (Gemma 4 E2B or Qwen3.5 0.8B) for the unforced public
+chat fixture; the control must emit `alpha cedar17 omega` and caller stop
+`cedar17` must leave exactly `alpha `, under single-piece and default batching:
+
+```bash
+dart run tool/testing/run_local_e2e.dart --scenario gguf-stop-sequences \
+  --model-path /path/to/chat.gguf --backend cpu
+```
+
+Repeat with `--backend metal` on macOS when available. Record the source commit,
+native runtime tag, printed model SHA-256, and native offload logs with results.

lib/src/backends/llama_cpp/llama_cpp_service.dart

@@ -23,6 +23,7 @@ import '../../core/template/media_placeholders.dart';
 import '../../core/models/inference/model_params.dart';
 import '../../core/template/chat_template_engine.dart';
 import 'load_param_helpers.dart';
+import 'stop_sequence_buffer.dart';
 import 'bindings.dart';
 import 'llama_cpp_raw_bindings.dart' as raw_bindings;
 
@@ -5399,7 +5400,7 @@ class LlamaCppService {
   ) async* {
     final cancelToken = Pointer<Int8>.fromAddress(cancelTokenAddress);
     int currentPos = startPos;
-    final accumulatedBytes = <int>[];
+    final stopBuffer = StopSequenceBuffer(stopSequences);
     final evalStopwatch = Stopwatch()..start();
     var sampleMicros = 0;
     var evalMicros = 0;
@@ -5429,17 +5430,10 @@ class LlamaCppService {
 
       if (n > 0) {
         final bytes = pieceBuf.asTypedList(n).toList();
-        yield bytes;
         generatedTokens++;
-
-        if (stopSequences.isNotEmpty) {
-          accumulatedBytes.addAll(bytes);
-          if (accumulatedBytes.length > 64) {
-            accumulatedBytes.removeRange(0, accumulatedBytes.length - 64);
-          }
-          final text = utf8.decode(accumulatedBytes, allowMalformed: true);
-          if (stopSequences.any((s) => text.endsWith(s))) break;
-        }
+        final visible = stopBuffer.add(bytes);
+        if (visible.isNotEmpty) yield visible;
+        if (stopBuffer.isStopped) break;
       }
 
       batch.n_tokens = 1;
@@ -5456,6 +5450,9 @@ class LlamaCppService {
       if (decodeStatus != 0) break;
     }
 
+    final remaining = stopBuffer.finish();
+    if (remaining.isNotEmpty) yield remaining;
+
     evalStopwatch.stop();
     ctx.lastPerfEvalMs = evalMicros / 1000.0;
     ctx.lastPerfSampleMs = sampleMicros / 1000.0;
@@ -5491,7 +5488,7 @@ class LlamaCppService {
 
     int currentPos = startPos;
     int? pendingSampledToken;
-    final accumulatedBytes = <int>[];
+    final stopBuffer = StopSequenceBuffer(stopSequences);
     final evalStopwatch = Stopwatch()..start();
     var sampleMicros = 0;
     var evalMicros = 0;
@@ -5536,17 +5533,9 @@ class LlamaCppService {
 
           if (n > 0) {
             final bytes = pieceBuf.asTypedList(n).toList();
-            yield bytes;
-            if (stopSequences.isNotEmpty) {
-              accumulatedBytes.addAll(bytes);
-              if (accumulatedBytes.length > 64) {
-                accumulatedBytes.removeRange(0, accumulatedBytes.length - 64);
-              }
-              final text = utf8.decode(accumulatedBytes, allowMalformed: true);
-              if (stopSequences.any((s) => text.endsWith(s))) {
-                shouldStop = true;
-              }
-            }
+            final visible = stopBuffer.add(bytes);
+            if (visible.isNotEmpty) yield visible;
+            shouldStop = stopBuffer.isStopped;
           }
 
           if (shouldStop) {
@@ -5850,20 +5839,9 @@ class LlamaCppService {
 
             if (n > 0) {
               final bytes = pieceBuf.asTypedList(n).toList();
-              yield bytes;
-              if (stopSequences.isNotEmpty) {
-                accumulatedBytes.addAll(bytes);
-                if (accumulatedBytes.length > 64) {
-                  accumulatedBytes.removeRange(0, accumulatedBytes.length - 64);
-                }
-                final text = utf8.decode(
-                  accumulatedBytes,
-                  allowMalformed: true,
-                );
-                if (stopSequences.any((s) => text.endsWith(s))) {
-                  shouldStop = true;
-                }
-              }
+              final visible = stopBuffer.add(bytes);
+              if (visible.isNotEmpty) yield visible;
+              shouldStop = stopBuffer.isStopped;
             }
 
             if (shouldStop || generatedTokens >= params.maxTokens) {
@@ -5880,6 +5858,8 @@ class LlamaCppService {
           }
         }
       }
+      final remaining = stopBuffer.finish();
+      if (remaining.isNotEmpty) yield remaining;
     } finally {
       malloc.free(draftPtr);
       malloc.free(idxPtr);

lib/src/backends/llama_cpp/stop_sequence_buffer.dart

@@ -0,0 +1,94 @@
+import 'dart:collection';
+import 'dart:convert';
+
+/// Holds only bytes that could still complete a caller stop sequence.
+///
+/// Matching UTF-8 bytes avoids decoding incomplete token pieces. A completed
+/// marker and everything after it are suppressed, even within the same piece.
+class StopSequenceBuffer {
+  /// Creates a per-generation buffer, ignoring empty and duplicate markers.
+  StopSequenceBuffer(List<String> stops)
+    : _stops = stops
+          .where((stop) => stop.isNotEmpty)
+          .toSet()
+          .map((stop) => _StopPattern(utf8.encode(stop)))
+          .toList(growable: false);
+
+  final List<_StopPattern> _stops;
+  final ListQueue<int> _pending = ListQueue<int>();
+
+  /// Whether a complete marker has been consumed.
+  bool get isStopped => _isStopped;
+  bool _isStopped = false;
+
+  /// Consumes a token piece and returns bytes safe to expose immediately.
+  List<int> add(List<int> bytes) {
+    if (_isStopped) return const [];
+    if (_stops.isEmpty) return bytes;
+    final previousLength = _pending.length;
+    _pending.addAll(bytes);
+    var stopIndex = _pending.length;
+    var retainedLength = 0;
+    for (final stop in _stops) {
+      for (var i = 0; i < bytes.length; i++) {
+        if (stop.advance(bytes[i])) {
+          final start = previousLength + i + 1 - stop.bytes.length;
+          if (start < stopIndex) stopIndex = start;
+        }
+      }
+      if (stop.matched > retainedLength) retainedLength = stop.matched;
+    }
+    if (stopIndex < _pending.length) {
+      final visible = _pending.take(stopIndex).toList();
+      _pending.clear();
+      _isStopped = true;
+      return visible;
+    }
+    final safeLength = _pending.length - retainedLength;
+    return List.generate(
+      safeLength,
+      (_) => _pending.removeFirst(),
+      growable: false,
+    );
+  }
+
+  /// Releases an unfinished marker prefix when generation ends without a match.
+  List<int> finish() {
+    final remaining = _pending.toList();
+    _pending.clear();
+    for (final stop in _stops) {
+      stop.matched = 0;
+    }
+    return remaining;
+  }
+}
+
+// KMP prefix state processes only newly received bytes. Repeated prefixes in
+// long caller markers must not cause rescans of the entire pending suffix on
+// every token. Work is linear in incoming bytes per stop (amortized).
+class _StopPattern {
+  _StopPattern(this.bytes) : fallback = List.filled(bytes.length, 0) {
+    var prefix = 0;
+    for (var i = 1; i < bytes.length; i++) {
+      while (prefix > 0 && bytes[i] != bytes[prefix]) {
+        prefix = fallback[prefix - 1];
+      }
+      if (bytes[i] == bytes[prefix]) prefix++;
+      fallback[i] = prefix;
+    }
+  }
+
+  final List<int> bytes;
+  final List<int> fallback;
+  int matched = 0;
+
+  bool advance(int byte) {
+    while (matched > 0 && byte != bytes[matched]) {
+      matched = fallback[matched - 1];
+    }
+    if (byte == bytes[matched]) matched++;
+    if (matched != bytes.length) return false;
+    matched = fallback[matched - 1];
+    return true;
+  }
+}

lib/src/core/models/inference/generation_params.dart

@@ -620,7 +620,12 @@ class GenerationParams {
   /// If null, a seed based on the current time will be used.
   final int? seed;
 
-  /// List of strings that, if generated, will immediately stop the generation process.
+  /// Strings that end generation when matched, excluding the marker from output.
+  ///
+  /// Empty strings are ignored. Native GGUF matches across token boundaries and
+  /// inside token pieces; an unfinished prefix is emitted if generation ends
+  /// without a full match. Exact entries in [preservedTokens] remain available
+  /// to the native chat parser instead of acting as text stops.
   final List<String> stopSequences;
 
   /// GBNF grammar string for structured output (e.g., "root ::= \"hello\" | \"world\"").

test/e2e/backends/gguf_stop_sequences_e2e_test.dart

@@ -0,0 +1,77 @@
+@TestOn('vm')
+@Tags(['local-only', 'e2e'])
+@Timeout(Duration(minutes: 5))
+library;
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:crypto/crypto.dart';
+import 'package:llamadart/llamadart.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('public GGUF chat suppresses caller marker and recovers', () async {
+    final model = Platform.environment['GGUF_STOP_MODEL'];
+    expect(model, isNotNull, reason: 'Set GGUF_STOP_MODEL to a chat GGUF');
+    final backendName = Platform.environment['GGUF_STOP_BACKEND'] ?? 'cpu';
+    final backend = GpuBackend.values.byName(backendName);
+    final engine = LlamaEngine(LlamaBackend());
+    addTearDown(engine.dispose);
+    await engine.setNativeLogLevel(LlamaLogLevel.info);
+    final checksum = await sha256.bind(File(model!).openRead()).first;
+    print(
+      jsonEncode({
+        'model_sha256': '$checksum',
+        'backend': backendName,
+        'dart': Platform.version,
+        'os': Platform.operatingSystemVersion,
+      }),
+    );
+    await engine.loadModel(
+      model,
+      modelParams: ModelParams(
+        contextSize: 1024,
+        gpuLayers: backend == GpuBackend.cpu ? 0 : 99,
+        preferredBackend: backend,
+        numberOfThreads: 4,
+        numberOfThreadsBatch: 4,
+      ),
+    );
+    Future<String> run(List<String> stops, int batching) async {
+      final text = StringBuffer();
+      await for (final chunk in engine.create(
+        [
+          LlamaChatMessage.fromText(
+            role: LlamaChatRole.user,
+            text: 'Reply with exactly: alpha cedar17 omega',
+          ),
+        ],
+        enableThinking: false,
+        params: GenerationParams(
+          temp: 0,
+          seed: 1,
+          maxTokens: 32,
+          stopSequences: stops,
+          streamBatchTokenThreshold: batching,
+          streamBatchByteThreshold: batching == 1 ? 1 : 512,
+        ),
+      )) {
+        for (final choice in chunk.choices) {
+          text.write(choice.delta.content ?? '');
+        }
+      }
+      return text.toString();
+    }
+
+    final control = await run([], 8);
+    print(jsonEncode({'control': control}));
+    expect(control, 'alpha cedar17 omega');
+    for (final batching in [1, 8]) {
+      final stopped = await run(['cedar17'], batching);
+      print(jsonEncode({'batch_tokens': batching, 'stopped': stopped}));
+      expect(stopped, 'alpha ');
+    }
+    expect(await run([], 8), control);
+  });
+}