diff --git a/CHANGELOG.md b/CHANGELOG.md index d388cbf8..cc0d5829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +* Preserve accented text, emoji, and other Unicode characters when detokenizing native GGUF tokens. + * Update native LiteRT-LM to `v0.17.0-5` with corrected Linux library loading, and enable explicit GPU selection on Linux x64 and Windows x64 while retaining CPU defaults. * Make zero-temperature native LiteRT-LM generation greedy to avoid corrupted GPU output. diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index 6a1acd41..73b86f2b 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -6019,7 +6019,8 @@ class LlamaCppService { final model = _models[modelHandle]; if (model == null) return ""; final vocab = llama_model_get_vocab(model.pointer); - final buffer = malloc(256); + // UTF-8 decoding requires unsigned bytes, including byte-fallback tokens. + final buffer = malloc(256); final bytes = []; for (final t in tokens) { final n = llama_token_to_piece(vocab, t, buffer.cast(), 256, 0, special); diff --git a/test/integration/backends/llama_cpp/tokenizer_integration_test.dart b/test/integration/backends/llama_cpp/tokenizer_integration_test.dart new file mode 100644 index 00000000..b36eac9d --- /dev/null +++ b/test/integration/backends/llama_cpp/tokenizer_integration_test.dart @@ -0,0 +1,70 @@ +@TestOn('vm') +@Timeout(Duration(minutes: 5)) +library; + +import 'package:llamadart/llamadart.dart'; +import 'package:test/test.dart'; + +import '../../../test_helper.dart'; + +void main() { + group('Native GGUF tokenizer', () { + late LlamaEngine engine; + + setUpAll(() async { + final model = await TestHelper.getTestModel(); + engine = LlamaEngine(LlamaBackend()); + addTearDown(engine.dispose); + await engine.loadModel( + model.path, + modelParams: const ModelParams( + contextSize: 128, + gpuLayers: 0, + preferredBackend: GpuBackend.cpu, + numberOfThreads: 1, + numberOfThreadsBatch: 1, + ), + ); + }); + + // The tiny Llama SentencePiece vocabulary adds one leading space. Keep + // that normalization explicit so trimming cannot hide damaged output. + const samples = { + 'ASCII and newline': 'Hello world\n123', + 'accented Latin': 'Montréal café', + 'emoji': '👋', + 'Korean': '한글', + 'mixed Unicode': 'Montréal 👋\n한글 café', + }; + for (final sample in samples.entries) { + test('round-trips ${sample.key}', () async { + final tokens = await engine.tokenize(sample.value, addSpecial: false); + expect(tokens, isNotEmpty); + expect(await engine.detokenize(tokens), ' ${sample.value}'); + }); + } + + test('joins UTF-8 bytes across token boundaries before decoding', () async { + final tokens = await engine.tokenize('👋', addSpecial: false); + final pieces = await Future.wait( + tokens.map((token) => engine.detokenize([token])), + ); + + // This vocabulary emits individual byte tokens for this emoji. A lone + // partial sequence is malformed, but the full token sequence is valid. + expect(pieces.join(), contains('\uFFFD')); + expect(await engine.detokenize(tokens), ' 👋'); + }); + + test('preserves special-token visibility', () async { + final tokens = await engine.tokenize('', addSpecial: false); + expect(tokens, hasLength(1)); + expect(await engine.detokenize(tokens), isEmpty); + expect(await engine.detokenize(tokens, special: true), ''); + }); + + test('detokenizes an empty token list', () async { + expect(await engine.detokenize([]), isEmpty); + }); + }); +} diff --git a/website/docs/changelog/recent-releases.md b/website/docs/changelog/recent-releases.md index e64f2e05..98dbe3a5 100644 --- a/website/docs/changelog/recent-releases.md +++ b/website/docs/changelog/recent-releases.md @@ -9,6 +9,8 @@ For canonical full release notes, use: ## Unreleased +* Preserve accented text, emoji, and other Unicode characters when detokenizing native GGUF tokens. + * Update native LiteRT-LM to `v0.17.0-5` with corrected Linux library loading, and enable explicit GPU selection on Linux x64 and Windows x64 while retaining CPU defaults. * Make zero-temperature native LiteRT-LM generation greedy to avoid corrupted GPU output.