From 5a434b79da43226c7078c64cb57c26213aa547d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=90=E1=BB=97=20T=E1=BA=A5t=20Ho=C3=A0ng=20H=C6=B0ng?=
 <hungdotmn@gmail.com>
Date: Fri, 4 Sep 2026 09:52:57 +0700
Subject: [PATCH] Register the seven H3 control tokens the vocabulary omits

MiniMax declares <d>, </d>, <|cutoff|>, <|lyrics_start|>, <|lyrics_end|>,
<|caption_start|> and <|caption_end|> in tokenizer_config.json under
"additional_special_tokens". They appear in neither the BPE vocabulary nor
tokenizer.json's "added_tokens" array, which stops at 151668, so the loader
never learned them and the BPE merges swallowed the markup instead.

The damage reaches past the tag itself. Against the released FL2VA tokenizer,
"<d>[English] Hello there." + "</d>" encoded as nine pieces

    90707 30768 22574 60 21927 1052 3918 67 29

where 90707 fuses "<d>[" into a single token and 3918 fuses ".</", so both the
dialogue marker and the language tag beside it reach the DiT as ordinary text.
The reference tokenizer appends the seven tokens after the last declared added
token and produces eight pieces instead:

    151669 58 22574 60 21927 1052 13 151670

Assign the same identifiers by continuing from the highest declared id, which
yields 151669-151675 for the released vocabulary and stays clear of the 151936
text-encoder embedding bound. Tokens already present in tokenizer.json are left
alone, so a future release that declares them properly keeps its own ids.

Verified against transformers 5.16.1 loading FL2VA/tokenizer: identifiers and
piece boundaries now match for the dialogue, cutoff and caption cases, while
"<|im_start|>" and the existing prose cases are unchanged. tests/test_tokenizer.c
covers the three new cases and passes against both the FL2VA and Ref2VA
tokenizers.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CbhnKLxBiCL4nUCUrbiKh
---
 h3_tokenizer.m         | 29 +++++++++++++++++++++++++++++
 tests/test_tokenizer.c | 11 +++++++++++
 2 files changed, 40 insertions(+)

diff --git a/h3_tokenizer.m b/h3_tokenizer.m
index 0d519fbd..16dc18dc 100644
--- a/h3_tokenizer.m
+++ b/h3_tokenizer.m
@@ -29,6 +29,19 @@ @interface H3Tokenizer : NSObject {
 @implementation H3Tokenizer
 @end
 
+/* MiniMax declares seven H3 control tokens in tokenizer_config.json under
+ * "additional_special_tokens" only. They are absent from tokenizer.json, so the
+ * reference tokenizer appends them after the last declared added token, giving
+ * <d> the identifier 151669 for the released vocabulary. Without them the BPE
+ * merges swallow the markup: "<d>[English] Hi.</d>" becomes nine ordinary
+ * pieces whose first token fuses the tag with the following bracket, so the
+ * dialogue, cutoff, lyric, and caption markers described in the official
+ * prompting guides never reach the DiT. */
+static NSString *const h3_extra_special_tokens[] = {
+    @"<d>", @"</d>", @"<|cutoff|>", @"<|lyrics_start|>", @"<|lyrics_end|>",
+    @"<|caption_start|>", @"<|caption_end|>",
+};
+
 static H3Tokenizer *TOK(const h3_tokenizer *tokenizer) {
     return (__bridge H3Tokenizer *)(void *)tokenizer;
 }
@@ -329,9 +342,20 @@ static int h3_added_match(H3Tokenizer *tokenizer, NSString *text,
         }
         NSArray *added = config[@"added_tokens"];
         if (!added) added = @[];
+        NSMutableSet<NSString *> *declared = [NSMutableSet set];
         for (NSDictionary *token in added) {
             maximum_id = MAX(maximum_id, [token[@"id"] unsignedIntegerValue]);
+            NSString *content = token[@"content"];
+            if (content) [declared addObject:content];
         }
+        NSMutableArray<NSString *> *extra_tokens = [NSMutableArray array];
+        for (size_t index = 0; index < sizeof(h3_extra_special_tokens) /
+                                       sizeof(h3_extra_special_tokens[0]); index++) {
+            NSString *content = h3_extra_special_tokens[index];
+            if (![declared containsObject:content]) [extra_tokens addObject:content];
+        }
+        NSUInteger first_extra_id = maximum_id + 1;
+        maximum_id += extra_tokens.count;
         NSMutableArray *inverse_vocab = [NSMutableArray arrayWithCapacity:maximum_id + 1];
         NSMutableArray *inverse_added = [NSMutableArray arrayWithCapacity:maximum_id + 1];
         for (NSUInteger index = 0; index <= maximum_id; index++) {
@@ -378,6 +402,11 @@ static int h3_added_match(H3Tokenizer *tokenizer, NSString *text,
             added_tokens[content] = identifier;
             inverse_added[identifier.unsignedIntegerValue] = content;
         }
+        for (NSUInteger index = 0; index < extra_tokens.count; index++) {
+            NSNumber *identifier = @(first_extra_id + index);
+            added_tokens[extra_tokens[index]] = identifier;
+            inverse_added[identifier.unsignedIntegerValue] = extra_tokens[index];
+        }
         tokenizer.addedTokens = added_tokens;
         tokenizer.inverseAddedTokens = inverse_added;
         tokenizer.addedAlternatives = [added_tokens.allKeys
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c
index 40ad4cdb..da2a671a 100644
--- a/tests/test_tokenizer.c
+++ b/tests/test_tokenizer.c
@@ -51,6 +51,17 @@ int main(int argc, char **argv) {
                sizeof(emoji) / sizeof(emoji[0]));
     const uint32_t special[] = {151644};
     check_case(tokenizer, "<|im_start|>", special, 1);
+    /* The seven H3 control tokens live in tokenizer_config.json rather than in
+     * tokenizer.json, and the reference tokenizer appends them after the last
+     * declared added token. */
+    const uint32_t dialogue[] = {151669, 58, 22574, 60, 21927, 1052, 13, 151670};
+    check_case(tokenizer, "<d>[English] Hello there.</d>", dialogue,
+               sizeof(dialogue) / sizeof(dialogue[0]));
+    const uint32_t cutoff[] = {151671};
+    check_case(tokenizer, "<|cutoff|>", cutoff, 1);
+    const uint32_t caption[] = {151674, 151675};
+    check_case(tokenizer, "<|caption_start|><|caption_end|>", caption,
+               sizeof(caption) / sizeof(caption[0]));
     const uint32_t cinematic[] = {32, 64665, 3265, 5239, 315, 264, 8866,
                                   1778, 11958, 4633, 10971, 13};
     check_case(tokenizer,