Skip to content

Register the seven H3 control tokens the vocabulary omits #56

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Tatlatat wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Register the seven H3 control tokens the vocabulary omits #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions h3_tokenizer.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,19 @@ @interface H3Tokenizer : NSObject {
@implementation H3Tokenizer
@end

/* MiniMax declares seven H3 control tokens in tokenizer_config.json under
* "additional_special_tokens" only. They are absent from tokenizer.json, so the
* reference tokenizer appends them after the last declared added token, giving
* <d> the identifier 151669 for the released vocabulary. Without them the BPE
* merges swallow the markup: "<d>[English] Hi.</d>" becomes nine ordinary
* pieces whose first token fuses the tag with the following bracket, so the
* dialogue, cutoff, lyric, and caption markers described in the official
* prompting guides never reach the DiT. */
static NSString *const h3_extra_special_tokens[] = {
@"<d>", @"</d>", @"<|cutoff|>", @"<|lyrics_start|>", @"<|lyrics_end|>",
@"<|caption_start|>", @"<|caption_end|>",
};

static H3Tokenizer *TOK(const h3_tokenizer *tokenizer) {
return (__bridge H3Tokenizer *)(void *)tokenizer;
}
Expand Down Expand Up @@ -329,9 +342,20 @@ static int h3_added_match(H3Tokenizer *tokenizer, NSString *text,
}
NSArray *added = config[@"added_tokens"];
if (!added) added = @[];
NSMutableSet<NSString *> *declared = [NSMutableSet set];
for (NSDictionary *token in added) {
maximum_id = MAX(maximum_id, [token[@"id"] unsignedIntegerValue]);
NSString *content = token[@"content"];
if (content) [declared addObject:content];
}
NSMutableArray<NSString *> *extra_tokens = [NSMutableArray array];
for (size_t index = 0; index < sizeof(h3_extra_special_tokens) /
sizeof(h3_extra_special_tokens[0]); index++) {
NSString *content = h3_extra_special_tokens[index];
if (![declared containsObject:content]) [extra_tokens addObject:content];
}
NSUInteger first_extra_id = maximum_id + 1;
maximum_id += extra_tokens.count;
NSMutableArray *inverse_vocab = [NSMutableArray arrayWithCapacity:maximum_id + 1];
NSMutableArray *inverse_added = [NSMutableArray arrayWithCapacity:maximum_id + 1];
for (NSUInteger index = 0; index <= maximum_id; index++) {
Expand Down Expand Up @@ -378,6 +402,11 @@ static int h3_added_match(H3Tokenizer *tokenizer, NSString *text,
added_tokens[content] = identifier;
inverse_added[identifier.unsignedIntegerValue] = content;
}
for (NSUInteger index = 0; index < extra_tokens.count; index++) {
NSNumber *identifier = @(first_extra_id + index);
added_tokens[extra_tokens[index]] = identifier;
inverse_added[identifier.unsignedIntegerValue] = extra_tokens[index];
}
tokenizer.addedTokens = added_tokens;
tokenizer.inverseAddedTokens = inverse_added;
tokenizer.addedAlternatives = [added_tokens.allKeys
Expand Down
11 changes: 11 additions & 0 deletions tests/test_tokenizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,17 @@ int main(int argc, char **argv) {
sizeof(emoji) / sizeof(emoji[0]));
const uint32_t special[] = {151644};
check_case(tokenizer, "<|im_start|>", special, 1);
/* The seven H3 control tokens live in tokenizer_config.json rather than in
* tokenizer.json, and the reference tokenizer appends them after the last
* declared added token. */
const uint32_t dialogue[] = {151669, 58, 22574, 60, 21927, 1052, 13, 151670};
check_case(tokenizer, "<d>[English] Hello there.</d>", dialogue,
sizeof(dialogue) / sizeof(dialogue[0]));
const uint32_t cutoff[] = {151671};
check_case(tokenizer, "<|cutoff|>", cutoff, 1);
const uint32_t caption[] = {151674, 151675};
check_case(tokenizer, "<|caption_start|><|caption_end|>", caption,
sizeof(caption) / sizeof(caption[0]));
const uint32_t cinematic[] = {32, 64665, 3265, 5239, 315, 264, 8866,
1778, 11958, 4633, 10971, 13};
check_case(tokenizer,
Expand Down