diff --git a/codec/regexp.gen.go b/codec/regexp.gen.go index a36ccef..bfb4f71 100644 --- a/codec/regexp.gen.go +++ b/codec/regexp.gen.go @@ -16,7 +16,7 @@ Capture(index = 0, unindex = -1) One(Ch = ') Atomic Alternate - Set(Set = [R-Tr-tſ]) + Set(Set = [STstſ]) Concatenate Set(Set = [Rr]) Set(Set = [Ee]) @@ -102,9 +102,9 @@ func splitRegexp_Execute(r *regexp2.Runner) error { alternation_starting_pos1 = pos // Branch 0 - // Node: Set(Set = [R-Tr-tſ]) - // Match [R-Tr-tſ]. - if len(slice) < 2 || ((slice[1] >= 128 || !helpers.IsInASCIIBitmap(slice[1], 0x0, 0x1c0000001c0000)) && (slice[1] < 128 || !set_4a1357005dba18ced5af0bde8202a98fab7a3500c675e3cb1c60597d7a36436a.CharIn(slice[1]))) { + // Node: Set(Set = [STstſ]) + // Match [STstſ]. + if len(slice) < 2 || ((slice[1] >= 128 || !helpers.IsInASCIIBitmap(slice[1], 0x0, 0x18000000180000)) && (slice[1] < 128 || !set_4a1357005dba18ced5af0bde8202a98fab7a3500c675e3cb1c60597d7a36436a.CharIn(slice[1]))) { goto AlternationBranch1 } diff --git a/tokenizer_test.go b/tokenizer_test.go index c3be00b..77b9b40 100644 --- a/tokenizer_test.go +++ b/tokenizer_test.go @@ -40,6 +40,7 @@ func TestCl100kBase(t *testing.T) { {text: "hello world", ids: []uint{15339, 256, 1917}}, {text: "supercalifragilistic", ids: []uint{13066, 3035, 278, 333, 4193, 321, 4633}}, {text: "We know what we are, but know not what we may be.", ids: []uint{1687, 1440, 1148, 584, 527, 11, 719, 1440, 539, 1148, 584, 1253, 387, 13}}, + {text: "'RE", ids: []uint{95253}}, } runTests(t, tok, tests)