From f8b72d50abbbaf6fb53bb6b9f5ffedb78b8051ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Thu, 3 Sep 2026 15:40:21 +0800 Subject: [PATCH] fix(text-chunking): split Default-mode CJK text at full-width punctuation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit split_word_ranges only cut words at ASCII spaces, so space-less CJK paragraphs parsed as a single word and never chunked at the codepoint budget — long Chinese text synthesized as one chunk in every TTS family using chunk_text_request (offline and streaming alike), regardless of text_chunk_size. Full-width CJK sentence/clause punctuation now forms word boundaries (the punctuation token itself carries the break flags); ASCII punctuation stays attached to its word, leaving Latin text unchanged. Adds text_chunking_test: CJK over/under budget, clause rollback, Latin sentence split, oversized single-word passthrough. --- CMakeLists.txt | 7 +++ src/framework/text/chunking.cpp | 29 ++++++++- tests/unittests/test_text_chunking.cpp | 85 ++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/test_text_chunking.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ffdb9418..01199cade 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2110,6 +2110,13 @@ if (ENGINE_BUILD_TESTS) COMMAND chinese_normalization_test ) + add_engine_unittest(text_chunking_test tests/unittests/test_text_chunking.cpp) + + add_test( + NAME text_chunking_test + COMMAND text_chunking_test + ) + add_engine_unittest(unicode_normalization_test tests/unittests/test_unicode_normalization.cpp) add_test( diff --git a/src/framework/text/chunking.cpp b/src/framework/text/chunking.cpp index c5641d707..d47fe489c 100644 --- a/src/framework/text/chunking.cpp +++ b/src/framework/text/chunking.cpp @@ -63,6 +63,17 @@ bool is_clause_break(std::string_view token) { token == u8"，" || token == u8"、" || token == u8"；" || token == u8"："; } +bool is_cjk_punctuation_delimiter(std::string_view token) noexcept { + // Full-width CJK sentence/clause punctuation (。！？，、；：) separates words + // even without ASCII spaces — CJK text has no inter-word spaces, so without + // this a whole paragraph parses as one word and Default-mode chunking can + // never split it at the codepoint budget (only TagAware/Japanese could). + // ASCII punctuation stays attached to its word so space-delimited Latin + // text is unaffected. + const auto leading = static_cast(token.front()); + return leading >= 0x80 && (is_sentence_break(token) || is_clause_break(token)); +} + bool is_tag_open(std::string_view token) { return token == "[" || token == "<"; } @@ -112,9 +123,25 @@ std::vector split_word_ranges(const std::vector & spans) { if (span_pos >= spans.size()) { break; } + // A CJK punctuation token forms a word of its own (attached to nothing), + // so a subsequent run never absorbs it and boundaries can land on it. + if (is_cjk_punctuation_delimiter(spans[span_pos].text)) { + words.push_back({ + span_pos, + span_pos + 1, + spans[span_pos].start, + spans[span_pos].end, + is_sentence_break(spans[span_pos].text), + is_clause_break(spans[span_pos].text), + }); + span_pos += 1; + continue; + } const size_t word_start = span_pos; size_t word_end = span_pos + 1; - while (word_end < spans.size() && !is_ascii_space(spans[word_end].text)) { + while (word_end < spans.size() && + !is_ascii_space(spans[word_end].text) && + !is_cjk_punctuation_delimiter(spans[word_end].text)) { ++word_end; } const auto last = spans[word_end - 1].text; diff --git a/tests/unittests/test_text_chunking.cpp b/tests/unittests/test_text_chunking.cpp new file mode 100644 index 000000000..f6d0927a6 --- /dev/null +++ b/tests/unittests/test_text_chunking.cpp @@ -0,0 +1,85 @@ +// Default-mode text chunking regression: CJK text (no ASCII spaces) must still +// split at the codepoint budget, with full-width punctuation as word boundaries. +#include "engine/framework/text/chunking.h" + +#include +#include +#include +#include + +namespace { + +void require(bool ok, const std::string & what) { + if (!ok) { + std::cerr << "FAIL: " << what << "\n"; + std::exit(1); + } +} + +void check_chunks(const std::string & text, int64_t budget, size_t min_chunks, const std::string & label) { + const auto chunks = engine::text::split_text_chunks(text, budget); + if (chunks.size() < min_chunks) { + std::cerr << "FAIL: " << label << " expected >= " << min_chunks << " chunks, got " << chunks.size() + << "\n"; + std::exit(1); + } + std::string joined; + for (const auto & c : chunks) { + joined += c; + } + require(joined == text, label + ": chunks must concatenate back to the trimmed input verbatim"); + for (const auto & c : chunks) { + require(!c.empty(), label + ": empty chunk"); + } +} + +} // namespace + +int main() { + // Regression: a 236-codepoint CJK paragraph with the 200-codepoint default + // budget used to come back as a single chunk, because split_word_ranges only + // cut at ASCII spaces and the whole paragraph parsed as one word. Must now + // split into at least two sentence-aligned chunks. + const std::string cjk_long = + "大家好，我是零一B语音合成模型，现在进行流式输出测试。这段文字比较长，目的是看模型能否把整段话分成多个音频块，边合成边推送。" + "今天的天气很好，适合外出散步，湖边的柳树已经抽出了新芽，水面倒映着蓝天白云。如果流式工作正常，客户端应该能很快收到第一个音频块。" + "第二段继续测试分块是否稳定，这里再补充一些内容，让总字数超过上限，这样模型必须把文字切开分多次合成。流式的意义在于长文本不必等全部算完。" + "最后再来一句收尾的话，确认整个流程完整结束，谢谢大家。"; + check_chunks(cjk_long, 200, 2, "CJK long text over budget splits"); + + // Under budget: still one chunk, verbatim. + check_chunks("短文本，不需要分块。", 200, 1, "short CJK text stays whole"); + + // Clause punctuation is a valid rollback boundary when the budget cuts a + // run of clauses: budget 12 lands inside the second clause run, so the + // first chunk must end at the 、 (clause) boundary, not mid-run. + { + const std::string text = "一二三四五六七八九十，一二三四五六七八九十。"; + const auto chunks = engine::text::split_text_chunks(text, 12); + require(chunks.size() == 2, "clause rollback produces two chunks"); + require(chunks[0] == "一二三四五六七八九十，", "first chunk ends at clause punctuation"); + require(chunks[1] == "一二三四五六七八九十。", "second chunk keeps the sentence"); + } + + // Latin text is unaffected: ASCII punctuation stays attached to its word + // and boundaries still fall on ASCII-space words / sentence breaks. + { + const std::string text = "Hello world. How are you today?"; + const auto chunks = engine::text::split_text_chunks(text, 20); + require(chunks.size() == 2, "latin splits at the sentence break"); + require(chunks[0] == "Hello world.", "first latin chunk ends at the period"); + require(chunks[1] == "How are you today?", "second latin chunk verbatim"); + } + + // A single oversized word without any boundary still passes through whole + // (pre-existing behavior: no hard character-level cut). + { + const std::string word(300, 'a'); + const auto chunks = engine::text::split_text_chunks(word, 200); + require(chunks.size() == 1, "unbreakable oversized word stays one chunk"); + require(chunks[0] == word, "oversized word verbatim"); + } + + std::cout << "text_chunking_test: all passed\n"; + return 0; +}