diff --git a/extraResources/sherpa/README.md b/extraResources/sherpa/README.md index f0c5664d..47c83b68 100644 --- a/extraResources/sherpa/README.md +++ b/extraResources/sherpa/README.md @@ -3,8 +3,10 @@ 封装 JS 复制自 npm `sherpa-onnx-node@1.13.2`(Apache-2.0)。 - `vendor/addon.js` 已替换为自定义加载器:从环境变量 `SHERPA_ONNX_LIB_DIR` - 用 `process.dlopen` 加载 `sherpa-onnx.node`(其余 `vendor/*.js` 原样保留, - 它们都经 `require('./addon.js')` 取原生模块)。 + 用 `process.dlopen` 加载 `sherpa-onnx.node`;vendor 文件都经 + `require('./addon.js')` 取原生模块。 +- `vendor/json-result.js` 由 `non-streaming-asr.js` 使用,处理原生识别结果中 + 的未转义控制字符;其余 `vendor/*.js` 原样保留。 - `vendor/addon-static-import.js` 在自定义加载器下不再被引用(保留以便升级对照)。 原生库**不在此处**,按需下载到 `userData/sherpa-onnx/current/` diff --git a/extraResources/sherpa/vendor/json-result.js b/extraResources/sherpa/vendor/json-result.js new file mode 100644 index 00000000..4d334b22 --- /dev/null +++ b/extraResources/sherpa/vendor/json-result.js @@ -0,0 +1,74 @@ +'use strict'; + +// Native sherpa results are JSON strings. Some Qwen outputs may contain raw +// C0 control characters inside the transcript text, which JSON.parse rejects. +function sanitizeJsonControlCharacters(jsonStr) { + let sanitized = ''; + let inString = false; + let escaped = false; + + for (const char of String(jsonStr)) { + const code = char.charCodeAt(0); + + if (inString) { + if (escaped) { + sanitized += char; + escaped = false; + continue; + } + if (char === '\\') { + sanitized += char; + escaped = true; + continue; + } + if (char === '"') { + sanitized += char; + inString = false; + continue; + } + if (code <= 0x1f) { + sanitized += escapeControlCharacter(char); + continue; + } + sanitized += char; + continue; + } + + if (char === '"') { + inString = true; + sanitized += char; + } else if (code > 0x1f || char === '\t' || char === '\n' || char === '\r') { + // Tabs, line feeds, and carriage returns are valid JSON whitespace + // outside strings. Other C0 controls are never valid JSON tokens. + sanitized += char; + } + } + + return sanitized; +} + +function escapeControlCharacter(char) { + switch (char) { + case '\b': + return '\\b'; + case '\f': + return '\\f'; + case '\n': + return '\\n'; + case '\r': + return '\\r'; + case '\t': + return '\\t'; + default: + return `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`; + } +} + +function parseJsonResult(jsonStr) { + return JSON.parse(sanitizeJsonControlCharacters(jsonStr)); +} + +module.exports = { + parseJsonResult, + sanitizeJsonControlCharacters, +}; diff --git a/extraResources/sherpa/vendor/non-streaming-asr.js b/extraResources/sherpa/vendor/non-streaming-asr.js index 44ae0f4c..e29b31cd 100644 --- a/extraResources/sherpa/vendor/non-streaming-asr.js +++ b/extraResources/sherpa/vendor/non-streaming-asr.js @@ -8,6 +8,7 @@ */ const addon = require('./addon.js'); +const { parseJsonResult } = require('./json-result.js'); /** * Internal symbol to mark async-created recognizers. @@ -127,7 +128,7 @@ class OfflineRecognizer { this.handle, stream.handle, ); - return JSON.parse(jsonStr); + return parseJsonResult(jsonStr); } /** @@ -137,7 +138,7 @@ class OfflineRecognizer { */ getResult(stream) { const jsonStr = addon.getOfflineStreamResultAsJson(stream.handle); - return JSON.parse(jsonStr); + return parseJsonResult(jsonStr); } } diff --git a/extraResources/sherpa/worker/sherpa-worker.js b/extraResources/sherpa/worker/sherpa-worker.js index b2467944..ec94360d 100644 --- a/extraResources/sherpa/worker/sherpa-worker.js +++ b/extraResources/sherpa/worker/sherpa-worker.js @@ -243,6 +243,104 @@ function ensureLoaded(req) { } } +// Qwen3-ASR 聊天模板:language Chinese正文 +// 逻辑与 main/helpers/engines/qwenText.ts 保持一致。 +const QWEN_TEMPLATE_LEFTOVER = + /^(?:language|chinese|english|none|zh|en|auto)$/i; + +const QWEN_LATIN_KEEP = new Set([ + 'ok', + 'okay', + 'yes', + 'no', + 'yeah', + 'yep', + 'nope', + 'right', + 'so', + 'well', + 'um', + 'uh', + 'ah', + 'oh', + 'wow', + 'next', + 'thanks', + 'hello', + 'hi', + 'bye', + 'alpha', + 'beta', + 'gamma', + 'delta', + 'epsilon', + 'theta', + 'lambda', + 'mu', + 'nu', + 'xi', + 'pi', + 'rho', + 'sigma', + 'tau', + 'phi', + 'psi', + 'chi', + 'eta', + 'omega', +]); + +function stripQwenCueDecorations(text) { + return text + .replace(/^[("'“‘]+/, '') + .replace(/[)"'”’.,!?。!?]+$/, '') + .trim(); +} + +function isQwenFormulaToken(token) { + return ( + /^[A-Za-z]$/.test(token) || + /^[A-Za-z]\d{1,3}$/.test(token) || + /^[A-Z]{2,4}$/.test(token) || + /^[A-Z]{2,4}\d{1,3}$/.test(token) + ); +} + +function isQwenHallucinatedCue(text) { + const token = stripQwenCueDecorations(text); + if (!token) return true; + if (QWEN_TEMPLATE_LEFTOVER.test(token)) return true; + if (/[\u4e00-\u9fff]/.test(text)) return false; + if (/\s/.test(token)) return false; + if (/^[\d.]+$/.test(token)) return false; + if (isQwenFormulaToken(token)) return false; + if (QWEN_LATIN_KEEP.has(token.toLowerCase())) return false; + return /^[A-Za-z][A-Za-z0-9]{1,23}$/.test(token); +} + +function sanitizeQwenAsrText(raw) { + if (typeof raw !== 'string' || raw.length === 0) return ''; + let text = raw; + if (/<\s*asr_text\s*>/i.test(text)) { + const parts = text.split(/<\s*asr_text\s*>/i); + text = parts[parts.length - 1] || ''; + } + text = text.replace(/<\s*\/\s*asr_text\s*>/gi, ''); + text = text + .replace(/^\*+\s*/, '') + .replace(/\s*\*+$/, '') + .trim(); + if ( + /^language\s*[::]?\s*[A-Z][A-Za-z]+(?:[\s-][A-Z][A-Za-z]+){0,2}$/.test( + text, + ) + ) { + return ''; + } + if (isQwenHallucinatedCue(text)) return ''; + return text; +} + function postCancelled(id) { cancelled.delete(id); channel.post({ @@ -273,7 +371,9 @@ async function transcribe(req) { const r = await recognizer.decodeAsync(stream); const start = seg.start / SAMPLE_RATE; const end = (seg.start + seg.samples.length) / SAMPLE_RATE; - const text = r && r.text ? r.text.trim() : ''; + const rawText = r && r.text ? r.text.trim() : ''; + const text = + req.modelType === 'qwen3_asr' ? sanitizeQwenAsrText(rawText) : rawText; if (text) segments.push({ start, end, text }); } }; diff --git a/main/helpers/engines/qwenEngine.ts b/main/helpers/engines/qwenEngine.ts index 5fa44bbf..18fa335e 100644 --- a/main/helpers/engines/qwenEngine.ts +++ b/main/helpers/engines/qwenEngine.ts @@ -22,6 +22,7 @@ import { } from '../subtitleTiming'; import { resplitSubtitleCues } from '../subtitleSegmentation'; import { buildQwenParams } from './qwenParams'; +import { sanitizeQwenAsrText } from './qwenText'; import { resolveEffectiveSettings } from './outcomePresets'; import type { TranscribeContext, TranscriptionEngineAdapter } from './types'; @@ -137,7 +138,13 @@ async function transcribeQwen(ctx: TranscribeContext): Promise { const subtitles = trimSubtitleTrailingSilence( resplitSubtitleCues( - (transcription?.segments || []).map(subtitleCueFromSegment), + (transcription?.segments || []) + .map((segment) => ({ + ...segment, + text: sanitizeQwenAsrText(segment.text), + })) + .filter((segment) => Boolean(segment.text)) + .map(subtitleCueFromSegment), formData as Record, ), tempAudioFile, diff --git a/main/helpers/engines/qwenText.ts b/main/helpers/engines/qwenText.ts new file mode 100644 index 00000000..14296247 --- /dev/null +++ b/main/helpers/engines/qwenText.ts @@ -0,0 +1,111 @@ +/** + * Qwen3-ASR 解码原文常带聊天模板前缀,例如: + * language Chinese所以 + * **language Chinese开玩笑 ** + * 剥掉语种头、asr_text 标签和装饰星号,只留听写正文。 + * + * 静音/幻听时还会整条吐出拉丁碎片(demand、imageUrl、Finds.)。 + * 第一性:没有汉字的单 token 默认不是中文课听写; + * 只保留公式字母、数学符号名、口语短回应和正常英文句子。 + */ + +const QWEN_TEMPLATE_LEFTOVER = + /^(?:language|chinese|english|none|zh|en|auto)$/i; + +/** 中文 STEM 课里单独出现也合理的拉丁短词。 */ +const QWEN_LATIN_KEEP = new Set([ + 'ok', + 'okay', + 'yes', + 'no', + 'yeah', + 'yep', + 'nope', + 'right', + 'so', + 'well', + 'um', + 'uh', + 'ah', + 'oh', + 'wow', + 'next', + 'thanks', + 'hello', + 'hi', + 'bye', + 'alpha', + 'beta', + 'gamma', + 'delta', + 'epsilon', + 'theta', + 'lambda', + 'mu', + 'nu', + 'xi', + 'pi', + 'rho', + 'sigma', + 'tau', + 'phi', + 'psi', + 'chi', + 'eta', + 'omega', +]); + +function stripCueDecorations(text: string): string { + return text + .replace(/^[("'“‘]+/, '') + .replace(/[)"'”’.,!?。!?]+$/, '') + .trim(); +} + +function isFormulaToken(token: string): boolean { + return ( + /^[A-Za-z]$/.test(token) || + /^[A-Za-z]\d{1,3}$/.test(token) || + /^[A-Z]{2,4}$/.test(token) || + /^[A-Z]{2,4}\d{1,3}$/.test(token) + ); +} + +function isQwenHallucinatedCue(text: string): boolean { + const token = stripCueDecorations(text); + if (!token) return true; + if (QWEN_TEMPLATE_LEFTOVER.test(token)) return true; + if (/[\u4e00-\u9fff]/.test(text)) return false; + if (/\s/.test(token)) return false; + if (/^[\d.]+$/.test(token)) return false; + if (isFormulaToken(token)) return false; + if (QWEN_LATIN_KEEP.has(token.toLowerCase())) return false; + return /^[A-Za-z][A-Za-z0-9]{1,23}$/.test(token); +} + +export function sanitizeQwenAsrText(raw: string | null | undefined): string { + if (typeof raw !== 'string' || raw.length === 0) return ''; + + let text = raw; + if (/<\s*asr_text\s*>/i.test(text)) { + const parts = text.split(/<\s*asr_text\s*>/i); + text = parts[parts.length - 1] ?? ''; + } + + text = text.replace(/<\s*\/\s*asr_text\s*>/gi, ''); + text = text + .replace(/^\*+\s*/, '') + .replace(/\s*\*+$/, '') + .trim(); + + if ( + /^language\s*[::]?\s*[A-Z][A-Za-z]+(?:[\s-][A-Z][A-Za-z]+){0,2}$/.test( + text, + ) + ) { + return ''; + } + + if (isQwenHallucinatedCue(text)) return ''; + return text; +} diff --git a/package.json b/package.json index 34938a60..98b8cf74 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:custom-languages": "tsc scripts/test-custom-languages.ts --outDir node_modules/.cache/custom-language-tests --module commonjs --moduleResolution node --target es2019 --esModuleInterop --skipLibCheck && node node_modules/.cache/custom-language-tests/scripts/test-custom-languages.js", "test:refine": "tsc scripts/test-refine-units.ts --outDir node_modules/.cache/refine-tests --module commonjs --moduleResolution node --target es2019 --esModuleInterop --skipLibCheck --resolveJsonModule && node node_modules/.cache/refine-tests/scripts/test-refine-units.js", "test:speaker-diarization": "node scripts/test-speaker-diarization-config.cjs && tsc scripts/test-speaker-diarization.ts --outDir node_modules/.cache/speaker-diarization-tests --module commonjs --moduleResolution node --target es2019 --esModuleInterop --skipLibCheck && node node_modules/.cache/speaker-diarization-tests/scripts/test-speaker-diarization.js", + "test:sherpa-json": "node scripts/test-sherpa-json-result.cjs", "test:proofread-speakers": "tsc scripts/test-proofread-speakers.ts --outDir node_modules/.cache/proofread-speaker-tests --module commonjs --moduleResolution node --target es2019 --esModuleInterop --skipLibCheck && node node_modules/.cache/proofread-speaker-tests/scripts/test-proofread-speakers.js", "test:manuscript": "tsc scripts/test-manuscript-matching.ts --outDir node_modules/.cache/manuscript-tests --module commonjs --moduleResolution node --target es2022 --esModuleInterop --skipLibCheck --resolveJsonModule && node node_modules/.cache/manuscript-tests/scripts/test-manuscript-matching.js", "longgap:gen": "tsc scripts/longgap/gen-audio.ts --outDir node_modules/.cache/longgap --module commonjs --moduleResolution node --target es2019 --esModuleInterop --skipLibCheck --resolveJsonModule && node node_modules/.cache/longgap/scripts/longgap/gen-audio.js", diff --git a/scripts/test-engine-units.ts b/scripts/test-engine-units.ts index 0b57c3ec..1962007d 100644 --- a/scripts/test-engine-units.ts +++ b/scripts/test-engine-units.ts @@ -107,6 +107,7 @@ import { progressPercent, } from '../main/helpers/sherpaOnnx/sherpaConfig'; import { buildQwenParams } from '../main/helpers/engines/qwenParams'; +import { sanitizeQwenAsrText } from '../main/helpers/engines/qwenText'; import { buildFireRedParams, clampFireRedMaxSpeech, @@ -1346,6 +1347,65 @@ eq( 'qwen: custom max_new_tokens passthrough', ); +// --- qwenText: 剥掉 Qwen3-ASR 聊天模板 --- +eq( + sanitizeQwenAsrText('language Chinese所以'), + '所以', + 'qwen: strips language Chinese prefix', +); +eq( + sanitizeQwenAsrText('**language Chinese开玩笑 **'), + '开玩笑', + 'qwen: strips markdown-wrapped language header', +); +eq( + sanitizeQwenAsrText('正态分布'), + '正态分布', + 'qwen: strips lone asr_text tag', +); +eq( + sanitizeQwenAsrText('language Chinese'), + '', + 'qwen: header-only cue becomes empty', +); +eq( + sanitizeQwenAsrText('language is important'), + 'language is important', + 'qwen: keeps ordinary speech starting with language', +); +eq( + sanitizeQwenAsrText('所以 x加y'), + '所以 x加y', + 'qwen: keeps clean transcript unchanged', +); +eq(sanitizeQwenAsrText(''), '', 'qwen: empty stays empty'); +eq(sanitizeQwenAsrText(undefined), '', 'qwen: undefined stays empty'); +eq( + sanitizeQwenAsrText('demand'), + '', + 'qwen: drops isolated english hallucination', +); +eq( + sanitizeQwenAsrText('detract.'), + '', + 'qwen: drops isolated english with period', +); +eq(sanitizeQwenAsrText('imageUrl'), '', 'qwen: drops camelCase identifier'); +eq(sanitizeQwenAsrText('picture.'), '', 'qwen: drops caption-like fragment'); +eq(sanitizeQwenAsrText('Finds.'), '', 'qwen: drops titlecase english fragment'); +eq(sanitizeQwenAsrText('aiyun'), '', 'qwen: drops random latin token'); +eq( + sanitizeQwenAsrText('language'), + '', + 'qwen: leftover language token is empty', +); +eq(sanitizeQwenAsrText('None'), '', 'qwen: leftover None token is empty'); +eq(sanitizeQwenAsrText('DXY'), 'DXY', 'qwen: keeps formula identifier'); +eq(sanitizeQwenAsrText('p'), 'p', 'qwen: keeps single-letter formula'); +eq(sanitizeQwenAsrText('EX1'), 'EX1', 'qwen: keeps formula with digits'); +eq(sanitizeQwenAsrText('sigma'), 'sigma', 'qwen: keeps math symbol name'); +eq(sanitizeQwenAsrText('OK'), 'OK', 'qwen: keeps short spoken interjection'); + // --- engineModels: fireRedAsr awareness --- const fireRedReady = { transcriptionEngine: 'fireRedAsr' as const, diff --git a/scripts/test-sherpa-json-result.cjs b/scripts/test-sherpa-json-result.cjs new file mode 100644 index 00000000..feeb1c76 --- /dev/null +++ b/scripts/test-sherpa-json-result.cjs @@ -0,0 +1,26 @@ +'use strict'; + +const assert = require('assert'); +const { + parseJsonResult, + sanitizeJsonControlCharacters, +} = require('../extraResources/sherpa/vendor/json-result.js'); + +const rawResult = '{"text":"first\nsecond\t\u0000end","tokens":["first"]}'; +assert.deepStrictEqual(parseJsonResult(rawResult), { + text: 'first\nsecond\t\u0000end', + tokens: ['first'], +}); + +const escapedResult = String.raw`{"text":"first\nsecond\t"}`; +assert.deepStrictEqual(parseJsonResult(escapedResult), { + text: 'first\nsecond\t', +}); + +assert.strictEqual(parseJsonResult('{"text":"ok"}\u0000\n').text, 'ok'); +assert.strictEqual( + sanitizeJsonControlCharacters('{"text":"a\u000bb"}'), + '{"text":"a\\u000bb"}', +); + +console.log('sherpa JSON result tests: 4 passed');