diff --git a/core-sidecar/CMakeLists.txt b/core-sidecar/CMakeLists.txt index 25c2577..e87d62a 100644 --- a/core-sidecar/CMakeLists.txt +++ b/core-sidecar/CMakeLists.txt @@ -68,6 +68,7 @@ find_package(CURL REQUIRED) add_executable(speech-core-tts-sidecar src/main.cpp src/audio_decode.cpp src/sidecar_text.cpp src/sidecar_sysinfo.cpp) target_include_directories(speech-core-tts-sidecar PRIVATE "${SPEECH_CORE_DIR}/include" + "${SPEECH_CORE_DIR}/src/models/litert" # Vendored LiteRT C API headers — needed since the transcribe command uses # speech-core's C++ LiteRT wrappers directly (litert_engine.h). "${SPEECH_CORE_DIR}/third_party/litert" diff --git a/core-sidecar/src/main.cpp b/core-sidecar/src/main.cpp index d527a9c..402ea2c 100644 --- a/core-sidecar/src/main.cpp +++ b/core-sidecar/src/main.cpp @@ -20,16 +20,20 @@ // transcribe — ASR a rendered take (Omnilingual CTC-300M) so the // host can grade it against the target text and // retry bad takes. Model dir comes per-request. +// transcribe_parakeet — user dictation / transcription with Parakeet TDT v3. #include #include #include +#include #include +#include "hf_download.h" #include "audio_decode.h" #include "sidecar_text.h" #include "sidecar_sysinfo.h" +#include #include #include #include @@ -40,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -138,6 +143,38 @@ static fs::path clips_cache_dir() { return dir; } +static std::string model_cache_root() { + if (const char* c = std::getenv("SONIQO_MODEL_CACHE_DIR"); c && *c) return c; + if (const char* c = std::getenv("SPEECH_CORE_CACHE_DIR"); c && *c) return c; +#if defined(_WIN32) + if (const char* la = std::getenv("LOCALAPPDATA"); la && *la) + return std::string(la) + "/speech-core"; +#elif defined(__APPLE__) + if (const char* h = std::getenv("HOME"); h && *h) + return std::string(h) + "/Library/Caches/speech-core"; +#else + if (const char* x = std::getenv("XDG_CACHE_HOME"); x && *x) + return std::string(x) + "/speech-core"; + if (const char* h = std::getenv("HOME"); h && *h) + return std::string(h) + "/.cache/speech-core"; +#endif + return "./speech-core-cache"; +} + +static std::string sanitize_repo_id(const std::string& id) { + std::string s; + for (char c : id) { + if (c == '/' || c == '\\') s += "__"; + else s += c; + } + return s; +} + +static std::string format_mb(uint64_t bytes) { + const uint64_t mb = 1024ull * 1024ull; + return std::to_string((bytes + mb - 1) / mb) + " MB"; +} + // --------------------------------------------------------------------------- // VoxCPM2 model state (one warm handle across calls) // --------------------------------------------------------------------------- @@ -343,6 +380,142 @@ static void handle_transcribe(const json::Dict& req, const std::string& id) { "\"text\":\"" + json_escape(res.text) + "\"}}"); } +// transcribe_parakeet — user-facing dictation/transcription. Uses the same +// logical Parakeet TDT v3 family as the macOS sidecar, backed by the LiteRT +// INT8 bundle on Windows/Linux. The bundle is downloaded on first use unless +// SONIQO_PARAKEET_BUNDLE_DIR points at a pre-provisioned directory. +static std::unique_ptr g_parakeet; +static std::string g_parakeet_dir; +static std::string g_parakeet_model_id; + +static void on_parakeet_download_progress(const std::string& file, int idx, int count, + uint64_t downloaded, uint64_t total) { + static std::string last_file; + static int last_pct = -1; + if (last_file != file) { + last_file = file; + last_pct = -1; + } + const double file_fraction = total ? static_cast(downloaded) / total : 0.0; + const double overall = count > 0 + ? ((static_cast(idx) + file_fraction) / static_cast(count)) * 100.0 + : file_fraction * 100.0; + const int pct = static_cast(overall + 0.5); + if (pct != last_pct && (pct % 5 == 0 || pct == 100)) { + last_pct = pct; + std::string detail = "Downloading Parakeet"; + if (total) detail += " " + format_mb(downloaded) + " / " + format_mb(total); + log_err("[sidecar] parakeet " + std::to_string(pct) + "% " + detail); + } +} + +static bool parakeet_bundle_exists(const std::string& dir) { + return fs::exists(fs::path(dir) / "parakeet-encoder.tflite") && + fs::exists(fs::path(dir) / "parakeet-decoder-joint.tflite") && + fs::exists(fs::path(dir) / "vocab.json"); +} + +static std::string ensure_parakeet_bundle(std::string& model_id_out) { + std::string dir = env_or("SONIQO_PARAKEET_BUNDLE_DIR", ""); + if (!dir.empty()) { + if (!parakeet_bundle_exists(dir)) { + throw std::runtime_error( + "Parakeet bundle not found at " + dir + + " (expected parakeet-encoder.tflite, parakeet-decoder-joint.tflite, vocab.json)"); + } + model_id_out = env_or("SONIQO_PARAKEET_MODEL_ID", + "soniqo/Parakeet-TDT-0.6B-v3-LiteRT-INT8"); + return dir; + } + + if (!speech_core::hf::download_supported()) { + throw std::runtime_error( + "SONIQO_PARAKEET_BUNDLE_DIR is unset and this speech-core build has no " + "model-download support (rebuild with -DSPEECH_CORE_WITH_HF_DOWNLOAD=ON)"); + } + + const std::string model_id = env_or("SONIQO_PARAKEET_MODEL_ID", + "soniqo/Parakeet-TDT-0.6B-v3-LiteRT-INT8"); + const std::string dir_out = model_cache_root() + "/" + sanitize_repo_id(model_id); + if (parakeet_bundle_exists(dir_out)) { + model_id_out = model_id; + return dir_out; + } + const std::vector files = { + "parakeet-encoder.tflite", + "parakeet-decoder-joint.tflite", + "vocab.json", + "config.json", + }; + log_err("[sidecar] ensuring Parakeet ASR bundle " + model_id + " …"); + speech_core::hf::download_bundle( + model_id, "main", files, dir_out, on_parakeet_download_progress); + if (!parakeet_bundle_exists(dir_out)) { + throw std::runtime_error("Parakeet bundle incomplete after download: " + dir_out); + } + model_id_out = model_id; + return dir_out; +} + +static std::string ensure_parakeet_model() { + try { + std::string model_id; + const std::string dir = ensure_parakeet_bundle(model_id); + if (g_parakeet && g_parakeet_dir == dir) return ""; + log_err("[sidecar] loading Parakeet LiteRT bundle from " + dir); + g_parakeet = std::make_unique( + dir + "/parakeet-encoder.tflite", + dir + "/parakeet-decoder-joint.tflite", + dir + "/vocab.json", + /*hw_accel=*/false); + g_parakeet_dir = dir; + g_parakeet_model_id = model_id; + log_err("[sidecar] parakeet ready"); + return ""; + } catch (const std::exception& e) { + g_parakeet.reset(); + return std::string("Parakeet ASR load failed: ") + e.what(); + } +} + +static void handle_transcribe_parakeet(const json::Dict& req, const std::string& id) { + const std::string audio_path = get(req, "audioPath"); + if (audio_path.empty()) { + emit_error(id, "transcribe_parakeet requires audioPath"); + return; + } + + const auto started = std::chrono::steady_clock::now(); + std::vector audio; + int rate = 0; + if (!load_audio_mono(audio_path, audio, rate) || audio.empty()) { + emit_error(id, "could not decode audio: " + audio_path); + return; + } + std::vector a16 = (rate == 16000) + ? std::move(audio) + : speech_core::Resampler::resample(audio.data(), audio.size(), rate, 16000); + + if (std::string err = ensure_parakeet_model(); !err.empty()) { + emit_error(id, err); + return; + } + auto res = g_parakeet->transcribe(a16.data(), a16.size(), 16000); + const double duration = static_cast(a16.size()) / 16000.0; + const double elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + char numeric[160]; + std::snprintf(numeric, sizeof(numeric), + "\"sampleRate\":16000,\"durationSec\":%.6f,\"elapsedSec\":%.6f", + duration, elapsed); + emit_line("{\"id\":\"" + json_escape(id) + "\",\"ok\":true,\"result\":{" + + "\"text\":\"" + json_escape(res.text) + "\"," + + "\"language\":\"" + json_escape(res.language) + "\"," + + "\"modelName\":\"parakeet-tdt-v3-0.6b-int8\"," + + "\"modelId\":\"" + json_escape(g_parakeet_model_id) + "\"," + + numeric + "}}"); +} + static void handle_synthesize(const json::Dict& req, const std::string& id) { const std::string ref_path = get(req, "referenceAudioPath"); const std::string text = get(req, "text"); @@ -485,6 +658,8 @@ int main() { handle_probe(req, id); } else if (command == "transcribe") { handle_transcribe(req, id); + } else if (command == "transcribe_parakeet") { + handle_transcribe_parakeet(req, id); } else { emit_error(id, "unknown command: " + command); } diff --git a/model-registry.json b/model-registry.json new file mode 100644 index 0000000..9c19570 --- /dev/null +++ b/model-registry.json @@ -0,0 +1,460 @@ +{ + "version": 1, + "ttsEngines": [ + { + "id": "voxcpm2", + "displayName": "VoxCPM2", + "modelName": "voxcpm2-mlx-bf16", + "modelId": "aufklarer/VoxCPM2-MLX-bf16", + "modelSize": "1.7B", + "runtime": "MLX", + "precision": "bf16", + "languages": [ + "ar", + "my", + "zh", + "da", + "nl", + "en", + "fi", + "fr", + "de", + "el", + "he", + "hi", + "id", + "it", + "ja", + "km", + "ko", + "lo", + "ms", + "no", + "pl", + "pt", + "ru", + "es", + "sw", + "sv", + "tl", + "th", + "tr", + "vi" + ], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": false, + "requiresLanguage": false, + "styleMode": "instruction", + "supportsInstruct": true, + "supportedMarkers": [ + "soft", + "warm", + "whispering", + "intense", + "excited", + "happy", + "calm", + "serious", + "surprised", + "sad", + "angry", + "dramatic", + "laughs" + ], + "needsTrim": true, + "sampleRate": 48000, + "usePolicy": "commercial-safe", + "readiness": "production", + "sidecarCommand": "synthesize_voxcpm2", + "macosOnly": false, + "platformOverrides": { + "linux": { + "modelName": "voxcpm2-litert-fp16", + "modelId": "soniqo/VoxCPM2-LiteRT", + "runtime": "LiteRT", + "precision": "fp16" + }, + "windows": { + "modelName": "voxcpm2-litert-fp16", + "modelId": "soniqo/VoxCPM2-LiteRT", + "runtime": "LiteRT", + "precision": "fp16" + } + } + }, + { + "id": "cosyvoice", + "displayName": "CosyVoice 3", + "modelName": "cosyvoice-3-0.5b-mlx-bf16", + "modelId": "aufklarer/CosyVoice3-0.5B-MLX-bf16", + "modelSize": "0.5B", + "runtime": "MLX", + "precision": "bf16", + "languages": ["en", "zh", "ja", "ko", "de", "es", "fr", "it", "ru"], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": true, + "requiresLanguage": true, + "styleMode": "instruction", + "supportsInstruct": true, + "supportedMarkers": [ + "soft", + "warm", + "whispering", + "intense", + "excited", + "happy", + "calm", + "serious", + "surprised", + "sad", + "angry", + "dramatic", + "laughs" + ], + "needsTrim": true, + "sampleRate": 24000, + "usePolicy": "commercial-safe", + "readiness": "production", + "sidecarCommand": "synthesize_cosyvoice", + "macosOnly": true + }, + { + "id": "qwen3", + "displayName": "Qwen3-TTS", + "modelName": "qwen3-tts-1.7b-mlx-bf16", + "modelId": "aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-bf16", + "modelSize": "1.7B", + "runtime": "MLX", + "precision": "bf16", + "languages": ["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": true, + "requiresLanguage": true, + "styleMode": "none", + "supportsInstruct": false, + "supportedMarkers": [], + "needsTrim": true, + "sampleRate": 24000, + "usePolicy": "commercial-safe", + "readiness": "legacy-fallback", + "sidecarCommand": "synthesize_icl", + "macosOnly": true + }, + { + "id": "chatterbox", + "displayName": "Chatterbox", + "modelName": "chatterbox-multilingual-mlx-fp16", + "modelId": "aufklarer/Chatterbox-Multilingual-MLX-fp16", + "modelSize": "0.5B", + "runtime": "MLX", + "precision": "fp16", + "languages": [ + "ar", + "da", + "de", + "el", + "en", + "es", + "fi", + "fr", + "hi", + "it", + "ja", + "ko", + "ms", + "nl", + "no", + "pl", + "pt", + "ru", + "sv", + "sw", + "tr", + "zh" + ], + "benchmarkLanguages": [ + "ar", + "da", + "de", + "el", + "en", + "es", + "fi", + "fr", + "hi", + "it", + "ja", + "ko", + "ms", + "nl", + "no", + "pl", + "pt", + "ru", + "sv", + "sw", + "tr", + "zh" + ], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": false, + "requiresLanguage": true, + "styleMode": "intensity", + "supportsInstruct": false, + "supportedMarkers": [ + "soft", + "warm", + "whispering", + "intense", + "excited", + "happy", + "calm", + "serious", + "surprised", + "sad", + "angry", + "dramatic", + "laughs" + ], + "needsTrim": true, + "sampleRate": 24000, + "usePolicy": "commercial-safe", + "readiness": "production", + "sidecarCommand": "synthesize_chatterbox", + "macosOnly": true + }, + { + "id": "omnivoice", + "displayName": "OmniVoice", + "modelName": "omnivoice-mlx-fp16", + "modelId": "aufklarer/OmniVoice-MLX-fp16", + "modelSize": "0.5B", + "runtime": "MLX", + "precision": "fp16", + "languages": ["en", "hi", "ru", "es", "fr", "de", "it", "pt", "zh", "ja", "ko"], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": false, + "requiresLanguage": true, + "styleMode": "controlled-vocabulary", + "supportsInstruct": false, + "supportedMarkers": [ + "whispering", + "excited", + "happy", + "calm", + "serious", + "sad", + "angry" + ], + "needsTrim": true, + "sampleRate": 24000, + "usePolicy": "commercial-safe", + "readiness": "production", + "sidecarCommand": "synthesize_omnivoice", + "macosOnly": true + }, + { + "id": "indic-mio", + "displayName": "Indic-Mio", + "modelName": "indic-mio-mlx-fp16", + "modelId": "aufklarer/Indic-Mio-MLX-fp16", + "modelSize": "0.6B", + "runtime": "MLX", + "precision": "fp16", + "languages": ["hi", "en"], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": false, + "requiresLanguage": false, + "styleMode": "suffix-tag", + "supportsInstruct": false, + "supportedMarkers": ["happy", "sad", "angry", "disgust", "fear", "surprise"], + "needsTrim": true, + "sampleRate": 24000, + "usePolicy": "commercial-safe", + "readiness": "production", + "sidecarCommand": "synthesize_indic_mio", + "macosOnly": true + }, + { + "id": "fish-audio", + "displayName": "Fish Audio S2 Pro", + "modelName": "fish-audio-s2-pro-mlx-fp16", + "modelId": "aufklarer/Fish-Audio-S2-Pro-MLX-fp16", + "modelSize": "S2 Pro", + "runtime": "MLX", + "precision": "fp16", + "languages": [ + "ja", + "en", + "zh", + "ko", + "es", + "pt", + "ar", + "ru", + "fr", + "de", + "sv", + "it", + "tr", + "no", + "nl", + "cy", + "eu", + "ca", + "da", + "gl", + "ta", + "hu", + "fi", + "pl", + "et", + "hi", + "la", + "ur", + "th", + "vi", + "jw", + "bn", + "yo", + "xsl", + "cs", + "sw", + "nn", + "he", + "ms", + "uk", + "id", + "kk", + "bg", + "lv", + "my", + "tl", + "sk", + "ne", + "fa", + "af", + "el", + "bo", + "hr", + "ro", + "sn", + "mi", + "yi", + "am", + "be", + "km", + "is", + "az", + "sd", + "br", + "sq", + "ps", + "mn", + "ht", + "ml", + "sr", + "sa", + "te", + "ka", + "bs", + "pa", + "lt", + "kn", + "si", + "hy", + "mr", + "as", + "gu", + "fo" + ], + "benchmarkLanguages": [ + "ar", + "my", + "zh", + "da", + "nl", + "en", + "fi", + "fr", + "de", + "el", + "he", + "hi", + "id", + "it", + "ja", + "km", + "ko", + "ms", + "no", + "pl", + "pt", + "ru", + "es", + "sw", + "sv", + "tl", + "th", + "tr", + "vi" + ], + "voiceProfileModes": ["reference-clone"], + "requiresReferenceAudio": true, + "requiresReferenceTranscript": true, + "requiresLanguage": false, + "styleMode": "bracket-tag", + "supportsInstruct": false, + "supportedMarkers": [ + "pause", + "emphasis", + "laughing", + "excited", + "angry", + "whisper", + "screaming", + "shouting", + "surprised", + "sad" + ], + "needsTrim": false, + "sampleRate": 44100, + "usePolicy": "research-only", + "readiness": "benchmark", + "sidecarCommand": "synthesize_fish_audio", + "macosOnly": true + } + ], + "asrModels": [ + { + "id": "parakeet-tdt-v3", + "displayName": "Parakeet TDT v3", + "modelName": "parakeet-tdt-v3-0.6b-int8", + "modelId": "aufklarer/Parakeet-TDT-v3-CoreML-INT8-30s", + "modelSize": "0.6B", + "languages": ["en"], + "runtime": "coreml", + "sampleRate": 16000, + "maxSegmentSec": 30, + "streaming": true, + "readiness": "production", + "sidecarCommand": "transcribe_parakeet", + "platformOverrides": { + "linux": { + "modelId": "soniqo/Parakeet-TDT-0.6B-v3-LiteRT-INT8", + "runtime": "litert" + }, + "windows": { + "modelId": "soniqo/Parakeet-TDT-0.6B-v3-LiteRT-INT8", + "runtime": "litert" + } + } + } + ] +} diff --git a/package.json b/package.json index ece493c..453f401 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "speech-studio", "private": true, - "version": "0.0.8", + "version": "0.0.9", "type": "module", "scripts": { "dev": "vite", diff --git a/scripts/tts_roundtrip_matrix.py b/scripts/tts_roundtrip_matrix.py new file mode 100644 index 0000000..76197c5 --- /dev/null +++ b/scripts/tts_roundtrip_matrix.py @@ -0,0 +1,804 @@ +#!/usr/bin/env python3 +"""Run a multilingual TTS -> ASR intelligibility matrix against the Swift sidecar. + +The matrix is intentionally a smoke benchmark: one short sentence per +model/language pair, synthesized through the sidecar protocol, transcribed by +the local `speech` CLI, then scored with token precision/recall and WER. +""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import os +import queue +import re +import shutil +import subprocess +import sys +import threading +import time +import unicodedata +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +SIDECAR = ROOT / "swift-sidecar/.build/debug/soniqo-tts-sidecar" +REGISTRY_PATH = ROOT / "model-registry.json" +REF_TEXT = ( + "Hello. This is a clean reference voice spoken slowly and clearly for " + "speech synthesis testing." +) + +TEXTS: dict[str, str] = { + "ar": "اليوم نختبر ما إذا كان الكلام واضحا وسهل الفهم.", + "da": "I dag tester vi, om talen er klar og let at forstå.", + "de": "Heute testen wir, ob die Sprache klar und leicht zu verstehen ist.", + "el": "Σήμερα ελέγχουμε αν η ομιλία είναι καθαρή και εύκολη στην κατανόηση.", + "en": "Today we test whether the speech is clear and easy to understand.", + "es": "Hoy probamos si el habla es clara y fácil de entender.", + "fi": "Tänään testaamme, onko puhe selkeää ja helppoa ymmärtää.", + "fr": "Aujourd'hui, nous testons si la parole est claire et facile à comprendre.", + "he": "היום אנחנו בודקים אם הדיבור ברור וקל להבנה.", + "hi": "आज हम जाँचते हैं कि आवाज़ साफ़ और समझने में आसान है या नहीं।", + "id": "Hari ini kami menguji apakah ucapan jelas dan mudah dipahami.", + "it": "Oggi testiamo se il parlato è chiaro e facile da capire.", + "ja": "今日は音声が明瞭で理解しやすいかをテストします。", + "km": "ថ្ងៃនេះយើងសាកល្បងថាសំឡេងច្បាស់ និងងាយស្រួលយល់ឬអត់។", + "ko": "오늘 우리는 음성이 명확하고 이해하기 쉬운지 테스트합니다.", + "lo": "ມື້ນີ້ພວກເຮົາທົດສອບວ່າສຽງຊັດເຈນ ແລະ ເຂົ້າໃຈງ່າຍຫຼືບໍ່.", + "ms": "Hari ini kami menguji sama ada pertuturan jelas dan mudah difahami.", + "my": "ဒီနေ့ အသံက ရှင်းလင်းပြီး နားလည်ရလွယ်မလွယ် စမ်းသပ်ပါမယ်။", + "nl": "Vandaag testen we of de spraak duidelijk en gemakkelijk te begrijpen is.", + "no": "I dag tester vi om talen er klar og lett å forstå.", + "pl": "Dzisiaj sprawdzamy, czy mowa jest wyraźna i łatwa do zrozumienia.", + "pt": "Hoje testamos se a fala é clara e fácil de entender.", + "ru": "Сегодня мы проверяем, является ли речь четкой и легкой для понимания.", + "sv": "I dag testar vi om talet är tydligt och lätt att förstå.", + "sw": "Leo tunajaribu kama hotuba iko wazi na ni rahisi kuelewa.", + "th": "วันนี้เราทดสอบว่าเสียงพูดชัดเจนและเข้าใจง่ายหรือไม่", + "tl": "Ngayon sinusubukan natin kung malinaw at madaling maunawaan ang pagsasalita.", + "tr": "Bugün konuşmanın net ve kolay anlaşılır olup olmadığını test ediyoruz.", + "vi": "Hôm nay chúng tôi kiểm tra xem giọng nói có rõ ràng và dễ hiểu không.", + "zh": "今天我们测试语音是否清晰且容易理解。", +} + +COSY_LANGUAGE = { + "zh": "chinese", + "en": "english", + "ja": "japanese", + "ko": "korean", + "de": "german", + "es": "spanish", + "fr": "french", + "it": "italian", + "ru": "russian", +} + + +@dataclass(frozen=True) +class EngineSpec: + id: str + display_name: str + model_id: str + precision: str + runtime: str + command: str + languages: tuple[str, ...] + benchmark_languages: tuple[str, ...] + requires_language: bool + reference_text_required: bool + + +def registry_platform_key() -> str: + if sys.platform == "darwin": + return "macos" + if sys.platform.startswith("win"): + return "windows" + return "linux" + + +def load_engine_specs(path: Path = REGISTRY_PATH) -> dict[str, EngineSpec]: + registry = json.loads(path.read_text(encoding="utf-8")) + platform = registry_platform_key() + engines: dict[str, EngineSpec] = {} + for raw in registry.get("ttsEngines", []): + item = dict(raw) + if item.get("macosOnly") and platform != "macos": + continue + override = (item.get("platformOverrides") or {}).get(platform) or {} + item.update(override) + engines[item["id"]] = EngineSpec( + id=item["id"], + display_name=item["displayName"], + model_id=item["modelId"], + precision=item["precision"], + runtime=item["runtime"], + command=item["sidecarCommand"], + languages=tuple(item["languages"]), + benchmark_languages=tuple(item.get("benchmarkLanguages") or item["languages"]), + requires_language=bool(item["requiresLanguage"]), + reference_text_required=bool(item["requiresReferenceTranscript"]), + ) + return engines + + +ENGINES: dict[str, EngineSpec] = load_engine_specs() + + +@dataclass +class Case: + engine: EngineSpec + language: str + text: str + + @property + def key(self) -> str: + return f"{self.engine.id}__{self.language}" + + +@dataclass +class Attempt: + case: Case + seed: int + ok: bool = False + error: str = "" + audio_path: str = "" + duration_sec: float = 0.0 + synth_sec: float = 0.0 + sample_rate: int = 0 + transcript: str = "" + asr_sec: float = 0.0 + asr_rtf: float = 0.0 + precision: float = 0.0 + recall: float = 0.0 + wer: float = 1.0 + accuracy: float = 0.0 + pass_intelligibility: bool = False + verdict: str = "failed" + regression: str = "" + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def stem(self) -> str: + return safe_name(f"{self.case.engine.id}__{self.case.language}__s{self.seed}") + + +def safe_name(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) + + +def run(cmd: list[str], *, timeout: float | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + + +def ensure_reference(outdir: Path) -> Path: + ref_path = outdir / "ref-samantha.wav" + if ref_path.exists(): + return ref_path + cp = run( + [ + "say", + "-v", + "Samantha", + "-o", + str(ref_path), + "--file-format=WAVE", + "--data-format=LEI16@22050", + REF_TEXT, + ], + timeout=30, + ) + if cp.returncode != 0: + raise RuntimeError(f"failed to generate reference with say:\n{cp.stdout}") + return ref_path + + +class Sidecar: + def __init__(self, binary: Path, log_path: Path): + if not binary.exists(): + raise FileNotFoundError(f"sidecar binary not found: {binary}") + self.log_file = log_path.open("w", encoding="utf-8") + self.proc = subprocess.Popen( + [str(binary)], + cwd=str(ROOT), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=self.log_file, + text=True, + bufsize=1, + ) + self._responses: queue.Queue[str] = queue.Queue() + self._reader = threading.Thread(target=self._read_stdout, daemon=True) + self._reader.start() + + def _read_stdout(self) -> None: + assert self.proc.stdout is not None + for line in self.proc.stdout: + self._responses.put(line) + + def request(self, payload: dict[str, Any], timeout: float) -> dict[str, Any]: + if self.proc.poll() is not None: + raise RuntimeError(f"sidecar exited with code {self.proc.returncode}") + assert self.proc.stdin is not None + self.proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") + self.proc.stdin.flush() + try: + line = self._responses.get(timeout=timeout) + except queue.Empty as exc: + raise TimeoutError(f"sidecar timeout for {payload.get('id')}") from exc + return json.loads(line) + + def close(self) -> None: + if self.proc.poll() is None: + self.proc.terminate() + try: + self.proc.wait(timeout=5) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait(timeout=5) + self.log_file.close() + + +def target_word_count(text: str) -> int: + tokens = tokenize(text) + return max(1, len(tokens)) + + +def clamp(value: int, lo: int, hi: int) -> int: + return max(lo, min(hi, value)) + + +def language_payload(engine: EngineSpec, language: str, routing: str) -> str | None: + if routing == "production" and not engine.requires_language: + return None + if engine.id == "cosyvoice": + return COSY_LANGUAGE.get(language, language) + if engine.id in {"omnivoice", "indic-mio"} and language == "hi": + return "hindi" + if engine.id == "indic-mio" and language == "en": + return "english" + return language + + +def synthesize( + sidecar: Sidecar, + case: Case, + seed: int, + ref_path: Path, + audio_dir: Path, + routing: str, + timeout: float, +) -> Attempt: + attempt = Attempt(case=case, seed=seed) + lang_dir = audio_dir / case.language + lang_dir.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + words = target_word_count(case.text) + max_tokens = clamp(words * 12 + 40, 60, 320) + min_stop_steps = clamp(words * 5 // 2, 8, max_tokens - 16) + payload: dict[str, Any] = { + "id": attempt.stem, + "command": case.engine.command, + "engine": case.engine.id, + "modelId": case.engine.model_id, + "text": case.text, + "voiceId": f"matrix-{case.engine.id}", + "referenceAudioPath": str(ref_path), + "seed": seed, + "cfgValue": 2.0, + "maxTokens": max_tokens, + "minStopSteps": min_stop_steps, + } + if case.engine.reference_text_required: + payload["referenceText"] = REF_TEXT + language = language_payload(case.engine, case.language, routing) + if language: + payload["language"] = language + attempt.metadata["languageSent"] = language or "" + attempt.metadata["maxTokens"] = max_tokens + attempt.metadata["minStopSteps"] = min_stop_steps + + try: + response = sidecar.request(payload, timeout=timeout) + except Exception as exc: # noqa: BLE001 - preserve error in report + attempt.error = str(exc) + attempt.synth_sec = time.monotonic() - started + return attempt + + attempt.synth_sec = time.monotonic() - started + if response.get("ok") is not True: + attempt.error = str(response.get("error") or "synthesis failed") + return attempt + result = response.get("result") or {} + src = result.get("audioPath") + if not src or not Path(src).exists(): + attempt.error = f"missing output audio: {src}" + return attempt + dest = lang_dir / f"{attempt.stem}.wav" + shutil.copyfile(src, dest) + attempt.audio_path = str(dest) + attempt.duration_sec = float(result.get("durationSec") or 0.0) + attempt.sample_rate = int(result.get("sampleRate") or 0) + attempt.ok = True + return attempt + + +def is_cjk_char(ch: str) -> bool: + code = ord(ch) + return ( + 0x3400 <= code <= 0x4DBF + or 0x4E00 <= code <= 0x9FFF + or 0x3040 <= code <= 0x30FF + or 0xAC00 <= code <= 0xD7AF + ) + + +def normalize_text(text: str) -> str: + text = unicodedata.normalize("NFKC", text).casefold() + out: list[str] = [] + for ch in text: + cat = unicodedata.category(ch) + if cat[0] in {"L", "N", "M"} or is_cjk_char(ch): + out.append(ch) + else: + out.append(" ") + return re.sub(r"\s+", " ", "".join(out)).strip() + + +def tokenize(text: str) -> list[str]: + normalized = normalize_text(text) + if not normalized: + return [] + cjk_count = sum(1 for ch in normalized if is_cjk_char(ch)) + if cjk_count >= max(2, len(normalized.replace(" ", "")) // 3): + return [ch for ch in normalized if not ch.isspace()] + return normalized.split() + + +def edit_distance(a: list[str], b: list[str]) -> int: + prev = list(range(len(b) + 1)) + for i, x in enumerate(a, 1): + cur = [i] + for j, y in enumerate(b, 1): + cur.append( + min( + prev[j] + 1, + cur[j - 1] + 1, + prev[j - 1] + (0 if x == y else 1), + ) + ) + prev = cur + return prev[-1] + + +def score(reference: str, hypothesis: str) -> tuple[float, float, float, float]: + ref = tokenize(reference) + hyp = tokenize(hypothesis) + if not ref: + return 0.0, 0.0, 1.0, 0.0 + overlap = sum((Counter(ref) & Counter(hyp)).values()) + precision = overlap / len(hyp) if hyp else 0.0 + recall = overlap / len(ref) + wer = edit_distance(ref, hyp) / len(ref) + accuracy = max(0.0, 1.0 - wer) + return precision, recall, wer, accuracy + + +def classify_intelligibility(attempt: Attempt) -> str: + if not attempt.ok or attempt.error or not attempt.transcript.strip(): + return "failed" + if attempt.precision >= 0.90 and attempt.recall >= 0.90 and attempt.wer <= 0.15: + return "exact" + if attempt.pass_intelligibility: + return "intelligible" + if attempt.precision >= 0.45 and attempt.recall >= 0.40 and attempt.accuracy >= 0.25: + return "word-drift" + return "failed" + + +def parse_asr_jsonl(stdout: str) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for line in stdout.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + if "file" in item: + out[str(item["file"])] = item + return out + + +def transcribe_by_language( + attempts: list[Attempt], + audio_dir: Path, + asr_dir: Path, + asr_engine: str, + asr_model: str, +) -> None: + by_lang: dict[str, list[Attempt]] = defaultdict(list) + for attempt in attempts: + if attempt.ok and attempt.audio_path: + by_lang[attempt.case.language].append(attempt) + + for language, items in sorted(by_lang.items()): + lang_audio_dir = audio_dir / language + lang_asr_dir = asr_dir / language + lang_asr_dir.mkdir(parents=True, exist_ok=True) + cmd = [ + "speech", + "transcribe-batch", + str(lang_audio_dir), + "--output-dir", + str(lang_asr_dir), + "--engine", + asr_engine, + "--language", + language, + "--jsonl", + ] + if asr_engine == "qwen3": + cmd.extend(["--model", asr_model]) + started = time.monotonic() + cp = run(cmd) + elapsed = time.monotonic() - started + if cp.returncode != 0: + for item in items: + item.error = item.error or f"ASR failed for language {language}: {cp.stdout[-500:]}" + item.verdict = classify_intelligibility(item) + continue + parsed = parse_asr_jsonl(cp.stdout) + for item in items: + row = parsed.get(item.stem) + if not row: + item.error = item.error or f"ASR missing result for {item.stem}" + item.verdict = classify_intelligibility(item) + continue + item.transcript = str(row.get("text") or "").strip() + item.asr_sec = float(row.get("time") or 0.0) + item.asr_rtf = float(row.get("rtf") or 0.0) + item.metadata["asrBatchWallSec"] = elapsed + p, r, wer, acc = score(item.case.text, item.transcript) + item.precision = p + item.recall = r + item.wer = wer + item.accuracy = acc + item.pass_intelligibility = ( + item.precision >= 0.55 + and item.recall >= 0.60 + and item.accuracy >= 0.45 + ) + item.verdict = classify_intelligibility(item) + + +def select_cases(models: list[str], languages: list[str]) -> list[Case]: + selected_engines = ENGINES.keys() if models == ["all"] else models + cases: list[Case] = [] + for engine_id in selected_engines: + if engine_id not in ENGINES: + raise ValueError(f"unknown engine {engine_id}; choose {', '.join(ENGINES)}") + engine = ENGINES[engine_id] + selected_langs = ( + engine.benchmark_languages if languages == ["all"] else tuple(languages) + ) + for language in selected_langs: + if language not in engine.languages: + continue + if language not in TEXTS: + raise ValueError(f"missing test text for language {language}") + cases.append(Case(engine=engine, language=language, text=TEXTS[language])) + return cases + + +def asr_model_descriptor(asr_engine: str, asr_model: str) -> tuple[str, str]: + if asr_engine == "qwen3": + model_ids = { + "0.6B": ("aufklarer/Qwen3-ASR-0.6B-MLX-4bit", "4bit"), + "0.6B-8bit": ("aufklarer/Qwen3-ASR-0.6B-MLX-8bit", "8bit"), + "1.7B": ("aufklarer/Qwen3-ASR-1.7B-MLX-8bit", "8bit"), + "1.7B-4bit": ("aufklarer/Qwen3-ASR-1.7B-MLX-4bit", "4bit"), + } + return model_ids.get(asr_model, (asr_model, "unknown")) + if asr_engine == "parakeet": + return ("aufklarer/Parakeet-TDT-v3-CoreML-INT8-30s", "int8") + return (asr_model or asr_engine, "unknown") + + +def write_reports( + attempts: list[Attempt], + outdir: Path, + baseline: dict[str, Any] | None, + asr_engine: str, + asr_model: str, +) -> None: + asr_model_id, asr_precision = asr_model_descriptor(asr_engine, asr_model) + rows = [] + for attempt in attempts: + key = f"{attempt.case.engine.id}/{attempt.case.language}" + tts_rtf = ( + attempt.synth_sec / attempt.duration_sec + if attempt.ok and attempt.duration_sec > 0 + else 0.0 + ) + if baseline and key in baseline: + base_acc = float(baseline[key].get("accuracy") or 0.0) + delta = attempt.accuracy - base_acc + if delta <= -0.15: + attempt.regression = f"regressed {delta * 100:.0f}pp" + else: + attempt.regression = f"{delta * 100:+.0f}pp" + rows.append( + { + "engine": attempt.case.engine.id, + "displayName": attempt.case.engine.display_name, + "modelId": attempt.case.engine.model_id, + "modelPrecision": attempt.case.engine.precision, + "runtime": attempt.case.engine.runtime, + "language": attempt.case.language, + "languageSent": attempt.metadata.get("languageSent", ""), + "seed": attempt.seed, + "ok": attempt.ok, + "pass": attempt.pass_intelligibility, + "verdict": attempt.verdict, + "precision": round(attempt.precision, 4), + "recall": round(attempt.recall, 4), + "wer": round(attempt.wer, 4), + "accuracy": round(attempt.accuracy, 4), + "durationSec": round(attempt.duration_sec, 3), + "synthSec": round(attempt.synth_sec, 3), + "ttsRtf": round(tts_rtf, 4), + "asrSec": round(attempt.asr_sec, 3), + "asrRtf": round(attempt.asr_rtf, 4), + "asrEngine": asr_engine, + "asrModel": asr_model, + "asrModelId": asr_model_id, + "asrPrecision": asr_precision, + "audioPath": attempt.audio_path, + "target": attempt.case.text, + "transcript": attempt.transcript, + "error": attempt.error, + "regression": attempt.regression, + } + ) + + csv_path = outdir / "tts-roundtrip-matrix.csv" + with csv_path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else []) + writer.writeheader() + writer.writerows(rows) + + json_path = outdir / "tts-roundtrip-matrix.json" + json_path.write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8") + + summary: dict[str, Any] = {} + for engine_id in ENGINES: + engine_rows = [r for r in rows if r["engine"] == engine_id] + if not engine_rows: + continue + warm_rows = engine_rows[1:] if len(engine_rows) > 1 else engine_rows + verdict_counts = Counter(str(r["verdict"]) for r in engine_rows) + summary[engine_id] = { + "modelId": ENGINES[engine_id].model_id, + "precision": ENGINES[engine_id].precision, + "runtime": ENGINES[engine_id].runtime, + "cases": len(engine_rows), + "passed": sum(1 for r in engine_rows if r["pass"]), + "passRate": sum(1 for r in engine_rows if r["pass"]) / len(engine_rows), + "verdicts": { + "exact": verdict_counts.get("exact", 0), + "intelligible": verdict_counts.get("intelligible", 0), + "word-drift": verdict_counts.get("word-drift", 0), + "failed": verdict_counts.get("failed", 0), + }, + "meanPrecision": sum(float(r["precision"]) for r in engine_rows) / len(engine_rows), + "meanRecall": sum(float(r["recall"]) for r in engine_rows) / len(engine_rows), + "meanWer": sum(float(r["wer"]) for r in engine_rows) / len(engine_rows), + "meanTtsRtf": sum(float(r["ttsRtf"]) for r in engine_rows) / len(engine_rows), + "warmMeanTtsRtf": sum(float(r["ttsRtf"]) for r in warm_rows) / len(warm_rows), + "meanAsrRtf": sum(float(r["asrRtf"]) for r in engine_rows) / len(engine_rows), + "exactLanguages": [r["language"] for r in engine_rows if r["verdict"] == "exact"], + "intelligibleLanguages": [ + r["language"] for r in engine_rows if r["verdict"] == "intelligible" + ], + "wordDriftLanguages": [ + r["language"] for r in engine_rows if r["verdict"] == "word-drift" + ], + "failedLanguages": [ + r["language"] for r in engine_rows if r["verdict"] == "failed" + ], + "failures": [ + f"{r['language']} acc={float(r['accuracy']) * 100:.0f}% p={float(r['precision']) * 100:.0f}%" + for r in engine_rows + if r["verdict"] == "failed" + ], + "languageResults": "; ".join( + f"{r['language']} {r['verdict']} " + f"P{float(r['precision']) * 100:.0f} " + f"R{float(r['recall']) * 100:.0f} " + f"WER{float(r['wer']) * 100:.0f}" + for r in engine_rows + ), + } + (outdir / "tts-roundtrip-summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + md = [ + "# TTS Roundtrip Matrix", + "", + f"Generated: {dt.datetime.now().isoformat(timespec='seconds')}", + "", + "Scores are ASR-derived intelligibility proxies: precision/recall are token overlap; WER is edit distance over normalized tokens or CJK characters.", + f"ASR scorer: `{asr_engine}` `{asr_model}` -> `{asr_model_id}` ({asr_precision}).", + "TTS RTF is wall-clock synthesis seconds divided by generated audio seconds. Warm RTF excludes the first row per engine because the sidecar loads that model lazily on first synthesis.", + "Verdicts: `exact` is a high-overlap transcript; `intelligible` passes the regression threshold with minor wording drift; `word-drift` is related/understandable but not exact enough for a pass; `failed` is empty, unintelligible, wrong-language, or unrelated output.", + "", + "| Engine | TTS artifact | Model precision | Roundtrip per language | Verdicts E/I/D/F | Cases | Passed | Pass rate | Mean ASR precision | Mean recall | Mean WER | TTS RTF | Warm TTS RTF | ASR RTF |", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for engine_id, item in summary.items(): + md.append( + "| {engine} | `{model}` | {model_precision} | {language_results} | {verdicts} | {cases} | {passed} | {pass_rate:.0%} | {precision:.0%} | {recall:.0%} | {wer:.0%} | {tts_rtf:.2f} | {warm_tts_rtf:.2f} | {asr_rtf:.2f} |".format( + engine=ENGINES[engine_id].display_name, + model=item["modelId"], + model_precision=item["precision"], + language_results=item["languageResults"], + verdicts="{exact}/{intelligible}/{drift}/{failed}".format( + exact=item["verdicts"]["exact"], + intelligible=item["verdicts"]["intelligible"], + drift=item["verdicts"]["word-drift"], + failed=item["verdicts"]["failed"], + ), + cases=item["cases"], + passed=item["passed"], + pass_rate=item["passRate"], + precision=item["meanPrecision"], + recall=item["meanRecall"], + wer=item["meanWer"], + tts_rtf=item["meanTtsRtf"], + warm_tts_rtf=item["warmMeanTtsRtf"], + asr_rtf=item["meanAsrRtf"], + ) + ) + md.extend( + [ + "", + "## Per Case", + "", + "| Engine | Model precision | Lang | Sent | Verdict | Pass | ASR precision | Recall | WER | Duration | Synth | TTS RTF | ASR RTF | Transcript | Error |", + "|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|---|", + ] + ) + for r in rows: + transcript = str(r["transcript"]).replace("|", "\\|") + if len(transcript) > 80: + transcript = transcript[:77] + "..." + error = str(r["error"]).replace("|", "\\|") + if len(error) > 80: + error = error[:77] + "..." + md.append( + "| {engine} | {model_precision} | {lang} | {sent} | {verdict} | {passed} | {precision:.0%} | {recall:.0%} | {wer:.0%} | {duration:.1f}s | {synth:.1f}s | {tts_rtf:.2f} | {asr_rtf:.2f} | {transcript} | {error} |".format( + engine=r["displayName"], + model_precision=r["modelPrecision"], + lang=r["language"], + sent=r["languageSent"] or "-", + verdict=r["verdict"], + passed="yes" if r["pass"] else "no", + precision=float(r["precision"]), + recall=float(r["recall"]), + wer=float(r["wer"]), + duration=float(r["durationSec"]), + synth=float(r["synthSec"]), + tts_rtf=float(r["ttsRtf"]), + asr_rtf=float(r["asrRtf"]), + transcript=transcript, + error=error, + ) + ) + (outdir / "tts-roundtrip-report.md").write_text("\n".join(md) + "\n", encoding="utf-8") + + +def load_baseline(path: Path | None) -> dict[str, Any] | None: + if not path: + return None + rows = json.loads(path.read_text(encoding="utf-8")) + return {f"{row['engine']}/{row['language']}": row for row in rows} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--models", default="all", help="Comma list or all") + parser.add_argument("--languages", default="all", help="Comma list or all") + parser.add_argument("--outdir", default="") + parser.add_argument("--sidecar", default=str(SIDECAR)) + parser.add_argument("--asr-engine", default="qwen3") + parser.add_argument("--asr-model", default="0.6B") + parser.add_argument("--seed", type=int, default=1000) + parser.add_argument("--timeout-sec", type=float, default=600) + parser.add_argument( + "--routing", + choices=("production", "declared"), + default="production", + help="production sends language only when requiresLanguage=true; declared sends it whenever the model-side API accepts one", + ) + parser.add_argument("--baseline-json", default="") + args = parser.parse_args() + + models = [x.strip() for x in args.models.split(",") if x.strip()] + languages = [x.strip() for x in args.languages.split(",") if x.strip()] + outdir = Path(args.outdir) if args.outdir else Path("/tmp/speech-studio-roundtrip") / dt.datetime.now().strftime("%Y%m%d-%H%M%S") + outdir.mkdir(parents=True, exist_ok=True) + audio_dir = outdir / "audio" + asr_dir = outdir / "asr" + audio_dir.mkdir(exist_ok=True) + asr_dir.mkdir(exist_ok=True) + + cases = select_cases(models, languages) + ref_path = ensure_reference(outdir) + print(f"[matrix] output: {outdir}") + print(f"[matrix] cases: {len(cases)}") + print(f"[matrix] reference: {ref_path}") + + attempts: list[Attempt] = [] + sidecar = Sidecar(Path(args.sidecar), outdir / "sidecar.stderr.log") + try: + for index, case in enumerate(cases, 1): + print( + f"[matrix] synth {index}/{len(cases)} {case.engine.id}/{case.language}", + flush=True, + ) + attempt = synthesize( + sidecar, + case, + args.seed, + ref_path, + audio_dir, + args.routing, + args.timeout_sec, + ) + if attempt.ok: + print( + f" ok {attempt.duration_sec:.2f}s audio in {attempt.synth_sec:.1f}s", + flush=True, + ) + else: + print(f" failed: {attempt.error}", flush=True) + attempts.append(attempt) + finally: + sidecar.close() + + print("[matrix] transcribing by language...", flush=True) + transcribe_by_language(attempts, audio_dir, asr_dir, args.asr_engine, args.asr_model) + write_reports( + attempts, + outdir, + load_baseline(Path(args.baseline_json)) if args.baseline_json else None, + args.asr_engine, + args.asr_model, + ) + + passed = sum(1 for item in attempts if item.pass_intelligibility) + print(f"[matrix] pass: {passed}/{len(attempts)}") + print(f"[matrix] report: {outdir / 'tts-roundtrip-report.md'}") + print(f"[matrix] csv: {outdir / 'tts-roundtrip-matrix.csv'}") + return 0 if passed == len(attempts) else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 900eb7f..2e6cb4c 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -3467,8 +3467,9 @@ dependencies = [ [[package]] name = "speech-studio" -version = "0.0.8" +version = "0.0.9" dependencies = [ + "base64 0.22.1", "chrono", "dirs 5.0.1", "serde", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 2ed3d76..ae0598f 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "speech-studio" -version = "0.0.8" +version = "0.0.9" description = "Speech Studio — voice cloning + scripted synthesis" authors = ["Soniqo"] edition = "2021" @@ -28,3 +28,4 @@ chrono = { version = "0.4", features = ["serde"] } dirs = "5" tauri-plugin-updater = "2" tauri-plugin-process = "2" +base64 = "0.22" diff --git a/src-tauri/resources/voices/hindi_fleurs.wav b/src-tauri/resources/voices/hindi_fleurs.wav new file mode 100644 index 0000000..2251c63 Binary files /dev/null and b/src-tauri/resources/voices/hindi_fleurs.wav differ diff --git a/src-tauri/resources/voices/hindi_fleurs_female.wav b/src-tauri/resources/voices/hindi_fleurs_female.wav new file mode 100644 index 0000000..873e31a Binary files /dev/null and b/src-tauri/resources/voices/hindi_fleurs_female.wav differ diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a526d43..76c3f61 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,11 +1,14 @@ +use base64::Engine; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use std::sync::{ atomic::{AtomicU32, Ordering}, - Mutex, + LazyLock, Mutex, }; +use std::time::Instant; use tauri::{Emitter, Manager, State}; use tauri_plugin_dialog::DialogExt; @@ -382,85 +385,259 @@ enum TtsEngine { FishAudio, } +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TtsEngineInfo { + id: TtsEngine, + #[serde(rename = "displayName")] + display_name: String, + #[serde(rename = "modelName")] + model_name: String, + #[serde(rename = "modelId")] + model_id: String, + #[serde(rename = "modelSize")] + model_size: String, + runtime: String, + precision: String, + languages: Vec, + #[serde( + rename = "benchmarkLanguages", + default, + skip_serializing_if = "Vec::is_empty" + )] + benchmark_languages: Vec, + #[serde(rename = "voiceProfileModes")] + voice_profile_modes: Vec, + #[serde(rename = "requiresReferenceAudio")] + requires_reference_audio: bool, + #[serde(rename = "requiresReferenceTranscript")] + requires_reference_transcript: bool, + #[serde(rename = "requiresLanguage")] + requires_language: bool, + #[serde(rename = "styleMode")] + style_mode: String, + #[serde(rename = "supportsInstruct")] + supports_instruct: bool, + #[serde(rename = "supportedMarkers")] + supported_markers: Vec, + #[serde(rename = "needsTrim")] + needs_trim: bool, + #[serde(rename = "sampleRate")] + sample_rate: u32, + #[serde(rename = "usePolicy")] + use_policy: String, + readiness: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TtsEngineRegistryEntry { + #[serde(flatten)] + info: TtsEngineInfo, + sidecar_command: String, + macos_only: bool, + #[serde(default)] + platform_overrides: HashMap, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum AsrModel { + ParakeetTdtV3, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct AsrModelInfo { + id: AsrModel, + #[serde(rename = "displayName")] + display_name: String, + #[serde(rename = "modelName")] + model_name: String, + #[serde(rename = "modelId")] + model_id: String, + #[serde(rename = "modelSize")] + model_size: String, + languages: Vec, + runtime: String, + #[serde(rename = "sampleRate")] + sample_rate: u32, + #[serde(rename = "maxSegmentSec")] + max_segment_sec: u32, + streaming: bool, + readiness: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct AsrModelRegistryEntry { + #[serde(flatten)] + info: AsrModelInfo, + sidecar_command: String, + #[serde(default)] + platform_overrides: HashMap, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ModelPlatformOverride { + model_name: Option, + model_id: Option, + runtime: Option, + precision: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ModelRegistry { + version: u32, + tts_engines: Vec, + asr_models: Vec, +} + +impl TtsEngineInfo { + fn apply_platform_override(&mut self, item: &ModelPlatformOverride) { + if let Some(value) = &item.model_name { + self.model_name = value.clone(); + } + if let Some(value) = &item.model_id { + self.model_id = value.clone(); + } + if let Some(value) = &item.runtime { + self.runtime = value.clone(); + } + if let Some(value) = &item.precision { + self.precision = value.clone(); + } + } +} + +impl AsrModelInfo { + fn apply_platform_override(&mut self, item: &ModelPlatformOverride) { + if let Some(value) = &item.model_name { + self.model_name = value.clone(); + } + if let Some(value) = &item.model_id { + self.model_id = value.clone(); + } + if let Some(value) = &item.runtime { + self.runtime = value.clone(); + } + } +} + +fn platform_registry_key() -> &'static str { + if cfg!(target_os = "macos") { + "macos" + } else if cfg!(target_os = "windows") { + "windows" + } else { + "linux" + } +} + +static MODEL_REGISTRY: LazyLock = LazyLock::new(|| { + let mut registry: ModelRegistry = + serde_json::from_str(include_str!("../../model-registry.json")) + .expect("model-registry.json must be valid"); + assert_eq!(registry.version, 1, "unsupported model registry version"); + + let platform = platform_registry_key(); + for entry in &mut registry.tts_engines { + if let Some(item) = entry.platform_overrides.get(platform) { + entry.info.apply_platform_override(item); + } + } + for entry in &mut registry.asr_models { + if let Some(item) = entry.platform_overrides.get(platform) { + entry.info.apply_platform_override(item); + } + } + registry +}); + +impl AsrModel { + fn registry_entry(self) -> &'static AsrModelRegistryEntry { + MODEL_REGISTRY + .asr_models + .iter() + .find(|entry| entry.info.id == self) + .expect("AsrModel missing from ASR_MODEL_REGISTRY") + } + + fn sidecar_command(self) -> &'static str { + self.registry_entry().sidecar_command.as_str() + } +} + impl TtsEngine { + fn registry_entry(self) -> &'static TtsEngineRegistryEntry { + MODEL_REGISTRY + .tts_engines + .iter() + .find(|entry| entry.info.id == self) + .expect("TtsEngine missing from TTS_ENGINE_REGISTRY") + } + fn sidecar_command(self) -> &'static str { - match self { - Self::VoxCPM2 => "synthesize_voxcpm2", - Self::CosyVoice => "synthesize_cosyvoice", - Self::Qwen3 => "synthesize_icl", - Self::Chatterbox => "synthesize_chatterbox", - Self::OmniVoice => "synthesize_omnivoice", - Self::IndicMio => "synthesize_indic_mio", - Self::FishAudio => "synthesize_fish_audio", - } + self.registry_entry().sidecar_command.as_str() } fn display_name(self) -> &'static str { - match self { - Self::VoxCPM2 => "VoxCPM2", - Self::CosyVoice => "CosyVoice 3", - Self::Qwen3 => "Qwen3-TTS", - Self::Chatterbox => "Chatterbox", - Self::OmniVoice => "OmniVoice", - Self::IndicMio => "Indic-Mio", - Self::FishAudio => "Fish Audio S2 Pro", - } + self.registry_entry().info.display_name.as_str() } fn requires_reference_transcript(self) -> bool { - matches!(self, Self::CosyVoice | Self::Qwen3 | Self::FishAudio) + self.registry_entry().info.requires_reference_transcript } - /// Whether synthesis needs a caller-chosen language. Chatterbox prepends a - /// `[lang]` token, so the Studio shows a language picker for it; the other - /// engines infer language from the text. + /// Whether synthesis needs a caller-chosen language. The Studio shows a + /// language picker only for engines that declare this in the registry. fn requires_language(self) -> bool { - matches!(self, Self::Chatterbox | Self::OmniVoice) + self.registry_entry().info.requires_language } /// How the engine applies inline emotion markers — drives the editor hint: /// - `instruction`: marker → an engine-specific style instruction. + /// - `controlled-vocabulary`: marker → fixed engine vocabulary only. /// - `intensity`: marker → an expressiveness level only (Chatterbox; not a /// specific emotion). /// - `suffix-tag`: marker is appended as an engine-specific suffix tag. /// - `bracket-tag`: marker is appended as an engine-specific bracket tag. /// - `none`: markers are stripped and ignored. fn style_mode(self) -> &'static str { - match self { - Self::VoxCPM2 | Self::CosyVoice | Self::OmniVoice => "instruction", - Self::Chatterbox => "intensity", - Self::IndicMio => "suffix-tag", - Self::FishAudio => "bracket-tag", - Self::Qwen3 => "none", - } + self.registry_entry().info.style_mode.as_str() } } fn normalize_sidecar_language(engine: TtsEngine, language: Option<&str>) -> Option { let trimmed = language.map(str::trim).filter(|value| !value.is_empty()); match engine { + TtsEngine::CosyVoice => trimmed.map(|value| match value.to_ascii_lowercase().as_str() { + "zh" | "zho" | "cmn" | "chinese" => "chinese".to_string(), + "en" | "eng" | "english" => "english".to_string(), + "ja" | "jpn" | "japanese" => "japanese".to_string(), + "ko" | "kor" | "korean" => "korean".to_string(), + "de" | "deu" | "ger" | "german" => "german".to_string(), + "es" | "spa" | "spanish" => "spanish".to_string(), + "fr" | "fra" | "fre" | "french" => "french".to_string(), + "it" | "ita" | "italian" => "italian".to_string(), + "ru" | "rus" | "russian" => "russian".to_string(), + _ => value.to_string(), + }), // The UI keeps BCP-47-ish ids because Chatterbox expects `[hi]`, but // OmniVoice gives cleaner Hindi output with the spelled language item. - TtsEngine::OmniVoice | TtsEngine::IndicMio => trimmed.map(|value| { - match value.to_ascii_lowercase().as_str() { + TtsEngine::OmniVoice | TtsEngine::IndicMio => { + trimmed.map(|value| match value.to_ascii_lowercase().as_str() { "hi" | "hin" => "hindi".to_string(), _ => value.to_string(), - } - }), + }) + } _ => trimmed.map(str::to_string), } } fn engine_is_supported(engine: TtsEngine) -> bool { - match engine { - TtsEngine::VoxCPM2 => true, - TtsEngine::CosyVoice => cfg!(target_os = "macos"), - TtsEngine::Qwen3 => cfg!(target_os = "macos"), - TtsEngine::Chatterbox => cfg!(target_os = "macos"), - TtsEngine::OmniVoice => cfg!(target_os = "macos"), - TtsEngine::IndicMio => cfg!(target_os = "macos"), - TtsEngine::FishAudio => cfg!(target_os = "macos"), - } + let entry = engine.registry_entry(); + !entry.macos_only || cfg!(target_os = "macos") } fn ensure_engine_supported(engine: TtsEngine) -> Result<(), String> { @@ -497,41 +674,31 @@ fn humanize_sidecar_error(engine: TtsEngine, error: String) -> String { error } -#[derive(Serialize)] -struct TtsEngineInfo { - id: TtsEngine, - #[serde(rename = "displayName")] - display_name: &'static str, - #[serde(rename = "requiresReferenceTranscript")] - requires_reference_transcript: bool, - #[serde(rename = "requiresLanguage")] - requires_language: bool, - #[serde(rename = "styleMode")] - style_mode: &'static str, +fn tts_engine_info(engine: TtsEngine) -> TtsEngineInfo { + engine.registry_entry().info.clone() } -fn tts_engine_info(engine: TtsEngine) -> TtsEngineInfo { - TtsEngineInfo { - id: engine, - display_name: engine.display_name(), - requires_reference_transcript: engine.requires_reference_transcript(), - requires_language: engine.requires_language(), - style_mode: engine.style_mode(), - } +fn asr_model_info(model: AsrModel) -> AsrModelInfo { + model.registry_entry().info.clone() } #[tauri::command] async fn available_tts_engines() -> Vec { - let mut engines = vec![tts_engine_info(TtsEngine::VoxCPM2)]; - if cfg!(target_os = "macos") { - engines.push(tts_engine_info(TtsEngine::CosyVoice)); - engines.push(tts_engine_info(TtsEngine::Qwen3)); - engines.push(tts_engine_info(TtsEngine::Chatterbox)); - engines.push(tts_engine_info(TtsEngine::OmniVoice)); - engines.push(tts_engine_info(TtsEngine::IndicMio)); - engines.push(tts_engine_info(TtsEngine::FishAudio)); - } - engines + MODEL_REGISTRY + .tts_engines + .iter() + .filter(|entry| engine_is_supported(entry.info.id)) + .map(|entry| entry.info.clone()) + .collect() +} + +#[tauri::command] +async fn available_asr_models() -> Vec { + MODEL_REGISTRY + .asr_models + .iter() + .map(|entry| entry.info.clone()) + .collect() } #[tauri::command] @@ -552,10 +719,12 @@ struct InitModelArgs { #[tauri::command] async fn init_model(manager: State<'_, SidecarManager>, args: InitModelArgs) -> Result<(), String> { ensure_engine_supported(args.engine)?; + let info = tts_engine_info(args.engine); let payload = serde_json::json!({ "id": format!("init-{}", uuid::Uuid::new_v4()), "command": "init_model", "engine": args.engine, + "modelId": info.model_id, }); let raw = manager.request(&payload)?; let env: SidecarResponse = serde_json::from_value(raw).map_err(|e| e.to_string())?; @@ -660,9 +829,7 @@ fn reference_access_error(action: &str, path: &Path, err: &std::io::Error) -> St std::io::ErrorKind::NotFound => { "The file is no longer at that path. Re-add the reference from its current location." } - _ => { - "Copy or export the reference audio to a normal local folder, then add it again." - } + _ => "Copy or export the reference audio to a normal local folder, then add it again.", }; format!( "Cannot {action} reference audio \"{}\": {err}. {hint}", @@ -906,8 +1073,8 @@ struct SynthesizeArgs { reference_audio_path: String, #[serde(rename = "referenceText")] reference_text: String, - /// Synthesis language id (Chatterbox `[lang]` token, e.g. "en"/"ar"/"hi"). - /// Optional; engines that infer language from text ignore it. + /// Synthesis language id for engines with `requiresLanguage=true`. + /// Optional; engines that infer or default language do not receive it. #[serde(default)] language: Option, } @@ -926,38 +1093,6 @@ struct SynthesizeResult { elapsed_sec: f64, } -/// How ASR-graded retry runs on this platform. macOS shells out to the -/// `speech` CLI (Parakeet, ships with the speech-swift toolchain). Windows and -/// Linux grade through the sidecar's `transcribe` command (Omnilingual -/// CTC-300M, multilingual), enabled by pointing SONIQO_STT_MODEL_DIR at a -/// directory holding omnilingual-ctc-300m.tflite + tokenizer.model. With -/// neither available we accept the first successful take. -enum Grader { - SpeechCli, - Sidecar(String), - None, -} - -fn resolve_grader() -> Grader { - if cfg!(target_os = "macos") { - return Grader::SpeechCli; - } - if let Ok(dir) = std::env::var("SONIQO_STT_MODEL_DIR") { - if std::path::Path::new(&dir) - .join("omnilingual-ctc-300m.tflite") - .exists() - && std::path::Path::new(&dir).join("tokenizer.model").exists() - { - return Grader::Sidecar(dir); - } - eprintln!( - "[synth] SONIQO_STT_MODEL_DIR set but model files missing: {}", - dir - ); - } - Grader::None -} - /// Split text into sentences on terminal punctuation, including the /// Devanagari danda/double-danda. The terminator stays attached to its /// sentence so the model gets a clean stop cue per chunk. @@ -1058,6 +1193,18 @@ fn chunk_text_for_synthesis(text: &str, max_words: usize) -> Vec { chunks } +fn synth_max_tokens(engine: TtsEngine, target_word_count: usize) -> usize { + if engine == TtsEngine::Qwen3 { + // Qwen3 frames are ~80 ms each. The generic cap lets short lines run + // to 96 frames (~7.7 s of audio, ~12 s wall time) whenever EOS fails. + // Keep enough headroom for slow speech, but prevent minute-long retry + // ladders for one short utterance. + (target_word_count.saturating_mul(4) + 20).clamp(40, 96) + } else { + (target_word_count.saturating_mul(12) + 40).clamp(60, 320) + } +} + /// One synthesis pass (seed/cfg retry ladder + optional ASR grading) for a /// single piece of text. Returns (audio_path, duration_sec) of the accepted /// take. Extracted from synthesize_clip so long-form chunking can call it @@ -1075,15 +1222,9 @@ fn synth_one_line( invocation_salt: &str, part_idx: usize, ) -> Result<(String, f64), String> { - // Seed ladder: both backends are deterministic per seed. VoxCPM2 also - // consumes cfgValue; CosyVoice ignores that optional field. - const SEED_LADDER: &[u64] = &[1000, 1001, 1002, 1010, 1011, 1012]; - const CFG_LADDER: &[f32] = &[2.0, 2.5, 3.0]; - // Token budget: ~12 steps/word + headroom (each step ≈ 50 ms of audio). - let grade_target = strip_style_markers_for_grading(text); - let target_word_count = canon_tokens(&grade_target).len(); - let max_tokens = (target_word_count.saturating_mul(12) + 40).clamp(60, 320); + let target_word_count = canon_tokens(&strip_style_markers_for_grading(text)).len(); + let max_tokens = synth_max_tokens(engine, target_word_count); // Floor under the model's stop signal: VoxCPM2 fires its stop token // prematurely on long non-Latin-script lines (a 19-word Hindi sentence // stops at ~40 steps ≈ 6 s, cutting the sentence). The model speaks @@ -1095,161 +1236,68 @@ fn synth_one_line( // so short chunks can still end naturally. A flat floor of 32 forced a // 6-word chunk (natural ≈ 15 steps) to ramble to 74 steps / 11.8 s. let min_stop_steps = (target_word_count.saturating_mul(5) / 2).clamp(8, max_tokens - 16); - - let grader = resolve_grader(); - // Non-Latin targets get a stricter accept rule (see below): the babble - // tail VoxCPM2 leaves on Hindi overshoots is short in ASR tokens (~2) but - // seconds long audibly, so the Latin-tuned suffix allowance is too loose. - let target_is_non_latin = grade_target - .chars() - .any(|c| !c.is_ascii() && c.is_alphabetic()); let sidecar_language = normalize_sidecar_language(engine, language); - - let mut best: Option<(String, Grade, u64, f64)> = None; - let mut last_error: Option = None; - - for (attempt_idx, &seed) in SEED_LADDER.iter().enumerate() { - let cfg = CFG_LADDER[attempt_idx.min(CFG_LADDER.len() - 1)]; - let payload = serde_json::json!({ - "id": format!("synth-{}-p{}-s{}-{}", clip_id, part_idx, seed, invocation_salt), - "command": engine.sidecar_command(), - "engine": engine, - "text": text, - "voiceId": voice_id, - "referenceAudioPath": reference_audio_path, - "referenceText": reference_text, - "language": sidecar_language.as_deref(), - "seed": seed, - "cfgValue": cfg, - "maxTokens": max_tokens, - "minStopSteps": min_stop_steps, - }); - - let raw = match manager.request(&payload) { - Ok(v) => v, - Err(e) => { - last_error = Some(format!("attempt {} (seed={}): {}", attempt_idx, seed, e)); - eprintln!("[synth] {}", last_error.as_ref().unwrap()); - continue; - } - }; - let env: SidecarResponse = match serde_json::from_value(raw) { - Ok(e) => e, - Err(e) => { - last_error = Some(format!("parse response: {}", e)); - eprintln!("[synth] {}", last_error.as_ref().unwrap()); - continue; - } - }; - if !env.ok { - last_error = Some(humanize_sidecar_error( - engine, - env.error.unwrap_or_else(|| "sidecar error".into()), - )); - eprintln!( - "[synth] clip {} part {} attempt {} (seed={}) failed: {}", - clip_id, - part_idx, - attempt_idx, - seed, - last_error.as_ref().unwrap() - ); - continue; - } - let result = env.result.unwrap_or_default(); - let audio_path = match result.get("audioPath").and_then(|v| v.as_str()) { - Some(p) => p.to_string(), - None => continue, - }; - let duration = result - .get("durationSec") - .and_then(|v| v.as_f64()) - .unwrap_or(0.0); - if let Err(e) = validate_synth_audio(&audio_path) { - last_error = Some(format!("attempt {} (seed={}): {}", attempt_idx, seed, e)); - eprintln!( - "[synth] clip {} part {} attempt {} (seed={}) rejected: {}", - clip_id, - part_idx, - attempt_idx, - seed, - last_error.as_ref().unwrap() - ); - continue; - } - - // No ASR grader on this platform — accept the first successful take - // rather than burning the whole seed ladder (each take is a full synth). - let graded = match &grader { - Grader::SpeechCli => asr_grade(&audio_path, &grade_target), - Grader::Sidecar(dir) => asr_grade_sidecar(manager, &audio_path, &grade_target, dir), - Grader::None => { - eprintln!( - "[synth] clip {} part {} accepted first take (seed={}, {:.2}s; grading unavailable on this platform)", - clip_id, part_idx, seed, duration - ); - return Ok((audio_path, duration)); - } - }; - let grade = graded.unwrap_or_else(|| Grade { - coverage: 0.0, - prefix_words: 0, - suffix_words: 0, - repeated_target_words: 0, - transcript: String::new(), - }); - eprintln!( - "[synth] clip {} part {} attempt {} (seed={}) {} cov={:.0}% pre={} suf={} rep={} ({:.2}s)", - clip_id, - part_idx, - attempt_idx, - seed, - if grade.is_clean_for(target_is_non_latin) { - "✓" - } else { - "✗" - }, - grade.coverage * 100.0, - grade.prefix_words, - grade.suffix_words, - grade.repeated_target_words, - duration, - ); - - // Keep the best attempt as fallback (composite score, not raw coverage). - let take_it = best - .as_ref() - .map(|(_, g, _, _)| g.score() < grade.score()) - .unwrap_or(true); - if take_it { - best = Some((audio_path.clone(), grade.clone(), seed, duration)); - } - - if grade.is_clean_for(target_is_non_latin) { - eprintln!( - "[synth] clip {} part {} accepted on attempt {} (seed={}, cov={:.0}%)", - clip_id, - part_idx, - attempt_idx, - seed, - grade.coverage * 100.0 - ); - return Ok((audio_path, duration)); - } - } - - if let Some((audio_path, grade, seed, duration)) = best { - eprintln!( - "[synth] clip {} part {} all attempts below threshold; returning best (seed={}, cov={:.0}%, score={:.2})", - clip_id, - part_idx, - seed, - grade.coverage * 100.0, - grade.score() - ); - return Ok((audio_path, duration)); + let model_id = engine.registry_entry().info.model_id.as_str(); + + // Single-shot for every engine. The 16-bit (fp16/bf16) models produce + // intelligible speech in one pass, so there is no seed/cfg retry ladder and + // no ASR grading gate — only a non-empty/non-silent audio guard. The old + // ladder graded takes with Parakeet on macOS, which cannot read non-Latin + // scripts: it scored good Hindi/CJK audio near 0% and burned every seed for + // nothing, and for Indic-Mio it re-ran the heavy WavLM speaker encoder on + // each attempt. cfgValue and minStopSteps are read only by the engines that + // use them (VoxCPM2, CosyVoice, …); the others ignore them. + // + // Seed varies per synthesis (derived from the per-call invocation_salt), so + // every Regenerate rolls a fresh take. That is the manual escape hatch for + // an unlucky single-shot render (e.g. a doubled onset word on one seed): + // the user re-rolls instead of an ASR gate auto-retrying. Matches how + // Voicebox handles it (regenerate → new random seed, no grading). A cached + // take keeps its salt, so it stays put until explicitly regenerated. + let seed = short_hash(invocation_salt) as u64; + let payload = serde_json::json!({ + "id": format!("synth-{}-p{}-s{}-{}", clip_id, part_idx, seed, invocation_salt), + "command": engine.sidecar_command(), + "engine": engine, + "modelId": model_id, + "text": text, + "voiceId": voice_id, + "referenceAudioPath": reference_audio_path, + "referenceText": reference_text, + "language": sidecar_language.as_deref(), + "seed": seed, + "cfgValue": 2.0, + "maxTokens": max_tokens, + "minStopSteps": min_stop_steps, + }); + let raw = manager.request(&payload)?; + let env: SidecarResponse = serde_json::from_value(raw).map_err(|e| e.to_string())?; + if !env.ok { + return Err(humanize_sidecar_error( + engine, + env.error.unwrap_or_else(|| "sidecar error".into()), + )); } - Err(last_error.unwrap_or_else(|| "all synth attempts failed".into())) + let result = env.result.unwrap_or_default(); + let audio_path = result + .get("audioPath") + .and_then(|v| v.as_str()) + .ok_or_else(|| "missing audioPath in sidecar response".to_string())? + .to_string(); + let duration = result + .get("durationSec") + .and_then(|v| v.as_f64()) + .unwrap_or(0.0); + validate_synth_audio(&audio_path)?; + eprintln!( + "[synth] clip {} part {} synthesized via {} single-shot (seed={}, {:.2}s)", + clip_id, + part_idx, + engine.sidecar_command(), + seed, + duration + ); + Ok((audio_path, duration)) } /// Words per synthesis chunk. VoxCPM2's AR quality drifts past ~15-20 s of @@ -1261,7 +1309,7 @@ const MAX_CHUNK_WORDS: usize = 14; const CHUNK_GAP_SEC: f64 = 0.28; fn trim_long_form_chunk_edges(engine: TtsEngine) -> bool { - !matches!(engine, TtsEngine::FishAudio) + engine.registry_entry().info.needs_trim } /// Trim leading/trailing low-energy tails from a rendered chunk. The model @@ -1859,360 +1907,6 @@ fn strip_style_markers_for_grading(text: &str) -> String { .to_string() } -/// Result of grading a synth take. Captures the four signals we use to decide -/// accept/reject: how much of the target text appears, and whether the -/// transcript is "shaped right" (no prefix junk, no trailing repetition). -#[derive(Debug, Clone)] -struct Grade { - /// Fraction of target words present in the transcript (via LCS). - coverage: f64, - /// Number of transcript words BEFORE the first aligned target word. - /// Detects reference-echo leak ("Capit, ruttering, quilt, JUST STAY…"). - prefix_words: usize, - /// Number of transcript words AFTER the last aligned target word. - /// Detects trailing garbage (model failed to EOS cleanly). - suffix_words: usize, - /// Count of target words that appear 2+ times in the transcript. - /// Detects line repetition (model regenerated the line after first EOS). - repeated_target_words: usize, - /// Raw ASR transcript, kept for logging only. - #[allow(dead_code)] - transcript: String, -} - -impl Grade { - /// Accept rule. Tuned against a 48-take sweep: - /// - Coverage ≥ 0.75 lets through clean substitutions (e.g. "end" → "and"). - /// - prefix ≤ 1 allows one short attack word ("And…", "Oh…") but blocks - /// reference-echo prefixes which are typically 3+ junk words. - /// - suffix ≤ 2 allows a trailing pause/period word but blocks tail - /// repetition or filler. - /// - repeated == 0 blocks line-repetition takes entirely. - fn is_clean(&self) -> bool { - self.is_clean_for(false) - } - - /// Accept rule with script-aware strictness. Non-Latin (e.g. Hindi) - /// targets use a lower coverage bar — the grading ASR (Omnilingual) mixes - /// Devanagari and Urdu script for Hindustani, and even skeleton-folded - /// matching (see tokens_match) drops some words — but a tighter suffix - /// bound: VoxCPM2's overshoot babble on Hindi decodes to only ~2 junk - /// tokens yet is seconds long audibly, so 2 trailing words is already a - /// broken take there. The prefix allowance is looser for non-Latin (≤2): - /// cross-script ASR reliably mishears a chunk's cold-start word as two - /// unalignable tokens (measured pre=2 on every seed of a clean Hindi - /// chunk), while real reference-echo prefixes are far longer (pre=31 on - /// the one bad take in the same ladder). - fn is_clean_for(&self, non_latin: bool) -> bool { - let (min_cov, max_prefix, max_suffix) = if non_latin { (0.6, 2, 1) } else { (0.75, 1, 2) }; - self.coverage >= min_cov - && self.prefix_words <= max_prefix - && self.suffix_words <= max_suffix - && self.repeated_target_words == 0 - } - - /// Composite score for ladder-exhausted fallback. Weights chosen so that - /// a clean cov=75% take beats a leaked cov=100% take; a repetition take - /// is heavily penalised because it doubles the audio length audibly. - fn score(&self) -> f64 { - self.coverage - - 0.1 * self.prefix_words as f64 - - 0.05 * self.suffix_words as f64 - - 0.2 * self.repeated_target_words as f64 - } -} - -/// Run `speech transcribe --engine parakeet` and grade the transcript against -/// the target. Returns None only if the ASR command itself fails — an empty -/// transcript is graded as 0% coverage, not None. -fn asr_grade(audio_path: &str, target: &str) -> Option { - let out = Command::new("speech") - .args(["transcribe", "--engine", "parakeet", audio_path]) - .output() - .ok()?; - if !out.status.success() { - eprintln!( - "[synth] parakeet failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - return None; - } - let transcript = String::from_utf8_lossy(&out.stdout) - .lines() - .find_map(|l| l.strip_prefix("Result: ").map(|s| s.trim().to_string())) - .unwrap_or_default(); - Some(grade_transcript(&transcript, target)) -} - -/// Grade via the sidecar's `transcribe` command (Omnilingual CTC-300M) — -/// the Windows/Linux counterpart of asr_grade's `speech` CLI shell-out. -fn asr_grade_sidecar( - manager: &SidecarManager, - audio_path: &str, - target: &str, - model_dir: &str, -) -> Option { - let payload = serde_json::json!({ - "id": format!("grade-{}", uuid::Uuid::new_v4().simple()), - "command": "transcribe", - "audioPath": audio_path, - "modelDir": model_dir, - }); - let raw = match manager.request(&payload) { - Ok(v) => v, - Err(e) => { - eprintln!("[synth] sidecar transcribe failed: {}", e); - return None; - } - }; - let env: SidecarResponse = serde_json::from_value(raw).ok()?; - if !env.ok { - eprintln!( - "[synth] sidecar transcribe error: {}", - env.error.unwrap_or_else(|| "unknown".into()) - ); - return None; - } - let result = env.result.unwrap_or_default(); - let transcript = result - .get("text") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - Some(grade_transcript(&transcript, target)) -} - -/// Fold a token to a script-agnostic consonant skeleton. Omnilingual decodes -/// Hindustani in mixed Devanagari/Urdu script (no language pin), so grading -/// "जोड़े" against "جوڑے" requires both to reduce to the same key. Both -/// scripts are phonetic: keep consonant classes, drop vowels/diacritics. -/// ASCII passes through unchanged. -fn fold_skeleton(token: &str) -> String { - let mut out = String::new(); - let chars: Vec = token.chars().collect(); - let mut i = 0; - while i < chars.len() { - let c = chars[i]; - // Decomposed nukta forms (base + U+093C) change the consonant class: - // ड़/ढ़ are flaps (= Urdu ڑ → 'r'), ज़ → 'j', फ़ → 'f'. Consume the pair. - if chars.get(i + 1) == Some(&'\u{093C}') { - let k = match c { - 'ड' | 'ढ' => Some('r'), - 'ज' => Some('j'), - 'फ' => Some('f'), - 'क' | 'ख' => Some('k'), - 'ग' => Some('g'), - _ => None, - }; - if let Some(k) = k { - out.push(k); - i += 2; - continue; - } - } - let k: Option = match c { - // Devanagari consonants (incl. precomposed nukta forms U+0958-095E; - // decomposed base+nukta also works — base maps, nukta drops below) - 'क' | 'ख' | '\u{0958}' | '\u{0959}' => Some('k'), - 'ग' | 'घ' | '\u{095A}' => Some('g'), - 'च' | 'छ' => Some('c'), - 'ज' | 'झ' | '\u{095B}' => Some('j'), - 'ट' | 'ठ' | 'त' | 'थ' => Some('t'), - 'ड' | 'ढ' | 'द' | 'ध' => Some('d'), - '\u{095C}' | '\u{095D}' => Some('r'), - 'ण' | 'न' | 'ङ' | 'ञ' => Some('n'), - 'प' => Some('p'), - 'फ' => Some('p'), - '\u{095E}' => Some('f'), - 'ब' | 'भ' => Some('b'), - 'म' => Some('m'), - 'य' => Some('y'), - 'र' => Some('r'), - 'ल' => Some('l'), - 'व' => Some('v'), - 'श' | 'ष' | 'स' => Some('s'), - 'ह' => Some('h'), - // Devanagari vowels, matras, anusvara, virama, nukta → drop - '\u{0900}'..='\u{0903}' - | '\u{0904}'..='\u{0914}' - | '\u{093A}'..='\u{094F}' - | '\u{0962}'..='\u{0963}' => None, - // Urdu/Arabic consonants - 'ک' | 'ق' | 'خ' => Some('k'), - 'گ' | 'غ' => Some('g'), - 'چ' => Some('c'), - 'ج' | 'ز' | 'ذ' | 'ض' | 'ظ' | 'ژ' => Some('j'), - 'ت' | 'ٹ' | 'ط' => Some('t'), - 'د' | 'ڈ' => Some('d'), - 'ڑ' => Some('r'), - 'ن' => Some('n'), - // ں (nun ghunna) is nasalisation — drops like Devanagari anusvara. - 'ں' => None, - 'پ' => Some('p'), - 'ف' => Some('f'), - 'ب' => Some('b'), - 'م' => Some('m'), - 'ی' | 'ئ' => Some('y'), - 'ر' => Some('r'), - 'ل' => Some('l'), - 'و' => Some('v'), - 'س' | 'ش' | 'ص' | 'ث' => Some('s'), - 'ہ' | 'ح' | 'ه' | 'ۂ' => Some('h'), - // ھ (heh doachashmee) marks aspiration in digraphs (ٹھ, کھ…) — - // drops so aspirated/unaspirated fold to the same class, matching - // how Devanagari ठ/ट both map to 't'. - 'ھ' => None, - 'ع' | 'ا' | 'آ' | 'أ' | 'إ' | 'ء' | 'ے' | 'ۓ' => None, - // Arabic tashkeel → drop - '\u{064B}'..='\u{0652}' => None, - other if other.is_ascii_alphanumeric() => Some(other), - _ => None, - }; - if let Some(k) = k { - out.push(k); - } - i += 1; - } - out -} - -/// Levenshtein distance over the (ASCII) folded skeletons. -fn edit_distance(a: &str, b: &str) -> usize { - let a: Vec = a.bytes().collect(); - let b: Vec = b.bytes().collect(); - let mut prev: Vec = (0..=b.len()).collect(); - for (i, &ca) in a.iter().enumerate() { - let mut cur = vec![i + 1]; - for (j, &cb) in b.iter().enumerate() { - let cost = if ca == cb { 0 } else { 1 }; - cur.push((prev[j] + cost).min(prev[j + 1] + 1).min(cur[j] + 1)); - } - prev = cur; - } - prev[b.len()] -} - -/// Token equivalence for grading. Exact match first; for non-ASCII tokens, -/// compare consonant skeletons with a small edit-distance tolerance — the -/// Urdu spelling of a Hindi word carries vowel letters (و/ی) that fold into -/// the skeleton, so an off-by-one consonant must still count as the same -/// word. ASCII-vs-ASCII never matches fuzzily (English grading unchanged). -fn tokens_match(a: &str, b: &str) -> bool { - if a == b { - return true; - } - if a.is_ascii() && b.is_ascii() { - return false; - } - let fa = fold_skeleton(a); - let fb = fold_skeleton(b); - if fa.is_empty() || fb.is_empty() { - return false; - } - if fa == fb { - return true; - } - let min_len = fa.len().min(fb.len()); - let tol = if min_len >= 6 { - 2 - } else { - usize::from(min_len >= 2) - }; - edit_distance(&fa, &fb) <= tol -} - -/// LCS-based grading: find the longest subsequence of `target` words that -/// appears (in order, with skips allowed) inside `transcript`. The position -/// of the first/last aligned transcript words tells us about prefix/suffix -/// junk. Repetition is detected separately as "did any target word appear 2+ -/// times in the transcript?". -fn grade_transcript(transcript: &str, target: &str) -> Grade { - let trans = canon_tokens(transcript); - let targ = canon_tokens(target); - if targ.is_empty() { - return Grade { - coverage: 0.0, - prefix_words: 0, - suffix_words: 0, - repeated_target_words: 0, - transcript: transcript.to_string(), - }; - } - if trans.is_empty() { - return Grade { - coverage: 0.0, - prefix_words: 0, - suffix_words: 0, - repeated_target_words: 0, - transcript: transcript.to_string(), - }; - } - - let n = trans.len(); - let m = targ.len(); - // dp[i][j] = LCS length over trans[0..i] vs targ[0..j]. Equivalence is - // tokens_match (script-folded fuzzy), not string equality, so a transcript - // in Urdu script still aligns against a Devanagari target. - let mut dp = vec![vec![0u32; m + 1]; n + 1]; - for i in 1..=n { - for j in 1..=m { - dp[i][j] = if tokens_match(&trans[i - 1], &targ[j - 1]) { - dp[i - 1][j - 1] + 1 - } else { - dp[i - 1][j].max(dp[i][j - 1]) - }; - } - } - let matched = dp[n][m] as usize; - if matched == 0 { - return Grade { - coverage: 0.0, - prefix_words: 0, - suffix_words: trans.len(), - repeated_target_words: 0, - transcript: transcript.to_string(), - }; - } - - // Backtrack to find FIRST and LAST aligned transcript indices. - let mut aligned: Vec = Vec::with_capacity(matched); - let (mut i, mut j) = (n, m); - while i > 0 && j > 0 { - if tokens_match(&trans[i - 1], &targ[j - 1]) && dp[i][j] == dp[i - 1][j - 1] + 1 { - aligned.push(i - 1); - i -= 1; - j -= 1; - } else if dp[i - 1][j] >= dp[i][j - 1] { - i -= 1; - } else { - j -= 1; - } - } - aligned.reverse(); - let first = aligned[0]; - let last = aligned[aligned.len() - 1] + 1; - - // Repeated target words: count target words appearing 2+ times in - // transcript. Uses a small set lookup since target is short. - let targ_set: std::collections::HashSet<&String> = targ.iter().collect(); - let mut counts: std::collections::HashMap<&String, usize> = std::collections::HashMap::new(); - for w in &trans { - if targ_set.contains(w) { - *counts.entry(w).or_insert(0) += 1; - } - } - let repeated = counts.values().filter(|&&c| c >= 2).count(); - - Grade { - coverage: matched as f64 / m as f64, - prefix_words: first, - suffix_words: trans.len() - last, - repeated_target_words: repeated, - transcript: transcript.to_string(), - } -} - fn canon_tokens(s: &str) -> Vec { s.to_lowercase() .chars() @@ -2240,11 +1934,9 @@ fn canon_tokens(s: &str) -> Vec { // ---------- demo seed ---------- // -// The demo uses real Qwen3-TTS ICL synthesis. Since Qwen3-TTS needs a -// reference audio + reference transcript per voice, and we don't ship any -// audio in this repo, we bootstrap the references by calling macOS `say` -// (Samantha / Daniel) into a temp WAV. Then we synthesize each demo line -// through the sidecar, which loads the Qwen3-TTS model on first call. +// The demo embeds human reference WAVs. Engines such as Qwen3-TTS and Fish +// Audio need the reference transcript to match that WAV closely, otherwise +// prompt words can leak into the synthesized take. // // First-ever invocation: downloads ~300MB of model weights from HuggingFace, // then ~2-5s per line. Subsequent invocations reuse the warm model. @@ -2277,6 +1969,9 @@ struct DemoSeed { clips: Vec, } +const DEMO_ANNA_REFERENCE_TEXT: &str = "The Hispaniola was rolling scuppers under in the ocean swell. The booms were tearing at the blocks. The rudder was banging."; +const DEMO_MAREK_REFERENCE_TEXT: &str = "It is a pretty little spot there, a green grass plateau running along by the water's edge and overhung by willows."; + fn short_hash(s: &str) -> u32 { // FNV-1a 32-bit. Stable across runs without pulling another crate. let mut h: u32 = 0x811c9dc5; @@ -2301,6 +1996,15 @@ fn clip_cache_dir() -> std::path::PathBuf { dir } +fn dictation_cache_dir() -> std::path::PathBuf { + let dir = dirs::cache_dir() + .unwrap_or_else(std::env::temp_dir) + .join("audio.soniqo.studio") + .join("dictation"); + let _ = std::fs::create_dir_all(&dir); + dir +} + fn wav_duration_sec(path: &std::path::Path) -> Result { use std::io::Read; let mut f = std::fs::File::open(path).map_err(|e| e.to_string())?; @@ -2318,6 +2022,132 @@ fn wav_duration_sec(path: &std::path::Path) -> Result { Ok(audio_bytes as f64 / byte_rate as f64) } +#[derive(Deserialize)] +struct SaveDictationAudioArgs { + #[serde(rename = "wavBase64")] + wav_base64: String, +} + +#[derive(Serialize)] +struct SaveDictationAudioResult { + #[serde(rename = "audioPath")] + audio_path: String, + #[serde(rename = "durationSec")] + duration_sec: f64, +} + +#[tauri::command] +async fn save_dictation_audio( + args: SaveDictationAudioArgs, +) -> Result { + let bytes = base64::engine::general_purpose::STANDARD + .decode(args.wav_base64.trim()) + .map_err(|e| format!("invalid dictation audio payload: {e}"))?; + const MAX_DICTATION_WAV_BYTES: usize = 100 * 1024 * 1024; + if bytes.len() > MAX_DICTATION_WAV_BYTES { + return Err("dictation recording is too large".into()); + } + if bytes.len() < 44 || &bytes[0..4] != b"RIFF" || &bytes[8..12] != b"WAVE" { + return Err("dictation audio must be a WAV file".into()); + } + + let path = + dictation_cache_dir().join(format!("dictation-{}.wav", uuid::Uuid::new_v4().simple())); + std::fs::write(&path, &bytes).map_err(|e| format!("could not write dictation audio: {e}"))?; + let duration_sec = wav_duration_sec(&path).unwrap_or(0.0); + Ok(SaveDictationAudioResult { + audio_path: path.to_string_lossy().to_string(), + duration_sec, + }) +} + +#[derive(Deserialize)] +struct TranscribeAudioArgs { + #[serde(rename = "audioPath")] + audio_path: String, + model: Option, + language: Option, +} + +#[derive(Serialize)] +struct TranscribeAudioResult { + text: String, + #[serde(rename = "modelName")] + model_name: String, + #[serde(rename = "modelId")] + model_id: String, + #[serde(rename = "durationSec")] + duration_sec: f64, + #[serde(rename = "elapsedSec")] + elapsed_sec: f64, + #[serde(skip_serializing_if = "Option::is_none")] + language: Option, +} + +#[tauri::command] +async fn transcribe_audio( + manager: State<'_, SidecarManager>, + args: TranscribeAudioArgs, +) -> Result { + let model = args.model.unwrap_or(AsrModel::ParakeetTdtV3); + let info = asr_model_info(model); + let audio_path = PathBuf::from(&args.audio_path); + let metadata = std::fs::metadata(&audio_path) + .map_err(|e| format!("cannot read dictation audio {}: {e}", audio_path.display()))?; + if !metadata.is_file() { + return Err(format!( + "dictation audio path is not a file: {}", + audio_path.display() + )); + } + + let started = Instant::now(); + let payload = serde_json::json!({ + "id": format!("asr-{}", uuid::Uuid::new_v4().simple()), + "command": model.sidecar_command(), + "audioPath": audio_path.to_string_lossy(), + "language": args.language, + }); + let raw = manager.request(&payload)?; + let env: SidecarResponse = serde_json::from_value(raw).map_err(|e| e.to_string())?; + if !env.ok { + return Err(env + .error + .unwrap_or_else(|| format!("{} transcription failed", info.display_name))); + } + + let result = env.result.unwrap_or_default(); + let text = result + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let language = result + .get("language") + .and_then(|v| v.as_str()) + .filter(|value| !value.is_empty()) + .map(str::to_string); + let duration_sec = result + .get("durationSec") + .and_then(|v| v.as_f64()) + .or_else(|| wav_duration_sec(&audio_path).ok()) + .unwrap_or(0.0); + let elapsed_sec = result + .get("elapsedSec") + .and_then(|v| v.as_f64()) + .unwrap_or_else(|| started.elapsed().as_secs_f64()); + + Ok(TranscribeAudioResult { + text, + model_name: info.model_name, + model_id: info.model_id, + duration_sec, + elapsed_sec, + language, + }) +} + #[derive(Serialize, Clone)] struct DemoProgress { phase: &'static str, @@ -2344,9 +2174,38 @@ fn emit_progress( ); } +fn demo_clip_seeds(cache_prefix: &str, lines: &[(usize, &str)]) -> Vec { + let cache_dir = clip_cache_dir(); + let mut clips = Vec::with_capacity(lines.len()); + for (idx, (speaker_idx, text)) in lines.iter().enumerate() { + let stable_id = format!( + "{}-s{}-l{}-{:x}", + cache_prefix, + speaker_idx, + idx, + short_hash(text) + ); + let cached_path = cache_dir.join(format!("{}.wav", stable_id)); + let (audio_path, duration_sec) = if cached_path.exists() { + let dur = wav_duration_sec(&cached_path).ok(); + (Some(cached_path.to_string_lossy().to_string()), dur) + } else { + (None, None) + }; + + clips.push(DemoClipSeed { + speaker_index: *speaker_idx, + audio_path, + duration_sec, + text: (*text).to_string(), + }); + } + clips +} + #[tauri::command] async fn seed_demo(app: tauri::AppHandle) -> Result { - eprintln!("[seed_demo] start (lazy mode: no Qwen3 synth, only `say` references)"); + eprintln!("[seed_demo] start (lazy mode: bundled references only)"); emit_progress(&app, "references", 0, 1, "Preparing reference voices…"); let dir = std::env::temp_dir().join("soniqo-demo"); std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir failed: {}", e))?; @@ -2358,17 +2217,17 @@ async fn seed_demo(app: tauri::AppHandle) -> Result { // clips, ~450 KB each, embedded via include_bytes!): // Anna → female reader, ~10s, literary narration. // Marek → male reader, ~9s, calm narration. - // Reference text was produced by Parakeet ASR over the same WAV — close - // enough to verbatim for ICL. + // Reference text must be exact for Qwen3-TTS ICL. A stale Anna transcript + // leaked prompt words into the synthesized take, while Marek stayed clean. let ref_specs: [(&[u8], &str, &str); 2] = [ ( include_bytes!("../resources/voices/anna.wav"), - "The Hispaniola was rolling scuppers under in the ocean swell. The booms were tearing at the blocks, ruddering.", + DEMO_ANNA_REFERENCE_TEXT, "ref-anna.wav", ), ( include_bytes!("../resources/voices/marek.wav"), - "It is a pretty little spot there, a green grass plateau running along by the water's edge and overhung by willows.", + DEMO_MAREK_REFERENCE_TEXT, "ref-marek.wav", ), ]; @@ -2390,9 +2249,7 @@ async fn seed_demo(app: tauri::AppHandle) -> Result { reference_text: (*ref_text).to_string(), }); } - eprintln!( - "[seed_demo] references ready, starting Qwen3-TTS synthesis (first call downloads model)" - ); + eprintln!("[seed_demo] references ready; synthesis is on demand"); // Step 2 — demo lines, each wrapped in a VoxCPM2 style marker. The // sidecar's extractFirstEmotionTag pulls the tag name out and passes it @@ -2413,26 +2270,55 @@ async fn seed_demo(app: tauri::AppHandle) -> Result { // attach its path so the user can immediately play it; otherwise leave // audio_path = None and let the user trigger synthesis explicitly via // Regenerate. No Qwen3 calls happen here — Load demo is ~instant. - let cache_dir = clip_cache_dir(); - let mut clips = Vec::with_capacity(lines.len()); - for (idx, (speaker_idx, text)) in lines.iter().enumerate() { - let stable_id = format!("demo-s{}-l{}-{:x}", speaker_idx, idx, short_hash(text)); - let cached_path = cache_dir.join(format!("{}.wav", stable_id)); - let (audio_path, duration_sec) = if cached_path.exists() { - let dur = wav_duration_sec(&cached_path).ok(); - (Some(cached_path.to_string_lossy().to_string()), dur) - } else { - (None, None) - }; + let clips = demo_clip_seeds("demo", &lines); - clips.push(DemoClipSeed { - speaker_index: *speaker_idx, - audio_path, - duration_sec, - text: (*text).to_string(), + Ok(DemoSeed { voices, clips }) +} + +#[tauri::command] +async fn seed_hindi_demo(app: tauri::AppHandle) -> Result { + eprintln!("[seed_hindi_demo] start"); + emit_progress(&app, "references", 0, 1, "Preparing Hindi reference voice…"); + let dir = std::env::temp_dir().join("soniqo-demo"); + std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir failed: {}", e))?; + + // Two FLEURS hi_in test-split speakers (CC-BY, PCM16 mono 16 kHz), one per + // gender. Reference text must be exact for voice-clone ICL — both + // transcripts are the raw FLEURS transcriptions, ASR-verified against the + // audio (a mismatched transcript leaks prompt words into the takes). + let ref_specs: [(&[u8], &str, &str); 2] = [ + ( + include_bytes!("../resources/voices/hindi_fleurs.wav"), + "लूना को साथी पहलवानों ने भी श्रद्धांजलि दी.", + "ref-hindi-fleurs.wav", + ), + ( + include_bytes!("../resources/voices/hindi_fleurs_female.wav"), + "यह शहर देश के बाकी शहरों से अलग है क्योंकि यह किसी अफ्रीकी शहर की बजाय अरब शहर लगता है.", + "ref-hindi-fleurs-female.wav", + ), + ]; + + let mut voices = Vec::with_capacity(ref_specs.len()); + for (bytes, ref_text, filename) in ref_specs.iter() { + let path = dir.join(filename); + std::fs::write(&path, bytes) + .map_err(|e| format!("write {} failed: {}", path.display(), e))?; + voices.push(DemoVoiceSeed { + reference_audio_path: path.to_string_lossy().to_string(), + reference_text: (*ref_text).to_string(), }); } + // Male (0) and female (1) speakers alternate, like the English demo. + let lines: [(usize, &str); 4] = [ + (0, "(happy) नमस्ते, आज हम हिंदी आवाज़ का परीक्षण कर रहे हैं।"), + (1, "(sad) यह पंक्ति शांत और भावुक सुनाई देनी चाहिए।"), + (0, "(angry) अब आवाज़ में थोड़ी तीव्रता और ज़ोर चाहिए।"), + (1, "(surprised) अंत में यह वाक्य साफ़ और उत्साहित होना चाहिए।"), + ]; + let clips = demo_clip_seeds("demo-hi", &lines); + Ok(DemoSeed { voices, clips }) } @@ -2819,6 +2705,7 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ ping_sidecar, available_tts_engines, + available_asr_models, init_model, interrupt_model_load, pick_video, @@ -2826,6 +2713,8 @@ pub fn run() { import_reference_audio, probe_reference, clone_voice, + save_dictation_audio, + transcribe_audio, synthesize_clip, export_project, list_projects, @@ -2833,6 +2722,7 @@ pub fn run() { load_project, delete_project, seed_demo, + seed_hindi_demo, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); @@ -2846,10 +2736,16 @@ mod tests { fn tts_engine_protocol_names_are_stable() { let cosy: TtsEngine = serde_json::from_str("\"cosyvoice\"").unwrap(); assert_eq!(cosy, TtsEngine::CosyVoice); + let parakeet: AsrModel = serde_json::from_str("\"parakeet-tdt-v3\"").unwrap(); + assert_eq!(parakeet, AsrModel::ParakeetTdtV3); assert_eq!( serde_json::to_string(&TtsEngine::VoxCPM2).unwrap(), "\"voxcpm2\"" ); + assert_eq!( + serde_json::to_string(&AsrModel::ParakeetTdtV3).unwrap(), + "\"parakeet-tdt-v3\"" + ); assert_eq!( serde_json::to_string(&TtsEngine::IndicMio).unwrap(), "\"indic-mio\"" @@ -2867,6 +2763,7 @@ mod tests { TtsEngine::FishAudio.sidecar_command(), "synthesize_fish_audio" ); + assert_eq!(parakeet.sidecar_command(), "transcribe_parakeet"); } #[test] @@ -2903,6 +2800,15 @@ mod tests { assert!(!TtsEngine::VoxCPM2.requires_reference_transcript()); } + #[test] + fn demo_anna_reference_transcript_matches_bundled_audio() { + assert!( + !DEMO_ANNA_REFERENCE_TEXT.contains("ruddering"), + "stale Anna transcript leaks reference text into Qwen3-TTS ICL output" + ); + assert!(DEMO_ANNA_REFERENCE_TEXT.ends_with("The rudder was banging.")); + } + #[test] fn chatterbox_engine_wiring() { let c: TtsEngine = serde_json::from_str("\"chatterbox\"").unwrap(); @@ -2917,11 +2823,202 @@ mod tests { assert_eq!(TtsEngine::VoxCPM2.style_mode(), "instruction"); assert_eq!(TtsEngine::IndicMio.style_mode(), "suffix-tag"); assert_eq!(TtsEngine::FishAudio.style_mode(), "bracket-tag"); + assert_eq!(TtsEngine::OmniVoice.style_mode(), "controlled-vocabulary"); assert_eq!(TtsEngine::Qwen3.style_mode(), "none"); } + #[test] + fn tts_engine_info_exposes_model_capabilities() { + let vox = tts_engine_info(TtsEngine::VoxCPM2); + assert_eq!(vox.languages.len(), 30); + assert!(vox.languages.iter().any(|language| language == "hi")); + assert!(vox.languages.iter().any(|language| language == "vi")); + assert!(!vox.requires_language); + assert_eq!(vox.model_name, "voxcpm2-mlx-bf16"); + assert_eq!(vox.model_id, "aufklarer/VoxCPM2-MLX-bf16"); + assert_eq!(vox.precision, "bf16"); + + let cosy = tts_engine_info(TtsEngine::CosyVoice); + assert!(cosy.requires_language); + assert_eq!( + cosy.languages, + ["en", "zh", "ja", "ko", "de", "es", "fr", "it", "ru"] + ); + assert!(cosy + .supported_markers + .iter() + .any(|marker| marker == "excited")); + + let qwen = tts_engine_info(TtsEngine::Qwen3); + assert_eq!(qwen.model_name, "qwen3-tts-1.7b-mlx-bf16"); + assert_eq!(qwen.model_id, "aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-bf16"); + assert_eq!(qwen.voice_profile_modes, ["reference-clone"]); + assert!(qwen.requires_reference_audio); + assert!(qwen.requires_reference_transcript); + assert!(qwen.requires_language); + assert!(!qwen.supports_instruct); + assert_eq!(qwen.style_mode, "none"); + assert!(qwen.languages.iter().any(|language| language == "en")); + assert!(qwen.languages.iter().any(|language| language == "ru")); + assert!(qwen.supported_markers.is_empty()); + assert_eq!(qwen.precision, "bf16"); + + let chatterbox = tts_engine_info(TtsEngine::Chatterbox); + assert_eq!(chatterbox.languages.len(), 22); + assert!(chatterbox.languages.iter().any(|language| language == "zh")); + assert!(chatterbox.languages.iter().any(|language| language == "ja")); + assert!(!chatterbox.languages.iter().any(|language| language == "he")); + assert!(chatterbox.languages.iter().any(|language| language == "ko")); + assert_eq!(chatterbox.precision, "fp16"); + + let omni = tts_engine_info(TtsEngine::OmniVoice); + assert_eq!(omni.model_name, "omnivoice-mlx-fp16"); + assert_eq!(omni.model_id, "aufklarer/OmniVoice-MLX-fp16"); + assert_eq!(omni.precision, "fp16"); + + let fish = tts_engine_info(TtsEngine::FishAudio); + assert_eq!(fish.style_mode, "bracket-tag"); + assert!(fish.languages.len() > 70); + assert!(fish + .benchmark_languages + .iter() + .any(|language| language == "hi")); + assert!(fish + .supported_markers + .iter() + .any(|marker| marker == "excited")); + assert_eq!(fish.use_policy, "research-only"); + assert!(!fish.needs_trim); + } + + #[test] + fn hindi_demo_reference_is_bundled_pcm_wav() { + let path = std::env::temp_dir().join(format!( + "speech-studio-hindi-ref-{}.wav", + uuid::Uuid::new_v4().simple() + )); + std::fs::write( + &path, + include_bytes!("../resources/voices/hindi_fleurs.wav"), + ) + .unwrap(); + + let (sample_rate, channels, bits) = read_wav_header(&path).unwrap(); + assert_eq!(sample_rate, 16_000); + assert_eq!(channels, 1); + assert_eq!(bits, 16); + + let duration = wav_duration_sec(&path).unwrap(); + assert!( + (3.7..4.0).contains(&duration), + "unexpected Hindi reference duration: {duration}" + ); + + let _ = std::fs::remove_file(path); + } + + #[test] + fn tts_registry_defaults_keep_16bit_precision_floor() { + for entry in MODEL_REGISTRY + .tts_engines + .iter() + .filter(|entry| engine_is_supported(entry.info.id)) + { + assert!( + matches!(entry.info.precision.as_str(), "bf16" | "fp16"), + "{} default precision should stay 16-bit, got {}", + entry.info.model_name, + entry.info.precision + ); + assert!( + !entry.info.model_id.contains("int8") + && !entry.info.model_id.contains("8bit") + && !entry.info.model_id.contains("4bit"), + "{} default model id should not be quantized: {}", + entry.info.model_name, + entry.info.model_id + ); + } + } + + #[test] + fn asr_registry_uses_parakeet_family_on_all_platforms() { + let parakeet = asr_model_info(AsrModel::ParakeetTdtV3); + assert_eq!(parakeet.model_name, "parakeet-tdt-v3-0.6b-int8"); + assert_eq!(parakeet.sample_rate, 16_000); + assert_eq!(parakeet.max_segment_sec, 30); + assert!(parakeet.streaming); + assert_eq!(parakeet.readiness, "production"); + assert_eq!(parakeet.languages, ["en"]); + assert_eq!( + parakeet.runtime, + if cfg!(target_os = "macos") { + "coreml" + } else { + "litert" + } + ); + assert_eq!( + parakeet.model_id, + if cfg!(target_os = "macos") { + "aufklarer/Parakeet-TDT-v3-CoreML-INT8-30s" + } else { + "soniqo/Parakeet-TDT-0.6B-v3-LiteRT-INT8" + } + ); + } + + #[test] + fn studio_registry_keeps_qwen_on_bf16() { + let ids: Vec = MODEL_REGISTRY + .tts_engines + .iter() + .map(|entry| entry.info.id) + .collect(); + assert_eq!( + ids, + vec![ + TtsEngine::VoxCPM2, + TtsEngine::CosyVoice, + TtsEngine::Qwen3, + TtsEngine::Chatterbox, + TtsEngine::OmniVoice, + TtsEngine::IndicMio, + TtsEngine::FishAudio, + ] + ); + + let qwen = TtsEngine::Qwen3.registry_entry(); + assert_eq!(qwen.info.model_name, "qwen3-tts-1.7b-mlx-bf16"); + assert_eq!( + qwen.info.model_id, + "aufklarer/Qwen3-TTS-12Hz-1.7B-Base-MLX-bf16" + ); + assert!(!qwen.info.model_name.contains("8bit")); + assert!(!qwen.info.model_id.contains("8bit")); + assert!(!qwen.info.model_name.contains("0.6b")); + assert_eq!(qwen.info.precision, "bf16"); + assert_eq!(qwen.info.style_mode, "none"); + } + + #[test] + fn qwen_synth_budget_is_bounded_for_short_lines() { + assert_eq!(synth_max_tokens(TtsEngine::Qwen3, 2), 40); + assert_eq!(synth_max_tokens(TtsEngine::Qwen3, 6), 44); + assert_eq!(synth_max_tokens(TtsEngine::Qwen3, 20), 96); + assert_eq!(synth_max_tokens(TtsEngine::CosyVoice, 6), 112); + } + #[test] fn sidecar_language_normalization_preserves_chatterbox_hi() { + assert_eq!( + normalize_sidecar_language(TtsEngine::CosyVoice, Some("ru")).as_deref(), + Some("russian") + ); + assert_eq!( + normalize_sidecar_language(TtsEngine::CosyVoice, Some("ja")).as_deref(), + Some("japanese") + ); assert_eq!( normalize_sidecar_language(TtsEngine::Chatterbox, Some("hi")).as_deref(), Some("hi") @@ -2984,6 +3081,21 @@ mod tests { ); } + #[test] + fn dictation_cache_dir_under_app_namespace() { + let dir = dictation_cache_dir(); + assert!( + dir.ends_with("dictation"), + "dictation dir should end with dictation: {:?}", + dir + ); + assert!( + dir.to_string_lossy().contains("audio.soniqo.studio"), + "dictation dir should be under the app namespace: {:?}", + dir + ); + } + // Simulates the installed-bundle layout: the sidecar binary sits next to // the main app binary while libLiteRt is staged in a separate resource dir. // The search path must include both so the loader finds the runtime. @@ -3131,94 +3243,6 @@ mod tests { assert!(trim_long_form_chunk_edges(TtsEngine::IndicMio)); } - #[test] - fn tokens_match_folds_devanagari_vs_urdu_script() { - // Omnilingual decodes Hindustani in mixed script; the same spoken word - // must align across spellings via the consonant skeleton. - assert!(tokens_match("जोड़े", "جوڑے")); - assert!(tokens_match("लाइन", "لائن")); - assert!(tokens_match("नीचे", "نیچے")); - // ASCII never matches fuzzily. - assert!(!tokens_match("line", "lane")); - } - - #[test] - fn grade_hindi_mixed_script_transcript_flags_babble_tail() { - // Real Omnilingual transcript of a take whose last second was AR - // babble ("یان کھن"); target is the chunk text in Devanagari. - let target = "गेम अपने आप सेव नहीं होता कि ठीक बीच में एक हॉरिजॉन्टल लाइन जोड़े।"; - let transcript = "گیم اپنے سیو نہیں ہوتا کہ ٹھیک بیچ میں ایک ہاریجانٹل لائن جوڑے یان کھن"; - let g = grade_transcript(transcript, target); - assert!( - g.coverage >= 0.6, - "cross-script coverage too low: {}", - g.coverage - ); - assert!( - g.suffix_words >= 2, - "babble tail not detected: suffix={}", - g.suffix_words - ); - assert!( - !g.is_clean_for(true), - "babble take must be rejected for non-Latin targets" - ); - } - - #[test] - fn grade_clean_take_passes() { - let g = grade_transcript( - "I knew you would make it, no matter what.", - "I knew you would make it, no matter what.", - ); - assert_eq!(g.coverage, 1.0); - assert_eq!(g.prefix_words, 0); - assert_eq!(g.suffix_words, 0); - assert_eq!(g.repeated_target_words, 0); - assert!(g.is_clean()); - } - - #[test] - fn grade_detects_reference_leak_prefix() { - // Clip 1 from the sweep: "Can be." prefix + line repeated twice. - let g = grade_transcript( - "Can be. I never thought we'd make it this far. Ruttering. I never thought we'd make it this far.", - "I never thought we'd make it this far.", - ); - assert_eq!(g.coverage, 1.0); // all target words present - assert!( - g.prefix_words >= 2, - "expected prefix junk, got {}", - g.prefix_words - ); - assert!( - g.repeated_target_words >= 4, - "expected repetition, got {}", - g.repeated_target_words - ); - assert!(!g.is_clean()); - } - - #[test] - fn grade_detects_short_take() { - let g = grade_transcript("Tonight.", "Then we end this together. Tonight."); - assert!(g.coverage < 0.5); - assert!(!g.is_clean()); - } - - #[test] - fn grade_accepts_minor_substitution() { - // "end" -> "and" — one-word substitution still passes if coverage stays - // above the 0.75 threshold. - let g = grade_transcript( - "Then we and this together. Tonight.", - "Then we end this together. Tonight.", - ); - assert!(g.coverage >= 0.75, "expected >=0.75, got {}", g.coverage); - assert_eq!(g.prefix_words, 0); - assert!(g.is_clean(), "should accept minor substitution"); - } - #[cfg(target_os = "macos")] #[test] fn colocate_metallib_simulates_bundle_layout() { @@ -3288,20 +3312,4 @@ mod tests { assert!(err < 1e-3, "mean abs error {err} too high"); } - #[test] - fn score_prefers_clean_low_cov_over_leaked_high_cov() { - // The whole point of the composite score: when ladder exhausts, prefer - // a clean cov=75% take over a cov=100% leak. - let clean = grade_transcript("I knew you would make.", "I knew you would make it."); - let leaked = grade_transcript( - "Capit ruttering quilt I knew you would make it", - "I knew you would make it.", - ); - assert!( - clean.score() > leaked.score(), - "clean.score={} leaked.score={}", - clean.score(), - leaked.score() - ); - } } diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index d6b81ba..71b7da9 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Speech Studio", - "version": "0.0.8", + "version": "0.0.9", "identifier": "audio.soniqo.studio", "build": { "beforeDevCommand": "pnpm dev", diff --git a/src-tauri/tests/sidecar.rs b/src-tauri/tests/sidecar.rs index bf11660..41f7622 100644 --- a/src-tauri/tests/sidecar.rs +++ b/src-tauri/tests/sidecar.rs @@ -680,7 +680,7 @@ fn bundled_voice_path(name: &str) -> PathBuf { } // Reference text for the bundled WAVs. Must match seed_demo in src/lib.rs. -const ANNA_REF_TEXT: &str = "The Hispaniola was rolling scuppers under in the ocean swell. The booms were tearing at the blocks, ruddering."; +const ANNA_REF_TEXT: &str = "The Hispaniola was rolling scuppers under in the ocean swell. The booms were tearing at the blocks. The rudder was banging."; const MAREK_REF_TEXT: &str = "It is a pretty little spot there, a green grass plateau running along by the water's edge and overhung by willows."; // Demo target lines. Must match `lines` in `seed_demo`. @@ -933,3 +933,273 @@ fn demo_references_e2e() { overall * 100.0 ); } + +// ----------------------------------------------------------------------------- +// Onset-junk + emotion-marker e2e tests. +// +// Qwen3 ICL and OmniVoice both opened some renders (reference-dependent) with +// a short low-level codec artifact before the first word — quiet hiss for +// Qwen3 on the Marek reference, a low hum for OmniVoice on Anna — followed by +// a silence gap before the phrase. The sidecar removes these via +// trimLeadingJunk before edge conditioning. These tests replay the exact +// production requests that used to produce audible junk (deterministic at +// seed 1000) and assert the first thing you hear is speech-loud. +// +// Indic-Mio styles via a closed suffix-tag vocabulary; the sidecar maps the +// Studio's inline "(happy)"-style marker onto it. Before that mapping the +// model read the marker aloud, so the third test pins "marker is styled, not +// spoken". +// +// All three load real models (cached under ~/.cache/huggingface after the +// first run) and need the `speech` CLI for Parakeet grading. Opt in via: +// cargo test --test sidecar -- --ignored --nocapture qwen_onset_is_speech_not_junk +// cargo test --test sidecar -- --ignored --nocapture omnivoice_onset_is_speech_not_junk +// cargo test --test sidecar -- --ignored --nocapture indic_mio_does_not_speak_emotion_marker +// ----------------------------------------------------------------------------- + +/// Read a sidecar-written WAV (44-byte header, 16-bit PCM mono) as f32 samples. +fn read_wav_samples(path: &str) -> (Vec, usize) { + use std::io::Read; + let mut f = std::fs::File::open(path).expect("open wav"); + let mut header = [0u8; 44]; + f.read_exact(&mut header).expect("wav header"); + let channels = u16::from_le_bytes([header[22], header[23]]) as usize; + let sample_rate = u32::from_le_bytes([header[24], header[25], header[26], header[27]]) as usize; + let bps = u16::from_le_bytes([header[34], header[35]]); + assert_eq!(bps, 16, "expected 16-bit PCM"); + assert_eq!(channels, 1, "expected mono"); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).expect("wav data"); + let samples = buf + .chunks_exact(2) + .map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0) + .collect(); + (samples, sample_rate) +} + +/// Group audio into energy islands (5 ms RMS windows above -60 dBFS, closed +/// by ≥50 ms of quiet) and return (first_island_peak_db, max_island_peak_db). +/// Mirrors the sidecar's trimLeadingJunk analysis: a clean render opens with +/// a speech-loud island; onset junk shows up as a first island far below the +/// loudest one (measured 10-28 dB down across both engines). +fn onset_island_peaks(samples: &[f32], sample_rate: usize) -> Option<(f64, f64)> { + let win = sample_rate * 5 / 1000; + if win == 0 || samples.len() < win * 8 { + return None; + } + let dbs: Vec = samples + .chunks_exact(win) + .map(|c| { + let e = c.iter().map(|s| (*s as f64) * (*s as f64)).sum::() / c.len() as f64; + if e > 0.0 { 10.0 * e.log10() } else { -120.0 } + }) + .collect(); + let mut peaks: Vec = Vec::new(); + let mut current: Option = None; + let mut quiet = 0; + for &db in &dbs { + if db > -60.0 { + current = Some(current.map_or(db, |p: f64| p.max(db))); + quiet = 0; + } else if let Some(peak) = current { + quiet += 1; + if quiet >= 10 { + peaks.push(peak); + current = None; + } + } + } + if let Some(peak) = current { + peaks.push(peak); + } + let first = *peaks.first()?; + let max = peaks.iter().cloned().fold(f64::MIN, f64::max); + Some((first, max)) +} + +fn parakeet_transcript(audio_path: &str) -> String { + let out = Command::new("speech") + .args(["transcribe", audio_path, "--engine", "parakeet"]) + .output() + .expect("run `speech transcribe` (install the speech CLI for e2e tests)"); + assert!( + out.status.success(), + "speech transcribe failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout) + .lines() + .find_map(|l| l.strip_prefix("Result: ")) + .map(|s| s.trim().to_string()) + .unwrap_or_default() +} + +/// Send one synthesis request, assert success, and return the output path. +fn synth_expect_ok(s: &mut SidecarHandle, payload: serde_json::Value) -> String { + let id = payload["id"].as_str().unwrap_or("").to_string(); + s.send(&payload); + let v = s.recv(); + assert_eq!(v["id"], serde_json::Value::String(id)); + assert_eq!( + v["ok"], + true, + "synthesis failed: {}", + v["error"].as_str().unwrap_or("(no error)") + ); + let path = v["result"]["audioPath"].as_str().expect("audioPath").to_string(); + let dur = v["result"]["durationSec"].as_f64().unwrap_or(0.0); + assert!( + (1.0..=15.0).contains(&dur), + "implausible duration {:.2}s for {}", + dur, + path + ); + path +} + +/// The first energy island must be speech, not a quiet junk prefix. 8 dB is +/// the sidecar detector's own quiet-junk bound: clean takes measured ≤4.7 dB +/// below the loudest island, junk takes ≥10.2 dB. +fn assert_onset_is_speech(audio_path: &str, target_text: &str) { + let (samples, sample_rate) = read_wav_samples(audio_path); + let (first_peak, max_peak) = + onset_island_peaks(&samples, sample_rate).expect("no energy islands in output"); + let deficit = max_peak - first_peak; + assert!( + deficit <= 8.0, + "onset junk: first island peaks {:.1} dB below the loudest ({:.1} vs {:.1} dBFS) in {}", + deficit, + first_peak, + max_peak, + audio_path + ); + let transcript = parakeet_transcript(audio_path); + let (_, _, cov) = token_coverage(target_text, &transcript); + assert!( + cov >= 0.6, + "coverage {:.0}% < 60% — transcript {:?} vs target {:?}", + cov * 100.0, + transcript, + target_text + ); +} + +#[test] +#[ignore] +fn qwen_onset_is_speech_not_junk() { + let marek = bundled_voice_path("marek"); + let mut s = SidecarHandle::spawn(); + // The exact production request (post-preprocess text, seed 1000) that + // used to open with ~100 ms of quiet hiss followed by half a second of + // silence before the phrase. + let path = synth_expect_ok( + &mut s, + serde_json::json!({ + "id": "e2e-qwen-onset", + "command": "synthesize_icl", + "engine": "qwen3", + "text": "Then we end this together, Tonight.", + "voiceId": "marek", + "referenceAudioPath": marek.to_string_lossy(), + "referenceText": MAREK_REF_TEXT, + "seed": 1000, + "maxTokens": 44, + }), + ); + assert_onset_is_speech(&path, "Then we end this together, Tonight."); +} + +#[test] +#[ignore] +fn omnivoice_onset_is_speech_not_junk() { + let anna = bundled_voice_path("anna"); + let mut s = SidecarHandle::spawn(); + // OmniVoice's variant of the same artifact: a ~200 ms low hum on the + // Anna reference before the first word. + let path = synth_expect_ok( + &mut s, + serde_json::json!({ + "id": "e2e-omni-onset", + "command": "synthesize_omnivoice", + "engine": "omnivoice", + "modelId": "aufklarer/OmniVoice-MLX-fp16", + "text": "(dramatic) I never thought we'd make it this far.", + "voiceId": "anna", + "referenceAudioPath": anna.to_string_lossy(), + "referenceText": ANNA_REF_TEXT, + "language": "en", + "seed": 1000, + }), + ); + assert_onset_is_speech(&path, "I never thought we'd make it this far."); +} + +#[test] +#[ignore] +fn indic_mio_does_not_speak_emotion_marker() { + let anna = bundled_voice_path("anna"); + let mut s = SidecarHandle::spawn(); + let target = "This line should sound bright and clear."; + let path = synth_expect_ok( + &mut s, + serde_json::json!({ + "id": "e2e-indic-marker", + "command": "synthesize_indic_mio", + "engine": "indic-mio", + "modelId": "aufklarer/Indic-Mio-MLX-fp16", + "text": format!("(happy) {}", target), + "voiceId": "indic-test", + "referenceAudioPath": anna.to_string_lossy(), + "referenceText": ANNA_REF_TEXT, + "language": "en", + "seed": 1000, + }), + ); + let transcript = parakeet_transcript(&path); + let (_, _, cov) = token_coverage(target, &transcript); + assert!( + cov >= 0.6, + "coverage {:.0}% < 60% — transcript {:?}", + cov * 100.0, + transcript + ); + assert!( + !tokenize(&transcript).contains(&"happy".to_string()), + "emotion marker was spoken aloud — transcript {:?}", + transcript + ); +} + +#[test] +fn sidecar_rejects_engine_mismatch() { + let mut s = SidecarHandle::spawn(); + s.send(&serde_json::json!({ + "id": "mismatch", + "command": "synthesize_omnivoice", + "engine": "qwen3", + "text": "hi", + "referenceAudioPath": "/nonexistent.wav", + })); + let v = s.recv(); + assert_eq!(v["ok"], false); + let err = v["error"].as_str().unwrap_or(""); + assert!( + err.contains("requires engine"), + "unexpected error: {}", + err + ); +} + +#[test] +fn sidecar_rejects_indic_mio_without_text() { + let mut s = SidecarHandle::spawn(); + s.send(&serde_json::json!({ + "id": "no-text", + "command": "synthesize_indic_mio", + "engine": "indic-mio", + })); + let v = s.recv(); + assert_eq!(v["ok"], false); + let err = v["error"].as_str().unwrap_or(""); + assert!(err.contains("requires text"), "unexpected error: {}", err); +} diff --git a/src/components/DictationPanel.tsx b/src/components/DictationPanel.tsx new file mode 100644 index 0000000..4f7ecf9 --- /dev/null +++ b/src/components/DictationPanel.tsx @@ -0,0 +1,260 @@ +import { useEffect, useState } from "react"; +import { Clipboard, FilePlus2, Loader2, Mic, Pause, Play, Square } from "lucide-react"; +import { convertFileSrc } from "@tauri-apps/api/core"; +import { availableAsrModels, type AsrModelInfo } from "../ipc/commands"; +import { newClip, useProjectStore } from "../state/projectStore"; +import type { SpeakerTrack } from "../types/project"; +import { Button } from "./ui/button"; +import { useI18n } from "../i18n/useI18n"; +import { useDictationRecorder } from "../hooks/useDictationRecorder"; + +interface DictationCapture { + id: string; + audioPath: string; + durationSec: number; + text: string; + elapsedSec: number; + createdAt: string; +} + +let currentDictationAudio: HTMLAudioElement | null = null; + +function formatSec(value: number): string { + return value.toFixed(value < 10 ? 1 : 0); +} + +function estimateClipDurationSec(text: string, fallbackSec: number): number { + const words = text.trim().split(/\s+/).filter(Boolean).length; + return Math.max(1.0, fallbackSec || words * 0.35 || 1.0); +} + +export function DictationPanel() { + const { messages: t } = useI18n(); + const addTrack = useProjectStore((s) => s.addTrack); + const addClip = useProjectStore((s) => s.addClip); + const updateClip = useProjectStore((s) => s.updateClip); + const select = useProjectStore((s) => s.select); + const { recording, busy, error, setError, start, stopAndTranscribe } = + useDictationRecorder(); + const [models, setModels] = useState([]); + const [flowMessage, setFlowMessage] = useState(null); + const [captures, setCaptures] = useState([]); + const [playingId, setPlayingId] = useState(null); + + const model = models[0]; + + useEffect(() => { + let cancelled = false; + void availableAsrModels() + .then((next) => { + if (!cancelled) setModels(next); + }) + .catch((e) => { + if (!cancelled) setError(String(e)); + }); + return () => { + cancelled = true; + if (currentDictationAudio) { + currentDictationAudio.pause(); + currentDictationAudio = null; + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + async function record() { + setFlowMessage(null); + await start(); + } + + async function finishRecording() { + setFlowMessage(null); + const result = await stopAndTranscribe(model?.id); + if (!result) return; + setCaptures((prev) => [ + { id: crypto.randomUUID(), createdAt: new Date().toISOString(), ...result }, + ...prev, + ]); + } + + function insertCapture(capture: DictationCapture) { + const text = capture.text.trim(); + if (!text) { + setError(t.dictation.emptyTranscript); + return; + } + const state = useProjectStore.getState(); + if (state.selection.kind === "clip") { + updateClip(state.selection.id, { + text, + renderedAudioPath: undefined, + locked: false, + }); + setFlowMessage(t.dictation.insertedClip); + return; + } + + const selectedTrackId = state.selection.kind === "track" ? state.selection.id : undefined; + const selectedTrack = selectedTrackId + ? state.project.tracks.find( + (track): track is SpeakerTrack => + track.kind === "speaker" && track.id === selectedTrackId, + ) + : undefined; + let targetTrack = selectedTrack ?? state.project.tracks.find( + (track): track is SpeakerTrack => track.kind === "speaker", + ); + if (!targetTrack) { + targetTrack = { + kind: "speaker", + id: crypto.randomUUID(), + name: t.defaults.speakerTrack(1), + clips: [], + }; + addTrack(targetTrack); + } + + const startSec = Math.max(0, state.transport.positionSec); + const endSec = startSec + estimateClipDurationSec(text, capture.durationSec); + const clip = newClip({ + trackId: targetTrack.id, + startSec, + endSec, + text, + }); + addClip(clip); + select({ kind: "clip", id: clip.id }); + setFlowMessage(t.dictation.insertedTimeline); + } + + function togglePlayback(capture: DictationCapture) { + if (playingId === capture.id) { + currentDictationAudio?.pause(); + currentDictationAudio = null; + setPlayingId(null); + return; + } + currentDictationAudio?.pause(); + const audio = new Audio(convertFileSrc(capture.audioPath)); + currentDictationAudio = audio; + audio.onended = () => { + if (currentDictationAudio === audio) currentDictationAudio = null; + setPlayingId(null); + }; + audio.onerror = () => setPlayingId(null); + audio.play().then(() => setPlayingId(capture.id)).catch(() => setPlayingId(null)); + } + + return ( +
+
+ + {t.dictation.title} + + + {model ? model.displayName : t.dictation.loadingModel} + +
+ +
+ +
+ {model ? t.dictation.modelMeta(model.runtime, model.modelSize) : t.dictation.modelPending} +
+
+ + {error && ( +
+ {error} +
+ )} + {flowMessage && ( +
+ {flowMessage} +
+ )} + +
+ {captures.length === 0 && ( +
+ {t.dictation.empty} +
+ )} + {captures.map((capture) => ( +
+
+ + {t.dictation.captureMeta( + formatSec(capture.durationSec), + formatSec(capture.elapsedSec), + )} + +
+ + + +
+
+

+ {capture.text || t.dictation.emptyTranscript} +

+
+ ))} +
+
+ ); +} diff --git a/src/components/Inspector.tsx b/src/components/Inspector.tsx index 6413a21..2a69f11 100644 --- a/src/components/Inspector.tsx +++ b/src/components/Inspector.tsx @@ -1,7 +1,6 @@ import { useState } from "react"; -import { convertFileSrc } from "@tauri-apps/api/core"; -import { Lock, Unlock, Trash2, RefreshCw, History, Loader2 } from "lucide-react"; -import { useProjectStore } from "../state/projectStore"; +import { Lock, Unlock, Trash2, RefreshCw, History, Loader2, ListPlus } from "lucide-react"; +import { useProjectStore, newClip } from "../state/projectStore"; import { ScriptEditor } from "./ScriptEditor"; import { synthesizeClip } from "../ipc/commands"; import { Button } from "./ui/button"; @@ -15,6 +14,8 @@ import { SelectValue, } from "./ui/select"; import { cn } from "@/lib/utils"; +import { clipAudioVersion } from "../lib/clipAudio"; +import { mediaFileSrc } from "../lib/mediaSrc"; import { dateLocale, type Messages } from "../i18n/messages"; import { useI18n } from "../i18n/useI18n"; @@ -62,8 +63,12 @@ export function Inspector() { const updateClip = useProjectStore((s) => s.updateClip); const assignVoiceToTrack = useProjectStore((s) => s.assignVoiceToTrack); const removeClip = useProjectStore((s) => s.removeClip); + const addClip = useProjectStore((s) => s.addClip); + const select = useProjectStore((s) => s.select); + const setPlaying = useProjectStore((s) => s.setPlaying); + const seek = useProjectStore((s) => s.seek); const engine = useProjectStore((s) => s.model.engine); - const language = useProjectStore((s) => s.model.language); + const modelLanguage = useProjectStore((s) => s.model.language); const activeEngine = useProjectStore((s) => s.model.engines.find((candidate) => candidate.id === s.model.engine), ); @@ -170,6 +175,34 @@ export function Inspector() { {track.sourcePath} )} + {track.kind === "speaker" && ( +
+ + +
+ )} ); @@ -257,6 +290,7 @@ export function Inspector() { const needsReferenceTranscript = activeEngine?.requiresReferenceTranscript ?? (engine === "cosyvoice" || engine === "qwen3" || engine === "fish-audio"); + const language = activeEngine?.requiresLanguage ? modelLanguage : undefined; const transcriptEngineName = activeEngine?.displayName ?? t.inspector.selectedEngine; const canRegenerate = !isRegenerating && @@ -267,6 +301,8 @@ export function Inspector() { async function regenerate() { if (!effectiveVoice || !effectiveVoice.referenceAudioPath) return; + setPlaying(false); + seek(current.startSec); setIsRegenerating(true); setRegenError(null); setRegenTiming(null); @@ -391,8 +427,8 @@ export function Inspector() {