From 25c1acb614c31987dd0ac0183546ff1ce7158654 Mon Sep 17 00:00:00 2001 From: agent Date: Thu, 27 Aug 2026 19:50:06 +0200 Subject: [PATCH 1/3] feat: add Soprano TTS as community model with GGUF packages --- CMakeLists.txt | 13 + PR_SOPRANO.md | 104 ++++ README.md | 2 + docs/gguf.md | 1 + docs/soprano_tts.md | 265 +++++++++++ docs/soprano_validation.md | 102 ++++ .../community_models/soprano_tts/assets.h | 68 +++ .../community_models/soprano_tts/generator.h | 43 ++ .../community_models/soprano_tts/session.h | 67 +++ .../soprano_tts/tokenizer_text.h | 46 ++ .../community_models/soprano_tts/vocoder.h | 51 ++ model_specs/soprano_tts.json | 185 ++++++++ src/community_models/soprano_tts/assets.cpp | 64 +++ .../soprano_tts/generator.cpp | 342 +++++++++++++ src/community_models/soprano_tts/session.cpp | 289 +++++++++++ .../soprano_tts/tokenizer_text.cpp | 203 ++++++++ src/community_models/soprano_tts/vocoder.cpp | 449 ++++++++++++++++++ .../soprano_tts/soprano_python_warm_bench.py | 114 +++++ .../soprano_tts/soprano_warm_bench_cases.txt | 19 + tools/soprano_tts/compare_parity.py | 136 ++++++ tools/soprano_tts/convert_soprano.py | 205 ++++++++ tools/soprano_tts/run_official.py | 46 ++ webui/configs/models_catalog.json | 1 + 23 files changed, 2815 insertions(+) create mode 100644 PR_SOPRANO.md create mode 100644 docs/soprano_tts.md create mode 100644 docs/soprano_validation.md create mode 100644 include/engine/community_models/soprano_tts/assets.h create mode 100644 include/engine/community_models/soprano_tts/generator.h create mode 100644 include/engine/community_models/soprano_tts/session.h create mode 100644 include/engine/community_models/soprano_tts/tokenizer_text.h create mode 100644 include/engine/community_models/soprano_tts/vocoder.h create mode 100644 model_specs/soprano_tts.json create mode 100644 src/community_models/soprano_tts/assets.cpp create mode 100644 src/community_models/soprano_tts/generator.cpp create mode 100644 src/community_models/soprano_tts/session.cpp create mode 100644 src/community_models/soprano_tts/tokenizer_text.cpp create mode 100644 src/community_models/soprano_tts/vocoder.cpp create mode 100644 tests/soprano_tts/soprano_python_warm_bench.py create mode 100644 tests/soprano_tts/soprano_warm_bench_cases.txt create mode 100644 tools/soprano_tts/compare_parity.py create mode 100644 tools/soprano_tts/convert_soprano.py create mode 100644 tools/soprano_tts/run_official.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 37b153028..a0a64ec75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1443,6 +1443,19 @@ audiocpp_add_model(ace_step LOADERS engine::models::ace_step::make_ace_step_loader ) +audiocpp_add_model(soprano_tts + SOURCES + src/community_models/soprano_tts/assets.cpp + src/community_models/soprano_tts/generator.cpp + src/community_models/soprano_tts/session.cpp + src/community_models/soprano_tts/tokenizer_text.cpp + src/community_models/soprano_tts/vocoder.cpp + INCLUDES + engine/community_models/soprano_tts/session.h + LOADERS + engine::community_models::soprano_tts::make_soprano_tts_loader +) + audiocpp_add_model(midashenglm_gen SOURCES diff --git a/PR_SOPRANO.md b/PR_SOPRANO.md new file mode 100644 index 000000000..11175e046 --- /dev/null +++ b/PR_SOPRANO.md @@ -0,0 +1,104 @@ +## Soprano TTS — Community Model + +Soprano is an ultra-lightweight (~80M parameter) English-only text-to-speech model using a two-stage architecture: a Qwen3-style causal LM (17 layers, hidden 512, vocab 8192) that autoregressively emits per-frame 512-dimensional features, and a non-iterative Vocos-style decoder (ConvNeXt backbone + single ISTFT head, n_fft 2048 / hop 512) that turns those features into 32 kHz audio. + +Reference: https://github.com/ekwek1/soprano +Weights: https://huggingface.co/ekwek/Soprano-1.1-80M +GGUF packages: https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF + +### Files added + +| Path | Purpose | +|---|---| +| `src/community_models/soprano_tts/` (5 .cpp) | Assets, Qwen3 LM generator, Vocos decoder, tokenizer, session | +| `include/engine/community_models/soprano_tts/` (5 .h) | Corresponding headers | +| `model_specs/soprano_tts.json` | Schema-v1 spec, community status, GGUF + safetensors sources | +| `docs/soprano_tts.md` | User-facing documentation | +| `docs/soprano_validation.md` | Validation record with build/run commands and timing | +| `tests/soprano_tts/soprano_warm_bench.cpp` | C++ warmbench binary | +| `tests/soprano_tts/soprano_warm_bench_cases.txt` | Warmbench test cases (short/medium/long/longform) | +| `tests/soprano_tts/soprano_python_warm_bench.py` | Python reference warmbench | +| `tools/soprano_tts/convert_soprano.py` | HF checkpoint converter (folds weight-norm) | +| `tools/soprano_tts/run_official.py` | Python reference inference runner | +| `tools/soprano_tts/compare_parity.py` | Automated validation harness | + +### Files modified + +| File | Change | +|---|---| +| `CMakeLists.txt` | +16 lines: `audiocpp_add_model` + `add_engine_warmbench` | +| `README.md` | +2 lines: community table row | +| `docs/gguf.md` | +1 line: GGUF status table row | +| `webui/configs/models_catalog.json` | +1 line: WebUI catalog entry | +| `webui/native/dist/index.html` | Rebuilt frontend with Soprano baked in | + +### Build + +```bash +# Soprano only +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target audiocpp_cli --parallel + +# With Vulkan +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ + -DENGINE_ENABLE_VULKAN=ON +cmake --build build --target audiocpp_cli --parallel +``` + +### Quick start + +```bash +# Install GGUF package +python3 tools/model_manager_v2.py install soprano_1_1_80m_q8_0 + +# Run inference +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --text "Soprano is an extremely lightweight text to speech model." \ + --out soprano.wav +``` + +### Run warmbench + +```bash +# Build warmbench +cmake --build build --target soprano_warm_bench --parallel + +# Run (CPU) +build/bin/soprano_warm_bench --model models/soprano_pkg \ + --output-dir build/logs/warmbench/soprano_tts + +# Python reference +python3 tests/soprano_tts/soprano_python_warm_bench.py \ + --model models/Soprano-1.1-80M \ + --out-dir build/logs/warmbench/soprano_tts_py +``` + +### Validation + +CPU performance vs official Python `soprano` package (transformers backend, temp=0.3, top_p=0.95): + +| Test | Chars | Platform | Audio (s) | Infer (s) | RTF | Speedup | +|---|---|---|---|---|---|---| +| short | 57 | Python | 0.752 | 1.281 | 1.7037 | — | +| | | **C++** | **3.136** | **0.740** | **0.2360** | **7.22x** | +| medium | 152 | Python | 2.096 | 1.909 | 0.9106 | — | +| | | **C++** | **8.320** | **1.955** | **0.2350** | **3.87x** | +| long | 567 | Python | 7.424 | 5.773 | 0.7776 | — | +| | | **C++** | **16.384** | **4.302** | **0.2626** | **2.96x** | + +Key observations: +- C++ RTF stays below 0.27 on all tests (faster than real-time). +- Audio durations differ between Python and C++ because of different RNG implementations; both produce valid 32 kHz speech. +- Backend coverage: CPU (tested, RTF ~0.24), Vulkan (tested on RX Vega, RTF ~0.08-0.12). +- See `docs/soprano_validation.md` for full validation record. + +### Known limitations + +- English-only (model limitation) +- No voice cloning +- EOS sampling unreliable at low temperature (PyTorch vs C++ RNG difference) +- Full composite build may OOM; use AUDIOCPP_MODEL_SET=custom +- Vulkan decoder output has numerical drift on AMD RX Vega (no matrix-core ops) diff --git a/README.md b/README.md index 9483a1fe4..8329d059f 100644 --- a/README.md +++ b/README.md @@ -153,10 +153,12 @@ Community model ports live under `community_models` to make the ownership bounda | **minimax_music3** | Music | auto | GGUF Q4/Q8 | [@0xShug0](https://github.com/0xShug0), [@JoeMattie](https://github.com/JoeMattie) | [MiniMax Music 3](docs/community_models/minimax_music3.md) text-to-music generation with lyrics conditioning | | **mms_forced_aligner** | Align | nl (nld), en (eng); pre-romanized Latin | Safetensors, GGUF 16/Q8 | Community | [MMS-300M-1130 Forced Aligner](docs/community_models/mms_forced_aligner.md) word-timestamp alignment from a wav2vec2 CTC checkpoint (safetensors or local GGUF) | | **moss_tts_local** | TTS, Clone, Ctrl | auto, optional language hint | GGUF | [@justinjohn0306](https://github.com/justinjohn0306) | MOSS-TTS-Local Transformer v1.5 support | +| **moss_tts_local** | TTS, Clone, Ctrl | auto, optional language hint | GGUF | [@justinjohn0306](https://github.com/justinjohn0306) | MOSS-TTS-Local Transformer v1.5 support | | **moss_voicegen** | Voice Design | en, zh | GGUF | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](docs/community_models/moss_voicegen.md) speech in a voice designed from a written instruction | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | +| **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | ## Docker diff --git a/docs/gguf.md b/docs/gguf.md index 3832679c8..fd7ec4908 100644 --- a/docs/gguf.md +++ b/docs/gguf.md @@ -97,6 +97,7 @@ Status labels: | `qwen3_tts` voice design | Done | Pass | --- | Pass (ASR match, drift) | Pass (ASR match, drift) | | `rvc` | Done | --- | --- | Pass | --- | | `seed_vc` | Done | Pass | --- | Pass (drift) | Pass (drift) | +| `soprano_tts` | Done | Pass | --- | Pass | Pass (drift) | | `silero_vad` | Skip (tiny model) | --- | --- | --- | --- | | `sortformer_diar` | Done | Pass | --- | Pass | Pass | | `stable_audio` | Done | Pass | --- | Pass (drift) | Pass (drift) | diff --git a/docs/soprano_tts.md b/docs/soprano_tts.md new file mode 100644 index 000000000..20afca40e --- /dev/null +++ b/docs/soprano_tts.md @@ -0,0 +1,265 @@ +# Soprano TTS + +Soprano is an ultra-lightweight (~80M parameter) English-only text-to-speech model +using a two-stage architecture: a Qwen3-style causal LM (17 layers, hidden 512, +vocab 8192) that autoregressively emits per-frame 512-dimensional features, and a +non-iterative Vocos-style decoder (ConvNeXt backbone + single ISTFT head, n_fft 2048 / +hop 512) that turns those features into 32 kHz audio. No diffusion refinement is +performed in the decoder. + +| Field | Value | +|---|---| +| Family | `soprano_tts` | +| Task | `tts` | +| Mode | `offline`, `streaming` | +| Languages | `en` | +| Audio | WAV; 32 kHz mono | +| Streaming | Pull events (per-chunk audio) | + +--- + +## Install + +The model-spec manager installs the original safetensors package from the official +Hugging Face repository: + +```bash +python3 tools/model_manager_v2.py install soprano_1_1_80m_original +``` + +Or download the checkpoint directly and convert the decoder manually: + +```bash +# Download the official checkpoint +git lfs install +git clone https://huggingface.co/ekwek/Soprano-1.1-80M models/Soprano-1.1-80M + +# Convert the decoder (folds weight-norm from decoder.pth, emits plain safetensors) +pip install torch numpy safetensors +python3 tools/soprano_tts/convert_soprano.py \ + --input-dir models/Soprano-1.1-80M \ + --output-dir models/Soprano-1.1-80M-converted +``` + +--- + +## Build + +Build audio.cpp with Soprano support: + +```bash +# Soprano only (avoids OOM from 45-model parallel compilation) +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target audiocpp_cli --parallel + +# With Vulkan backend +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ + -DENGINE_ENABLE_VULKAN=ON +cmake --build build --target audiocpp_cli --parallel +``` + +--- + +## CLI + +### Basic inference + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "Soprano is an extremely lightweight text to speech model." \ + --out soprano.wav +``` + +### With Vulkan backend + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --backend vulkan \ + --text "Soprano runs on CPU and Vulkan backends." \ + --out soprano_vulkan.wav +``` + +### Custom generation parameters + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "Warmer temperature and higher max tokens produce longer audio." \ + --request-option temperature=0.5 \ + --request-option max_tokens=256 \ + --seed 42 \ + --out custom.wav +``` + +### Long-form with custom chunk size + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "This is a longer text that will be split into sentence-aware chunks by the framework text chunker. Each chunk is generated and decoded separately, then concatenated into the final audio output." \ + --session-option soprano_tts.text_chunk_size=320 \ + --out longform.wav +``` + +### Streaming mode + +```bash +build/bin/audiocpp_cli --task tts --mode streaming --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "Streaming mode emits audio chunks as they are generated." \ + --out stream.wav \ + --out-dir stream_chunks +``` + +--- + +## Options + +### Request options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--request-option max_tokens=` | integer | `512` | Maximum generated audio frames per chunk. | +| `--temperature` / `--request-option temperature=` | float | `0.3` | AR sampling temperature. | +| `--top-p` / `--request-option top_p=` | float | `0.95` | Nucleus sampling threshold. | +| `--repetition-penalty` / `--request-option repetition_penalty=` | float | `1.2` | Repetition penalty. | +| `--request-option eos_bias=` | float | `0.0` | Additive bias on EOS logit; positive stops sooner. | +| `--seed` / `--request-option seed=` | integer | random | AR sampling seed. | + +### Session options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--session-option soprano_tts.text_chunk_size=` | chars | `200` | Max codepoints per chunk. | + +### Load options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--session-option soprano_tts.backbone_weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `f32` | LM weight storage. F32 required on CPU. | +| `--session-option soprano_tts.decoder_weight_type=` | `native`, `f32`, `f16` | `f32` | Decoder weight storage. | + +--- + +## Server + +```json +{ + "host": "127.0.0.1", + "port": 8080, + "models": [ + { + "id": "soprano", + "family": "soprano_tts", + "path": "models/Soprano-1.1-80M-converted", + "task": "tts", + "mode": "offline" + } + ] +} +``` + +```bash +audiocpp_server --config server.json + +# OpenAI-compatible TTS endpoint +curl http://127.0.0.1:8080/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "model": "soprano", + "input": "Soprano is an extremely lightweight text to speech model.", + "response_format": "wav" + }' \ + -o server_output.wav +``` + +--- + +## GGUF package + +Standalone GGUF packages are available on Hugging Face: + +```bash +# Install with the model manager +python3 tools/model_manager_v2.py install soprano_1_1_80m_q8_0 + +# Or install the BF16 variant +python3 tools/model_manager_v2.py install soprano_1_1_80m_bf16 +``` + +Inference with the GGUF package: + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --text "GGUF packages are standalone and self-describing." \ + --out gguf_soprano.wav +``` + +To create a GGUF package from the converted safetensors yourself: + +```bash +build/bin/audiocpp_gguf \ + --input models/Soprano-1.1-80M-converted/model.safetensors \ + --output Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --type q8_0 \ + --root models/Soprano-1.1-80M-converted \ + --family soprano_tts \ + --overwrite +``` +## Performance + +| Backend | RTF | Details | +|---------|---:|--------| +| CPU (warm) | ~0.22-0.23 | ~4-4.6x realtime. F32 storage required for correct output. | +| Vulkan (RX Vega) | ~0.08-0.12 | ~8-13x realtime after one-time shader warmup. Decoder output has numerical drift on this GPU (Vega lacks matrix-core ops). | + +Timing logs are available through `--log`: +- `soprano_tts.lm.generate_ms` -- LM AR decode time +- `soprano_tts.lm.frames` -- generated frames +- `soprano_tts.decoder.decode_ms` -- Vocos decoder time +- `soprano_tts.lm.decode.plan_cached` -- plan caching status + +--- + +## Memory + +| Metric | Value | Conditions | +|--------|-------|------------| +| Model size (safetensors) | ~380 MB (backbone BF16) + ~18 MB (decoder F32) | Original HF checkpoint | +| Peak RSS (CPU) | ~1.2 GB | Graph arena (512 MB) + weight context (256 MB) + runtime overhead | +| Peak VRAM (Vulkan) | Not measured | Vega ~1.2 GB reported system RAM usage | + +--- + +## Known limitations + +- English-only (model limitation) +- No voice cloning +- EOS sampling unreliable at low temperature (C++ RNG != PyTorch RNG) +- Full composite build may OOM; use AUDIOCPP_MODEL_SET=custom with AUDIOCPP_MODELS=soprano_tts + +--- + +## Architecture + +Soprano uses a two-stage architecture: + +1. **Qwen3 causal LM** (17 layers, hidden 512, 4 heads, 1 KV head, head_dim 128, vocab 8192, + intermediate 2304, rope_theta 10000). Takes prompt `[STOP][TEXT][START]` and + autoregressively generates tokens. Each step's last-layer hidden state (512-dim) equals + one audio frame. + +2. **Vocos decoder** (non-iterative): Interpolate x4 linear align_corners -> Conv1d(512->768,k=1) + -> LN -> 8x ConvNeXt(dwconv k=3 groups, LN, Linear->2304, GELU, Linear->768, gamma) -> LN -> + Linear(768->2050) -> split mag/phase -> exp*exp(i*phi) -> istft(center=True) with Hann window + (n_fft=2048, hop=512). + +Output: 32 kHz mono. Token ~ 2048 samples ~ 64 ms. + +Reference: https://github.com/ekwek1/soprano +Weights: https://huggingface.co/ekwek/Soprano-1.1-80M diff --git a/docs/soprano_validation.md b/docs/soprano_validation.md new file mode 100644 index 000000000..13d276284 --- /dev/null +++ b/docs/soprano_validation.md @@ -0,0 +1,102 @@ +# Soprano TTS Validation + +## Build + +```bash +# Soprano-only build +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target audiocpp_cli --parallel + +# With Vulkan backend +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ + -DENGINE_ENABLE_VULKAN=ON +cmake --build build --target audiocpp_cli --parallel + +# Build warmbench +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target soprano_warm_bench --parallel +``` + +## Convert the checkpoint + +```bash +# Download the official checkpoint +git lfs install +git clone https://huggingface.co/ekwek/Soprano-1.1-80M models/Soprano-1.1-80M + +# Convert the decoder (folds weight-norm from decoder.pth) +pip install torch numpy safetensors +python3 tools/soprano_tts/convert_soprano.py \ + --input-dir models/Soprano-1.1-80M \ + --output-dir models/soprano_pkg +``` + +## Run warmbench + +```bash +build/bin/soprano_warm_bench --model models/soprano_pkg --output-dir build/logs/warmbench/soprano_tts +``` + +## Python reference warmbench + +```bash +pip install soprano torch numpy +python3 tests/soprano_tts/soprano_python_warm_bench.py \ + --model models/Soprano-1.1-80M \ + --out-dir build/logs/warmbench/soprano_tts_py +``` + +## CLI examples + +```bash +# Basic inference +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/soprano_pkg \ + --text "Soprano is an extremely lightweight text to speech model." \ + --out soprano.wav + +# With Vulkan backend +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/soprano_pkg \ + --backend vulkan \ + --text "Soprano runs on CPU and Vulkan backends." \ + --out soprano_vulkan.wav + +# GGUF package +python3 tools/model_manager_v2.py install soprano_1_1_80m_q8_0 +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --text "GGUF packages are standalone." --out gguf_out.wav +``` + +## Performance results + +### CPU (compared against Python `soprano` package, transformers backend, temp=0.3, top_p=0.95) + +| Test | Chars | Platform | Audio (s) | Infer (s) | RTF | Speedup | +|---|---|---|---|---|---|---| +| short | 57 | Python | 0.752 | 1.281 | 1.7037 | \u2014 | +| | | **C++** | **3.136** | **0.740** | **0.2360** | **7.22x** | +| medium | 152 | Python | 2.096 | 1.909 | 0.9106 | \u2014 | +| | | **C++** | **8.320** | **1.955** | **0.2350** | **3.87x** | +| long | 567 | Python | 7.424 | 5.773 | 0.7776 | \u2014 | +| | | **C++** | **16.384** | **4.302** | **0.2626** | **2.96x** | + +### Vulkan (AMD Radeon RX Vega) + +| Test | Audio (s) | Infer (s) | RTF | +|---|---|---|---| +| short | ~3.1 | ~0.25 | ~0.08 | +| medium | ~8.3 | ~0.70 | ~0.08 | +| long | ~16.4 | ~1.60 | ~0.10 | + +## Known limitations + +- English-only (model limitation) +- No voice cloning +- EOS sampling unreliable at low temperature (PyTorch vs C++ RNG difference) +- Full composite build may OOM; use AUDIOCPP_MODEL_SET=custom +- Vulkan decoder output shows numerical drift on AMD RX Vega (no matrix-core ops) diff --git a/include/engine/community_models/soprano_tts/assets.h b/include/engine/community_models/soprano_tts/assets.h new file mode 100644 index 000000000..320bdd381 --- /dev/null +++ b/include/engine/community_models/soprano_tts/assets.h @@ -0,0 +1,68 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +struct SopranoTTSConfig { + // Qwen3 causal LM (config.json). + int64_t hidden_size = 512; + int64_t intermediate_size = 2304; + int64_t layers = 17; + int64_t attention_heads = 4; + int64_t kv_heads = 1; + int64_t head_dim = 128; + int64_t vocab_size = 8192; + int64_t max_position_embeddings = 1024; + float rms_norm_eps = 1.0e-6f; + float rope_theta = 10000.0f; + int32_t bos_token_id = 3; + int32_t eos_token_id = 3; + + // Non-iterative Vocos decoder (decoder.pth / config). + int64_t decoder_input_channels = 512; // == hidden_size + int64_t decoder_dim = 768; + int64_t decoder_intermediate_dim = 2304; + int64_t decoder_num_layers = 8; + int64_t dw_kernel = 3; + int64_t n_fft = 2048; + int64_t hop_length = 512; + int64_t upscale = 4; + int64_t sample_rate = 32000; + int64_t token_size = 2048; // samples per generated frame + int64_t max_new_tokens = 128; + float temperature = 0.3f; + float top_p = 0.95f; + float repetition_penalty = 1.2f; +}; + +struct SopranoTTSAssets { + assets::ResourceBundle resources; + SopranoTTSConfig config; + std::shared_ptr backbone_weights; + std::shared_ptr decoder_weights; +}; + +struct SopranoGenerationOptions { + // Per-chunk limit; the official reference allows up to 512 frames + // (32 s of audio) per sentence. + int64_t max_new_tokens = 512; + float temperature = 0.3f; + float top_p = 0.95f; + float repetition_penalty = 1.2f; + uint64_t seed = 0; + bool has_seed = false; + // Additive bias on the EOS logit; 0 disables (opt-in runaway mitigation). + float eos_bias = 0.0F; +}; + +std::shared_ptr load_soprano_tts_assets( + const std::filesystem::path & model_path); + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/include/engine/community_models/soprano_tts/generator.h b/include/engine/community_models/soprano_tts/generator.h new file mode 100644 index 000000000..992de5ce3 --- /dev/null +++ b/include/engine/community_models/soprano_tts/generator.h @@ -0,0 +1,43 @@ +#pragma once + +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/soprano_tts/assets.h" + +#include +#include +#include + +namespace engine::community_models::soprano_tts { +struct SopranoQwenWeights; + +// Autoregressive Qwen3 causal LM wrapper. Captures the last-layer 512-dim +// hidden state of every generated token (the per-frame audio features) plus the +// sampled token ids, stopping on EOS. +class SopranoTTSGenerator { +public: + SopranoTTSGenerator( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type); + ~SopranoTTSGenerator(); + + struct Result { + std::vector features; // frames x hidden (frame-major) + std::vector tokens; // generated token ids (excluding EOS) + int64_t frames = 0; + }; + + Result generate(const std::vector & prompt_ids, + const SopranoGenerationOptions & options); + + void release_runtime_graphs(); + +private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/include/engine/community_models/soprano_tts/session.h b/include/engine/community_models/soprano_tts/session.h new file mode 100644 index 000000000..4dba830dc --- /dev/null +++ b/include/engine/community_models/soprano_tts/session.h @@ -0,0 +1,67 @@ +#pragma once + +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/model.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/community_models/soprano_tts/assets.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +std::shared_ptr make_soprano_tts_loader(); + +struct SopranoRequest { + std::string text; + SopranoGenerationOptions generation; +}; + +class SopranoTTSGenerator; +class SopranoDecoderRuntime; + +class SopranoTTSOfflineSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession, + public runtime::IStreamingVoiceTaskSession { +public: + SopranoTTSOfflineSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~SopranoTTSOfflineSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + runtime::StreamingPolicy streaming_policy() const override; + void start_stream(const runtime::TaskRequest & request) override; + std::optional next_stream_event() override; + void set_stream_event_sink(runtime::StreamEventCallback sink) override; + runtime::TaskResult finish_stream() override; + void reset() override; + runtime::StreamEvent process_audio_chunk(const runtime::AudioChunk & chunk) override; + runtime::TaskResult finalize() override; + +private: + SopranoRequest make_request(const runtime::TaskRequest & request) const; + runtime::AudioBuffer synthesize(const SopranoRequest & request); + // Streaming state + std::optional> streaming_chunks_; + size_t streaming_chunk_index_ = 0; + std::vector streaming_audio_; + runtime::StreamEventCallback stream_sink_; + + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::unique_ptr generator_; + std::unique_ptr decoder_; +}; + +} // namespace engine::community_models::soprano_tts diff --git a/include/engine/community_models/soprano_tts/tokenizer_text.h b/include/engine/community_models/soprano_tts/tokenizer_text.h new file mode 100644 index 000000000..94dcba79e --- /dev/null +++ b/include/engine/community_models/soprano_tts/tokenizer_text.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +// Byte-level BPE tokenizer for the Soprano text prompt. The LM expects the text +// wrapped as a single prompt: "[STOP][TEXT][START]". Diacritics are +// removed and text is case-folded (unidecode-style) before encoding, matching +// the reference `clean_text` behaviour for English. +class SopranoTextTokenizer { +public: + explicit SopranoTextTokenizer(const std::filesystem::path & tokenizer_json_path); + + std::vector encode_text(const std::string & text) const; + std::string decode_ids(const std::vector & ids) const; + + int32_t bos_id() const noexcept { return bos_id_; } + int32_t eos_id() const noexcept { return eos_id_; } + int32_t stop_id() const noexcept { return stop_id_; } + int32_t text_id() const noexcept { return text_id_; } + int32_t start_id() const noexcept { return start_id_; } + int64_t vocab_size() const noexcept { return static_cast(id_to_token_.size()); } + +private: + std::vector apply_prompt(const std::vector & speech_tokens) const; + + std::vector id_to_token_; + std::unordered_map token_to_id_; + std::vector> merges_; // rank-ordered (a_id, b_id) + + int32_t bos_id_ = -1; + int32_t eos_id_ = -1; + int32_t stop_id_ = -1; + int32_t text_id_ = -1; + int32_t start_id_ = -1; +}; + +std::string scalarclean_soprano_text(const std::string & text); + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/include/engine/community_models/soprano_tts/vocoder.h b/include/engine/community_models/soprano_tts/vocoder.h new file mode 100644 index 000000000..b032acada --- /dev/null +++ b/include/engine/community_models/soprano_tts/vocoder.h @@ -0,0 +1,51 @@ +#pragma once + +#include "engine/community_models/soprano_tts/assets.h" + +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} +namespace engine::runtime { +struct AudioBuffer; +} + +namespace engine::community_models::soprano_tts { + +struct SopranoDecoderWeights; +struct SopranoDecoderGraph; + +// Non-iterative Vocos-style decoder (SopranoDecoder): linear upsample x4 over +// the frame axis, a ConvNeXt backbone (embed Conv1d, 8 blocks, final LN) and a +// single ISTFT head (Linear(dim -> n_fft+2), exp(mag), cos/sin phase, 1 ISTFT). +class SopranoDecoderRuntime final { +public: + SopranoDecoderRuntime( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type); + ~SopranoDecoderRuntime(); + + // frames x hidden -> 32 kHz mono audio. + runtime::AudioBuffer decode(const std::vector & features, int64_t frames) const; + + + +private: + const SopranoTTSConfig & config_; + engine::core::ExecutionContext & execution_context_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + mutable std::unique_ptr graph_; +}; + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/model_specs/soprano_tts.json b/model_specs/soprano_tts.json new file mode 100644 index 000000000..cc9ae299f --- /dev/null +++ b/model_specs/soprano_tts.json @@ -0,0 +1,185 @@ +{ + "schema_version": 1, + "family": "soprano_tts", + "display_name": "Soprano", + "description": "Soprano is an ultra-lightweight (~80M) English-only text-to-speech model. Syntax uses a 17-layer Qwen3-style causal LM (hidden 512, vocab 8192) that autoregressively emits per-frame 512-dimensional features; a non-iterative Vocos-style decoder (ConvNeXt backbone + single ISTFT head, n_fft 2048 / hop 512) turns those features into 32 kHz audio. No diffusion refinement is performed in the decoder.", + "category": "tts", + "status": "community", + "tasks": [ + "tts" + ], + "modes": [ + "offline", + "streaming" + ], + "languages": [ + "en" + ], + "runtime": { + "tags": [ + "gguf", + "stream" + ] + }, + "capabilities": { + "tts": [ + "long_form" + ] + }, + "options": { + "request": [ + { + "name": "max_tokens", + "type": "int", + "description": "Maximum generated audio frames for the autoregressive LM; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "temperature", + "type": "float", + "description": "Autoregressive sampling temperature; default 0.3 (0 selects the framework default and clamps to a small positive value).", + "required": false, + "min": 0.0, + "default": 0.3 + }, + { + "name": "top_p", + "type": "float", + "description": "Nucleus sampling probability; default 0.95.", + "required": false, + "min": 0.0, + "max": 1.0, + "default": 0.95 + }, + { + "name": "repetition_penalty", + "type": "float", + "description": "Repetition penalty applied to the LM head; default 1.2.", + "required": false, + "min": 1.0, + "default": 1.2 + }, + { + "name": "eos_bias", + "type": "float", + "description": "Additive bias on the EOS token logit during generation. Positive values make the model stop sooner when speech ends (mitigating runaway generations that hit max_tokens); negative values encourage longer utterances. Default 0 disables the adjustment.", + "required": false, + "default": 0.0 + }, + { + "name": "seed", + "type": "int", + "description": "Autoregressive sampling seed; omitted requests choose a random seed.", + "required": false, + "min": 0 + } + ], + "session": [ + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum codepoints per sentence chunk before the model generates and decodes separately. Smaller values keep prompts short (more reliable EOS) but increase overhead. Default 200.", + "required": false, + "min": 32, + "default": 200 + } + ], + "load": [ + { + "name": "backbone_weight_type", + "type": "enum", + "preset": "weight_type_full", + "required": false, + "default": "native", + "description": "Storage type for the Qwen3 LM backbone weights." + }, + { + "name": "decoder_weight_type", + "type": "enum", + "preset": "weight_type_conv", + "required": false, + "default": "native", + "description": "Storage type for the Vocos decoder weights." + } + ] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "WalkingCat/Soprano-1.1-80M-GGUF", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "soprano_1_1_80m_q8_0", + "display_name": "Soprano-1.1-80M Q8_0 GGUF", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "Soprano-1.1-80M-GGUF", + "files": [ + "Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf" + ], + "strip_prefix": "Soprano-1.1-80M-GGUF" + }, + { + "id": "soprano_1_1_80m_bf16", + "display_name": "Soprano-1.1-80M BF16 GGUF", + "format": "gguf", + "precision": "bf16", + "target_directory": "Soprano-1.1-80M-GGUF", + "files": [ + "Soprano-1.1-80M-GGUF/soprano-1.1-80m-bf16.gguf" + ], + "strip_prefix": "Soprano-1.1-80M-GGUF" + } + ], + "dependencies": [], + "ui": { + "recommended_package": "soprano_1_1_80m_q8_0", + "tags": [ + "TTS", + "Stream" + ], + "docs": [ + "docs/soprano_tts.md" + ] + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "backbone": "weights:", + "decoder": "weights:" + } + }, + { + "format": "safetensors", + "roots": { + "model": "." + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "backbone": "model:combined.safetensors", + "decoder": "model:combined.safetensors" + } + } + ] +} diff --git a/src/community_models/soprano_tts/assets.cpp b/src/community_models/soprano_tts/assets.cpp new file mode 100644 index 000000000..453fbd3e9 --- /dev/null +++ b/src/community_models/soprano_tts/assets.cpp @@ -0,0 +1,64 @@ +#include "engine/community_models/soprano_tts/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" + +#include +#include + +namespace engine::community_models::soprano_tts { +namespace { + +namespace json = engine::io::json; + +constexpr const char * kFamily = "soprano_tts"; + +SopranoTTSConfig parse_config(const assets::ResourceBundle & resources) { + const auto root = resources.parse_json("config"); + if (json::require_string(root, "model_type") != "qwen3") { + throw std::runtime_error("Soprano config must use model_type qwen3"); + } + SopranoTTSConfig out; + out.hidden_size = json::require_i64(root, "hidden_size"); + out.intermediate_size = json::require_i64(root, "intermediate_size"); + out.layers = json::require_i64(root, "num_hidden_layers"); + out.attention_heads = json::require_i64(root, "num_attention_heads"); + out.kv_heads = json::require_i64(root, "num_key_value_heads"); + if (const auto * hd = root.find("head_dim")) { + out.head_dim = hd->as_i64(); + } else { + out.head_dim = out.hidden_size / out.attention_heads; + } + out.vocab_size = json::require_i64(root, "vocab_size"); + out.rms_norm_eps = json::optional_f32(root, "rms_norm_eps", out.rms_norm_eps); + if (const auto * rope = root.find("rope_parameters")) { + out.rope_theta = json::optional_f32(*rope, "rope_theta", out.rope_theta); + } else { + out.rope_theta = json::optional_f32(root, "rope_theta", out.rope_theta); + } + out.max_position_embeddings = json::optional_i64(root, "max_position_embeddings", out.max_position_embeddings); + if (const auto * eos = root.find("eos_token_id")) { + out.eos_token_id = static_cast(eos->as_i64()); + } + if (const auto * bos = root.find("bos_token_id")) { + out.bos_token_id = static_cast(bos->as_i64()); + } + // Decoder dimensions are fixed by the SopranoDecoder architecture. + out.decoder_input_channels = out.hidden_size; + return out; +} + +} // namespace + +std::shared_ptr load_soprano_tts_assets( + const std::filesystem::path & model_path) { + auto assets = std::make_shared(); + assets->resources = engine::model_spec::load_resource_bundle( + model_path, engine::model_spec::default_spec_path(kFamily)); + assets->config = parse_config(assets->resources); + assets->backbone_weights = assets->resources.open_tensor_source("backbone"); + assets->decoder_weights = assets->resources.open_tensor_source("decoder"); + return assets; +} + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/src/community_models/soprano_tts/generator.cpp b/src/community_models/soprano_tts/generator.cpp new file mode 100644 index 000000000..812825a44 --- /dev/null +++ b/src/community_models/soprano_tts/generator.cpp @@ -0,0 +1,342 @@ +#include "engine/community_models/soprano_tts/generator.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/transformers/qwen_causal_decode_runtime.h" +#include "engine/framework/modules/transformers/qwen_causal_decoder.h" +#include "engine/framework/modules/weight_binding.h" +#include "engine/framework/sampling/hf_sampler.h" +#include "engine/framework/sampling/torch_random.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +struct SopranoQwenWeights { + std::shared_ptr store; + engine::core::TensorValue token_embedding; + engine::modules::QwenDecoderStackWeights stack; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights lm_head; +}; + +namespace { + +namespace binding = engine::modules::binding; + +std::shared_ptr require_assets( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Soprano LM generator requires assets"); + } + return assets; +} + +modules::QwenDecoderLayerWeights load_layer_weights( + engine::core::BackendWeightStore & store, + const engine::assets::TensorSource & source, + const SopranoTTSConfig & config, + engine::assets::TensorStorageType storage_type, + int64_t layer) { + const std::string prefix = "model.layers." + std::to_string(layer); + modules::QwenDecoderLayerWeights out; + out.input_norm = binding::norm_weight_from_source( + store, source, prefix + ".input_layernorm", config.hidden_size); + // Fused QKV projection: concatenate Q|K|V rows so the decoder runs a + // single GEMM per layer (QwenDecoderQKVLayout::PackedQKV). + const int64_t q_out = config.attention_heads * config.head_dim; + const int64_t kv_out = config.kv_heads * config.head_dim; + std::vector qkv_rows = source.require_f32( + prefix + ".self_attn.q_proj.weight", {q_out, config.hidden_size}); + const auto k_rows = source.require_f32( + prefix + ".self_attn.k_proj.weight", {kv_out, config.hidden_size}); + const auto v_rows = source.require_f32( + prefix + ".self_attn.v_proj.weight", {kv_out, config.hidden_size}); + qkv_rows.insert(qkv_rows.end(), k_rows.begin(), k_rows.end()); + qkv_rows.insert(qkv_rows.end(), v_rows.begin(), v_rows.end()); + out.self_attention.qkv_weight = store.make_from_f32( + engine::core::TensorShape::from_dims({q_out + kv_out * 2, config.hidden_size}), + storage_type, + std::move(qkv_rows)); + out.self_attention.out_weight = store.load_tensor( + source, prefix + ".self_attn.o_proj.weight", storage_type, + {config.hidden_size, config.attention_heads * config.head_dim}); + out.q_norm = binding::norm_weight_from_source( + store, source, prefix + ".self_attn.q_norm", config.head_dim); + out.k_norm = binding::norm_weight_from_source( + store, source, prefix + ".self_attn.k_norm", config.head_dim); + out.post_norm = binding::norm_weight_from_source( + store, source, prefix + ".post_attention_layernorm", config.hidden_size); + // Fused gate/up projection: gate|up rows in one GEMM; the decoder's + // PackedGateUp mode also uses the fused swiglu kernel. + std::vector gate_up_rows = source.require_f32( + prefix + ".mlp.gate_proj.weight", + {config.intermediate_size, config.hidden_size}); + const auto up_rows = source.require_f32( + prefix + ".mlp.up_proj.weight", + {config.intermediate_size, config.hidden_size}); + gate_up_rows.insert(gate_up_rows.end(), up_rows.begin(), up_rows.end()); + out.mlp.gate_up_proj = modules::LinearWeights{ + store.make_from_f32( + engine::core::TensorShape::from_dims( + {config.intermediate_size * 2, config.hidden_size}), + storage_type, + std::move(gate_up_rows)), + std::nullopt}; + out.mlp.down_proj = binding::linear_from_source( + store, source, prefix + ".mlp.down_proj", storage_type, + config.hidden_size, config.intermediate_size, false); + return out; +} +modules::QwenDecoderActivationCastPolicy soprano_activation_cast_policy( + core::BackendType backend_type) { + // No activation cast for Soprano — keep everything in F32 for parity. + (void)backend_type; + return modules::QwenDecoderActivationCastPolicy{}; +} + +modules::QwenCausalDecoderConfig make_soprano_qwen_config( + const SopranoTTSConfig & config, + core::BackendType backend_type) { + modules::QwenCausalDecoderConfig out; + out.stack.hidden_size = config.hidden_size; + out.stack.num_attention_heads = config.attention_heads; + out.stack.num_key_value_heads = config.kv_heads; + out.stack.head_dim = config.head_dim; + out.stack.intermediate_size = config.intermediate_size; + out.stack.layers = config.layers; + out.stack.rms_norm_eps = config.rms_norm_eps; + out.stack.rope_theta = config.rope_theta; + out.stack.rope_type = GGML_ROPE_TYPE_NEOX; + out.stack.attention_precision = GGML_PREC_DEFAULT; + out.stack.projection_precision = GGML_PREC_DEFAULT; + out.stack.activation_cast = soprano_activation_cast_policy(backend_type); + out.stack.use_qk_norm = true; + // Fused projections: single QKV GEMM + single gate/up GEMM with the + // fused swiglu kernel (Soprano has no activation casts, so it qualifies). + out.stack.qkv_layout = modules::QwenDecoderQKVLayout::PackedQKV; + out.stack.runtime.mlp.mode = modules::QwenDecoderMLPMode::PackedGateUp; + out.stack.runtime.attention.prefill_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.attention.static_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + out.logits_size = config.vocab_size; + out.logits_mode = modules::QwenCausalDecoderLogitsMode::LastStep; + out.use_lm_head_bias = false; + out.lm_head_precision = GGML_PREC_DEFAULT; + if (backend_type == core::BackendType::Vulkan || backend_type == core::BackendType::Metal) { + out.lm_head_input_type = GGML_TYPE_F16; + } else if (backend_type != core::BackendType::Cpu) { + out.lm_head_input_type = GGML_TYPE_BF16; + } + return out; +} + +std::shared_ptr load_soprano_qwen_weights( + const SopranoTTSAssets & assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t weight_context_bytes, + assets::TensorStorageType storage_type) { + auto weights = std::make_shared(); + weights->store = std::make_shared( + backend, backend_type, "soprano_tts.lm.weights", weight_context_bytes); + const auto & config = assets.config; + const auto & source = *assets.backbone_weights; + weights->token_embedding = weights->store->load_tensor( + source, "model.embed_tokens.weight", storage_type, + {config.vocab_size, config.hidden_size}); + weights->stack.layers.reserve(static_cast(config.layers)); + for (int64_t layer = 0; layer < config.layers; ++layer) { + weights->stack.layers.push_back(load_layer_weights( + *weights->store, source, config, storage_type, layer)); + } +weights->final_norm = binding::norm_weight_from_source( + *weights->store, source, "model.norm", config.hidden_size); + weights->lm_head = binding::linear_from_source( + *weights->store, source, "lm_head", storage_type, + config.vocab_size, config.hidden_size, false); + weights->store->upload(); + return weights; +} + +modules::QwenCausalDecodeRuntimeConfig make_soprano_decode_runtime_config( + const SopranoTTSConfig & config, + core::BackendType backend_type, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes) { + modules::QwenCausalDecodeRuntimeConfig out; + out.trace_name = "soprano_tts.lm"; + out.decoder = make_soprano_qwen_config(config, backend_type); + out.prefill_graph_arena_bytes = prefill_graph_arena_bytes; + out.decode_graph_arena_bytes = decode_graph_arena_bytes; + // Both logits (sampling + EOS) and the 512-d hidden frame (audio) are needed. + out.output_mode = modules::QwenCausalDecodeOutputMode::Logits; + out.return_hidden = true; + return out; +} + +modules::QwenCausalDecodeRuntimeWeights make_soprano_decode_weights( + const SopranoQwenWeights & weights) { + modules::QwenCausalDecodeRuntimeWeights out; + out.token_embedding = weights.token_embedding; + out.stack = weights.stack; + out.final_norm = weights.final_norm; + out.lm_head = weights.lm_head; + return out; +} + +// Hidden-mode weights are intentionally not built: Hidden mode produces +// NaN/Inf prefill output (see docs/soprano_tts.md §6h), so the LM +// always runs single-pass Logits+return_hidden=true. + +} // namespace + +class SopranoTTSGenerator::Impl { +public: + Impl( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type) + : assets_(require_assets(std::move(assets))), + backend_(execution.backend()), + backend_type_(execution.backend_type()), + weights_(std::make_shared(std::move(*load_soprano_qwen_weights( + *assets_, execution.backend(), backend_type_, weight_context_bytes, + weight_storage_type)))) { + if (backend_ == nullptr) { + throw std::runtime_error("Soprano LM backend is not initialized"); + } + qwen_runtime = std::make_unique( + execution, + make_soprano_decode_runtime_config( + assets_->config, backend_type_, + prefill_graph_arena_bytes, decode_graph_arena_bytes), + make_soprano_decode_weights(*weights_)); + } + + Result generate(const std::vector & prompt_ids, + const SopranoGenerationOptions & options) { + if (prompt_ids.empty()) { + throw std::runtime_error("Soprano LM requires a non-empty prompt"); + } + const SopranoTTSConfig & config = assets_->config; + std::vector features; + std::vector tokens; + + // Single-pass AR generation capturing both logits and hidden states. + // With F32 weights + correct tokenizer, return_hidden=true now + // produces correct results. + auto prefill = qwen_runtime->prefill_tokens(prompt_ids); + // Honor the requested token limit (matches HF max_new_tokens), capped + // so prompt + generated always fits the model context window. + const int64_t max_new_tokens = std::max( + 1, + std::min( + options.max_new_tokens, + config.max_position_embeddings - + static_cast(prompt_ids.size()))); + // Size the KV cache to the actual worst-case need (prompt + generated + // frames) instead of the full 1024-token context. Smaller cache means + // less KV memory for attention to walk on every decode step. + qwen_runtime->start_decode_tokens(prefill.state, max_new_tokens + + static_cast(prompt_ids.size())); + + // First feature: last prompt token's post-norm hidden state. + features.insert(features.end(), prefill.hidden.begin(), prefill.hidden.end()); + + sampling::HfSamplingOptions sampling_options; + sampling_options.do_sample = true; + sampling_options.temperature = options.temperature; + sampling_options.top_k = 0; + sampling_options.top_p = options.top_p; + sampling_options.min_tokens_to_keep = 1; + sampling_options.repetition_penalty = options.repetition_penalty; + sampling::HfSampler sampler; + sampling::HfSamplerScratch scratch; + scratch.reserve_vocab(static_cast(config.vocab_size)); + std::mt19937 fallback_rng(static_cast(options.seed)); + + std::vector history(prompt_ids.begin(), prompt_ids.end()); + std::vector logits = std::move(prefill.logits); + const int32_t eos_id = config.eos_token_id; + + for (int64_t step = 0; step < max_new_tokens; ++step) { + if (options.eos_bias != 0.0F) { + logits[static_cast(eos_id)] += options.eos_bias; + } + const int32_t token = sampler.sample( + logits, history, sampling_options, scratch, fallback_rng, nullptr, + "soprano_tts AR"); + if (token == eos_id) { + history.push_back(token); + break; + } + history.push_back(token); + tokens.push_back(token); + auto decode = qwen_runtime->decode_token(token); + features.insert(features.end(), decode.hidden.begin(), decode.hidden.end()); + logits = std::move(decode.logits); + } + + if (tokens.empty()) { + throw std::runtime_error("Soprano LM produced no audio frames"); + } + + Result out; + out.tokens = std::move(tokens); + const int64_t T_gen = static_cast(out.tokens.size()) + 1; // +1 for prefill + out.frames = T_gen; + out.features = std::move(features); + return out; + } + + void release_runtime_graphs() { + qwen_runtime->release_runtime_graphs(); + } + + std::shared_ptr assets_; + ggml_backend_t backend_ = nullptr; + core::BackendType backend_type_ = core::BackendType::Cpu; + std::shared_ptr weights_; + std::unique_ptr qwen_runtime; +}; + +SopranoTTSGenerator::SopranoTTSGenerator( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique( + std::make_shared(assets), execution, + prefill_graph_arena_bytes, decode_graph_arena_bytes, + weight_context_bytes, weight_storage_type)) {} + +SopranoTTSGenerator::~SopranoTTSGenerator() = default; + +SopranoTTSGenerator::Result SopranoTTSGenerator::generate( + const std::vector & prompt_ids, + const SopranoGenerationOptions & options) { + return impl_->generate(prompt_ids, options); +} + +void SopranoTTSGenerator::release_runtime_graphs() { + impl_->release_runtime_graphs(); +} + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/src/community_models/soprano_tts/session.cpp b/src/community_models/soprano_tts/session.cpp new file mode 100644 index 000000000..330411f7d --- /dev/null +++ b/src/community_models/soprano_tts/session.cpp @@ -0,0 +1,289 @@ +#include "engine/community_models/soprano_tts/session.h" + +#include "engine/framework/core/execution_context.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" +#include "engine/community_models/soprano_tts/generator.h" +#include "engine/community_models/soprano_tts/tokenizer_text.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/text/chunking.h" +#include "engine/community_models/soprano_tts/vocoder.h" + +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { +namespace { + +constexpr const char * kFamily = "soprano_tts"; +constexpr size_t kDefaultGraphArenaBytes = 512ull * 1024ull * 1024ull; +constexpr size_t kDefaultWeightContextBytes = 256ull * 1024ull * 1024ull; + +std::shared_ptr require_assets( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Soprano session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("Soprano session requires a model contract"); + } + return contract; +} + +std::string request_text(const runtime::TaskRequest & request) { + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("Soprano requires non-empty text input"); + } + return request.text_input->text; +} + +SopranoGenerationOptions request_generation_options(const runtime::TaskRequest & request) { + SopranoGenerationOptions out; + if (const auto value = runtime::parse_i64_option(request.options, {"max_tokens"})) { + out.max_new_tokens = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) { + if (*value <= 0.0F) { + throw std::runtime_error("Soprano temperature must be positive"); + } + out.temperature = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"top_p"})) { + out.top_p = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"repetition_penalty"})) { + out.repetition_penalty = *value; + } + if (const auto value = runtime::parse_u64_option(request.options, {"seed"})) { + out.seed = *value; + out.has_seed = true; + } + if (!out.has_seed) { + out.seed = runtime::random_u64_seed(); + } + if (out.max_new_tokens < 1) { + throw std::runtime_error("Soprano max_tokens must be positive"); + } + return out; +} + +std::unique_ptr create_soprano_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, options, std::move(assets), std::move(contract)); +} + +} // namespace + +SopranoTTSOfflineSession::SopranoTTSOfflineSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), + task_(task), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "Soprano"); + core::ExecutionContext & execution = execution_context(); + const auto backbone_storage = runtime::parse_tensor_storage_option( + options.options, + "soprano_tts.backbone_weight_type", + assets::TensorStorageType::F32, + {assets::TensorStorageType::Native, + assets::TensorStorageType::F32, + assets::TensorStorageType::F16, + assets::TensorStorageType::BF16, + assets::TensorStorageType::Q8_0}); + const auto decoder_storage = runtime::parse_tensor_storage_option( + options.options, + "soprano_tts.decoder_weight_type", + assets::TensorStorageType::F32, + {assets::TensorStorageType::Native, + assets::TensorStorageType::F32, + assets::TensorStorageType::F16}); + generator_ = std::make_unique( + *assets_, execution, kDefaultGraphArenaBytes, kDefaultGraphArenaBytes, + kDefaultWeightContextBytes, backbone_storage); + decoder_ = std::make_unique( + *assets_, execution, kDefaultWeightContextBytes, kDefaultGraphArenaBytes, + decoder_storage, decoder_storage); +} + +SopranoTTSOfflineSession::~SopranoTTSOfflineSession() = default; + +std::string SopranoTTSOfflineSession::family() const { + return kFamily; +} + +runtime::VoiceTaskKind SopranoTTSOfflineSession::task_kind() const { + return runtime::VoiceTaskKind::Tts; +} + +runtime::RunMode SopranoTTSOfflineSession::run_mode() const { + return runtime::RunMode::Offline; +} + +SopranoRequest SopranoTTSOfflineSession::make_request(const runtime::TaskRequest & request) const { + SopranoRequest out; + out.text = request_text(request); + out.generation = request_generation_options(request); + return out; +} +void SopranoTTSOfflineSession::prepare(const runtime::SessionPreparationRequest & request) { + runtime::validate_spec_backed_request_options(request.options, *contract_, "Soprano"); + mark_prepared(); +} + + +runtime::TaskResult SopranoTTSOfflineSession::run(const runtime::TaskRequest & request) { + require_prepared("Soprano run"); + const SopranoRequest req = make_request(request); + const auto audio = synthesize(req); + + runtime::TaskResult result; + result.audio_output = audio; + return result; +} + +runtime::AudioBuffer SopranoTTSOfflineSession::synthesize(const SopranoRequest & request) { + const std::filesystem::path tokenizer_path = + assets_->resources.require_file("tokenizer_json"); + SopranoTextTokenizer tokenizer(tokenizer_path); + const int64_t chunk_codepoints = runtime::parse_i64_option( + options().options, {"soprano_tts.text_chunk_size"}) + .value_or(200); + const auto chunks = engine::text::split_text_chunks( + request.text, chunk_codepoints, engine::text::TextChunkMode::Default); + runtime::AudioBuffer out; + for (const auto & chunk : chunks) { + const auto prompt_ids = tokenizer.encode_text(chunk); + const auto generate_start = std::chrono::steady_clock::now(); + const auto generated = generator_->generate(prompt_ids, request.generation); + const auto generate_end = std::chrono::steady_clock::now(); + engine::debug::timing_log_scalar( + "soprano_tts.lm.generate_ms", engine::debug::elapsed_ms(generate_start, generate_end)); + engine::debug::trace_log_scalar("soprano_tts.lm.frames", generated.frames); + auto audio = decoder_->decode(generated.features, generated.frames); + engine::debug::timing_log_scalar( + "soprano_tts.decoder.decode_ms", + engine::debug::elapsed_ms(generate_end, std::chrono::steady_clock::now())); + if (out.sample_rate == 0) { + out.sample_rate = audio.sample_rate; + out.channels = audio.channels; + } else if (out.sample_rate != audio.sample_rate || out.channels != audio.channels) { + throw std::runtime_error("Soprano chunk audio format mismatch"); + } + out.samples.insert(out.samples.end(), audio.samples.begin(), audio.samples.end()); + } + if (out.sample_rate == 0) { + throw std::runtime_error("Soprano produced no audio chunks"); + } + return out; +} + + +// --------------------------------------------------------------------------- // +// Streaming interface +// --------------------------------------------------------------------------- // +runtime::StreamingPolicy SopranoTTSOfflineSession::streaming_policy() const { + runtime::StreamingPolicy policy; + policy.input = runtime::StreamingInputKind::None; + policy.output = runtime::StreamingOutputKind::PullEvents; + return policy; +} +void SopranoTTSOfflineSession::start_stream(const runtime::TaskRequest & request) { + require_prepared("Soprano start_stream"); + runtime::validate_spec_backed_request_options(request.options, *contract_, "Soprano"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("Soprano start_stream requires a streaming session"); + } + reset(); + const auto parsed = make_request(request); + const auto chunk_codepoints = runtime::parse_i64_option( + options().options, {"soprano_tts.text_chunk_size"}).value_or(200); + streaming_chunks_ = engine::text::split_text_chunks( + parsed.text, chunk_codepoints, engine::text::TextChunkMode::Default); + streaming_chunk_index_ = 0; + streaming_audio_.clear(); +} +std::optional SopranoTTSOfflineSession::next_stream_event() { + if (!streaming_chunks_.has_value()) { + throw std::runtime_error("Soprano streaming has not been started"); + } + if (streaming_chunk_index_ >= streaming_chunks_->size()) { + return std::nullopt; + } + const auto & chunk_text = (*streaming_chunks_)[streaming_chunk_index_]; + const std::filesystem::path tokenizer_path = assets_->resources.require_file("tokenizer_json"); + SopranoTextTokenizer tokenizer(tokenizer_path); + SopranoRequest soprano_req; + soprano_req.text = chunk_text; + const auto audio = synthesize(soprano_req); + streaming_audio_.push_back(audio); + runtime::StreamEvent event; + event.named_audio_outputs.push_back({ + "chunk_" + std::to_string(streaming_chunk_index_), + audio, + {}, + }); + if (stream_sink_) { + stream_sink_(event); + } + ++streaming_chunk_index_; + return event; +} +void SopranoTTSOfflineSession::set_stream_event_sink(runtime::StreamEventCallback sink) { + stream_sink_ = std::move(sink); +} +runtime::TaskResult SopranoTTSOfflineSession::finish_stream() { + if (!streaming_chunks_.has_value()) { + throw std::runtime_error("Soprano streaming has not been started"); + } + runtime::TaskResult result; + runtime::AudioBuffer merged; + for (const auto & chunk_audio : streaming_audio_) { + if (merged.sample_rate == 0) { + merged = chunk_audio; + } else { + runtime::append_audio_buffer(merged, chunk_audio); + } + } + result.audio_output = std::move(merged); + reset(); + return result; +} +void SopranoTTSOfflineSession::reset() { + streaming_chunks_.reset(); + streaming_chunk_index_ = 0; + streaming_audio_.clear(); +} +runtime::StreamEvent SopranoTTSOfflineSession::process_audio_chunk(const runtime::AudioChunk & chunk) { + (void)chunk; + throw std::runtime_error("Soprano is a TTS model and does not accept audio input"); +} +runtime::TaskResult SopranoTTSOfflineSession::finalize() { + return runtime::TaskResult{}; +} + + +std::shared_ptr make_soprano_tts_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_soprano_tts_assets; + config.create_session = create_soprano_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::community_models::soprano_tts diff --git a/src/community_models/soprano_tts/tokenizer_text.cpp b/src/community_models/soprano_tts/tokenizer_text.cpp new file mode 100644 index 000000000..c77ca3139 --- /dev/null +++ b/src/community_models/soprano_tts/tokenizer_text.cpp @@ -0,0 +1,203 @@ +#include "engine/community_models/soprano_tts/tokenizer_text.h" + +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { +namespace { + +namespace json = engine::io::json; + +int32_t token_id(const std::unordered_map & token_to_id, + const std::string & token) { + const auto it = token_to_id.find(token); + if (it == token_to_id.end()) { + throw std::runtime_error("Soprano tokenizer missing token: " + token); + } + return it->second; +} + +std::string utf8_to_lower_ascii(std::string_view input) { + // English-focused fold + diacritic strip (unidecode-like). For the prompt + // tokens Soprano was trained on (ASCII letters/digits/punctuation) a + // byte-level fold is sufficient. + std::string out; + out.reserve(input.size()); + for (const unsigned char c : input) { + out.push_back(static_cast(std::tolower(c))); + } + return out; +} + +std::vector pre_tokenize(const std::string & text) { + // GPT-2-style byte-level pre-tokenizer: split on whitespace and digits, + // keeping punctuation attached. + std::vector parts; + std::string cur; + for (const char ch : text) { + if (std::isspace(static_cast(ch))) { + // Flush current word, then keep the space as its own piece. + if (!cur.empty()) { + parts.push_back(std::move(cur)); + cur.clear(); + } + parts.emplace_back(1, ' '); + } else { + cur.push_back(ch); + } + } + if (!cur.empty()) parts.push_back(std::move(cur)); + return parts; +} + +} // namespace + +SopranoTextTokenizer::SopranoTextTokenizer(const std::filesystem::path & path) { + const auto root = json::parse_file(path); + const auto & model = root.require("model"); + const auto & vocab = model.require("vocab"); + id_to_token_.resize(vocab.as_object().size()); + for (const auto & entry : vocab.as_object()) { + const auto id = static_cast(entry.second.as_i64()); + if (id >= 0 && static_cast(id) < id_to_token_.size()) { + id_to_token_[static_cast(id)] = entry.first; + } + token_to_id_.emplace(entry.first, id); + } + for (const auto & added : root.require("added_tokens").as_array()) { + const auto token = json::require_string(added, "content"); + const auto id = json::require_i64(added, "id"); + token_to_id_[token] = static_cast(id); + } + if (const auto * merges = model.find("merges")) { + int32_t rank = 0; + for (const auto & merge : merges->as_array()) { + // Soprano ships merges as a list of two-element ["a","b"] arrays. + const auto & pair = merge.as_array(); + if (pair.size() < 2) { + continue; + } + const auto a = token_to_id_.find(pair[0].as_string()); + const auto b = token_to_id_.find(pair[1].as_string()); + if (a != token_to_id_.end() && b != token_to_id_.end()) { + merges_.emplace_back(a->second, b->second); + ++rank; + } + } + } + stop_id_ = token_id(token_to_id_, "[STOP]"); + text_id_ = token_id(token_to_id_, "[TEXT]"); + start_id_ = token_id(token_to_id_, "[START]"); + // config bos/eos are both the STOP token id (3). + const auto eos_it = token_to_id_.find("[STOP]"); + eos_id_ = (eos_it != token_to_id_.end()) ? eos_it->second : 3; + bos_id_ = eos_id_; +} +std::vector SopranoTextTokenizer::encode_text(const std::string & raw) const { + const std::string text = scalarclean_soprano_text(raw); + std::vector tokens; + const auto pieces = pre_tokenize(text); + for (const auto & piece : pieces) { + // Character-level: look up each character directly in the vocab. + std::vector word; + word.reserve(piece.size()); + for (const char ch : piece) { + std::string ch_str(1, ch); + const auto it = token_to_id_.find(ch_str); + if (it != token_to_id_.end()) { + word.push_back(it->second); + } else { + // Unknown character → [UNK] + const auto unk = token_to_id_.find("[UNK]"); + if (unk != token_to_id_.end()) { + word.push_back(unk->second); + } + } + } + // BPE: repeatedly apply the lowest-rank adjacent merge. + for (;;) { + int64_t best_rank = -1; + size_t best_pos = 0; + for (size_t i = 0; i + 1 < word.size(); ++i) { + for (size_t r = 0; r < merges_.size(); ++r) { + if (merges_[r].first == word[i] && merges_[r].second == word[i + 1]) { + if (best_rank < 0 || static_cast(r) < best_rank) { + best_rank = static_cast(r); + best_pos = i; + } + break; + } + } + } + if (best_rank < 0) { + break; + } + const auto & a_str = id_to_token_[static_cast(word[best_pos])]; + const auto & b_str = id_to_token_[static_cast(word[best_pos + 1])]; + const std::string merged_str = a_str + b_str; + const auto it = token_to_id_.find(merged_str); + if (it == token_to_id_.end()) { + break; + } + word[best_pos] = it->second; + word.erase(word.begin() + static_cast(best_pos + 1)); + } + tokens.insert(tokens.end(), word.begin(), word.end()); + } + return apply_prompt(tokens); +} + +std::vector SopranoTextTokenizer::apply_prompt(const std::vector & speech) const { + std::vector out; + out.reserve(speech.size() + 4); + out.push_back(stop_id_); + out.push_back(text_id_); + out.insert(out.end(), speech.begin(), speech.end()); + out.push_back(start_id_); + return out; +} + +std::string SopranoTextTokenizer::decode_ids(const std::vector & ids) const { + std::string out; + for (const int32_t id : ids) { + if (id <= 0 || static_cast(id) >= id_to_token_.size()) { + continue; + } + const auto & token = id_to_token_[static_cast(id)]; + if (token.size() == 1) { + const int byte = static_cast(token[0]) - 1; + if (byte >= 0) { + out.push_back(static_cast(byte)); + } else { + out += token; + } + } else { + out += token; + } + } + return out; +} + +std::string scalarclean_soprano_text(const std::string & text) { + std::string out; + bool prev_space = false; + for (const char ch : text) { + if (std::isspace(static_cast(ch))) { + if (!prev_space && !out.empty()) { + out.push_back(' '); + } + prev_space = true; + } else { + out.push_back(static_cast(std::tolower(static_cast(ch)))); + prev_space = false; + } + } + return out; +} + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/src/community_models/soprano_tts/vocoder.cpp b/src/community_models/soprano_tts/vocoder.cpp new file mode 100644 index 000000000..a05221f6e --- /dev/null +++ b/src/community_models/soprano_tts/vocoder.cpp @@ -0,0 +1,449 @@ +#include "engine/community_models/soprano_tts/vocoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/fft.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +struct SopranoConvNeXtBlockWeights { + engine::modules::DepthwiseConv1dWeights dwconv; + engine::modules::NormWeights norm; + engine::modules::LinearWeights pwconv1; + engine::modules::LinearWeights pwconv2; + engine::core::TensorValue gamma; +}; + +struct SopranoDecoderWeights { + std::shared_ptr store; + engine::modules::Conv1dWeights embed; + engine::modules::NormWeights norm; + std::vector convnext; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights head_out; + std::vector istft_window; +}; + +namespace { + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +engine::core::TensorValue scale_last_dim( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input, + const engine::core::TensorValue & scale) { + const auto view = engine::core::reshape_tensor( + ctx, scale, engine::core::TensorShape::from_dims({1, 1, scale.shape.dims[0]})); + const auto repeated = engine::modules::RepeatModule({input.shape}).build(ctx, view); + return engine::modules::MulModule{}.build(ctx, input, repeated); +} + +} // namespace +std::shared_ptr load_decoder_weights( + ggml_backend_t backend, + engine::core::BackendType backend_type, + const engine::assets::TensorSource & source, + const SopranoTTSConfig & config, + size_t weight_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) { + auto weights = std::make_shared(); + weights->store = std::make_shared( + backend, backend_type, "soprano_tts.decoder.weights", weight_context_bytes); + weights->embed = engine::modules::binding::conv1d_from_source( + *weights->store, source, "decoder.embed", conv_storage_type, + config.decoder_dim, config.decoder_input_channels, 1, true); + weights->norm = engine::modules::binding::norm_from_source( + *weights->store, source, "decoder.norm", config.decoder_dim); + weights->convnext.reserve(static_cast(config.decoder_num_layers)); + for (int64_t layer = 0; layer < config.decoder_num_layers; ++layer) { + const std::string prefix = "decoder.convnext." + std::to_string(layer); + SopranoConvNeXtBlockWeights block; + block.dwconv = engine::modules::binding::depthwise_conv1d_from_source( + *weights->store, source, prefix + ".dwconv", conv_storage_type, + config.decoder_dim, static_cast(config.dw_kernel), true); + block.norm = engine::modules::binding::norm_from_source( + *weights->store, source, prefix + ".norm", config.decoder_dim); + block.pwconv1 = engine::modules::binding::linear_from_source( + *weights->store, source, prefix + ".pwconv1", matmul_storage_type, + config.decoder_intermediate_dim, config.decoder_dim, true); + block.pwconv2 = engine::modules::binding::linear_from_source( + *weights->store, source, prefix + ".pwconv2", matmul_storage_type, + config.decoder_dim, config.decoder_intermediate_dim, true); + block.gamma = weights->store->load_f32_tensor( + source, prefix + ".gamma", {config.decoder_dim}); + weights->convnext.push_back(std::move(block)); + } + weights->final_norm = engine::modules::binding::norm_from_source( + *weights->store, source, "decoder.final_layer_norm", config.decoder_dim); + weights->head_out = engine::modules::binding::linear_from_source( + *weights->store, source, "decoder.head.out", matmul_storage_type, + config.n_fft + 2, config.decoder_dim, true); + weights->istft_window = source.require_f32("decoder.head.istft.window", {config.n_fft}); + weights->store->upload(); + return weights; +} + +engine::core::TensorValue build_convnext_block( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input_bct, + const SopranoConvNeXtBlockWeights & weights, + const SopranoTTSConfig & config) { + auto hidden = engine::modules::DepthwiseConv1dModule({ + static_cast(config.decoder_dim), static_cast(config.dw_kernel), + 1, static_cast(config.dw_kernel / 2), 1, + weights.dwconv.bias.has_value(), + }).build(ctx, input_bct, weights.dwconv); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.decoder_dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.norm); + hidden = engine::modules::LinearModule({ + config.decoder_dim, config.decoder_intermediate_dim, true, GGML_PREC_F32, + }).build(ctx, hidden, weights.pwconv1); + hidden = engine::modules::GeluModule({engine::modules::GeluApproximation::ExactErf}).build(ctx, hidden); + hidden = engine::modules::LinearModule({ + config.decoder_intermediate_dim, config.decoder_dim, true, GGML_PREC_F32, + }).build(ctx, hidden, weights.pwconv2); + hidden = scale_last_dim(ctx, hidden, weights.gamma); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + return engine::modules::AddModule{}.build(ctx, input_bct, hidden); +} + +engine::core::TensorValue build_decoder_head( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & feat_bct, + const SopranoDecoderWeights & weights, + const SopranoTTSConfig & config, + int64_t output_frames) { + // SopranoDecoder: interpolate upscale (4*(T-1)+1 frames), embed, ConvNeXt, + // then a single ISTFT head projection (Linear(dim -> n_fft+2)). + engine::core::TensorValue hidden; + // Use align_corners=True to match F.interpolate(mode='linear', align_corners=True). + // Interpolate1dModule::Linear does NOT set ALIGN_CORNERS, so call ggml directly. + { + const auto contiguous = engine::core::ensure_backend_addressable_layout(ctx, feat_bct); + auto output_shape = feat_bct.shape; + output_shape.dims[output_shape.rank - 1] = output_frames; + ggml_tensor * interp = ggml_interpolate( + ctx.ggml, + contiguous.tensor, + output_frames, + contiguous.tensor->ne[1], + contiguous.tensor->ne[2], + contiguous.tensor->ne[3], + static_cast(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS)); + hidden = engine::core::wrap_tensor(interp, output_shape, GGML_TYPE_F32); + } + hidden = engine::modules::Conv1dModule({ + config.decoder_input_channels, config.decoder_dim, 1, 1, 0, 1, + weights.embed.bias.has_value(), + }).build(ctx, hidden, weights.embed); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.decoder_dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.norm); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + for (const auto & block : weights.convnext) { + hidden = build_convnext_block(ctx, hidden, block, config); + } + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.decoder_dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.final_norm); + // Keep channel-last (…, T2, 768) for the head.out linear. + hidden = engine::modules::LinearModule({ + config.decoder_dim, config.n_fft + 2, true, GGML_PREC_F32, + }).build(ctx, hidden, weights.head_out); + return hidden; +} +namespace { + +// Reconstruct audio from head output (log-magnitude|phase halves) with a single +// non-iterative ISTFT pass, mirroring the SopranoDecoder head. +// Matches torch.istft(spec, n_fft, hop, win, window, center=True): +// - Produces (frames-1)*hop_length + n_fft raw samples +// - Trims n_fft//2 from each side → (frames-1)*hop_length output samples +std::vector istft_center_from_head( + const std::vector & head, + int64_t frames, + const SopranoTTSConfig & config, + const std::vector & window, + size_t threads) { + const int64_t freq_bins = config.n_fft / 2 + 1; + const int64_t out_dim = config.n_fft + 2; + if (static_cast(head.size()) != frames * out_dim) { + throw std::runtime_error("Soprano decoder head output shape mismatch"); + } + if (static_cast(window.size()) != config.n_fft) { + throw std::runtime_error("Soprano decoder ISTFT window shape mismatch"); + } + std::vector> spectrum(static_cast(frames * freq_bins)); + const int omp_threads = static_cast(std::max(1, threads)); +#ifdef _OPENMP +#pragma omp parallel for num_threads(omp_threads) if (frames >= 8) +#endif + for (int64_t frame = 0; frame < frames; ++frame) { + const float * row = head.data() + static_cast(frame * out_dim); + for (int64_t freq = 0; freq < freq_bins; ++freq) { + float mag = std::min(std::exp(row[freq]), 100.0F); + // Zero out first and last freq bins (matching reference bugfix) + if (freq == 0 || freq == freq_bins - 1) { + mag = 0.0F; + } + const float phase = row[freq_bins + freq]; + spectrum[static_cast(frame * freq_bins + freq)] = { + mag * std::cos(phase), mag * std::sin(phase)}; + } + } + std::vector framed(static_cast(frames * config.n_fft), 0.0F); + engine::audio::real_fft_inverse( + {static_cast(frames), static_cast(config.n_fft)}, + { + static_cast(freq_bins * static_cast(sizeof(std::complex))), + static_cast(sizeof(std::complex)), + }, + { + static_cast(config.n_fft * static_cast(sizeof(float))), + static_cast(sizeof(float)), + }, + 1, spectrum.data(), framed.data(), + 1.0F / static_cast(config.n_fft), threads); + + // No output trimming: match torch.istft with center=True which produces + // (frames-1)*hop_length + n_fft samples. + const int64_t output_size = (frames - 1) * config.hop_length + config.n_fft; + std::vector folded(static_cast(output_size), 0.0F); + std::vector envelope(static_cast(output_size), 0.0F); + // OLA parallelized over contiguous output blocks; each block gathers the + // overlapping window contributions frame-by-frame in ascending order, so + // the per-sample accumulation order matches the serial version exactly. + { + const int64_t block = 4096; + const int64_t nblocks = (output_size + block - 1) / block; +#ifdef _OPENMP +#pragma omp parallel for num_threads(omp_threads) if (nblocks > 1) +#endif + for (int64_t b = 0; b < nblocks; ++b) { + const int64_t b0 = b * block; + const int64_t b1 = std::min(output_size, b0 + block); + int64_t f0 = (b0 - config.n_fft) / config.hop_length + 1; + if (f0 < 0) { + f0 = 0; + } + int64_t f1 = (b1 - 1) / config.hop_length; + if (f1 >= frames) { + f1 = frames - 1; + } + for (int64_t frame = f0; frame <= f1; ++frame) { + const int64_t start = frame * config.hop_length; + int64_t i0 = b0 - start; + if (i0 < 0) { + i0 = 0; + } + int64_t i1 = b1 - start; + if (i1 > config.n_fft) { + i1 = config.n_fft; + } + const float * src = framed.data() + static_cast(frame * config.n_fft); + for (int64_t i = i0; i < i1; ++i) { + const float w = window[static_cast(i)]; + folded[static_cast(start + i)] += src[i] * w; + envelope[static_cast(start + i)] += w * w; + } + } + } + } + if (output_size <= 0) { + throw std::runtime_error("Soprano decoder ISTFT produced non-positive output size"); + } + // torch.istft with center=True: trim n_fft//2 from each side. + // Final output: (frames-1)*hop_length samples. + const int64_t pad = config.n_fft / 2; + const int64_t samples = output_size - 2 * pad; + if (samples <= 0) { + throw std::runtime_error("Soprano decoder ISTFT produced non-positive samples after trim"); + } + std::vector audio(static_cast(samples), 0.0F); + for (int64_t i = 0; i < samples; ++i) { + const int64_t src = i + pad; + const float denom = envelope[static_cast(src)]; + if (denom > 1.0e-11F) { + audio[static_cast(i)] = folded[static_cast(src)] / denom; + } + } + return audio; +} + +} // namespace + +struct SopranoDecoderGraph { + SopranoDecoderGraph( + ggml_backend_t backend, + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SopranoTTSConfig & config, + std::shared_ptr weights, + int64_t frames_in) + : backend(backend), + weights(std::move(weights)), + frames(frames_in), + input_channels(config.decoder_input_channels), + head_dim(config.n_fft + 2), + output_frames(config.upscale * (frames_in - 1) + 1), + config(&config) { + if (backend == nullptr || this->weights == nullptr) { + throw std::runtime_error("Soprano decoder graph requires backend and weights"); + } + if (frames_in <= 0) { + throw std::runtime_error("Soprano decoder graph requires positive frame count"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + ctx.reset(ggml_init(params)); + if (ctx == nullptr) { + throw std::runtime_error("failed to initialize soprano decoder graph context"); + } + engine::core::ModuleBuildContext build_ctx{ctx.get(), "soprano_tts.decoder", backend_type}; + input = engine::core::make_tensor( + build_ctx, GGML_TYPE_F32, + engine::core::TensorShape::from_dims({1, config.decoder_input_channels, frames})).tensor; + auto feat = engine::core::wrap_tensor( + input, + engine::core::TensorShape::from_dims({1, config.decoder_input_channels, frames}), + GGML_TYPE_F32); + auto head = build_decoder_head(build_ctx, feat, *this->weights, config, output_frames); + head = engine::core::ensure_backend_addressable_layout(build_ctx, head); + output = head.tensor; + ggml_set_output(output); + graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, output); + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate soprano decoder graph"); + } + } + + ~SopranoDecoderGraph() { + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + gallocr = nullptr; + } + } + + bool matches(const SopranoDecoderWeights & other, int64_t other_frames) const noexcept { + return weights.get() == &other && frames == other_frames; + } + + std::vector run( + const std::vector & features, + int64_t frame_count, + const std::vector & window, + size_t threads) { + std::vector bct(static_cast(input_channels * frame_count), 0.0F); + for (int64_t frame = 0; frame < frame_count; ++frame) { + for (int64_t c = 0; c < input_channels; ++c) { + bct[static_cast(c * frame_count + frame)] = + features[static_cast(frame * input_channels + c)]; + } + } + ggml_backend_tensor_set(input, bct.data(), 0, bct.size() * sizeof(float)); + const ggml_status status = engine::core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Soprano decoder graph compute failed"); + } + std::vector head(static_cast(output_frames * head_dim), 0.0F); + ggml_backend_tensor_get(output, head.data(), 0, head.size() * sizeof(float)); + + return istft_center_from_head(head, output_frames, *this->config, window, threads); + } + + + ggml_backend_t backend = nullptr; + std::shared_ptr weights; + int64_t frames = 0; + int64_t input_channels = 0; + int64_t head_dim = 0; + int64_t output_frames = 0; + const SopranoTTSConfig * config = nullptr; + std::unique_ptr ctx; + ggml_tensor * input = nullptr; + ggml_tensor * output = nullptr; + ggml_cgraph * graph = nullptr; + ggml_gallocr_t gallocr = nullptr; +}; + +SopranoDecoderRuntime::SopranoDecoderRuntime( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) + : config_(assets.config), + execution_context_(execution_context), + graph_context_bytes_(graph_context_bytes), + weights_(load_decoder_weights( + execution_context.backend(), + execution_context.backend_type(), + *assets.decoder_weights, + assets.config, + weight_context_bytes, + matmul_storage_type, + conv_storage_type)) {} + +SopranoDecoderRuntime::~SopranoDecoderRuntime() = default; + +runtime::AudioBuffer SopranoDecoderRuntime::decode( + const std::vector & features, + int64_t frames) const { + if (frames <= 0 || static_cast(features.size()) != frames * config_.decoder_input_channels) { + throw std::runtime_error("Soprano decoder requires consistent feature frames"); + } + if (graph_ == nullptr || !graph_->matches(*weights_, frames)) { + graph_ = std::make_unique( + execution_context_.backend(), + execution_context_.backend_type(), + graph_context_bytes_, + config_, + weights_, + frames); + } + auto audio = graph_->run( + features, frames, weights_->istft_window, + static_cast(execution_context_.config().threads)); + runtime::AudioBuffer out; + out.sample_rate = static_cast(config_.sample_rate); + out.channels = 1; + out.samples = std::move(audio); + return out; +} + +// --------------------------------------------------------------------------- // assembly helpers live below; see session.cpp and +// the CMake target for the LM generator + full pipeline. +// --------------------------------------------------------------------------- // +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/tests/soprano_tts/soprano_python_warm_bench.py b/tests/soprano_tts/soprano_python_warm_bench.py new file mode 100644 index 000000000..e765a2a77 --- /dev/null +++ b/tests/soprano_tts/soprano_python_warm_bench.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Soprano TTS Python warm bench — runs the official Python reference and collects timing. + +Usage: + python3 tests/soprano_tts/soprano_python_warm_bench.py --model models/Soprano-1.1-80M --out-dir build/logs/warmbench/soprano_tts_py +""" +from __future__ import annotations + +import argparse +import json +import os +import struct +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +def parse_wav_duration(path: str) -> float: + with open(path, "rb") as f: + data = f.read() + sr = struct.unpack(" 0 else 0.0 + +def load_cases(path: str) -> dict[str, list[str]]: + catalog: dict[str, list[str]] = {} + current_section = "" + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1] + elif current_section: + catalog.setdefault(current_section, []).append(line) + return catalog + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", default="models/Soprano-1.1-80M") + ap.add_argument("--cases", default="tests/soprano_tts/soprano_warm_bench_cases.txt") + ap.add_argument("--out-dir", default="build/logs/warmbench/soprano_tts_py") + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + + model_path = os.path.join(REPO_ROOT, args.model) + cases_path = os.path.join(REPO_ROOT, args.cases) + out_dir = os.path.join(REPO_ROOT, args.out_dir) + os.makedirs(out_dir, exist_ok=True) + + from soprano import SopranoTTS + + print("=== Soprano Python WarmBench ===") + print(f"Model: {model_path}") + print(f"Output: {out_dir}") + + # Load model + t0 = time.time() + model = SopranoTTS(backend="auto", device="cpu", model_path=model_path) + load_time = time.time() - t0 + print(f"Load: {load_time:.2f}s") + + # Load cases + catalog = load_cases(cases_path) + + # Warmup + print("\nWarmup...") + model.infer("At sunrise the studio monitors clicked on, and the first calibration phrase rolled across the room with steady timing.") + print("Warmup complete\n") + + # Run cases + results = [] + for section, texts in catalog.items(): + print(f"Case: {section} ({len(texts)} texts)") + for i, text in enumerate(texts): + case_name = f"{section}_{i}" + out_path = os.path.join(out_dir, f"{case_name}.wav") + + t0 = time.time() + model.infer(text, out_path) + infer_time = time.time() - t0 + + audio_dur = parse_wav_duration(out_path) + rtf = infer_time / audio_dur if audio_dur > 0 else 0 + + results.append({ + "name": case_name, + "infer_time_s": round(infer_time, 3), + "audio_duration_s": round(audio_dur, 3), + "rtf": round(rtf, 4), + }) + + print(f" {case_name}: {infer_time*1000:.0f} ms infer, {audio_dur:.3f} s audio, RTF={rtf:.4f}") + + # Summary + print("\n" + "=" * 60) + print(f"{'Name':<20} {'Infer (s)':<12} {'Audio (s)':<12} {'RTF':<10}") + print("-" * 54) + for r in results: + print(f"{r['name']:<20} {r['infer_time_s']:<12.3f} {r['audio_duration_s']:<12.3f} {r['rtf']:<10.4f}") + + if args.json: + print(json.dumps({"load_time_s": round(load_time, 3), "results": results}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/soprano_tts/soprano_warm_bench_cases.txt b/tests/soprano_tts/soprano_warm_bench_cases.txt new file mode 100644 index 000000000..7ae07dd8b --- /dev/null +++ b/tests/soprano_tts/soprano_warm_bench_cases.txt @@ -0,0 +1,19 @@ +[short] +Soprano is an extremely lightweight text to speech model. +Soft lanterns flickered near the rails. +Rain traced silver lines across the window. + +[medium] +The quick brown fox jumps over the lazy dog. This sentence contains every letter of the alphabet. It has been used for typing practice for many decades. +Morning barges drifted past the bridge as the clerk reread the notice and tucked the blue envelope in her coat. + +[long] +The field of text-to-speech synthesis has advanced significantly in recent years. Modern systems can generate highly natural and expressive speech that is nearly indistinguishable from human recordings. These systems use deep neural networks to model the complex relationship between text and audio. Soprano is one such system, designed to be lightweight and efficient while maintaining high quality output. + +[longform] +A week ago a friend invited a couple of other couples over for dinner. Eventually, the food, but not the wine, was cleared off the table for what turned out to be some fierce Scrabbling. Heeding the strategy of going for the shorter, more valuable word over the longer cheaper word, our final play was Bon, which, as luck would have it, happens to be a Japanese Buddhist festival, and not, as I had originally asserted while laying the tiles on the board, one half of a chocolate-covered cherry treat. Anyway, the strategy worked. My team only lost by 53 points instead of 58. Just the day before, our host had written of the challenges of writing short. In journalism, my friend's chosen trade, and mostly my own too, Mark Twain\'s observation undoubtedly applies: I didn\'t have time to write a short letter, so I wrote a long one instead. The principle holds across genres, in letters, reporting, and other writing. It is harder to be concise than to blather. Good writing is boiled down, not baked full of air like a souffle. No matter how yummy souffles may be. +Silver carts wait by the pier. +Quiet wagons gather by the station. +Morning barges drifted past the bridge as the clerk reread the notice and tucked the blue envelope in her coat. +Rain taps softly on the glass roof. +Small lamps glow along the arcade. diff --git a/tools/soprano_tts/compare_parity.py b/tools/soprano_tts/compare_parity.py new file mode 100644 index 000000000..8fd45fc7e --- /dev/null +++ b/tools/soprano_tts/compare_parity.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Compare Soprano TTS outputs between Python reference and audio.cpp C++ implementation. + +Usage: + python3 tools/soprano_tts/compare_parity.py + +Requires: + pip install numpy + Official checkpoint in models/Soprano-1.1-80M/ + Converted package in models/soprano_pkg/ + audiocpp_cli at build/bin/Release/audiocpp_cli.exe +""" +import subprocess, os, sys, json, struct, time +import numpy as np + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +TEXTS = { + "short": "Soprano is an extremely lightweight text to speech model.", + "medium": ( + "The quick brown fox jumps over the lazy dog. " + "This sentence contains every letter of the alphabet. " + "It has been used for typing practice for many decades." + ), + "long": ( + "The field of text-to-speech synthesis has advanced significantly in recent years. " + "Modern systems can generate highly natural and expressive speech that is nearly " + "indistinguishable from human recordings. These systems use deep neural networks " + "to model the complex relationship between text and audio. Soprano is one such " + "system, designed to be lightweight and efficient while maintaining high quality " + "output." + ), +} + + +def parse_wav(path): + with open(path, "rb") as f: + data = f.read() + sr = struct.unpack(" 0 else 0 + + cp_t, cp_p = run_cpp(name, text) + s, r = parse_wav(cp_p) + cp_dur = len(s) / r if r > 0 else 0 + cp_rtf = round(cp_t["infer_time_s"] / cp_dur, 4) if cp_dur > 0 else 0 + + print(f" C++: {cp_dur:.3f}s audio in {cp_t['infer_time_s']:.3f}s (RTF={cp_rtf:.4f})") + results.append({ + "test": name, "chars": len(text), + "cpp": {"audio_s": round(cp_dur, 3), "infer_s": cp_t["infer_time_s"], "rtf": cp_rtf} + }) + + print("\n" + "=" * 60) + print("RTF of audio.cpp C++ implementation on CPU:") + print(f"{'Test':<8} {'Chars':<8} {'Audio(s)':<12} {'Infer(s)':<12} {'RTF':<10}") + print("-" * 50) + for r in results: + c = r["cpp"] + print(f"{r['test']:<8} {r['chars']:<8} {c['audio_s']:<12.3f} {c['infer_s']:<12.3f} {c['rtf']:<10.4f}") + print() + print("Note: Outputs differ from Python reference because PyTorch and C++ use") + print("different random number generators for sampling. Both produce valid speech.") + + +if __name__ == "__main__": + main() diff --git a/tools/soprano_tts/convert_soprano.py b/tools/soprano_tts/convert_soprano.py new file mode 100644 index 000000000..a3633c8e1 --- /dev/null +++ b/tools/soprano_tts/convert_soprano.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +""" +convert_soprano.py -- Convert Soprano-1.1-80M for audio.cpp's ``soprano_tts`` family. + +The HF checkpoint ships two artifacts: + * ``model.safetensors`` - the Qwen3-style causal LM (no weight norm). + * ``decoder.pth`` - a PyTorch ``SopranoDecoder`` that applies + ``torch.nn.utils.weight_norm`` on its conv/linear + weights. audio.cpp cannot evaluate weight-norm at + load time, so this script *folds* it and emits a + plain ``decoder.safetensors``. + +Output layout (matches ``model_specs/soprano_tts.json`` ``sources``): + + / + config.json (passthrough) + generation_config.json (generated) + tokenizer.json (passthrough) + model.safetensors (passthrough) + decoder.safetensors (weight-norm folded) +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +# --------------------------------------------------------------------------- # +# Weight-norm folding +# --------------------------------------------------------------------------- # +def _torch_available() -> bool: + try: + import torch # noqa: F401 + return True + except Exception: + return False + + +def load_pt_checkpoint(path: Path): + """Load ``decoder.pth`` state dict (requires torch to read the pickle).""" + if _torch_available(): + import torch + return torch.load(path, map_location="cpu", weights_only=True) + raise RuntimeError( + "Soprano conversion requires torch to read decoder.pth (weight-norm " + "state). Install torch or convert on a host that has it." + ) + + +def fold_weight_norm(state: dict) -> dict: + """Fold ``torch.nn.utils.weight_norm`` ``weight_g`` / ``weight_v`` pairs. + + Emits the folded ``weight`` and removes the ``weight_g``/``weight_v`` keys, + matching AudioCpp's assumption of plain weight tensors. + """ + folded = dict(state) + to_fold = [] + for key in list(state.keys()): + if not key.endswith(".weight_g"): + continue + base = key[: -len(".weight_g")] + v_key = base + ".weight_v" + g_key = base + ".weight_g" + if v_key not in state: + continue + g = state[g_key] + v = state[v_key] + if hasattr(v, "is_cuda") and v.is_cuda: + g = g.to("cpu") + v = v.to("cpu") + norm = (v * v).sum(dim=list(range(1, v.dim())), keepdim=True).sqrt() + norm = norm.clamp_min(1e-12) + folded_w = (v / norm) * g + to_fold.append((base + ".weight", g_key, v_key, folded_w)) + for weight_key, g_key, v_key, w in to_fold: + folded[weight_key] = w.detach().float() + del folded[g_key] + del folded[v_key] + return folded + + +# ---------------------------------------------------------------------------- # +# Decoder tensor renaming (Soprano nn.Module -> audio.cpp binding keys) +# ---------------------------------------------------------------------------- # +def rename_decoder_keys(state: dict) -> dict: + """Map SopranoDecoder state names onto the keys the audio.cpp decoder + loader expects (the same prefixes used by the `vevo2` Vocoder): + + decoder.embed Conv1d(512 -> 768, k=1, pad 0, bias=true) + decoder.norm LayerNorm(768) + decoder.convnext..dwconv DepthwiseConv1d(k=3, groups=768, bias=false) + decoder.convnext..norm LayerNorm(768) + decoder.convnext..pwconv1 Linear(768 -> 2304, bias=true) + decoder.convnext..pwconv2 Linear(2304 -> 768, bias=true) + decoder.convnext..gamma layer-scale, [768] + decoder.final_layer_norm LayerNorm(768) + decoder.head.out Linear(768 -> n_fft+2, bias=true) + """ + out = {} + for key, value in state.items(): + new = _map_key(key) + if new: + out[new] = value + return out + + +def _map_key(key: str): + """Map a Soprano state-dict key to the audio.cpp 'decoder.' namespace. + + Soprano names: ``decoder.embed.weight``, ``decoder.norm.weight``, + ``decoder.convnext.N.dwconv.weight``, ``decoder.convnext.N.gamma``, + ``decoder.final_layer_norm.weight``, ``decoder.head.out.weight``. + """ + parts = key.split(".") + # Drop a redundant `decoder.` / `model.` front prefix if present. + while len(parts) >= 2 and parts[0] in ("decoder", "model") and parts[1] in ("decoder",): + parts = parts[2:] + key = ".".join(parts) + if key.startswith("decoder."): + return key + return "decoder." + key + + +# ---------------------------------------------------------------------------- # +# Safetensors writer (torch-free) +# ---------------------------------------------------------------------------- # +def _write_safetensors(tensors: dict, path: Path) -> None: + import numpy as np + header = {} + data = bytearray() + for name, tensor in tensors.items(): + if tensor.dtype in (torch_dtype_f16(),): + arr = np.asarray(tensor.detach().cpu()).view("int16") if False else None + raw = tensor.detach().float().numpy().astype(" int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--input-dir", required=True, help="HF Soprano-1.1-80M directory") + ap.add_argument("--output-dir", "--out", required=True) + args = ap.parse_args() + + src = Path(args.input_dir) + out = Path(args.output_dir) + out.mkdir(parents=True, exist_ok=True) + + for p in ("config.json", "tokenizer.json", "model.safetensors", "decoder.pth"): + if not (src / p).exists(): + raise SystemExit(f"missing input file: {src / p}") + + # Pass-through sidecars. + for name in ("config.json", "tokenizer.json"): + (out / name).write_bytes((src / name).read_bytes()) + (out / "generation_config.json").write_text( + json.dumps({"max_new_tokens": 512, "bos_token_id": 3, "eos_token_id": 3}, + indent=2) + "\n", encoding="utf-8") + + # LM safetensors is passed through byte-for-byte. + (out / "model.safetensors").write_bytes((src / "model.safetensors").read_bytes()) + + # Fold + split decoder. + state = load_pt_checkpoint(src / "decoder.pth") + state = fold_weight_norm(state) + renamed = rename_decoder_keys(state) + if not renamed: + raise SystemExit("decoder.pth contained no recognised tensors") + _write_safetensors(renamed, out / "decoder.safetensors") + + print(f"[convert_soprano] wrote {out}") + print(f"[convert_soprano] decoder tensors: {len(renamed)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/tools/soprano_tts/run_official.py b/tools/soprano_tts/run_official.py new file mode 100644 index 000000000..35af60578 --- /dev/null +++ b/tools/soprano_tts/run_official.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Run official SopranoTTS for ground-truth comparison. +Usage: + python3 run_official.py --text "..." --out output.wav +""" +import argparse, json, os, time, wave + +def run_inference(text: str, out_path: str, model_path: str = "models/Soprano-1.1-80M"): + from soprano import SopranoTTS + + load_t0 = time.time() + model = SopranoTTS(backend="auto", device="cpu", model_path=model_path) + load_time = time.time() - load_t0 + + infer_t0 = time.time() + out = model.infer(text, out_path) + infer_time = time.time() - infer_t0 + + with wave.open(out_path, "r") as wf: + audio_dur = wf.getnframes() / wf.getframerate() + + result = { + "text": text, + "text_chars": len(text), + "load_time_s": round(load_time, 3), + "infer_time_s": round(infer_time, 3), + "audio_duration_s": round(audio_dur, 3), + "rtf": round(infer_time / audio_dur, 4) if audio_dur > 0 else 0, + "output": out_path, + "output_bytes": os.path.getsize(out_path), + } + return result + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--text", default="Soprano is an extremely lightweight text to speech model.") + ap.add_argument("--out", default="soprano_official.wav") + ap.add_argument("--model", default="models/Soprano-1.1-80M") + ap.add_argument("--json", action="store_true", help="Output JSON") + args = ap.parse_args() + result = run_inference(args.text, args.out, args.model) + if args.json: + print(json.dumps(result, indent=2)) + else: + for k, v in result.items(): + print(f"{k}: {v}") diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 80d46e57e..924c2fd99 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -15,6 +15,7 @@ { "id": "qwen3-tts-1.7b", "display_name": "Qwen3-TTS 1.7B Base (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_base", "min_vram_gb": 8 }, { "id": "qwen3-tts-1.7b-custom", "display_name": "Qwen3-TTS 1.7B CustomVoice (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-CustomVoice", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_custom_voice", "min_vram_gb": 8 }, { "id": "miotts", "display_name": "MioTTS 1.7B (tts; needs MioCodec)", "family": "miotts", "path": "models/MioTTS-1.7B", "task": "tts", "mode": "offline", "download_id": "miotts_1_7b", "min_vram_gb": 8 }, + { "id": "soprano-tts", "display_name": "Soprano TTS (tts)", "family": "soprano_tts", "path": "models/Soprano-1.1-80M-GGUF", "task": "tts", "mode": "offline", "download_id": "soprano_1_1_80m_q8_0", "min_vram_gb": 1 }, { "id": "voxcpm2", "display_name": "VoxCPM2 (tts)", "family": "voxcpm2", "path": "models/VoxCPM2", "task": "tts", "mode": "offline", "download_id": "voxcpm2", "session_options": { "voxcpm2.weight_type": "q8_0" }, "min_vram_gb": 6 }, { "id": "voxcpm1", "display_name": "VoxCPM1 0.5B (tts + clone)", "family": "voxcpm1", "path": "models/VoxCPM1-GGUF", "task": "tts", "mode": "offline", "download_id": "voxcpm1_0.5b_q8_0", "min_vram_gb": 4 }, { "id": "vibevoice", "display_name": "VibeVoice 1.5B/7B (tts, long-form/multi-speaker)", "family": "vibevoice", "path": "models/VibeVoice-1.5B", "task": "tts", "mode": "offline", "download_id": "vibevoice_1_5b", "min_vram_gb": 7 }, From 5f4576d44e86b0fe9a7c4dab5eb5a4d20632c780 Mon Sep 17 00:00:00 2001 From: Arda <126098145+drzsdrtfg@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:50:45 +0200 Subject: [PATCH 2/3] Delete PR_SOPRANO.md --- PR_SOPRANO.md | 104 -------------------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 PR_SOPRANO.md diff --git a/PR_SOPRANO.md b/PR_SOPRANO.md deleted file mode 100644 index 11175e046..000000000 --- a/PR_SOPRANO.md +++ /dev/null @@ -1,104 +0,0 @@ -## Soprano TTS — Community Model - -Soprano is an ultra-lightweight (~80M parameter) English-only text-to-speech model using a two-stage architecture: a Qwen3-style causal LM (17 layers, hidden 512, vocab 8192) that autoregressively emits per-frame 512-dimensional features, and a non-iterative Vocos-style decoder (ConvNeXt backbone + single ISTFT head, n_fft 2048 / hop 512) that turns those features into 32 kHz audio. - -Reference: https://github.com/ekwek1/soprano -Weights: https://huggingface.co/ekwek/Soprano-1.1-80M -GGUF packages: https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF - -### Files added - -| Path | Purpose | -|---|---| -| `src/community_models/soprano_tts/` (5 .cpp) | Assets, Qwen3 LM generator, Vocos decoder, tokenizer, session | -| `include/engine/community_models/soprano_tts/` (5 .h) | Corresponding headers | -| `model_specs/soprano_tts.json` | Schema-v1 spec, community status, GGUF + safetensors sources | -| `docs/soprano_tts.md` | User-facing documentation | -| `docs/soprano_validation.md` | Validation record with build/run commands and timing | -| `tests/soprano_tts/soprano_warm_bench.cpp` | C++ warmbench binary | -| `tests/soprano_tts/soprano_warm_bench_cases.txt` | Warmbench test cases (short/medium/long/longform) | -| `tests/soprano_tts/soprano_python_warm_bench.py` | Python reference warmbench | -| `tools/soprano_tts/convert_soprano.py` | HF checkpoint converter (folds weight-norm) | -| `tools/soprano_tts/run_official.py` | Python reference inference runner | -| `tools/soprano_tts/compare_parity.py` | Automated validation harness | - -### Files modified - -| File | Change | -|---|---| -| `CMakeLists.txt` | +16 lines: `audiocpp_add_model` + `add_engine_warmbench` | -| `README.md` | +2 lines: community table row | -| `docs/gguf.md` | +1 line: GGUF status table row | -| `webui/configs/models_catalog.json` | +1 line: WebUI catalog entry | -| `webui/native/dist/index.html` | Rebuilt frontend with Soprano baked in | - -### Build - -```bash -# Soprano only -cmake -B build -DCMAKE_BUILD_TYPE=Release \ - -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts -cmake --build build --target audiocpp_cli --parallel - -# With Vulkan -cmake -B build -DCMAKE_BUILD_TYPE=Release \ - -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ - -DENGINE_ENABLE_VULKAN=ON -cmake --build build --target audiocpp_cli --parallel -``` - -### Quick start - -```bash -# Install GGUF package -python3 tools/model_manager_v2.py install soprano_1_1_80m_q8_0 - -# Run inference -build/bin/audiocpp_cli --task tts --family soprano_tts \ - --model models/Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ - --text "Soprano is an extremely lightweight text to speech model." \ - --out soprano.wav -``` - -### Run warmbench - -```bash -# Build warmbench -cmake --build build --target soprano_warm_bench --parallel - -# Run (CPU) -build/bin/soprano_warm_bench --model models/soprano_pkg \ - --output-dir build/logs/warmbench/soprano_tts - -# Python reference -python3 tests/soprano_tts/soprano_python_warm_bench.py \ - --model models/Soprano-1.1-80M \ - --out-dir build/logs/warmbench/soprano_tts_py -``` - -### Validation - -CPU performance vs official Python `soprano` package (transformers backend, temp=0.3, top_p=0.95): - -| Test | Chars | Platform | Audio (s) | Infer (s) | RTF | Speedup | -|---|---|---|---|---|---|---| -| short | 57 | Python | 0.752 | 1.281 | 1.7037 | — | -| | | **C++** | **3.136** | **0.740** | **0.2360** | **7.22x** | -| medium | 152 | Python | 2.096 | 1.909 | 0.9106 | — | -| | | **C++** | **8.320** | **1.955** | **0.2350** | **3.87x** | -| long | 567 | Python | 7.424 | 5.773 | 0.7776 | — | -| | | **C++** | **16.384** | **4.302** | **0.2626** | **2.96x** | - -Key observations: -- C++ RTF stays below 0.27 on all tests (faster than real-time). -- Audio durations differ between Python and C++ because of different RNG implementations; both produce valid 32 kHz speech. -- Backend coverage: CPU (tested, RTF ~0.24), Vulkan (tested on RX Vega, RTF ~0.08-0.12). -- See `docs/soprano_validation.md` for full validation record. - -### Known limitations - -- English-only (model limitation) -- No voice cloning -- EOS sampling unreliable at low temperature (PyTorch vs C++ RNG difference) -- Full composite build may OOM; use AUDIOCPP_MODEL_SET=custom -- Vulkan decoder output has numerical drift on AMD RX Vega (no matrix-core ops) From d8aca14c7a63b09ed04a637fd0efea898d76cd9d Mon Sep 17 00:00:00 2001 From: Anon Date: Sat, 29 Aug 2026 09:15:43 +0200 Subject: [PATCH 3/3] fix: soprano_tts review issues; warm bench; measured perf docs - converter emits spec-matched combined.safetensors (backbone + folded decoder incl. ISTFT window); loader requires the window strictly - register soprano_warm_bench target; document ENGINE_BUILD_WARMBENCH - parse eos_bias from request options - streaming follows NeuTTS pattern: run_mode reports task mode, options parsed once in start_stream and reused by next_stream_event - accept bare spec-declared session/load option names (text_chunk_size, backbone_weight_type, decoder_weight_type) - add soprano warm bench and document measured CPU/Vulkan performance including F16/Q8_0 backbone storage types Note: the soprano warm bench --warmup/--iterations paths depend on the QwenCausalDecodeRuntime prefill re-feed fix (#331). --- CMakeLists.txt | 4 + docs/soprano_tts.md | 47 ++- docs/soprano_validation.md | 5 +- .../community_models/soprano_tts/session.h | 10 +- src/community_models/soprano_tts/session.cpp | 68 ++-- tests/soprano_tts/soprano_warm_bench.cpp | 317 ++++++++++++++++++ tools/soprano_tts/convert_soprano.py | 99 ++++-- 7 files changed, 476 insertions(+), 74 deletions(-) create mode 100644 tests/soprano_tts/soprano_warm_bench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a0a64ec75..ca308ba4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1914,6 +1914,10 @@ if (ENGINE_BUILD_WARMBENCH) add_engine_warmbench(qwen3_tts_warm_bench tests/qwen3_tts/qwen3_tts_warm_bench.cpp) add_engine_warmbench(seed_vc_warm_bench tests/seed_vc/seed_vc_warm_bench.cpp) add_engine_warmbench(silero_vad_warm_bench tests/silero_vad/silero_vad_warm_bench.cpp) + add_engine_warmbench(soprano_warm_bench tests/soprano_tts/soprano_warm_bench.cpp) + target_compile_definitions(soprano_warm_bench PRIVATE + ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}" + ) add_engine_warmbench(sortformer_diar_warm_bench tests/sortformer_diar/sortformer_diar_warm_bench.cpp) add_engine_warmbench(supertonic_warm_bench tests/supertonic/supertonic_warm_bench.cpp) add_engine_warmbench(vevo2_warm_bench tests/vevo2/vevo2_warm_bench.cpp) diff --git a/docs/soprano_tts.md b/docs/soprano_tts.md index 20afca40e..e9ece6099 100644 --- a/docs/soprano_tts.md +++ b/docs/soprano_tts.md @@ -34,7 +34,7 @@ Or download the checkpoint directly and convert the decoder manually: git lfs install git clone https://huggingface.co/ekwek/Soprano-1.1-80M models/Soprano-1.1-80M -# Convert the decoder (folds weight-norm from decoder.pth, emits plain safetensors) +# Convert (folds weight-norm from decoder.pth, emits combined.safetensors) pip install torch numpy safetensors python3 tools/soprano_tts/convert_soprano.py \ --input-dir models/Soprano-1.1-80M \ @@ -101,7 +101,7 @@ build/bin/audiocpp_cli --task tts --family soprano_tts \ build/bin/audiocpp_cli --task tts --family soprano_tts \ --model models/Soprano-1.1-80M-converted \ --text "This is a longer text that will be split into sentence-aware chunks by the framework text chunker. Each chunk is generated and decoded separately, then concatenated into the final audio output." \ - --session-option soprano_tts.text_chunk_size=320 \ + --session-option text_chunk_size=320 \ --out longform.wav ``` @@ -134,14 +134,14 @@ build/bin/audiocpp_cli --task tts --mode streaming --family soprano_tts \ | Option | Values | Default | Meaning | |---|---|---:|---| -| `--session-option soprano_tts.text_chunk_size=` | chars | `200` | Max codepoints per chunk. | +| `--session-option text_chunk_size=` | chars | `200` | Max codepoints per chunk. | ### Load options | Option | Values | Default | Meaning | |---|---|---:|---| -| `--session-option soprano_tts.backbone_weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `f32` | LM weight storage. F32 required on CPU. | -| `--session-option soprano_tts.decoder_weight_type=` | `native`, `f32`, `f16` | `f32` | Decoder weight storage. | +| `--session-option backbone_weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `f32` | LM weight storage. `f16`/`q8_0` are faster (see Performance). | +| `--session-option decoder_weight_type=` | `native`, `f32`, `f16` | `f32` | Decoder weight storage. | --- @@ -204,7 +204,7 @@ To create a GGUF package from the converted safetensors yourself: ```bash build/bin/audiocpp_gguf \ - --input models/Soprano-1.1-80M-converted/model.safetensors \ + --input models/Soprano-1.1-80M-converted/combined.safetensors \ --output Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ --type q8_0 \ --root models/Soprano-1.1-80M-converted \ @@ -213,10 +213,37 @@ build/bin/audiocpp_gguf \ ``` ## Performance -| Backend | RTF | Details | -|---------|---:|--------| -| CPU (warm) | ~0.22-0.23 | ~4-4.6x realtime. F32 storage required for correct output. | -| Vulkan (RX Vega) | ~0.08-0.12 | ~8-13x realtime after one-time shader warmup. Decoder output has numerical drift on this GPU (Vega lacks matrix-core ops). | +Measured on an Intel i5-10400 (6C/12T) CPU and an AMD Radeon RX Vega (8 GB) GPU, +Release build, warm cache, short/medium sentences: + +| Backend | Backbone storage | RTF | Details | +|---------|------------------|----:|--------| +| CPU | F32 (default) | ~0.23 | ~4.3x realtime; LM decode dominates (~12.3 ms/frame) | +| CPU | F16 | ~0.16 | ~6x realtime; output statistically identical to F32 | +| CPU | Q8_0 | ~0.12 | ~8x realtime; sampling diverges slightly from F32 | +| Vulkan | F32 (default) | ~0.15-0.21 | ~5-7x realtime after one-time shader warmup | +| Vulkan | F16 | ~0.11-0.13 | ~8x realtime | +| Vulkan | Q8_0 | ~0.11-0.13 | same as F16; long-form text amortizes to ~0.08 | + +The LM decode step is memory-bandwidth bound: halving weight traffic (F16) +speeds it up ~1.6x on CPU. Storage types are selected per session (see below); +F32 remains the bit-exact reference, while F16 measured numerically identical +output for this checkpoint, and Q8_0 trades a small sampling drift for the +fastest inference. + +### Tuning storage types + +``` +# CPU: F16 backbone (recommended) +build/bin/audiocpp_cli --task tts --family soprano_tts --model models/soprano-1.1-80m-converted \ + --text "..." --session-option backbone_weight_type=f16 --out out.wav + +# CPU: Q8_0 backbone (fastest) +build/bin/audiocpp_cli --task tts --family soprano_tts --model models/soprano-1.1-80m-converted \ + --text "..." --session-option backbone_weight_type=q8_0 --out out.wav + +# GPU: pre-quantized GGUF packages already run at the q8_0 rate +``` Timing logs are available through `--log`: - `soprano_tts.lm.generate_ms` -- LM AR decode time diff --git a/docs/soprano_validation.md b/docs/soprano_validation.md index 13d276284..7202670bc 100644 --- a/docs/soprano_validation.md +++ b/docs/soprano_validation.md @@ -14,9 +14,10 @@ cmake -B build -DCMAKE_BUILD_TYPE=Release \ -DENGINE_ENABLE_VULKAN=ON cmake --build build --target audiocpp_cli --parallel -# Build warmbench +# Build warmbench (the target is gated behind ENGINE_BUILD_WARMBENCH) cmake -B build -DCMAKE_BUILD_TYPE=Release \ - -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ + -DENGINE_BUILD_WARMBENCH=ON cmake --build build --target soprano_warm_bench --parallel ``` diff --git a/include/engine/community_models/soprano_tts/session.h b/include/engine/community_models/soprano_tts/session.h index 4dba830dc..903c9be35 100644 --- a/include/engine/community_models/soprano_tts/session.h +++ b/include/engine/community_models/soprano_tts/session.h @@ -51,10 +51,12 @@ class SopranoTTSOfflineSession final : public runtime::RuntimeSessionBase, private: SopranoRequest make_request(const runtime::TaskRequest & request) const; runtime::AudioBuffer synthesize(const SopranoRequest & request); - // Streaming state - std::optional> streaming_chunks_; - size_t streaming_chunk_index_ = 0; - std::vector streaming_audio_; + // Streaming state (NeuTTS-style: start_stream parses the full request + // once, next_stream_event consumes the stored per-chunk requests). + std::vector streaming_requests_; + size_t streaming_index_ = 0; + std::vector streaming_chunks_; + bool streaming_started_ = false; runtime::StreamEventCallback stream_sink_; runtime::TaskSpec task_; diff --git a/src/community_models/soprano_tts/session.cpp b/src/community_models/soprano_tts/session.cpp index 330411f7d..59731b389 100644 --- a/src/community_models/soprano_tts/session.cpp +++ b/src/community_models/soprano_tts/session.cpp @@ -62,6 +62,9 @@ SopranoGenerationOptions request_generation_options(const runtime::TaskRequest & if (const auto value = runtime::parse_finite_float_option(request.options, {"repetition_penalty"})) { out.repetition_penalty = *value; } + if (const auto value = runtime::parse_finite_float_option(request.options, {"eos_bias"})) { + out.eos_bias = *value; + } if (const auto value = runtime::parse_u64_option(request.options, {"seed"})) { out.seed = *value; out.has_seed = true; @@ -97,9 +100,11 @@ SopranoTTSOfflineSession::SopranoTTSOfflineSession( contract_(require_contract(std::move(contract))) { runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "Soprano"); core::ExecutionContext & execution = execution_context(); + // Spec-declared session/load options are validated against their bare + // names, so only those spellings are accepted (no prefixed aliases). const auto backbone_storage = runtime::parse_tensor_storage_option( options.options, - "soprano_tts.backbone_weight_type", + "backbone_weight_type", assets::TensorStorageType::F32, {assets::TensorStorageType::Native, assets::TensorStorageType::F32, @@ -108,7 +113,7 @@ SopranoTTSOfflineSession::SopranoTTSOfflineSession( assets::TensorStorageType::Q8_0}); const auto decoder_storage = runtime::parse_tensor_storage_option( options.options, - "soprano_tts.decoder_weight_type", + "decoder_weight_type", assets::TensorStorageType::F32, {assets::TensorStorageType::Native, assets::TensorStorageType::F32, @@ -132,7 +137,7 @@ runtime::VoiceTaskKind SopranoTTSOfflineSession::task_kind() const { } runtime::RunMode SopranoTTSOfflineSession::run_mode() const { - return runtime::RunMode::Offline; + return task_.mode; } SopranoRequest SopranoTTSOfflineSession::make_request(const runtime::TaskRequest & request) const { @@ -149,6 +154,9 @@ void SopranoTTSOfflineSession::prepare(const runtime::SessionPreparationRequest runtime::TaskResult SopranoTTSOfflineSession::run(const runtime::TaskRequest & request) { require_prepared("Soprano run"); + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("Soprano run requires an offline session"); + } const SopranoRequest req = make_request(request); const auto audio = synthesize(req); @@ -162,7 +170,7 @@ runtime::AudioBuffer SopranoTTSOfflineSession::synthesize(const SopranoRequest & assets_->resources.require_file("tokenizer_json"); SopranoTextTokenizer tokenizer(tokenizer_path); const int64_t chunk_codepoints = runtime::parse_i64_option( - options().options, {"soprano_tts.text_chunk_size"}) + options().options, {"text_chunk_size"}) .value_or(200); const auto chunks = engine::text::split_text_chunks( request.text, chunk_codepoints, engine::text::TextChunkMode::Default); @@ -210,50 +218,57 @@ void SopranoTTSOfflineSession::start_stream(const runtime::TaskRequest & request throw std::runtime_error("Soprano start_stream requires a streaming session"); } reset(); - const auto parsed = make_request(request); + // Parse the request (and its sampling options) once, then keep one full + // request per text chunk so next_stream_event honors the user options. const auto chunk_codepoints = runtime::parse_i64_option( - options().options, {"soprano_tts.text_chunk_size"}).value_or(200); - streaming_chunks_ = engine::text::split_text_chunks( - parsed.text, chunk_codepoints, engine::text::TextChunkMode::Default); - streaming_chunk_index_ = 0; - streaming_audio_.clear(); + options().options, {"text_chunk_size"}).value_or(200); + const auto chunks = engine::text::split_text_chunks( + request_text(request), chunk_codepoints, engine::text::TextChunkMode::Default); + SopranoRequest parsed; + parsed.generation = request_generation_options(request); + streaming_requests_.reserve(chunks.size()); + for (const auto & chunk : chunks) { + SopranoRequest chunk_request; + chunk_request.text = chunk; + chunk_request.generation = parsed.generation; + streaming_requests_.push_back(std::move(chunk_request)); + } + if (streaming_requests_.empty()) { + throw std::runtime_error("Soprano streaming text chunking produced no segments"); + } + streaming_started_ = true; } std::optional SopranoTTSOfflineSession::next_stream_event() { - if (!streaming_chunks_.has_value()) { + if (!streaming_started_) { throw std::runtime_error("Soprano streaming has not been started"); } - if (streaming_chunk_index_ >= streaming_chunks_->size()) { + if (streaming_index_ >= streaming_requests_.size()) { return std::nullopt; } - const auto & chunk_text = (*streaming_chunks_)[streaming_chunk_index_]; - const std::filesystem::path tokenizer_path = assets_->resources.require_file("tokenizer_json"); - SopranoTextTokenizer tokenizer(tokenizer_path); - SopranoRequest soprano_req; - soprano_req.text = chunk_text; - const auto audio = synthesize(soprano_req); - streaming_audio_.push_back(audio); + auto audio = synthesize(streaming_requests_[streaming_index_]); runtime::StreamEvent event; event.named_audio_outputs.push_back({ - "chunk_" + std::to_string(streaming_chunk_index_), + "chunk_" + std::to_string(streaming_index_), audio, {}, }); + streaming_chunks_.push_back(std::move(audio)); if (stream_sink_) { stream_sink_(event); } - ++streaming_chunk_index_; + ++streaming_index_; return event; } void SopranoTTSOfflineSession::set_stream_event_sink(runtime::StreamEventCallback sink) { stream_sink_ = std::move(sink); } runtime::TaskResult SopranoTTSOfflineSession::finish_stream() { - if (!streaming_chunks_.has_value()) { + if (!streaming_started_) { throw std::runtime_error("Soprano streaming has not been started"); } runtime::TaskResult result; runtime::AudioBuffer merged; - for (const auto & chunk_audio : streaming_audio_) { + for (const auto & chunk_audio : streaming_chunks_) { if (merged.sample_rate == 0) { merged = chunk_audio; } else { @@ -265,9 +280,10 @@ runtime::TaskResult SopranoTTSOfflineSession::finish_stream() { return result; } void SopranoTTSOfflineSession::reset() { - streaming_chunks_.reset(); - streaming_chunk_index_ = 0; - streaming_audio_.clear(); + streaming_requests_.clear(); + streaming_index_ = 0; + streaming_chunks_.clear(); + streaming_started_ = false; } runtime::StreamEvent SopranoTTSOfflineSession::process_audio_chunk(const runtime::AudioChunk & chunk) { (void)chunk; diff --git a/tests/soprano_tts/soprano_warm_bench.cpp b/tests/soprano_tts/soprano_warm_bench.cpp new file mode 100644 index 000000000..d1b10c28d --- /dev/null +++ b/tests/soprano_tts/soprano_warm_bench.cpp @@ -0,0 +1,317 @@ +#include "engine/framework/audio/wav_writer.h" + +#include "../core/audio_task_warm_bench.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct SopranoBenchRequest { + std::string id; + engine::runtime::TaskRequest request; +}; + +std::string option_text(const engine::io::json::Value & value) { + if (value.is_bool()) { + return value.as_bool() ? "true" : "false"; + } + if (value.is_number()) { + return engine::io::json::stringify_number(value.as_number()); + } + return value.as_string(); +} + +std::string optional_string(const engine::io::json::Value & object, const std::string & key) { + const auto * value = object.find(key); + return value == nullptr || value->is_null() ? std::string{} : value->as_string(); +} + +std::string required_string(const engine::io::json::Value & object, const std::string & key) { + const auto value = optional_string(object, key); + if (value.empty()) { + throw std::runtime_error("Soprano warmbench request missing field: " + key); + } + return value; +} + +void set_optional_option( + engine::runtime::TaskRequest & request, + const engine::io::json::Value & object, + const std::string & source, + const std::string & target) { + const auto * value = object.find(source); + if (value != nullptr && !value->is_null()) { + request.options[target] = option_text(*value); + } +} + +SopranoBenchRequest make_request(const engine::io::json::Value & object, const std::string & fallback_id) { + SopranoBenchRequest out; + out.id = optional_string(object, "id"); + if (out.id.empty()) { + out.id = fallback_id; + } + out.request.text_input = engine::runtime::Transcript{required_string(object, "text"), "en"}; + set_optional_option(out.request, object, "max_tokens", "max_tokens"); + set_optional_option(out.request, object, "temperature", "temperature"); + set_optional_option(out.request, object, "top_p", "top_p"); + set_optional_option(out.request, object, "repetition_penalty", "repetition_penalty"); + set_optional_option(out.request, object, "eos_bias", "eos_bias"); + set_optional_option(out.request, object, "seed", "seed"); + return out; +} + +std::vector parse_requests(const std::string & request_sequence_json) { + if (request_sequence_json.empty()) { + throw std::runtime_error("Soprano warmbench requires --request-sequence-json"); + } + const auto root = engine::io::json::parse(request_sequence_json); + std::vector requests; + int index = 0; + for (const auto & item : root.as_array()) { + requests.push_back(make_request(item, "request_" + std::to_string(index++))); + } + if (requests.empty()) { + throw std::runtime_error("Soprano warmbench request sequence is empty"); + } + return requests; +} + +// Parses the shared ``[section]`` case catalog (same file the Python reference +// warmbench consumes: tests/soprano_tts/soprano_warm_bench_cases.txt). +std::vector parse_cases( + const std::filesystem::path & cases_path, + const std::vector & sections) { + std::ifstream file(cases_path); + if (!file) { + throw std::runtime_error("Soprano warmbench cannot open cases file: " + cases_path.string()); + } + std::vector requests; + std::string section; + std::string line; + while (std::getline(file, line)) { + while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) { + line.pop_back(); + } + if (line.empty()) { + continue; + } + if (line.front() == '[' && line.back() == ']') { + section = line.substr(1, line.size() - 2); + continue; + } + if (section.empty()) { + continue; + } + if (!sections.empty() && + std::find(sections.begin(), sections.end(), section) == sections.end()) { + continue; + } + SopranoBenchRequest request; + request.id = section + "_" + std::to_string( + std::count_if(requests.begin(), requests.end(), + [§ion](const SopranoBenchRequest & item) { + return item.id.rfind(section + "_", 0) == 0; + })); + request.request.text_input = engine::runtime::Transcript{line, "en"}; + requests.push_back(std::move(request)); + } + if (requests.empty()) { + throw std::runtime_error("Soprano warmbench cases file produced no requests"); + } + return requests; +} + +std::optional parse_warmup_request(const std::string & request_json) { + if (request_json.empty()) { + return std::nullopt; + } + return make_request(engine::io::json::parse(request_json), "warmup"); +} + +engine::io::json::Value audio_summary_json(const engine::runtime::AudioBuffer & audio) { + if (audio.samples.empty()) { + throw std::runtime_error("Soprano warmbench received empty audio output"); + } + double sum = 0.0; + double abs_sum = 0.0; + double sq_sum = 0.0; + float min_value = audio.samples.front(); + float max_value = audio.samples.front(); + for (const float sample : audio.samples) { + sum += static_cast(sample); + abs_sum += std::abs(static_cast(sample)); + sq_sum += static_cast(sample) * static_cast(sample); + min_value = std::min(min_value, sample); + max_value = std::max(max_value, sample); + } + const auto channels = std::max(1, audio.channels); + const double frames = static_cast(audio.samples.size() / static_cast(channels)); + const double count = static_cast(audio.samples.size()); + return engine::io::json::Value::make_object({ + {"sample_rate", engine::tools::number(static_cast(audio.sample_rate))}, + {"channels", engine::tools::number(static_cast(audio.channels))}, + {"samples", engine::tools::number(count)}, + {"frames", engine::tools::number(frames)}, + {"duration_sec", engine::tools::number(audio.sample_rate > 0 ? frames / audio.sample_rate : 0.0)}, + {"sum", engine::tools::number(sum)}, + {"mean_abs", engine::tools::number(abs_sum / count)}, + {"rms", engine::tools::number(std::sqrt(sq_sum / count))}, + {"min", engine::tools::number(min_value)}, + {"max", engine::tools::number(max_value)}, + }); +} + +engine::io::json::Value step_json( + const engine::runtime::TaskResult & result, + const SopranoBenchRequest & request, + int request_index, + double wall_ms, + const std::filesystem::path & audio_path) { + if (!result.audio_output.has_value()) { + throw std::runtime_error("Soprano warmbench expected audio output"); + } + engine::io::json::Value::Object stem{ + {"name", engine::tools::string("audio")}, + {"summary", audio_summary_json(*result.audio_output)}, + }; + if (!audio_path.empty()) { + stem.emplace("audio", engine::tools::string(audio_path.string())); + } + const auto & audio = *result.audio_output; + const double frames = static_cast( + audio.samples.size() / static_cast(std::max(1, audio.channels))); + const double duration_sec = audio.sample_rate > 0 ? frames / audio.sample_rate : 0.0; + const double rtf = duration_sec > 0.0 ? wall_ms / 1000.0 / duration_sec : 0.0; + return engine::io::json::Value::make_object({ + {"request_index", engine::tools::number(static_cast(request_index))}, + {"id", engine::tools::string(request.id)}, + {"text_length", engine::tools::number(static_cast(request.request.text_input->text.size()))}, + {"stems", engine::io::json::Value::make_array({engine::io::json::Value::make_object(std::move(stem))})}, + {"metrics", engine::io::json::Value::make_object({ + {"wall_ms", engine::tools::number(wall_ms)}, + {"rtf", engine::tools::number(rtf)}, + })}, + }); +} + +} // namespace + +int main(int argc, char ** argv) { + try { +#ifndef ENGINE_REPO_ROOT +#error "soprano_warm_bench requires the ENGINE_REPO_ROOT compile definition (provided by its CMake target)" +#endif + const std::filesystem::path repo_root = ENGINE_REPO_ROOT; + const std::filesystem::path model_path = engine::tools::arg_value(argc, argv, "--model", "models/soprano_pkg"); + const std::string backend_name = engine::tools::arg_value(argc, argv, "--backend", "cpu"); + const int device = engine::tools::int_arg(argc, argv, "--device", 0); + const int threads = engine::tools::int_arg(argc, argv, "--threads", 8); + const int warmup = engine::tools::int_arg(argc, argv, "--warmup", 0); + const int iterations = engine::tools::int_arg(argc, argv, "--iterations", 1); + const std::filesystem::path output_dir = engine::tools::arg_value(argc, argv, "--output-dir", ""); + const std::filesystem::path timing_path = + engine::tools::arg_value(argc, argv, "--timing-file", "build/logs/warmbench/soprano_tts_timing.log"); + const auto warmup_request = + parse_warmup_request(engine::tools::arg_value(argc, argv, "--warmup-request-json", "")); + const auto request_sequence_json = engine::tools::arg_value(argc, argv, "--request-sequence-json", ""); + const std::filesystem::path cases_path = engine::tools::arg_value( + argc, argv, "--cases", (repo_root / "tests/soprano_tts/soprano_warm_bench_cases.txt").string()); + const auto sections = engine::tools::split_csv(engine::tools::arg_value(argc, argv, "--sections", "")); + + if (!timing_path.parent_path().empty()) { + std::filesystem::create_directories(timing_path.parent_path()); + } + engine::tools::set_process_env("ENGINE_TRACE_ENABLED", "0"); + engine::tools::set_process_env("ENGINE_TIMING_ENABLED", "1"); + engine::tools::set_process_env("ENGINE_TIMING_FILE", timing_path.string()); + engine::debug::configure_logging(engine::debug::LoggingConfig{true, timing_path.string()}); + + auto registry = engine::runtime::make_default_registry(); + engine::runtime::ModelLoadRequest load_request; + load_request.model_path = model_path; + load_request.family_hint = "soprano_tts"; + auto model = registry.load(load_request); + + engine::runtime::SessionOptions options; + options.backend.type = engine::tools::parse_backend(backend_name); + options.backend.device = device; + options.backend.threads = threads; + for (const auto & [key, value] : engine::tools::session_option_args(argc, argv)) { + options.options.insert_or_assign(key, value); + } + + auto session_base = model->create_task_session( + {engine::runtime::VoiceTaskKind::Tts, engine::runtime::RunMode::Offline}, + options); + auto * session = dynamic_cast(session_base.get()); + if (session == nullptr) { + throw std::runtime_error("loaded Soprano session is not offline-capable"); + } + + const auto requests = request_sequence_json.empty() + ? parse_cases(cases_path, sections) + : parse_requests(request_sequence_json); + const auto & prepare_request = warmup_request.has_value() ? warmup_request->request : requests.front().request; + session->prepare(engine::runtime::build_preparation_request(prepare_request)); + for (int i = 0; i < warmup; ++i) { + (void) session->run(prepare_request); + } + if (!output_dir.empty()) { + std::filesystem::create_directories(output_dir); + } + + engine::io::json::Value::Array steps; + steps.reserve(requests.size()); + for (size_t request_index = 0; request_index < requests.size(); ++request_index) { + engine::runtime::TaskResult last_result; + double total_ms = 0.0; + for (int iteration = 0; iteration < std::max(1, iterations); ++iteration) { + const auto started = std::chrono::steady_clock::now(); + last_result = session->run(requests[request_index].request); + const auto ended = std::chrono::steady_clock::now(); + total_ms += std::chrono::duration(ended - started).count(); + } + if (!last_result.audio_output.has_value()) { + throw std::runtime_error("Soprano warmbench expected audio output"); + } + const double wall_ms = total_ms / static_cast(std::max(1, iterations)); + std::filesystem::path audio_path; + if (!output_dir.empty()) { + audio_path = output_dir / (requests[request_index].id + ".wav"); + const auto & audio = *last_result.audio_output; + engine::audio::write_pcm16_wav(audio_path, audio.sample_rate, audio.channels, audio.samples); + } + std::cout << "soprano.request[" << request_index << "].id=" << requests[request_index].id << "\n"; + std::cout << "soprano.request[" << request_index << "].wall_ms=" << wall_ms << "\n"; + steps.push_back(step_json( + last_result, + requests[request_index], + static_cast(request_index), + wall_ms, + audio_path)); + } + + const auto summary = engine::io::json::Value::make_object({ + {"family", engine::tools::string("soprano_tts")}, + {"backend", engine::tools::string(backend_name)}, + {"mode", engine::tools::string("offline")}, + {"sequence_steps", engine::io::json::Value::make_array(std::move(steps))}, + }); + std::cout << "summary_json=" << engine::io::json::stringify(summary) << "\n"; + return 0; + } catch (const std::exception & ex) { + std::cerr << "soprano_warm_bench failed: " << ex.what() << "\n"; + return 1; + } +} diff --git a/tools/soprano_tts/convert_soprano.py b/tools/soprano_tts/convert_soprano.py index a3633c8e1..f6f8e32bb 100644 --- a/tools/soprano_tts/convert_soprano.py +++ b/tools/soprano_tts/convert_soprano.py @@ -8,23 +8,26 @@ * ``decoder.pth`` - a PyTorch ``SopranoDecoder`` that applies ``torch.nn.utils.weight_norm`` on its conv/linear weights. audio.cpp cannot evaluate weight-norm at - load time, so this script *folds* it and emits a - plain ``decoder.safetensors``. + load time, so this script *folds* it. -Output layout (matches ``model_specs/soprano_tts.json`` ``sources``): +Output layout (matches ``model_specs/soprano_tts.json`` ``sources``, which maps +both the ``backbone`` and ``decoder`` tensor sources to a single file): / config.json (passthrough) generation_config.json (generated) tokenizer.json (passthrough) - model.safetensors (passthrough) - decoder.safetensors (weight-norm folded) + combined.safetensors (backbone tensors byte-copied from + model.safetensors + weight-norm-folded decoder + tensors, incl. the ISTFT window from + decoder.pth) """ from __future__ import annotations import argparse import json +import struct from pathlib import Path @@ -73,7 +76,7 @@ def fold_weight_norm(state: dict) -> dict: v = v.to("cpu") norm = (v * v).sum(dim=list(range(1, v.dim())), keepdim=True).sqrt() norm = norm.clamp_min(1e-12) - folded_w = (v / norm) * g + folded_w = (v / norm) * g.reshape(norm.shape) to_fold.append((base + ".weight", g_key, v_key, folded_w)) for weight_key, g_key, v_key, w in to_fold: folded[weight_key] = w.detach().float() @@ -90,7 +93,7 @@ def rename_decoder_keys(state: dict) -> dict: loader expects (the same prefixes used by the `vevo2` Vocoder): decoder.embed Conv1d(512 -> 768, k=1, pad 0, bias=true) - decoder.norm LayerNorm(768) + decoder.norm LayerN decoder.convnext..dwconv DepthwiseConv1d(k=3, groups=768, bias=false) decoder.convnext..norm LayerNorm(768) decoder.convnext..pwconv1 Linear(768 -> 2304, bias=true) @@ -98,6 +101,7 @@ def rename_decoder_keys(state: dict) -> dict: decoder.convnext..gamma layer-scale, [768] decoder.final_layer_norm LayerNorm(768) decoder.head.out Linear(768 -> n_fft+2, bias=true) + decoder.head.istft.window periodic Hann window, [n_fft] """ out = {} for key, value in state.items(): @@ -112,7 +116,8 @@ def _map_key(key: str): Soprano names: ``decoder.embed.weight``, ``decoder.norm.weight``, ``decoder.convnext.N.dwconv.weight``, ``decoder.convnext.N.gamma``, - ``decoder.final_layer_norm.weight``, ``decoder.head.out.weight``. + ``decoder.final_layer_norm.weight``, ``decoder.head.out.weight``, + ``decoder.head.istft.window``. """ parts = key.split(".") # Drop a redundant `decoder.` / `model.` front prefix if present. @@ -125,22 +130,45 @@ def _map_key(key: str): # ---------------------------------------------------------------------------- # -# Safetensors writer (torch-free) +# Safetensors reader / writer # ---------------------------------------------------------------------------- # -def _write_safetensors(tensors: dict, path: Path) -> None: +def read_safetensors_entries(path: Path) -> dict: + """Read a safetensors file as raw entries (torch-free). + + Returns ``{name: {"dtype": str, "shape": [int], "raw": bytes}}``; tensor + payloads are byte-copied verbatim so backbone dtypes (BF16 etc.) survive. + """ + blob = path.read_bytes() + if len(blob) < 8: + raise SystemExit(f"invalid safetensors file: {path}") + (header_len,) = struct.unpack(" tuple[str, list, bytes]: + """Convert a torch tensor into an (dtype, shape, raw) F32 entry.""" import numpy as np + + arr = tensor.detach().float().cpu().numpy().astype(" None: + """Write ``{name: (dtype, shape, raw)}`` entries as one safetensors file.""" header = {} data = bytearray() - for name, tensor in tensors.items(): - if tensor.dtype in (torch_dtype_f16(),): - arr = np.asarray(tensor.detach().cpu()).view("int16") if False else None - raw = tensor.detach().float().numpy().astype(" None: fh.write(bytes(data)) -def torch_dtype_f16(): - import torch - return torch.float16 - - # ---------------------------------------------------------------------------- # # main # ---------------------------------------------------------------------------- # @@ -185,21 +208,33 @@ def main() -> int: json.dumps({"max_new_tokens": 512, "bos_token_id": 3, "eos_token_id": 3}, indent=2) + "\n", encoding="utf-8") - # LM safetensors is passed through byte-for-byte. - (out / "model.safetensors").write_bytes((src / "model.safetensors").read_bytes()) - - # Fold + split decoder. + # Backbone tensors are byte-copied verbatim (keeps BF16 storage). + entries = { + name: (info["dtype"], info["shape"], info["raw"]) + for name, info in read_safetensors_entries(src / "model.safetensors").items() + } + backbone_count = len(entries) + + # Fold + rename decoder, then merge into the same file under the + # "decoder." namespace (audio.cpp distinguishes the two sources by key). + # The ISTFT window (head.istft.window) is part of the checkpoint state + # dict and passes through like every other decoder tensor; the loader + # requires it, so a checkpoint without it fails loudly at inference time. state = load_pt_checkpoint(src / "decoder.pth") state = fold_weight_norm(state) renamed = rename_decoder_keys(state) if not renamed: raise SystemExit("decoder.pth contained no recognised tensors") - _write_safetensors(renamed, out / "decoder.safetensors") + for name, tensor in renamed.items(): + entries[name] = _entry_to_f32(tensor) + + write_safetensors(entries, out / "combined.safetensors") print(f"[convert_soprano] wrote {out}") - print(f"[convert_soprano] decoder tensors: {len(renamed)}") + print(f"[convert_soprano] combined.safetensors: {backbone_count} backbone tensors " + f"+ {len(renamed)} decoder tensors") return 0 if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main())