From c5abdcbd287d30dd3ebb2dbc6afdecc54a0cbe4c Mon Sep 17 00:00:00 2001 From: niksedk Date: Fri, 4 Sep 2026 17:58:59 +0200 Subject: [PATCH] higgs_audio_tts: give the codec decoder right-hand context at the end of the stream The Higgs codec decoder is a non-causal conv stack, so the last frames of a stream are decoded against zero padding on their right instead of real context. That decodes as a rising hiss over the final ~300 ms, clearly audible on short utterances ("Hello, how are you doing?" with a cloned voice ends in a swell that is cut off). This is the same mechanism as the periodic click at chunk seams (#429) but at the end of the stream, where the chunked path's context frames do not apply. Repeat the last frame kCodecTailContextFrames (8) times before decoding and trim the extra samples again. 16 and 32 frames give sample-identical output, so 8 is enough. decode_codes() keeps its contract (same sample count as before); the previous body becomes decode_codes_impl(). Measured on Metal (M4), Q8_0, voice clone, same seed before/after, RMS of the last three 40 ms frames and peak of the last 100 ms: short, seed 1: -39 -44 -34 dBFS, peak 3990 -> -41 -48 -46, peak 864 short, seed 2: -39 -46 -32 dBFS, peak 3951 -> -40 -47 -41, peak 1121 long (11.4 s): -48 -53 -46 dBFS -> -48 -53 -53, peak 334 Co-Authored-By: Claude Fable 5.1 --- include/engine/models/higgs_audio_tts/codec.h | 5 +++ src/models/higgs_audio_tts/codec.cpp | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/include/engine/models/higgs_audio_tts/codec.h b/include/engine/models/higgs_audio_tts/codec.h index 28516eea5..88a7bcd92 100644 --- a/include/engine/models/higgs_audio_tts/codec.h +++ b/include/engine/models/higgs_audio_tts/codec.h @@ -117,6 +117,11 @@ class HiggsCodecRuntime { void release_runtime_graphs(); private: + HiggsCodecDecodeOutput decode_codes_impl( + const std::vector & codes, + int64_t frames, + int64_t codebooks) const; + std::shared_ptr assets_; ggml_backend_t backend_ = nullptr; core::BackendType backend_type_ = core::BackendType::Cpu; diff --git a/src/models/higgs_audio_tts/codec.cpp b/src/models/higgs_audio_tts/codec.cpp index 33204ad08..157ac9eaf 100644 --- a/src/models/higgs_audio_tts/codec.cpp +++ b/src/models/higgs_audio_tts/codec.cpp @@ -64,6 +64,12 @@ constexpr int64_t kCodecPadSamples = kCodecHopLength / 2; constexpr int64_t kCodecDecodeWindowFrames = 128; constexpr int64_t kCodecDecodeContextFrames = 32; constexpr int64_t kCodecFullDecodeMaxFrames = 512; +// The decoder is a non-causal conv stack, so the last frames of a stream see zero padding +// on their right instead of real context and decode as a rising hiss over the final +// ~300 ms (worst on short utterances). Repeating the last frame past the end gives those +// frames context; the extra audio is trimmed again. Eight frames is enough — 16 and 32 +// produce identical samples. +constexpr int64_t kCodecTailContextFrames = 8; constexpr int64_t kResidualDilations[] = {1, 3, 9}; struct GgmlContextDeleter { @@ -1549,6 +1555,40 @@ HiggsCodecDecodeOutput HiggsCodecRuntime::decode_codes(const std::vector(codes.size()) != frames * codebooks) { throw std::runtime_error("Higgs TTS codec decode code count mismatch"); } + if (kCodecTailContextFrames <= 0) { + return decode_codes_impl(codes, frames, codebooks); + } + + std::vector padded_codes; + padded_codes.reserve(static_cast((frames + kCodecTailContextFrames) * codebooks)); + padded_codes.insert(padded_codes.end(), codes.begin(), codes.end()); + const auto last_frame_begin = codes.end() - static_cast(codebooks); + for (int64_t i = 0; i < kCodecTailContextFrames; ++i) { + padded_codes.insert(padded_codes.end(), last_frame_begin, codes.end()); + } + + auto out = decode_codes_impl(padded_codes, frames + kCodecTailContextFrames, codebooks); + const int64_t keep_samples = frames * kCodecHopLength; + if (keep_samples <= 0 || keep_samples > static_cast(out.values.size())) { + throw std::runtime_error("Higgs TTS codec tail-context decode produced too few samples"); + } + out.values.resize(static_cast(keep_samples)); + out.samples = keep_samples; + return out; +} + +HiggsCodecDecodeOutput HiggsCodecRuntime::decode_codes_impl(const std::vector & codes, + int64_t frames, + int64_t codebooks) const { + if (frames <= 0) { + throw std::runtime_error("Higgs TTS codec decode requires positive frame count"); + } + if (codebooks != kCodecCodebooks) { + throw std::runtime_error("Higgs TTS codec decode requires exactly 8 codebooks"); + } + if (static_cast(codes.size()) != frames * codebooks) { + throw std::runtime_error("Higgs TTS codec decode code count mismatch"); + } engine::debug::trace_log_scalar("higgs_audio_tts.codec.decode.input_frames", frames); engine::debug::trace_log_scalar("higgs_audio_tts.codec.decode.input_codebooks", codebooks); engine::debug::trace_log_i32("higgs_audio_tts.codec.decode.input_codes",