From 66ef3ef14b5c44139aa8ee369a9e6ec22e0366ca Mon Sep 17 00:00:00 2001 From: KahazaTester Date: Tue, 25 Aug 2026 14:27:04 -0400 Subject: [PATCH] Fix DiffSinger rendering, Singer auto-detection & Language Fallback --- .../DiffSinger/DiffSingerBasePhonemizer.cs | 25 ++-- .../DiffSingerSpeakerEmbedManager.cs | 118 +++++++++++++----- .../Phonemizers/DiffSingerG2pPhonemizer.cs | 16 ++- OpenUtauMobile/Helpers/LocalizationManager.cs | 2 +- .../ViewModels/ClassicSingerSetupViewModel.cs | 50 +++++++- 5 files changed, 168 insertions(+), 43 deletions(-) diff --git a/OpenUtau.Core/DiffSinger/DiffSingerBasePhonemizer.cs b/OpenUtau.Core/DiffSinger/DiffSingerBasePhonemizer.cs index 82d6d0e3..4a92c7ab 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerBasePhonemizer.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerBasePhonemizer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -182,13 +182,24 @@ string[] GetSymbols(Note note) { return new string[] { }; } - string GetSpeakerAtIndex(Note note, int index){ + string GetSpeakerAtIndex(Note note, int index) { + if (dsConfig.speakers == null) return ""; var attr = note.phonemeAttributes?.FirstOrDefault(attr => attr.index == index) ?? default; var speaker = singer.Subbanks - .Where(subbank => subbank.Color == attr.voiceColor && subbank.toneSet.Contains(note.tone)) - .FirstOrDefault(); - if(speaker is null) { - return ""; + .FirstOrDefault(subbank => subbank.Color == attr.voiceColor && subbank.toneSet.Contains(note.tone)); + if (speaker is null) { + //Fall back to the first subbank matching the voice color + speaker = singer.Subbanks + .FirstOrDefault(subbank => subbank.Color == attr.voiceColor); + } + if (speaker is null) { + //Fall back to the first defined subbank + speaker = singer.Subbanks.FirstOrDefault(); + } + if (speaker is null) { + throw new Exception( + $"No subbanks defined for singer \"{singer.Name}\". " + + "Please check the singer's configuration."); } return speaker.Suffix; } @@ -433,7 +444,7 @@ protected override void ProcessPart(Note[][] phrase) { Note[] word = phrase[wordIndex]; var noteResult = new List>(); if (!wordFound[wordIndex]){ - //partResult[word[0].position] = noteResult; + partResult[word[0].position] = noteResult; continue; } if (word[0].lyric.StartsWith("+")) { diff --git a/OpenUtau.Core/DiffSinger/DiffSingerSpeakerEmbedManager.cs b/OpenUtau.Core/DiffSinger/DiffSingerSpeakerEmbedManager.cs index 979d49dd..c9ba066f 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerSpeakerEmbedManager.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerSpeakerEmbedManager.cs @@ -5,6 +5,7 @@ using Microsoft.ML.OnnxRuntime.Tensors; using NumSharp; +using Serilog; using OpenUtau.Core.Render; @@ -21,10 +22,11 @@ public DiffSingerSpeakerEmbedManager(DsConfig dsConfig, string rootPath) { this.dsConfig = dsConfig; this.rootPath = rootPath; } + public NDArray loadSpeakerEmbed(string speaker) { string path = Path.Join(rootPath, speaker + ".emb"); if(File.Exists(path)) { - var reader = new BinaryReader(File.OpenRead(path)); + using var reader = new BinaryReader(File.OpenRead(path)); return np.array(Enumerable.Range(0, dsConfig.hiddenSize) .Select(i => reader.ReadSingle())); } else { @@ -49,7 +51,7 @@ public NDArray getSpeakerEmbeds() { public bool IsVoiceColorCurve(string abbr, out int subBankId) { subBankId = 0; - if (abbr.StartsWith(VoiceColorHeader) && int.TryParse(abbr.Substring(2), out subBankId)) {; + if (abbr.StartsWith(VoiceColorHeader) && int.TryParse(abbr.Substring(2), out subBankId)) { subBankId -= 1; return true; } else { @@ -57,67 +59,123 @@ public bool IsVoiceColorCurve(string abbr, out int subBankId) { } } - public int getSpeakerIndexBySuffix(string suffix){ + static readonly HashSet warnedMissingSpeakerSuffixes = new(); + + public int getSpeakerIndexBySuffix(string suffix) { var speakerIndex = dsConfig.speakers.IndexOf(suffix); - if(speakerIndex == -1){ - speakerIndex = 0; + if (speakerIndex >= 0) { + return speakerIndex; + } + speakerIndex = dsConfig.speakers.FindIndex(s => { + var spSegs = s.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var sfSegs = suffix.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return sfSegs.Length <= spSegs.Length + && spSegs[^sfSegs.Length..].SequenceEqual(sfSegs); + }); + if (speakerIndex >= 0) { + return speakerIndex; } - return speakerIndex; + if (dsConfig.speakers == null || dsConfig.speakers.Count == 0) { + throw new InvalidOperationException( + "Subbanks are defined in character.yaml but \"speakers\" is empty in dsconfig.yaml."); + } + var fallback = dsConfig.speakers[0]; + var warnKey = $"{rootPath}|{suffix}|{fallback}"; + lock (warnedMissingSpeakerSuffixes) { + if (warnedMissingSpeakerSuffixes.Add(warnKey)) { + Log.Warning( + "Speaker suffix \"{Suffix}\" not found in dsConfig.speakers ({Candidates}). Falling back to \"{Fallback}\".", + suffix, + string.Join(", ", dsConfig.speakers), + fallback); + } + } + return 0; } //used by phonemizer (duration model) - public Tensor PhraseSpeakerEmbedByPhone(string[] speakerByPhone){ + public Tensor PhraseSpeakerEmbedByPhone(string[] speakerByPhone) { var hiddenSize = dsConfig.hiddenSize; var speakerEmbeds = getSpeakerEmbeds(); var totalPhones = speakerByPhone.Length; - NDArray spkCurves = np.zeros(totalPhones, dsConfig.speakers.Count); - foreach(int phoneId in Enumerable.Range(0,totalPhones)) { + var result = new float[totalPhones * hiddenSize]; + + for (int phoneId = 0; phoneId < totalPhones; phoneId++) { var spkId = getSpeakerIndexBySuffix(speakerByPhone[phoneId]); - spkCurves[phoneId, spkId] = 1; + var embed = speakerEmbeds[":", spkId].ToArray(); + var dest = result.AsSpan(phoneId * hiddenSize, hiddenSize); + embed.AsSpan().CopyTo(dest); } - var spkEmbedResult = np.dot(spkCurves, speakerEmbeds.T); - var spkEmbedTensor = new DenseTensor(spkEmbedResult.ToArray(), - new int[] { totalPhones, hiddenSize }) + + return new DenseTensor(result, new int[] { totalPhones, hiddenSize }) .Reshape(new int[] { 1, totalPhones, hiddenSize }); - return spkEmbedTensor; } //used by variance, pitch and acoustic - public Tensor PhraseSpeakerEmbedByFrame(RenderPhrase phrase, IList durations, float frameMs, int totalFrames, int headFrames, int tailFrames){ + public Tensor PhraseSpeakerEmbedByFrame(RenderPhrase phrase, IList durations, float frameMs, int totalFrames, int headFrames, int tailFrames) { var singer = phrase.singer; var hiddenSize = dsConfig.hiddenSize; var speakerEmbeds = getSpeakerEmbeds(); - //get default speaker for each phoneme + // Per-frame CLR / phoneme suffix is always weight 1.0 ("100%"). + // Voice-color curves add on top; then weights are normalized to a convex mix. + // Example: CLR=A and cl_B=100% → A:B = 1:1 (not pure B). var headDefaultSpk = getSpeakerIndexBySuffix(phrase.phones[0].suffix); var tailDefaultSpk = getSpeakerIndexBySuffix(phrase.phones[^1].suffix); var defaultSpkByFrame = Enumerable.Repeat(headDefaultSpk, headFrames).ToList(); defaultSpkByFrame.AddRange(Enumerable.Range(0, phrase.phones.Length) - .SelectMany(phIndex => Enumerable.Repeat(getSpeakerIndexBySuffix(phrase.phones[phIndex].suffix), durations[phIndex+1]))); + .SelectMany(phIndex => Enumerable.Repeat( + getSpeakerIndexBySuffix(phrase.phones[phIndex].suffix), + durations[phIndex + 1]))); defaultSpkByFrame.AddRange(Enumerable.Repeat(tailDefaultSpk, tailFrames)); //get speaker curves NDArray spkCurves = np.zeros(totalFrames, dsConfig.speakers.Count); - foreach(var curve in phrase.curves) { - if(IsVoiceColorCurve(curve.Item1,out int subBankId) && subBankId < singer.Subbanks.Count) { + foreach (var curve in phrase.curves) { + if (IsVoiceColorCurve(curve.Item1, out int subBankId) && subBankId < singer.Subbanks.Count) { var spkId = getSpeakerIndexBySuffix(singer.Subbanks[subBankId].Suffix); spkCurves[":", spkId] += DiffSingerUtils.SampleCurve(phrase, curve.Item2, 0, frameMs, totalFrames, headFrames, tailFrames, x => x * 0.01f) .Select(f => (float)f).ToArray(); } } - foreach(int frameId in Enumerable.Range(0,totalFrames)) { - //standarization - var spkSum = spkCurves[frameId, ":"].ToArray().Sum(); - if (spkSum > 1) { - spkCurves[frameId, ":"] /= spkSum; - } else { - spkCurves[frameId, defaultSpkByFrame[frameId]] += 1 - spkSum; + + int speakerCount = dsConfig.speakers.Count; + var result = new float[totalFrames * hiddenSize]; + var weights = new float[speakerCount]; + for (int frameId = 0; frameId < totalFrames; frameId++) { + Array.Clear(weights, 0, speakerCount); + int clrSpkId = defaultSpkByFrame[frameId]; + weights[clrSpkId] = 1f; + for (int spk = 0; spk < speakerCount; spk++) { + weights[spk] += (float)spkCurves[frameId, spk]; + } + + float weightSum = 0f; + for (int spk = 0; spk < speakerCount; spk++) { + if (weights[spk] < 0f) { + weights[spk] = 0f; + } + weightSum += weights[spk]; + } + if (weightSum < 1e-8f) { + weights[clrSpkId] = 1f; + weightSum = 1f; + } + + var dest = result.AsSpan(frameId * hiddenSize, hiddenSize); + dest.Clear(); + for (int spk = 0; spk < speakerCount; spk++) { + float w = weights[spk] / weightSum; + if (w < 1e-8f) { + continue; + } + var embed = speakerEmbeds[":", spk].ToArray(); + for (int j = 0; j < dest.Length; j++) { + dest[j] += w * embed[j]; + } } } - var spkEmbedResult = np.dot(spkCurves, speakerEmbeds.T); - var spkEmbedTensor = new DenseTensor(spkEmbedResult.ToArray(), - new int[] { totalFrames, hiddenSize }) + return new DenseTensor(result, new int[] { totalFrames, hiddenSize }) .Reshape(new int[] { 1, totalFrames, hiddenSize }); - return spkEmbedTensor; } } } diff --git a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs index c1648c33..f3d88ed8 100644 --- a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs +++ b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs @@ -1,4 +1,4 @@ -using Serilog; +using Serilog; using System; using System.Collections.Generic; using System.IO; @@ -52,6 +52,7 @@ protected override IG2p LoadG2p(string rootPath, bool useLangId = false) { // Load dictionary from singer folder. G2pDictionary.Builder g2pBuilder = new G2pDictionary.Builder(); var replacements = new Dictionary(); + bool dictFound = false; foreach(var dictionaryName in dictionaryNames){ string dictionaryPath = Path.Combine(rootPath, dictionaryName); if (File.Exists(dictionaryPath)) { @@ -66,13 +67,20 @@ protected override IG2p LoadG2p(string rootPath, bool useLangId = false) { phonemeSymbols[symbol.symbol.Trim()] = true; } } - Log.Error("Loaded symbols: " + string.Join(", ", phonemeSymbols.Keys)); + Log.Information("Loaded symbols: " + string.Join(", ", phonemeSymbols.Keys)); } catch (Exception e) { Log.Error(e, $"Failed to load {dictionaryPath}"); + throw new Exception($"Failed to load {dictionaryPath}", e); } + dictFound = true; break; } } + if(!dictFound){ + var triedPaths = string.Join(", ", dictionaryNames.Select(n => Path.Combine(rootPath, n))); + throw new FileNotFoundException( + $"No dictionary file found. Tried: {triedPaths}"); + } //SP and AP should always be vowel g2pBuilder.AddSymbol("SP", true); g2pBuilder.AddSymbol("AP", true); @@ -89,9 +97,9 @@ protected override IG2p LoadG2p(string rootPath, bool useLangId = false) { foreach(var c in GetBaseG2pConsonants()){ phonemeSymbols[c]=false; } - if(useLangId){ + var langCode = GetLangCode(); + if(!string.IsNullOrEmpty(langCode)){ //For diffsinger multi dict voicebanks, the replacements of g2p phonemes default to the / - var langCode = GetLangCode(); foreach(var ph in GetBaseG2pVowels().Concat(GetBaseG2pConsonants())){ if(!replacements.ContainsKey(ph)){ replacements[ph]=langCode + "/" + ph; diff --git a/OpenUtauMobile/Helpers/LocalizationManager.cs b/OpenUtauMobile/Helpers/LocalizationManager.cs index 5a934ff6..4aa96b19 100644 --- a/OpenUtauMobile/Helpers/LocalizationManager.cs +++ b/OpenUtauMobile/Helpers/LocalizationManager.cs @@ -21,7 +21,7 @@ public static class LocalizationManager public const string FollowSystemLanguageCode = "system"; private const string ExplicitLanguageFallbackCode = "en"; - private const string SystemLanguageFallbackCode = "zh-Hans"; + private const string SystemLanguageFallbackCode = "en"; /// Supported languages (language code => display name). public static readonly IReadOnlyList<(string Code, string DisplayName)> AvailableLanguages = diff --git a/OpenUtauMobile/ViewModels/ClassicSingerSetupViewModel.cs b/OpenUtauMobile/ViewModels/ClassicSingerSetupViewModel.cs index cfc97efa..b7185e95 100644 --- a/OpenUtauMobile/ViewModels/ClassicSingerSetupViewModel.cs +++ b/OpenUtauMobile/ViewModels/ClassicSingerSetupViewModel.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.ObjectModel; using System.IO; using System.Linq; @@ -99,6 +99,22 @@ public ClassicSingerSetupViewModel(MainViewModel navigator) : base(navigator) VoicebankConfig? config = LoadCharacterYaml(ArchiveFilePath); MissingInfo = config == null || string.IsNullOrEmpty(config.SingerType); + if (!MissingInfo) + { + // character.yaml has a SingerType — use it as default selection. + var declaredType = config!.SingerType; + if (SingerTypes.Contains(declaredType)) + { + SingerType = declaredType; + } + } + else + { + // No SingerType in yaml (or no yaml at all). + // Mirror VoicebankLoader.LoadInfo heuristic: scan archive for config files. + SingerType = DetectSingerTypeFromArchive(ArchiveFilePath); + } + if (!string.IsNullOrEmpty(config?.TextFileEncoding)) { try @@ -291,6 +307,38 @@ private bool IsEncrypted(string archiveFilePath) } } + /// + /// Scans archive entries for dsconfig.yaml or enuconfig.yaml to infer singer type. + /// Mirrors the legacy detection heuristic in VoicebankLoader.LoadInfo. + /// + private string DetectSingerTypeFromArchive(string archiveFilePath) + { + try + { + using (IArchive archive = ArchiveFactory.Open(archiveFilePath)) + { + bool hasDsconfig = archive.Entries.Any(e => + Path.GetFileName(e.Key) == "dsconfig.yaml"); + if (hasDsconfig) + { + return "diffsinger"; + } + + bool hasEnuconfig = archive.Entries.Any(e => + Path.GetFileName(e.Key) == "enuconfig.yaml"); + if (hasEnuconfig) + { + return "enunu"; + } + } + } + catch + { + // Fall through to default + } + return "utau"; + } + private async Task InstallAsync() { string archiveFilePath = ArchiveFilePath;