From f1747fbb4926d5f26be404901de9899429517679 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 21:41:15 -0400 Subject: [PATCH 01/30] Add files via upload --- melo/ConvertAudiotoWav.bat | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 melo/ConvertAudiotoWav.bat diff --git a/melo/ConvertAudiotoWav.bat b/melo/ConvertAudiotoWav.bat new file mode 100644 index 000000000..dc2636e47 --- /dev/null +++ b/melo/ConvertAudiotoWav.bat @@ -0,0 +1,46 @@ +@echo off +setlocal enabledelayedexpansion + +rem Specify input and output folders here +set "INPUT_FOLDER=data\example\audios" +set "OUTPUT_FOLDER=data\example\wavs" + +rem Create output folder if it doesn't exist +if not exist "%OUTPUT_FOLDER%" mkdir "%OUTPUT_FOLDER%" + +rem Initialize counter for sequential naming +set /a counter=1 + +rem Loop through common audio formats that ffmpeg supports +for %%F in ( + "%INPUT_FOLDER%\*.mp3" + "%INPUT_FOLDER%\*.m4a" + "%INPUT_FOLDER%\*.wav" + "%INPUT_FOLDER%\*.ogg" + "%INPUT_FOLDER%\*.flac" + "%INPUT_FOLDER%\*.aac" + "%INPUT_FOLDER%\*.wma" + "%INPUT_FOLDER%\*.aiff" + "%INPUT_FOLDER%\*.aifc" + "%INPUT_FOLDER%\*.opus" + "%INPUT_FOLDER%\*.ape" + "%INPUT_FOLDER%\*.wv" + "%INPUT_FOLDER%\*.m4b" + "%INPUT_FOLDER%\*.mp2" + "%INPUT_FOLDER%\*.mp4" + "%INPUT_FOLDER%\*.mpc" + "%INPUT_FOLDER%\*.mka" + "%INPUT_FOLDER%\*.ac3" + "%INPUT_FOLDER%\*.dts" + "%INPUT_FOLDER%\*.amr" + "%INPUT_FOLDER%\*.au" + "%INPUT_FOLDER%\*.mid" +) do ( + rem Convert the file using ffmpeg + ffmpeg -i "%%F" -acodec pcm_s16le -ar 44100 "%OUTPUT_FOLDER%\!counter!.wav" + + rem Increment the counter + set /a counter+=1 +) + +echo Conversion complete. Check the '%OUTPUT_FOLDER%' folder for the converted files. \ No newline at end of file From c7b87d1bdbc53fb602214cb98d9fe7ea442e36b7 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:05:53 -0400 Subject: [PATCH 02/30] Add files via upload --- melo/trim_model.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 melo/trim_model.py diff --git a/melo/trim_model.py b/melo/trim_model.py new file mode 100644 index 000000000..7c9b3b085 --- /dev/null +++ b/melo/trim_model.py @@ -0,0 +1,82 @@ +import os +import torch +import argparse +from pathlib import Path + +def clean_checkpoint(input_path, output_path=None): + """ + Convert a full MeloTTS training checkpoint to an inference-only checkpoint. + + Args: + input_path (str): Path to the full checkpoint + output_path (str, optional): Path for the clean checkpoint + """ + print(f"Loading checkpoint: {input_path}") + + # Load the checkpoint + checkpoint = torch.load(input_path, map_location='cpu') + + print("\nCheckpoint contents:") + print("Keys in checkpoint:", list(checkpoint.keys())) + + # Create clean checkpoint keeping only necessary components + clean_checkpoint = { + 'model': checkpoint['model'], # Keep the model state + 'iteration': checkpoint['iteration'], # Keep track of training iteration + 'learning_rate': checkpoint['learning_rate'], # Keep the learning rate + # Exclude the optimizer state + } + + # Generate output path if not provided + if output_path is None: + input_path = Path(input_path) + output_path = input_path.parent / f"{input_path.stem}_clean{input_path.suffix}" + + # Save the clean checkpoint + print(f"\nSaving clean checkpoint to: {output_path}") + torch.save(clean_checkpoint, output_path) + + # Print size comparison + original_size = os.path.getsize(input_path) / (1024 * 1024) # MB + clean_size = os.path.getsize(output_path) / (1024 * 1024) # MB + + print(f"\nSize comparison:") + print(f"Original checkpoint: {original_size:.2f} MB") + print(f"Clean checkpoint: {clean_size:.2f} MB") + print(f"Size reduction: {((original_size - clean_size) / original_size * 100):.1f}%") + + # Verify the clean checkpoint was saved properly + if os.path.getsize(output_path) == 0: + raise Exception("Error: Output file is empty!") + + # Load the clean checkpoint to verify it's valid + try: + test_load = torch.load(output_path, map_location='cpu') + print("\nVerification successful - cleaned checkpoint contains:") + print("Keys in cleaned checkpoint:", list(test_load.keys())) + except Exception as e: + raise Exception(f"Error verifying cleaned checkpoint: {str(e)}") + +def main(): + parser = argparse.ArgumentParser(description='Convert MeloTTS training checkpoint to inference-only checkpoint') + parser.add_argument('input_path', type=str, help='Path to the full checkpoint file') + parser.add_argument('--output_path', type=str, default=None, + help='Path for the clean checkpoint (optional)') + parser.add_argument('--debug', action='store_true', + help='Print additional debugging information') + + args = parser.parse_args() + + try: + clean_checkpoint(args.input_path, args.output_path) + print("\nCheckpoint cleaned successfully!") + except Exception as e: + print(f"\nError occurred: {str(e)}") + import traceback + traceback.print_exc() + return 1 + + return 0 + +if __name__ == "__main__": + main() \ No newline at end of file From 7d1ea7782a1badf08cc5a9764bb443ac019531cf Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:09:33 -0400 Subject: [PATCH 03/30] Add files via upload --- melo/trim_model.bat | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 melo/trim_model.bat diff --git a/melo/trim_model.bat b/melo/trim_model.bat new file mode 100644 index 000000000..771b0f3d8 --- /dev/null +++ b/melo/trim_model.bat @@ -0,0 +1,3 @@ +conda activate melotts-win + +python trim_model.py "Path\To\Model\G_xxxx.pth" --output_path "Path\To\Save\Model\G_xxxx_cleaned.pth" \ No newline at end of file From c7259b9e0361b04dd3308004a1d18e27e8335469 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:12:56 -0400 Subject: [PATCH 04/30] Update preprocess_text.py adjust default settings for training/resume training and config file creation --- melo/preprocess_text.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/melo/preprocess_text.py b/melo/preprocess_text.py index 8fdf87295..5e01ff81e 100644 --- a/melo/preprocess_text.py +++ b/melo/preprocess_text.py @@ -149,10 +149,16 @@ def main( config["num_tones"] = num_tones config["symbols"] = symbols + # Added lines: + config["train"]["skip_optimizer"] = False # Ensure skip_optimizer is False by default. Seems this must be false to be able to resume training later. + config["train"]["n_ckpts_to_keep"] = 5 # Add checkpoint keeping parameter + config["train"]["eval_interval"] = 200 # How often checkpoints are saved. Set default eval_interval to 200, removing this line will default it to 1000 + config["train"]["batch_size"] = 12 # Set default batch_size to 12, removing this line will default it to 6 + with open(out_config_path, "w", encoding="utf-8") as f: json.dump(config, f, indent=2, ensure_ascii=False) logger.info("Preprocessing completed successfully") if __name__ == "__main__": - main() \ No newline at end of file + main() From 31d770c388f53b62e6126b0f93e4724c00d05f2c Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:16:17 -0400 Subject: [PATCH 05/30] Update transcript.py default to "medium" whisper model (for balance between speed and quality), as "base" is good and faster but tends to have more transcription mistakes. --- melo/transcript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/melo/transcript.py b/melo/transcript.py index f17f02809..8111eaf7a 100644 --- a/melo/transcript.py +++ b/melo/transcript.py @@ -8,7 +8,7 @@ LANGUAGE_MODEL = "EN-default" # Load the whisper model -model = whisper.load_model("base") +model = whisper.load_model("medium") # Get the list of WAV files in the input directory wav_files = [file for file in os.listdir(INPUT_FOLDER) if file.endswith(".wav")] @@ -37,4 +37,4 @@ line += "\n" transcript_file.write(line) -print(f"Transcription complete. Check '{OUTPUT_FILE}' for results.") \ No newline at end of file +print(f"Transcription complete. Check '{OUTPUT_FILE}' for results.") From 896be6e501923e17584930c9ac623adf746ea84a Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:39:09 -0400 Subject: [PATCH 06/30] Add files via upload --- melo/transcript_fast.py | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 melo/transcript_fast.py diff --git a/melo/transcript_fast.py b/melo/transcript_fast.py new file mode 100644 index 000000000..bb7797694 --- /dev/null +++ b/melo/transcript_fast.py @@ -0,0 +1,42 @@ +import os +os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' +from faster_whisper import WhisperModel + +# Configuration variables +INPUT_FOLDER = "data\\example\\wavs" +OUTPUT_FILE = "data\\example\\metadata.list" +LANGUAGE_CODE = "EN" +LANGUAGE_MODEL = "EN-default" + +# Initialize the faster-whisper model +model = WhisperModel("medium", device="cuda", compute_type="int8_float16") + +# Get the list of WAV files in the input directory +wav_files = [file for file in os.listdir(INPUT_FOLDER) if file.endswith(".wav")] + +# Sort the WAV files in numeric order +wav_files = sorted(wav_files, key=lambda x: int(os.path.splitext(x)[0])) + +# Open a text file for writing the transcripts +with open(OUTPUT_FILE, "w", encoding="utf-8") as transcript_file: + # Prepare the output path with forward slashes + output_path = INPUT_FOLDER.replace('\\', '/') + + # Iterate through each WAV file + for i, wav_file in enumerate(wav_files): + print(f"Transcribing: {wav_file}") + # Construct the full path to the WAV file + wav_path = os.path.join(INPUT_FOLDER, wav_file) + + # Transcribe using faster-whisper + segments, info = model.transcribe(wav_path) + # Get the full text from all segments + transcribed_text = " ".join([segment.text for segment in segments]).strip() + + # Write the result to the transcript file in the MeloTTS format + line = f"{output_path}/{wav_file}|{LANGUAGE_MODEL}|{LANGUAGE_CODE}|{transcribed_text}" + if i < len(wav_files) - 1: + line += "\n" + transcript_file.write(line) + +print(f"Transcription complete. Check '{OUTPUT_FILE}' for results.") \ No newline at end of file From bc854b1c4658eff35fb52c54b338b50211372308 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:53:44 -0400 Subject: [PATCH 07/30] Update transcript_fast.py add cmd arguments --- melo/transcript_fast.py | 79 ++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/melo/transcript_fast.py b/melo/transcript_fast.py index bb7797694..6d253b094 100644 --- a/melo/transcript_fast.py +++ b/melo/transcript_fast.py @@ -1,42 +1,57 @@ import os +import argparse os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' from faster_whisper import WhisperModel -# Configuration variables -INPUT_FOLDER = "data\\example\\wavs" -OUTPUT_FILE = "data\\example\\metadata.list" -LANGUAGE_CODE = "EN" -LANGUAGE_MODEL = "EN-default" +def transcribe_folder(input_folder, output_file): + LANGUAGE_CODE = "EN" + LANGUAGE_MODEL = "EN-default" -# Initialize the faster-whisper model -model = WhisperModel("medium", device="cuda", compute_type="int8_float16") + # Initialize the faster-whisper model + model = WhisperModel("medium", device="cuda", compute_type="int8_float16") -# Get the list of WAV files in the input directory -wav_files = [file for file in os.listdir(INPUT_FOLDER) if file.endswith(".wav")] + # Get the list of WAV files in the input directory + wav_files = [file for file in os.listdir(input_folder) if file.endswith(".wav")] -# Sort the WAV files in numeric order -wav_files = sorted(wav_files, key=lambda x: int(os.path.splitext(x)[0])) + # Sort the WAV files in numeric order + wav_files = sorted(wav_files, key=lambda x: int(os.path.splitext(x)[0])) -# Open a text file for writing the transcripts -with open(OUTPUT_FILE, "w", encoding="utf-8") as transcript_file: - # Prepare the output path with forward slashes - output_path = INPUT_FOLDER.replace('\\', '/') - - # Iterate through each WAV file - for i, wav_file in enumerate(wav_files): - print(f"Transcribing: {wav_file}") - # Construct the full path to the WAV file - wav_path = os.path.join(INPUT_FOLDER, wav_file) - - # Transcribe using faster-whisper - segments, info = model.transcribe(wav_path) - # Get the full text from all segments - transcribed_text = " ".join([segment.text for segment in segments]).strip() + # Open a text file for writing the transcripts + with open(output_file, "w", encoding="utf-8") as transcript_file: + # Prepare the output path with forward slashes + output_path = input_folder.replace('\\', '/') - # Write the result to the transcript file in the MeloTTS format - line = f"{output_path}/{wav_file}|{LANGUAGE_MODEL}|{LANGUAGE_CODE}|{transcribed_text}" - if i < len(wav_files) - 1: - line += "\n" - transcript_file.write(line) + # Iterate through each WAV file + for i, wav_file in enumerate(wav_files): + print(f"Transcribing: {wav_file}") + wav_path = os.path.join(input_folder, wav_file) + + segments, info = model.transcribe(wav_path) + transcribed_text = " ".join([segment.text for segment in segments]).strip() + + line = f"{output_path}/{wav_file}|{LANGUAGE_MODEL}|{LANGUAGE_CODE}|{transcribed_text}" + if i < len(wav_files) - 1: + line += "\n" + transcript_file.write(line) -print(f"Transcription complete. Check '{OUTPUT_FILE}' for results.") \ No newline at end of file + print(f"Transcription complete. Check '{output_file}' for results.") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Transcribe WAV files using faster-whisper') + parser.add_argument('--wavs_path', type=str, default="data\\example\\wavs", + help='Path to the directory containing WAV files (default: data\\example\\wavs)') + parser.add_argument('--metadata_path', type=str, default="data\\example\\metadata.list", + help='Path for the output metadata.list file (default: data\\example\\metadata.list)') + + args = parser.parse_args() + + # Check if the input directory exists + if not os.path.exists(args.wavs_path): + print(f"Error: Directory '{args.wavs_path}' does not exist.") + exit(1) + + # Create output directory if it doesn't exist + os.makedirs(os.path.dirname(args.metadata_path), exist_ok=True) + + # Run transcription + transcribe_folder(args.wavs_path, args.metadata_path) From bea99313516ce33ca662d76853d36d646b1a0791 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:57:00 -0400 Subject: [PATCH 08/30] Add files via upload --- melo/transcript_fast.bat | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 melo/transcript_fast.bat diff --git a/melo/transcript_fast.bat b/melo/transcript_fast.bat new file mode 100644 index 000000000..8bc6fd23a --- /dev/null +++ b/melo/transcript_fast.bat @@ -0,0 +1,4 @@ +conda activate melotts-win + +python transcript_fast.py + From 739d25b89cc6b9555e90ad57d76ceafe71638beb Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:58:40 -0400 Subject: [PATCH 09/30] Update requirements.txt added faster-whisper --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6df4ce360..c674afb69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ langid==1.1.6 tqdm tensorboard==2.16.2 loguru==0.7.2 +faster-whisper==0.9.0 From 35dd0c07cce82cbe7cfdd835925ad63c8353bb9b Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:06:59 -0400 Subject: [PATCH 10/30] Update ConvertAudiotoWav.bat --- melo/ConvertAudiotoWav.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/melo/ConvertAudiotoWav.bat b/melo/ConvertAudiotoWav.bat index dc2636e47..ca893831c 100644 --- a/melo/ConvertAudiotoWav.bat +++ b/melo/ConvertAudiotoWav.bat @@ -2,7 +2,7 @@ setlocal enabledelayedexpansion rem Specify input and output folders here -set "INPUT_FOLDER=data\example\audios" +set "INPUT_FOLDER=data\example\audio" set "OUTPUT_FOLDER=data\example\wavs" rem Create output folder if it doesn't exist @@ -43,4 +43,4 @@ for %%F in ( set /a counter+=1 ) -echo Conversion complete. Check the '%OUTPUT_FOLDER%' folder for the converted files. \ No newline at end of file +echo Conversion complete. Check the '%OUTPUT_FOLDER%' folder for the converted files. From 7350be03e7da73654a00220531245cb958d37dc9 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:38:44 -0400 Subject: [PATCH 11/30] Update README.md --- README.md | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 05f19e9c2..28214871a 100644 --- a/README.md +++ b/README.md @@ -26,25 +26,38 @@ If you have trouble doing the download with the `python -m unidic download` you pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 ``` +3. Prepare faster-whisper (optional for fast transcribing of audio files): + - If you have cuda/cublas errors, download this `https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z`, extract and place the 5 dll files directly into the `MeloTTS-Windows/melo/` folder + 4. Run using: ``` melo-ui ``` # Local Training on Windows +## Preparing Dataset 1. In the `melo/data/example` folder, delete the example `metadata.list` file. -2. If you need to convert mp3 to wav, create a folder called `mp3s` in the example folder and copy all your mp3 files into the `mp3s` folder -3. With a conda window activated with the enviroment open in the `melo` folder, run `ConvertMp3toWav.bat` from the conda prompt. This will create a folder `data/example/wavs` with all of the converted wav files. -4. Create a transcript file by running `python transcript.py` which will create a `data/example/metadata.list` file. -5. Run `python preprocess_text.py --metadata data/example/metadata.list` to create the `train.list`, `config.json`, among other files in the `data/example` folder. -6. Modify `config.json` to change the batch size, epochs, learning rate, etc. -7. From the conda prompt run `train.bat` to start the training. -8. File will be created within the `data/example/config` folder with the checkpoints and other logging information. -9. To test out a checkpoint, run: `python infer.py --text "this is a test" -m "C:\ai\MeloTTS-Windows\melo\data\example\config\G_0.pth" -o output` changing the G_0 to the checkpoint you want to test with G_1000, G2000, etc. -10. When you want to use a checkpoint from the UI, create a `melo/custom` folder and copy the .pth and `config.json` file over from the `data/example/config`, rename the .pth to a user-friendly name, and launch the UI to see it in the custom voice dropdown. -11. To see the tensorboard, install `pip install tensorflow` -12. Run `tensorboard --logdir=data\example\config` -13. This will give you the local URL to view the tensorboard. +2. MeloTTS expects wav audio files (with a sample rate of 44100Hz). If you need to convert audio to wav format (with 44100Hz sample rate), create a folder called `audio` in the example folder and copy all your audio files into the `audio` folder +4. With a conda window activated with the enviroment open in the `melo` folder, run `ConvertAudiotoWav.bat` from the conda prompt. This will create a folder `data/example/wavs` with all of the converted wav files. +5. Create a transcript file by running `transcript_fast.bat` which will create a `data/example/metadata.list` file using faster-whisper. Alternately, you can run `python transcript.py` to use the original whisper. +6. Run `python preprocess_text.py --metadata data/example/metadata.list` to create the `train.list`, `config.json`, among other files in the `data/example` folder. +7. Modify `config.json` to change the batch size, epochs, learning rate, etc. + - ⚠️ **Important, If you plan to Resume Training Later:** + - The `eval_interval` setting determines how frequently your model is saved during training + - For example, if `eval_interval=1000`, the model saves only once every 1000 epochs + - If you stop training between save points, any progress since the last save will be lost + - For safer training sessions that you may need to resume later, use a smaller `eval_interval` value + - You can also adjust `n_ckpts_to_keep` to limit the max models kept (if `n_ckpts_to_keep=5`, it will delete the oldest models when their are more than 5 saved models) +## Start Training +1. From the conda prompt run `train.bat` to start the training. +2. File will be created within the `data/example/config` folder with the checkpoints and other logging information. +3. To test out a checkpoint, run: `python infer.py --text "this is a test" -m "C:\ai\MeloTTS-Windows\melo\data\example\config\G_0.pth" -o output` changing the G_0 to the checkpoint you want to test with G_1000, G2000, etc. +4. When you want to use a checkpoint from the UI, create a `melo/custom` folder and copy the .pth and `config.json` file over from the `data/example/config`, rename the .pth to a user-friendly name, and launch the UI to see it in the custom voice dropdown. +5. To see the tensorboard, install `pip install tensorflow` +6. Run `tensorboard --logdir=data\example\config` +7. This will give you the local URL to view the tensorboard. +## Resuming Training +1. From the conda prompt run `train.bat` again to resume the training. The training will resume from the newest G_XXXX.pth file. # Original Readme:
From 22fd00061805564ef4a390544e4d059e89e33772 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:43:29 -0400 Subject: [PATCH 12/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 28214871a..5115a7fc6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ pip3 install torch torchvision torchaudio --index-url https://download.pytorch.o ``` 3. Prepare faster-whisper (optional for fast transcribing of audio files): - - If you have cuda/cublas errors, download this `https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z`, extract and place the 5 dll files directly into the `MeloTTS-Windows/melo/` folder + - Download cuda/cublas here [https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z](https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z), extract and place the 5 dll files directly into the `MeloTTS-Windows/melo/` folder 4. Run using: ``` From 5aa498e043b5c848112dd0bbf02501c3f5b6ab6d Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:53:02 -0400 Subject: [PATCH 13/30] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5115a7fc6..e68d678ef 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ melo-ui 7. This will give you the local URL to view the tensorboard. ## Resuming Training 1. From the conda prompt run `train.bat` again to resume the training. The training will resume from the newest G_XXXX.pth file. +## Trimming Model +You can trim your model to make it a way smaller filesize (which will make it load faster during the model loading process). When testing, this made the model filesize about 66% smaller. Note the created trimmed model is for inference-only(using the model just to generate audio from text) and you won't be able to train it further. +1. Open `trim_model.bat` file in a text editor to change the directory of your G_XXXX.pth file and the save location, save the changes, then run `trim_model.bat` to create a trimmed model for inference only. # Original Readme:
From 841f6e5211fdf1fff492f2981baa0bf06f5384d5 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Mon, 28 Oct 2024 02:56:39 -0400 Subject: [PATCH 14/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e68d678ef..c20dc34aa 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ melo-ui 7. Modify `config.json` to change the batch size, epochs, learning rate, etc. - ⚠️ **Important, If you plan to Resume Training Later:** - The `eval_interval` setting determines how frequently your model is saved during training - - For example, if `eval_interval=1000`, the model saves only once every 1000 epochs + - For example, if `eval_interval=1000`, the model saves only once every 1000 steps - If you stop training between save points, any progress since the last save will be lost - For safer training sessions that you may need to resume later, use a smaller `eval_interval` value - You can also adjust `n_ckpts_to_keep` to limit the max models kept (if `n_ckpts_to_keep=5`, it will delete the oldest models when their are more than 5 saved models) From bbd7b7ee78b9fd8e48526a4b1df6d0d78b8b22c3 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:41:32 -0400 Subject: [PATCH 15/30] Update config.json changed defaults for template to help with resuming training --- melo/configs/config.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/melo/configs/config.json b/melo/configs/config.json index f93ce6660..8f7a3335a 100644 --- a/melo/configs/config.json +++ b/melo/configs/config.json @@ -1,7 +1,7 @@ { "train": { "log_interval": 200, - "eval_interval": 1000, + "eval_interval": 200, "seed": 52, "epochs": 10000, "learning_rate": 0.0003, @@ -10,7 +10,7 @@ 0.99 ], "eps": 1e-09, - "batch_size": 6, + "batch_size": 12, "fp16_run": false, "lr_decay": 0.999875, "segment_size": 16384, @@ -18,7 +18,8 @@ "warmup_epochs": 0, "c_mel": 45, "c_kl": 1.0, - "skip_optimizer": true + "skip_optimizer": false, + "keep_ckpts": 5 }, "data": { "training_files": "", From 7ab914defa10969e3679dd487437393a34ad4f45 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:42:30 -0400 Subject: [PATCH 16/30] Update preprocess_text.py removed changes, changed config.json template instead --- melo/preprocess_text.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/melo/preprocess_text.py b/melo/preprocess_text.py index 5e01ff81e..c190a5116 100644 --- a/melo/preprocess_text.py +++ b/melo/preprocess_text.py @@ -149,12 +149,6 @@ def main( config["num_tones"] = num_tones config["symbols"] = symbols - # Added lines: - config["train"]["skip_optimizer"] = False # Ensure skip_optimizer is False by default. Seems this must be false to be able to resume training later. - config["train"]["n_ckpts_to_keep"] = 5 # Add checkpoint keeping parameter - config["train"]["eval_interval"] = 200 # How often checkpoints are saved. Set default eval_interval to 200, removing this line will default it to 1000 - config["train"]["batch_size"] = 12 # Set default batch_size to 12, removing this line will default it to 6 - with open(out_config_path, "w", encoding="utf-8") as f: json.dump(config, f, indent=2, ensure_ascii=False) From aa7b043237307af8c0f924e323cd7faadbc7b2c5 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:19:25 -0400 Subject: [PATCH 17/30] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c20dc34aa..fbe1ab622 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ pip3 install torch torchvision torchaudio --index-url https://download.pytorch.o 3. Prepare faster-whisper (optional for fast transcribing of audio files): - Download cuda/cublas here [https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z](https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z), extract and place the 5 dll files directly into the `MeloTTS-Windows/melo/` folder + - to prevent conflicts, run this from the conda window `pip install transformers==4.30.2 huggingface_hub==0.16.4` 4. Run using: ``` From 44a5bb2f7b3c0f24f4e5fb893ff71c2046581d65 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:21:57 -0400 Subject: [PATCH 18/30] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fbe1ab622..426ddd579 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,11 @@ pip3 install torch torchvision torchaudio --index-url https://download.pytorch.o 3. Prepare faster-whisper (optional for fast transcribing of audio files): - Download cuda/cublas here [https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z](https://github.com/Purfview/whisper-standalone-win/releases/download/libs/cuBLAS.and.cuDNN_CUDA11_win_v2.7z), extract and place the 5 dll files directly into the `MeloTTS-Windows/melo/` folder - - to prevent conflicts, run this from the conda window `pip install transformers==4.30.2 huggingface_hub==0.16.4` + - To install faster-whisper (and prevent conflicts with it) run this from the conda window: +``` +pip install faster-whisper==0.9.0 +pip install transformers==4.30.2 huggingface_hub==0.16.4 +``` 4. Run using: ``` From dff2d0053c53fedccde8a8bee18b8bb127e27019 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:22:16 -0400 Subject: [PATCH 19/30] Update requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c674afb69..6df4ce360 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,4 +28,3 @@ langid==1.1.6 tqdm tensorboard==2.16.2 loguru==0.7.2 -faster-whisper==0.9.0 From 8108d5a115e5b86f23574320d241328e3bdb3508 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Thu, 31 Oct 2024 03:52:39 -0400 Subject: [PATCH 20/30] Add files via upload --- melo/test_melotts.py | 140 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 melo/test_melotts.py diff --git a/melo/test_melotts.py b/melo/test_melotts.py new file mode 100644 index 000000000..70c3a7460 --- /dev/null +++ b/melo/test_melotts.py @@ -0,0 +1,140 @@ +from melo.api import TTS +import sounddevice as sd +import numpy as np +from glob import glob +import os + +def get_ckpt_files(folder_path): + return [os.path.basename(f) for f in glob(os.path.join(folder_path, '*.pth'))] + +def get_config_path(ckpt_path): + """Get the corresponding config path for a checkpoint file""" + base_path = os.path.splitext(ckpt_path)[0] + config_options = [ + f"{base_path}.config.json", # try g_1000.config.json + f"{base_path}.json", # try g_1000.json + ] + + for config_path in config_options: + if os.path.exists(config_path): + return config_path + return None + +def load_custom_model(ckpt_file): + ckpt_path = os.path.join("custom", ckpt_file) + config_path = get_config_path(ckpt_path) + + if config_path is None: + print(f"Error: No config file found for {ckpt_file}") + print("Expected either:") + print(f"- {os.path.splitext(ckpt_file)[0]}.config.json") + print(f"- {os.path.splitext(ckpt_file)[0]}.json") + return None + + try: + return TTS(language="EN", config_path=config_path, ckpt_path=ckpt_path) + except Exception as e: + print(f"Error loading custom model: {e}") + return None + + +def get_speaker_names(model): + """Get mapping of speaker IDs to names""" + return model.hps.data.spk2id + +def list_available_models(): + models = { + 'EN': TTS(language='EN'), + 'ES': TTS(language='ES'), + 'FR': TTS(language='FR'), + 'ZH': TTS(language='ZH'), + 'JP': TTS(language='JP'), + 'KR': TTS(language='KR'), + } + return models + +def select_speaker(model): + """Let user select a speaker from available options""" + speakers = get_speaker_names(model) + + print("\nAvailable speakers:") + for name, id in speakers.items(): + print(f"- {name} (ID: {id})") + + while True: + speaker = input("\nEnter speaker name or ID: ") + # Check if input is a number + if speaker.isdigit(): + id_num = int(speaker) + # Check if ID exists in values + for name, spk_id in speakers.items(): + if spk_id == id_num: + return id_num + print("Invalid speaker ID. Please try again.") + # Check if input is a speaker name + elif speaker in speakers: + return speakers[speaker] + print("Invalid input. Enter either speaker name or ID number.") + +def main(): + print("Loading models...") + models = list_available_models() + + # Check for custom models + custom_folder = os.path.join("custom") + custom_models = get_ckpt_files(custom_folder) + + print("\nAvailable languages:") + for lang in models.keys(): + print(f"- {lang}") + + if custom_models: + print("\nAvailable custom models:") + for model in custom_models: + print(f"- {model}") + + use_custom = input("\nUse custom model? (y/n): ").lower() == 'y' + + if use_custom: + if not custom_models: + print("No custom models found in 'custom' directory") + return + + print("\nSelect custom model:") + for i, model in enumerate(custom_models): + print(f"{i+1}. {model}") + + try: + choice = int(input("Enter number: ")) - 1 + model = load_custom_model(custom_models[choice]) + if not model: + return + speaker_id = select_speaker(model) + except (ValueError, IndexError): + print("Invalid selection") + return + else: + lang = input("\nSelect language (EN/ES/FR/ZH/JP/KR): ").upper() + if lang not in models: + print("Invalid language selected") + return + + model = models[lang] + speaker_id = select_speaker(model) + + print("\nEnter text to speak (Type 'quit' to exit)") + while True: + text = input("\nText (Type 'quit' to exit): ") + if text.lower() == 'quit': + break + + try: + audio = model.tts_to_file(text, speaker_id, pbar=None, quiet=True) + sd.play(audio, model.hps.data.sampling_rate) + sd.wait() + + except Exception as e: + print(f"Error generating speech: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file From d1e9f92b54c1c0bf5c6052f4d8a0de0cf1c04a41 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Thu, 31 Oct 2024 03:56:42 -0400 Subject: [PATCH 21/30] Update and rename test_melotts.py to try.py --- melo/{test_melotts.py => try.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename melo/{test_melotts.py => try.py} (96%) diff --git a/melo/test_melotts.py b/melo/try.py similarity index 96% rename from melo/test_melotts.py rename to melo/try.py index 70c3a7460..5bb74760b 100644 --- a/melo/test_melotts.py +++ b/melo/try.py @@ -137,4 +137,4 @@ def main(): print(f"Error generating speech: {e}") if __name__ == "__main__": - main() \ No newline at end of file + main() From c7464457521e02dfef137bd60544f954934d9d97 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Fri, 1 Nov 2024 22:58:44 -0400 Subject: [PATCH 22/30] Add files via upload --- melo/trim_models.py | 144 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 melo/trim_models.py diff --git a/melo/trim_models.py b/melo/trim_models.py new file mode 100644 index 000000000..af6672178 --- /dev/null +++ b/melo/trim_models.py @@ -0,0 +1,144 @@ +import os +import torch +import argparse +from pathlib import Path + +def clean_checkpoint(input_path): + """ + Convert a full MeloTTS training checkpoint to an inference-only checkpoint. + + Args: + input_path (Path): Path to the full checkpoint + """ + print(f"Processing checkpoint: {input_path}") + + # Load the checkpoint + checkpoint = torch.load(input_path, map_location='cpu') + + print("\nCheckpoint contents:") + print("Keys in checkpoint:", list(checkpoint.keys())) + + # Create clean checkpoint keeping only necessary components + clean_checkpoint = { + 'model': checkpoint['model'], # Keep the model state + 'iteration': checkpoint['iteration'], # Keep track of training iteration + 'learning_rate': checkpoint['learning_rate'], # Keep the learning rate + # Exclude the optimizer state + } + + # Generate output path with _trimmed suffix + output_path = input_path.parent / f"{input_path.stem}_trimmed{input_path.suffix}" + + # Save the clean checkpoint + print(f"\nSaving clean checkpoint to: {output_path}") + torch.save(clean_checkpoint, output_path) + + # Print size comparison + original_size = os.path.getsize(input_path) / (1024 * 1024) # MB + clean_size = os.path.getsize(output_path) / (1024 * 1024) # MB + + print(f"\nSize comparison:") + print(f"Original checkpoint: {original_size:.2f} MB") + print(f"Clean checkpoint: {clean_size:.2f} MB") + print(f"Size reduction: {((original_size - clean_size) / original_size * 100):.1f}%") + + # Verify the clean checkpoint was saved properly + if os.path.getsize(output_path) == 0: + raise Exception("Error: Output file is empty!") + + # Load the clean checkpoint to verify it's valid + try: + test_load = torch.load(output_path, map_location='cpu') + print("\nVerification successful - cleaned checkpoint contains:") + print("Keys in cleaned checkpoint:", list(test_load.keys())) + except Exception as e: + raise Exception(f"Error verifying cleaned checkpoint: {str(e)}") + +def should_process_file(filename): + """ + Check if the file should be processed based on filename patterns. + + Args: + filename (str): Name of the file to check + + Returns: + bool: True if file should be processed, False otherwise + """ + excluded_prefixes = ['D_', 'DUR_'] + return not any(filename.startswith(prefix) for prefix in excluded_prefixes) + +def process_directory(directory_path, debug=False): + """ + Process all eligible .pth files in the given directory. + + Args: + directory_path (str): Path to directory containing checkpoints + debug (bool): Whether to print debug information + """ + # Clean up the path string and convert to Path object + directory_path = directory_path.strip().strip('"').strip("'") + directory = Path(directory_path).resolve() + + if debug: + print(f"Attempting to process directory: {directory}") + + if not directory.exists(): + raise ValueError(f"Directory does not exist: {directory}") + + if not directory.is_dir(): + raise ValueError(f"Path is not a directory: {directory}") + + # Find all .pth files in the directory that don't start with excluded prefixes + pth_files = [f for f in directory.glob("*.pth") if should_process_file(f.name)] + + if not pth_files: + print(f"No eligible .pth files found in {directory}") + return + + print(f"Found {len(pth_files)} eligible .pth files to process") + + # Print files that will be skipped if debug is enabled + if debug: + all_pth_files = list(directory.glob("*.pth")) + skipped_files = [f for f in all_pth_files if not should_process_file(f.name)] + if skipped_files: + print("\nSkipping the following files:") + for f in skipped_files: + print(f"- {f.name}") + + success_count = 0 + for pth_file in pth_files: + try: + print(f"\nProcessing {pth_file.name}...") + clean_checkpoint(pth_file) + success_count += 1 + except Exception as e: + print(f"Error processing {pth_file.name}: {str(e)}") + if debug: + import traceback + traceback.print_exc() + + print(f"\nProcessing complete!") + print(f"Successfully processed {success_count} out of {len(pth_files)} files") + +def main(): + parser = argparse.ArgumentParser(description='Convert MeloTTS training checkpoints to inference-only checkpoints') + parser.add_argument('directory', type=str, help='Directory containing checkpoint files') + parser.add_argument('--debug', action='store_true', + help='Print additional debugging information') + + args = parser.parse_args() + + try: + process_directory(args.directory, args.debug) + except Exception as e: + print(f"\nError occurred: {str(e)}") + if args.debug: + import traceback + traceback.print_exc() + return 1 + + return 0 + +if __name__ == "__main__": + main() \ No newline at end of file From bf02388cd231fad2c39d237d8e25f2cfe6c16810 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Fri, 1 Nov 2024 22:59:25 -0400 Subject: [PATCH 23/30] Update and rename trim_model.bat to trim_models.bat --- melo/trim_model.bat | 3 --- melo/trim_models.bat | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 melo/trim_model.bat create mode 100644 melo/trim_models.bat diff --git a/melo/trim_model.bat b/melo/trim_model.bat deleted file mode 100644 index 771b0f3d8..000000000 --- a/melo/trim_model.bat +++ /dev/null @@ -1,3 +0,0 @@ -conda activate melotts-win - -python trim_model.py "Path\To\Model\G_xxxx.pth" --output_path "Path\To\Save\Model\G_xxxx_cleaned.pth" \ No newline at end of file diff --git a/melo/trim_models.bat b/melo/trim_models.bat new file mode 100644 index 000000000..9e11e4f51 --- /dev/null +++ b/melo/trim_models.bat @@ -0,0 +1,3 @@ +conda activate melotts-win + +python trim_models.py "Path\To\Models\" From 3d729fdb38754e35cbac3f892c86064a4c8411e1 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Fri, 1 Nov 2024 23:00:01 -0400 Subject: [PATCH 24/30] Delete melo/trim_model.py --- melo/trim_model.py | 82 ---------------------------------------------- 1 file changed, 82 deletions(-) delete mode 100644 melo/trim_model.py diff --git a/melo/trim_model.py b/melo/trim_model.py deleted file mode 100644 index 7c9b3b085..000000000 --- a/melo/trim_model.py +++ /dev/null @@ -1,82 +0,0 @@ -import os -import torch -import argparse -from pathlib import Path - -def clean_checkpoint(input_path, output_path=None): - """ - Convert a full MeloTTS training checkpoint to an inference-only checkpoint. - - Args: - input_path (str): Path to the full checkpoint - output_path (str, optional): Path for the clean checkpoint - """ - print(f"Loading checkpoint: {input_path}") - - # Load the checkpoint - checkpoint = torch.load(input_path, map_location='cpu') - - print("\nCheckpoint contents:") - print("Keys in checkpoint:", list(checkpoint.keys())) - - # Create clean checkpoint keeping only necessary components - clean_checkpoint = { - 'model': checkpoint['model'], # Keep the model state - 'iteration': checkpoint['iteration'], # Keep track of training iteration - 'learning_rate': checkpoint['learning_rate'], # Keep the learning rate - # Exclude the optimizer state - } - - # Generate output path if not provided - if output_path is None: - input_path = Path(input_path) - output_path = input_path.parent / f"{input_path.stem}_clean{input_path.suffix}" - - # Save the clean checkpoint - print(f"\nSaving clean checkpoint to: {output_path}") - torch.save(clean_checkpoint, output_path) - - # Print size comparison - original_size = os.path.getsize(input_path) / (1024 * 1024) # MB - clean_size = os.path.getsize(output_path) / (1024 * 1024) # MB - - print(f"\nSize comparison:") - print(f"Original checkpoint: {original_size:.2f} MB") - print(f"Clean checkpoint: {clean_size:.2f} MB") - print(f"Size reduction: {((original_size - clean_size) / original_size * 100):.1f}%") - - # Verify the clean checkpoint was saved properly - if os.path.getsize(output_path) == 0: - raise Exception("Error: Output file is empty!") - - # Load the clean checkpoint to verify it's valid - try: - test_load = torch.load(output_path, map_location='cpu') - print("\nVerification successful - cleaned checkpoint contains:") - print("Keys in cleaned checkpoint:", list(test_load.keys())) - except Exception as e: - raise Exception(f"Error verifying cleaned checkpoint: {str(e)}") - -def main(): - parser = argparse.ArgumentParser(description='Convert MeloTTS training checkpoint to inference-only checkpoint') - parser.add_argument('input_path', type=str, help='Path to the full checkpoint file') - parser.add_argument('--output_path', type=str, default=None, - help='Path for the clean checkpoint (optional)') - parser.add_argument('--debug', action='store_true', - help='Print additional debugging information') - - args = parser.parse_args() - - try: - clean_checkpoint(args.input_path, args.output_path) - print("\nCheckpoint cleaned successfully!") - except Exception as e: - print(f"\nError occurred: {str(e)}") - import traceback - traceback.print_exc() - return 1 - - return 0 - -if __name__ == "__main__": - main() \ No newline at end of file From cb56e4768941fc3725a81a77ae9b131261322834 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Fri, 1 Nov 2024 23:01:19 -0400 Subject: [PATCH 25/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 426ddd579..b89c9426e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ melo-ui 1. From the conda prompt run `train.bat` again to resume the training. The training will resume from the newest G_XXXX.pth file. ## Trimming Model You can trim your model to make it a way smaller filesize (which will make it load faster during the model loading process). When testing, this made the model filesize about 66% smaller. Note the created trimmed model is for inference-only(using the model just to generate audio from text) and you won't be able to train it further. -1. Open `trim_model.bat` file in a text editor to change the directory of your G_XXXX.pth file and the save location, save the changes, then run `trim_model.bat` to create a trimmed model for inference only. +1. Open `trim_models.bat` file in a text editor to change the directory to your G_XXXX.pth files and the save location, save the changes, then run `trim_models.bat` to create a trimmed model for inference only. # Original Readme:
From 122a662fe28640786339aace5d1c1e1b960415a7 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sat, 2 Nov 2024 00:25:30 -0400 Subject: [PATCH 26/30] Add files via upload --- melo/resource_usage_test.py | 167 ++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 melo/resource_usage_test.py diff --git a/melo/resource_usage_test.py b/melo/resource_usage_test.py new file mode 100644 index 000000000..56429e3b4 --- /dev/null +++ b/melo/resource_usage_test.py @@ -0,0 +1,167 @@ +import os +import time +import psutil +import GPUtil +import torch +from melo.api import TTS +import numpy as np + +def get_system_usage(): + """Get current system resource usage""" + cpu_percent = psutil.cpu_percent(interval=0.1) + ram = psutil.virtual_memory() + ram_usage = ram.used / (1024 ** 3) # Convert to GB + + gpu_stats = None + if torch.cuda.is_available(): + try: + gpus = GPUtil.getGPUs() + if gpus: + gpu = gpus[0] # Get first GPU + gpu_stats = { + 'gpu_load': gpu.load * 100, # Convert to percentage + 'gpu_memory_used': gpu.memoryUsed, + 'gpu_memory_total': gpu.memoryTotal + } + except Exception as e: + print(f"Error getting GPU stats: {e}") + + return { + 'cpu_percent': cpu_percent, + 'ram_used_gb': ram_usage, + 'gpu_stats': gpu_stats + } + +def run_tts_test(text, device, output_filename, speaker_id=0): + """Run TTS test on specified device and monitor resources""" + print(f"\nRunning TTS test on {device.upper()}...") + + # Clear CUDA cache and ensure clean state + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + # Initialize model on specified device + model = TTS(language="EN", device=device) + + # Get initial resource usage + initial_usage = get_system_usage() + print(f"Initial resource usage:") + print(f"CPU: {initial_usage['cpu_percent']:.1f}%") + print(f"RAM: {initial_usage['ram_used_gb']:.1f} GB") + if initial_usage['gpu_stats']: + print(f"GPU: {initial_usage['gpu_stats']['gpu_load']:.1f}%") + print(f"VRAM: {initial_usage['gpu_stats']['gpu_memory_used']:.1f} MB / " + f"{initial_usage['gpu_stats']['gpu_memory_total']:.1f} MB") + + # Run TTS + start_time = time.time() + try: + # Force BERT models to the correct device before TTS + if device == "cuda": + from melo.text.english_bert import model as bert_model + if bert_model is not None: + bert_model.to("cuda") + + audio = model.tts_to_file( + text=text, + speaker_id=speaker_id, + output_path=output_filename, + quiet=True + ) + + # Get peak resource usage + peak_usage = get_system_usage() + + end_time = time.time() + duration = end_time - start_time + + print(f"\nPeak resource usage:") + print(f"CPU: {peak_usage['cpu_percent']:.1f}%") + print(f"RAM: {peak_usage['ram_used_gb']:.1f} GB") + if peak_usage['gpu_stats']: + print(f"GPU: {peak_usage['gpu_stats']['gpu_load']:.1f}%") + print(f"VRAM: {peak_usage['gpu_stats']['gpu_memory_used']:.1f} MB / " + f"{peak_usage['gpu_stats']['gpu_memory_total']:.1f} MB") + + print(f"\nConversion completed in {duration:.2f} seconds") + print(f"Output saved to: {output_filename}") + + except Exception as e: + print(f"Error during TTS conversion: {e}") + + # Clean up + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + +def main(): + # Create output directory + os.makedirs("tts_outputs", exist_ok=True) + + # Test text + text = "This is a test of the MeloTTS system in both GPU and CPU modes. How's the performance?" + + # Get starting resource usage (before using MeloTTS for inference) + peak_usage = get_system_usage() + + print(f"\nPeak resource usage before MeloTTS inference:") + print(f"--CPU: {peak_usage['cpu_percent']:.1f}%") + print(f"--RAM: {peak_usage['ram_used_gb']:.1f} GB") + if peak_usage['gpu_stats']: + print(f"--GPU: {peak_usage['gpu_stats']['gpu_load']:.1f}%") + print(f"--VRAM: {peak_usage['gpu_stats']['gpu_memory_used']:.1f} MB / " + f"{peak_usage['gpu_stats']['gpu_memory_total']:.1f} MB") + + # Run CPU tests + print("\n=== CPU Test - First Run (Warmup) ===") + run_tts_test( + text=text, + device="cpu", + output_filename="tts_outputs/cpu_output_1.wav" + ) + + print("\n=== CPU Test - Second Run (Measurement) ===") + run_tts_test( + text=text, + device="cpu", + output_filename="tts_outputs/cpu_output_2.wav" + ) + + print("\n=== CPU Test - Third Run (Extra Measurement) ===") + run_tts_test( + text=text, + device="cpu", + output_filename="tts_outputs/cpu_output_3.wav" + ) + + # Run GPU tests if available + if torch.cuda.is_available(): + # Force CUDA synchronization and cache clear before GPU test + torch.cuda.synchronize() + torch.cuda.empty_cache() + + print("\n=== GPU Test - First Run (Warmup) ===") + run_tts_test( + text=text, + device="cuda", + output_filename="tts_outputs/gpu_output_1.wav" + ) + + print("\n=== GPU Test - Second Run (Measurement) ===") + run_tts_test( + text=text, + device="cuda", + output_filename="tts_outputs/gpu_output_2.wav" + ) + + print("\n=== GPU Test - Third Run (Extra Measurement) ===") + run_tts_test( + text=text, + device="cuda", + output_filename="tts_outputs/gpu_output_3.wav" + ) + else: + print("\nGPU not available for testing") + +if __name__ == "__main__": + main() \ No newline at end of file From 040083bb5a1b0fef20913b07e11cb2962ddf0ebe Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sat, 2 Nov 2024 00:26:42 -0400 Subject: [PATCH 27/30] Update resource_usage_test.py --- melo/resource_usage_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melo/resource_usage_test.py b/melo/resource_usage_test.py index 56429e3b4..655298467 100644 --- a/melo/resource_usage_test.py +++ b/melo/resource_usage_test.py @@ -1,3 +1,4 @@ +#this script requires you to use: pip install psutil GPUtil import os import time import psutil @@ -164,4 +165,4 @@ def main(): print("\nGPU not available for testing") if __name__ == "__main__": - main() \ No newline at end of file + main() From c5e860fc539be3b538d40f32105aafe9b034d54a Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sat, 2 Nov 2024 02:38:55 -0400 Subject: [PATCH 28/30] Add files via upload --- melo/gpu_cpu_switching_test.py | 187 +++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 melo/gpu_cpu_switching_test.py diff --git a/melo/gpu_cpu_switching_test.py b/melo/gpu_cpu_switching_test.py new file mode 100644 index 000000000..ade87f077 --- /dev/null +++ b/melo/gpu_cpu_switching_test.py @@ -0,0 +1,187 @@ +import torch +import gc +from melo.api import TTS +import os +import melo.text.english_bert as bert +import psutil +import GPUtil + +def get_memory_usage(): + """Get current RAM and VRAM usage""" + ram = psutil.virtual_memory() + ram_usage = ram.used / (1024 ** 3) # Convert to GB + + gpu_memory = None + if torch.cuda.is_available(): + try: + gpus = GPUtil.getGPUs() + if gpus: + gpu = gpus[0] # Get first GPU + gpu_memory = { + 'used': gpu.memoryUsed, # MB + 'total': gpu.memoryTotal # MB + } + except Exception as e: + print(f"Error getting GPU stats: {e}") + + return ram_usage, gpu_memory + +def clear_gpu_memory(): + """Clear both GPU and RAM memory more aggressively""" + # First clear CUDA memory + if torch.cuda.is_available(): + for obj in gc.get_objects(): + try: + if torch.is_tensor(obj): + if obj.is_cuda: + del obj + except Exception: + pass + torch.cuda.synchronize() + torch.cuda.empty_cache() + + # Force Python garbage collection + gc.collect() + + # Optional: Force more aggressive garbage collection + for _ in range(2): + gc.collect() + + # Print memory stats to verify clearing + ram = psutil.virtual_memory() + print(f"\nAfter clearing - RAM Usage: {ram.used / (1024 ** 3):.2f} GB") + if torch.cuda.is_available(): + gpus = GPUtil.getGPUs() + if gpus: + gpu = gpus[0] + print(f"After clearing - VRAM Usage: {gpu.memoryUsed:.2f} MB / {gpu.memoryTotal:.2f} MB") + +def reset_bert_model(device): + """Reset the global BERT model in english_bert.py only when switching devices""" + current_device = None + + # Print memory usage before reset + ram_before, vram_before = get_memory_usage() + print(f"\nMemory before BERT reset:") + print(f"RAM Usage: {ram_before:.2f} GB") + if vram_before: + print(f"VRAM Usage: {vram_before['used']:.2f} MB / {vram_before['total']:.2f} MB") + + # Check current device of BERT model if it exists + if bert.model is not None: + current_device = next(bert.model.parameters()).device.type + + # If the model is already on the correct device, return early + if (current_device == 'cuda' and device == 'cuda') or \ + (current_device == 'cpu' and device == 'cpu'): + return + + # Explicitly move model to CPU before deletion if it's on CUDA + if current_device == 'cuda': + bert.model.cpu() + + # Delete the model and clear memory + del bert.model + clear_gpu_memory() + bert.model = None + + # Create proper dummy input for BERT initialization + dummy_text = "Hello world" + tokens = bert.tokenizer(dummy_text, return_tensors="pt") + word2ph = [1] * tokens["input_ids"].shape[1] + _ = bert.get_bert_feature(dummy_text, word2ph, device) + + # Print memory usage after reset + ram_after, vram_after = get_memory_usage() + print(f"\nMemory after BERT reset:") + print(f"RAM Usage: {ram_after:.2f} GB") + if vram_after: + print(f"VRAM Usage: {vram_after['used']:.2f} MB / {vram_after['total']:.2f} MB") + + print(f"BERT model reloaded: {current_device} -> {device}") + +import time # Add this to your imports + +def generate_speech(model, text, spk_id, device, output_path): + """Generate speech with specified device""" + try: + # Print memory usage before model movement + ram_before, vram_before = get_memory_usage() + print(f"\nMemory before moving model to {device}:") + print(f"RAM Usage: {ram_before:.2f} GB") + if vram_before: + print(f"VRAM Usage: {vram_before['used']:.2f} MB / {vram_before['total']:.2f} MB") + + # If moving from CUDA to CPU, first move model to CPU then clear CUDA memory + if device == "cpu" and next(model.parameters()).is_cuda: + model.cpu() + clear_gpu_memory() + else: + model = model.to(device) + + reset_bert_model(device) + spk_id = torch.tensor([spk_id], device=device) + + # Print memory usage after model movement + ram_after, vram_after = get_memory_usage() + print(f"\nMemory after moving model to {device}:") + print(f"RAM Usage: {ram_after:.2f} GB") + if vram_after: + print(f"VRAM Usage: {vram_after['used']:.2f} MB / {vram_after['total']:.2f} MB") + + # Add timing measurement + start_time = time.time() + model.tts_to_file(text, spk_id, output_path) + end_time = time.time() + conversion_time = end_time - start_time + print(f"\n----Text-to-speech conversion time on {device}: {conversion_time:.2f} seconds") + + finally: + clear_gpu_memory() + + +def alternate_gpu_cpu_inference(ckpt_path, text, language="EN", output_dir="outputs"): + """Alternate between GPU and CPU inference""" + os.makedirs(output_dir, exist_ok=True) + + # Initialize model + config_path = os.path.join(os.path.dirname(ckpt_path), 'config.json') + + # Sequence of devices to test + devices = ["cuda", "cpu", "cuda", "cpu"] + current_model = None + current_device = None + + for i, device in enumerate(devices): + # Skip if CUDA not available for GPU inference + if device == "cuda" and not torch.cuda.is_available(): + print(f"CUDA not available, skipping GPU inference {i+1}") + continue + + print(f"\nGenerating speech using {device.upper()} - Round {i+1}") + + # Only create new model instance if switching devices or first run + if current_model is None or current_device != device: + if current_model is not None: + del current_model + clear_gpu_memory() + + current_model = TTS(language=language, config_path=config_path, + ckpt_path=ckpt_path, device=device) + current_device = device + + # Get first speaker ID + spk_id = list(current_model.hps.data.spk2id.values())[0] + + output_path = os.path.join(output_dir, f"output_{i+1}_{device}.wav") + + # Generate speech + generate_speech(current_model, text, spk_id, device, output_path) + print(f"Generated: {output_path}") + +if __name__ == "__main__": + # Example usage + ckpt_path = "C:/Users/lyria/Documents/Scripts/python/melotts/melotts/MeloTTS-Windows/melo/data/example_fused3voices_7wavs/output_3voices_7wavs/G_2000.pth" + text = "This is a test of alternating between GPU and CPU inference." + + alternate_gpu_cpu_inference(ckpt_path, text) \ No newline at end of file From 0ec5a5dd2127dbe47ec762db4596083b5517f6c7 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sat, 2 Nov 2024 02:39:26 -0400 Subject: [PATCH 29/30] Rename resource_usage_test.py to test_resource_usage.py --- melo/{resource_usage_test.py => test_resource_usage.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename melo/{resource_usage_test.py => test_resource_usage.py} (100%) diff --git a/melo/resource_usage_test.py b/melo/test_resource_usage.py similarity index 100% rename from melo/resource_usage_test.py rename to melo/test_resource_usage.py From 34a724b99be9182a004d6204db9d17754838c500 Mon Sep 17 00:00:00 2001 From: EliseWindbloom <109670213+EliseWindbloom@users.noreply.github.com> Date: Sat, 2 Nov 2024 02:39:57 -0400 Subject: [PATCH 30/30] Update and rename gpu_cpu_switching_test.py to test_gpu_cpu_switching.py --- melo/{gpu_cpu_switching_test.py => test_gpu_cpu_switching.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename melo/{gpu_cpu_switching_test.py => test_gpu_cpu_switching.py} (96%) diff --git a/melo/gpu_cpu_switching_test.py b/melo/test_gpu_cpu_switching.py similarity index 96% rename from melo/gpu_cpu_switching_test.py rename to melo/test_gpu_cpu_switching.py index ade87f077..2b6c3c37d 100644 --- a/melo/gpu_cpu_switching_test.py +++ b/melo/test_gpu_cpu_switching.py @@ -184,4 +184,4 @@ def alternate_gpu_cpu_inference(ckpt_path, text, language="EN", output_dir="outp ckpt_path = "C:/Users/lyria/Documents/Scripts/python/melotts/melotts/MeloTTS-Windows/melo/data/example_fused3voices_7wavs/output_3voices_7wavs/G_2000.pth" text = "This is a test of alternating between GPU and CPU inference." - alternate_gpu_cpu_inference(ckpt_path, text) \ No newline at end of file + alternate_gpu_cpu_inference(ckpt_path, text)