From 6ebdde58f038fd7dcd6b6bee35b4754ad36223f2 Mon Sep 17 00:00:00 2001 From: mrq Date: Tue, 7 Feb 2023 20:55:56 +0000 Subject: [PATCH] (finally) added the CVVP model weigh slider, latents export more data too for weighing against CVVP --- README.md | 3 +++ app.py | 41 +++++++++++++++++++++++++---------------- tortoise/api.py | 9 +++++++-- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1cb6298..4576acb 100755 --- a/README.md +++ b/README.md @@ -168,6 +168,9 @@ Below are settings that override the default launch arguments. Some of these req Below are an explanation of experimental flags. Messing with these might impact performance, as these are exposed only if you know what you are doing. * `Half-Precision`: (attempts to) hint to PyTorch to auto-cast to float16 (half precision) for compute. Disabled by default, due to it making computations slower. * `Conditional Free`: a quality boosting improvement at the cost of some performance. Enabled by default, as I think the penaly is negligible in the end. +* `CVVP Weight`: governs how much weight the CVVP model should influence candidates. The original documentation mentions this is deprecated as it does not really influence things, but you're still free to play around with it. + Currently, setting requires regenerating your voice latents, as I forgot to have it return some extra data that weighing against the CVVP model uses. Oops. + Setting this to 1 leads to bad behavior. ## Example(s) diff --git a/app.py b/app.py index b6aaa7a..5c5d40a 100755 --- a/app.py +++ b/app.py @@ -18,7 +18,7 @@ from tortoise.utils.audio import load_audio, load_voice, load_voices from tortoise.utils.text import split_and_recombine_text -def generate(text, delimiter, emotion, prompt, voice, mic_audio, seed, candidates, num_autoregressive_samples, diffusion_iterations, temperature, diffusion_sampler, breathing_room, experimentals, progress=gr.Progress(track_tqdm=True)): +def generate(text, delimiter, emotion, prompt, voice, mic_audio, seed, candidates, num_autoregressive_samples, diffusion_iterations, temperature, diffusion_sampler, breathing_room, cvvp_weight, experimentals, progress=gr.Progress(track_tqdm=True)): if voice != "microphone": voices = [voice] else: @@ -35,7 +35,7 @@ def generate(text, delimiter, emotion, prompt, voice, mic_audio, seed, candidate if voice_samples is not None: sample_voice = voice_samples[0] - conditioning_latents = tts.get_conditioning_latents(voice_samples, progress=progress, max_chunk_size=args.cond_latent_max_chunk_size) + conditioning_latents = tts.get_conditioning_latents(voice_samples, return_mels=True, progress=progress, max_chunk_size=args.cond_latent_max_chunk_size) if voice != "microphone": torch.save(conditioning_latents, f'./tortoise/voices/{voice}/cond_latents.pth') voice_samples = None @@ -45,6 +45,10 @@ def generate(text, delimiter, emotion, prompt, voice, mic_audio, seed, candidate if seed == 0: seed = None + if conditioning_latents is not None and len(conditioning_latents) == 2 and cvvp_weight > 0: + print("Requesting weighing against CVVP weight, but voice latents are missing some extra data. Please regenerate your voice latents.") + cvvp_weight = 0 + start_time = time.time() settings = { @@ -66,6 +70,7 @@ def generate(text, delimiter, emotion, prompt, voice, mic_audio, seed, candidate 'progress': progress, 'half_p': "Half Precision" in experimentals, 'cond_free': "Conditioning-Free" in experimentals, + 'cvvp_amount': cvvp_weight, } if delimiter == "\\n": @@ -159,6 +164,7 @@ def generate(text, delimiter, emotion, prompt, voice, mic_audio, seed, candidate 'temperature': temperature, 'diffusion_sampler': diffusion_sampler, 'breathing_room': breathing_room, + 'cvvp_weight': cvvp_weight, 'experimentals': experimentals, 'time': time.time()-start_time, } @@ -244,20 +250,21 @@ def import_generate_settings(file="./config/generate.json"): return None return ( - settings['text'], - settings['delimiter'], - settings['emotion'], - settings['prompt'], - settings['voice'], - settings['mic_audio'], - settings['seed'], - settings['candidates'], - settings['num_autoregressive_samples'], - settings['diffusion_iterations'], - settings['temperature'], - settings['diffusion_sampler'], - settings['breathing_room'], - settings['experimentals'], + None if 'text' not in settings else settings['text'], + None if 'delimiter' not in settings else settings['delimiter'], + None if 'emotion' not in settings else settings['emotion'], + None if 'prompt' not in settings else settings['prompt'], + None if 'voice' not in settings else settings['voice'], + None if 'mic_audio' not in settings else settings['mic_audio'], + None if 'seed' not in settings else settings['seed'], + None if 'candidates' not in settings else settings['candidates'], + None if 'num_autoregressive_samples' not in settings else settings['num_autoregressive_samples'], + None if 'diffusion_iterations' not in settings else settings['diffusion_iterations'], + None if 'temperature' not in settings else settings['temperature'], + None if 'diffusion_sampler' not in settings else settings['diffusion_sampler'], + None if 'breathing_room' not in settings else settings['breathing_room'], + None if 'cvvp_weight' not in settings else settings['cvvp_weight'], + None if 'experimentals' not in settings else settings['experimentals'], ) def curl(url): @@ -436,6 +443,7 @@ def main(): experimentals = gr.CheckboxGroup(["Half Precision", "Conditioning-Free"], value=["Conditioning-Free"], label="Experimental Flags") + cvvp_weight = gr.Slider(value=0, minimum=0, maximum=1, label="CVVP Weight") check_updates_now = gr.Button(value="Check for Updates") @@ -463,6 +471,7 @@ def main(): temperature, diffusion_sampler, breathing_room, + cvvp_weight, experimentals, ] diff --git a/tortoise/api.py b/tortoise/api.py index 27c72dd..cf62a42 100755 --- a/tortoise/api.py +++ b/tortoise/api.py @@ -124,7 +124,7 @@ def format_conditioning(clip, cond_length=132300, device='cuda', sampling_rate=2 elif gap > 0: rand_start = random.randint(0, gap) clip = clip[:, rand_start:rand_start + cond_length] - mel_clip = TorchMelSpectrogram(sampling_rate=sample_rate)(clip.unsqueeze(0)).squeeze(0) + mel_clip = TorchMelSpectrogram(sampling_rate=sampling_rate)(clip.unsqueeze(0)).squeeze(0) return mel_clip.unsqueeze(0).to(device) @@ -469,7 +469,11 @@ class TextToSpeech: if voice_samples is not None: auto_conditioning, diffusion_conditioning, auto_conds, _ = self.get_conditioning_latents(voice_samples, return_mels=True, verbose=True) elif conditioning_latents is not None: - auto_conditioning, diffusion_conditioning = conditioning_latents + latent_tuple = conditioning_latents + if len(latent_tuple) == 2: + auto_conditioning, diffusion_conditioning = conditioning_latents + else: + auto_conditioning, diffusion_conditioning, auto_conds, _ = conditioning_latents else: auto_conditioning, diffusion_conditioning = self.get_random_conditioning_latents() auto_conditioning = auto_conditioning.to(self.device) @@ -539,6 +543,7 @@ class TextToSpeech: clip_results.append(cvvp * cvvp_amount + clvp * (1-cvvp_amount)) else: clip_results.append(clvp) + clip_results = torch.cat(clip_results, dim=0) samples = torch.cat(samples, dim=0) best_results = samples[torch.topk(clip_results, k=k).indices]