diff --git a/api.py b/api.py index 204c91f..978358e 100644 --- a/api.py +++ b/api.py @@ -140,6 +140,13 @@ class TextToSpeech: average_conditioning_embeddings=True).cpu().eval() self.autoregressive.load_state_dict(torch.load('.models/autoregressive_diverse.pth')) + self.autoregressive_for_latents = UnifiedVoice(max_mel_tokens=604, max_text_tokens=402, max_conditioning_inputs=2, layers=30, + model_dim=1024, + heads=16, number_text_tokens=256, start_text_token=255, checkpointing=False, + train_solo_embeddings=False, + average_conditioning_embeddings=True).cpu().eval() + self.autoregressive_for_latents.load_state_dict(torch.load('.models/autoregressive_diverse.pth')) + self.clip = VoiceCLIP(dim_text=512, dim_speech=512, dim_latent=512, num_text_tokens=256, text_enc_depth=12, text_seq_len=350, text_heads=8, num_speech_tokens=8192, speech_enc_depth=12, speech_heads=8, speech_seq_len=430, @@ -221,11 +228,11 @@ class TextToSpeech: # The diffusion model actually wants the last hidden layer from the autoregressive model as conditioning # inputs. Re-produce those for the top results. This could be made more efficient by storing all of these # results, but will increase memory usage. - self.autoregressive = self.autoregressive.cuda() - best_latents = self.autoregressive(conds, text, torch.tensor([text.shape[-1]], device=conds.device), best_results, + self.autoregressive_for_latents = self.autoregressive_for_latents.cuda() + best_latents = self.autoregressive_for_latents(conds, text, torch.tensor([text.shape[-1]], device=conds.device), best_results, torch.tensor([best_results.shape[-1]*self.autoregressive.mel_length_compression], device=conds.device), return_latent=True, clip_inputs=False) - self.autoregressive = self.autoregressive.cpu() + self.autoregressive_for_latents = self.autoregressive_for_latents.cpu() print("Performing vocoding..") wav_candidates = [] diff --git a/eval_multiple.py b/eval_multiple.py index 9f1919d..529d3bd 100644 --- a/eval_multiple.py +++ b/eval_multiple.py @@ -6,32 +6,35 @@ from api import TextToSpeech from utils.audio import load_audio if __name__ == '__main__': - fname = 'Y:\\libritts\\test-clean\\transcribed-brief-w2v.tsv' - outpath = 'D:\\tmp\\tortoise-tts-eval\\diverse_new_decoder_1' + fname = 'Y:\\clips\\books2\\subset512-oco.tsv' + stop_after = 128 + outpath_base = 'D:\\tmp\\tortoise-tts-eval\\diverse' outpath_real = 'D:\\tmp\\tortoise-tts-eval\\real' - os.makedirs(outpath, exist_ok=True) os.makedirs(outpath_real, exist_ok=True) with open(fname, 'r', encoding='utf-8') as f: lines = [l.strip().split('\t') for l in f.readlines()] - recorder = open(os.path.join(outpath, 'transcript.tsv'), 'w', encoding='utf-8') tts = TextToSpeech() - for e, line in enumerate(lines): - transcript = line[0] - if len(transcript) > 120: - continue # We need to support this, but cannot yet. - path = os.path.join(os.path.dirname(fname), line[1]) - cond_audio = load_audio(path, 22050) - torchaudio.save(os.path.join(outpath_real, os.path.basename(line[1])), cond_audio, 22050) - sample = tts.tts(transcript, [cond_audio, cond_audio], num_autoregressive_samples=256, k=1, - repetition_penalty=2.0, length_penalty=2, temperature=.5, top_p=.5, - diffusion_temperature=.7, cond_free_k=2, diffusion_iterations=100) + for k in range(4): + outpath = f'{outpath_base}_{k}' + os.makedirs(outpath, exist_ok=True) + recorder = open(os.path.join(outpath, 'transcript.tsv'), 'w', encoding='utf-8') + for e, line in enumerate(lines): + if e >= stop_after: + break + transcript = line[0] + path = os.path.join(os.path.dirname(fname), line[1]) + cond_audio = load_audio(path, 22050) + torchaudio.save(os.path.join(outpath_real, os.path.basename(line[1])), cond_audio, 22050) + sample = tts.tts(transcript, [cond_audio, cond_audio], num_autoregressive_samples=128, k=1, + repetition_penalty=2.0, length_penalty=2, temperature=.5, top_p=.5, + diffusion_temperature=.7, cond_free_k=2, diffusion_iterations=70) - down = torchaudio.functional.resample(sample, 24000, 22050) - fout_path = os.path.join(outpath, os.path.basename(line[1])) - torchaudio.save(fout_path, down.squeeze(0), 22050) + down = torchaudio.functional.resample(sample, 24000, 22050) + fout_path = os.path.join(outpath, os.path.basename(line[1])) + torchaudio.save(fout_path, down.squeeze(0), 22050) - recorder.write(f'{transcript}\t{fout_path}\n') - recorder.flush() - recorder.close() \ No newline at end of file + recorder.write(f'{transcript}\t{fout_path}\n') + recorder.flush() + recorder.close() \ No newline at end of file diff --git a/read.py b/read.py index 08841f0..639dd7d 100644 --- a/read.py +++ b/read.py @@ -30,24 +30,13 @@ if __name__ == '__main__': # These are voices drawn randomly from the training set. You are free to substitute your own voices in, but testing # has shown that the model does not generalize to new voices very well. preselected_cond_voices = { - # Male voices - 'dotrice': ['voices/dotrice/1.wav', 'voices/dotrice/2.wav'], - 'harris': ['voices/harris/1.wav', 'voices/harris/2.wav'], - 'lescault': ['voices/lescault/1.wav', 'voices/lescault/2.wav'], - 'otto': ['voices/otto/1.wav', 'voices/otto/2.wav'], - 'obama': ['voices/obama/1.wav', 'voices/obama/2.wav'], - 'carlin': ['voices/carlin/1.wav', 'voices/carlin/2.wav'], - # Female voices - 'atkins': ['voices/atkins/1.wav', 'voices/atkins/2.wav'], - 'grace': ['voices/grace/1.wav', 'voices/grace/2.wav'], - 'kennard': ['voices/kennard/1.wav', 'voices/kennard/2.wav'], - 'mol': ['voices/mol/1.wav', 'voices/mol/2.wav'], - 'lj': ['voices/lj/1.wav', 'voices/lj/2.wav'], + 'emma_stone': ['voices/emma_stone/1.wav','voices/emma_stone/2.wav','voices/emma_stone/3.wav'], + 'tom_hanks': ['voices/tom_hanks/1.wav','voices/tom_hanks/2.wav','voices/tom_hanks/3.wav'], } parser = argparse.ArgumentParser() parser.add_argument('-textfile', type=str, help='A file containing the text to read.', default="data/riding_hood.txt") - parser.add_argument('-voice', type=str, help='Use a preset conditioning voice (defined above). Overrides cond_path.', default='dotrice') + parser.add_argument('-voice', type=str, help='Use a preset conditioning voice (defined above). Overrides cond_path.', default='emma_stone') parser.add_argument('-num_samples', type=int, help='How many total outputs the autoregressive transformer should produce.', default=512) parser.add_argument('-batch_size', type=int, help='How many samples to process at once in the autoregressive model.', default=16) parser.add_argument('-output_path', type=str, help='Where to store outputs.', default='results/longform/') diff --git a/sweep.py b/sweep.py index f62246d..13b40fc 100644 --- a/sweep.py +++ b/sweep.py @@ -24,19 +24,24 @@ def permutations(args): if __name__ == '__main__': - fname = 'Y:\\libritts\\test-clean\\transcribed-brief-w2v.tsv' - outpath_base = 'D:\\tmp\\tortoise-tts-eval\\std_sweep3' + fname = 'Y:\\clips\\books2\\subset512-oco.tsv' + stop_after = 128 + outpath_base = 'D:\\tmp\\tortoise-tts-eval\\sweep' outpath_real = 'D:\\tmp\\tortoise-tts-eval\\real' arg_ranges = { - 'top_p': [.3,.4,.5,.6], - 'temperature': [.5, .6], + 'top_p': [.5, 1], + 'temperature': [.5, 1], + 'diffusion_temperature': [.6, 1], + 'cond_free_k': [0, 1, 4], + 'repetition_penalty': [1.0, 2.0] } cfgs = permutations(arg_ranges) shuffle(cfgs) for cfg in cfgs: - outpath = os.path.join(outpath_base, f'{cfg["top_p"]}_{cfg["temperature"]}') + cfg_desc = '_'.join([f'{k}-{v}' for k,v in cfg.items()]) + outpath = os.path.join(outpath_base, f'{cfg_desc}') os.makedirs(outpath, exist_ok=True) os.makedirs(outpath_real, exist_ok=True) with open(fname, 'r', encoding='utf-8') as f: @@ -45,15 +50,14 @@ if __name__ == '__main__': recorder = open(os.path.join(outpath, 'transcript.tsv'), 'w', encoding='utf-8') tts = TextToSpeech() for e, line in enumerate(lines): + if e >= stop_after: + break transcript = line[0] - if len(transcript) > 120: - continue # We need to support this, but cannot yet. path = os.path.join(os.path.dirname(fname), line[1]) cond_audio = load_audio(path, 22050) torchaudio.save(os.path.join(outpath_real, os.path.basename(line[1])), cond_audio, 22050) - sample = tts.tts(transcript, [cond_audio, cond_audio], num_autoregressive_samples=256, k=1, diffusion_iterations=200, - repetition_penalty=2.0, length_penalty=2, temperature=.5, top_p=.5, - diffusion_temperature=.7, cond_free_k=2, **cfg) + sample = tts.tts(transcript, [cond_audio, cond_audio], num_autoregressive_samples=256, + k=1, diffusion_iterations=70, length_penalty=1.0, **cfg) down = torchaudio.functional.resample(sample, 24000, 22050) fout_path = os.path.join(outpath, os.path.basename(line[1])) torchaudio.save(fout_path, down.squeeze(0), 22050)