From d29ea0df5ef50143a372ecbb6151b3f76a03a9f0 Mon Sep 17 00:00:00 2001 From: James Betker Date: Tue, 14 Jun 2022 15:19:52 -0600 Subject: [PATCH] Update ADF to be compatible with classical mel spectrograms --- .../audio/music/transformer_diffusion12.py | 2 +- codes/trainer/eval/audio_diffusion_fid.py | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/codes/models/audio/music/transformer_diffusion12.py b/codes/models/audio/music/transformer_diffusion12.py index 0359767b..f63f53f1 100644 --- a/codes/models/audio/music/transformer_diffusion12.py +++ b/codes/models/audio/music/transformer_diffusion12.py @@ -472,7 +472,7 @@ def test_vqvae_model(): ) quant_weights = torch.load('D:\\dlas\\experiments\\retrained_dvae_8192_clips.pth') model.quantizer.load_state_dict(quant_weights, strict=True) - #torch.save(model.state_dict(), 'sample.pth') + torch.save(model.state_dict(), 'sample.pth') print_network(model) o = model(clip, ts, cond) diff --git a/codes/trainer/eval/audio_diffusion_fid.py b/codes/trainer/eval/audio_diffusion_fid.py index b9f61e1f..742c58d5 100644 --- a/codes/trainer/eval/audio_diffusion_fid.py +++ b/codes/trainer/eval/audio_diffusion_fid.py @@ -208,15 +208,16 @@ class AudioDiffusionFid(evaluator.Evaluator): def perform_diffusion_tfd(self, audio, codes, text): SAMPLE_RATE = 24000 audio_resampled = torchaudio.functional.resample(audio, 22050, SAMPLE_RATE).unsqueeze(0) - mel = wav_to_univnet_mel(audio_resampled, do_normalization=True) - gen_mel = self.diffuser.p_sample_loop(self.model, mel.shape, - model_kwargs={'truth_mel': mel, + vmel = wav_to_mel(audio) + umel = wav_to_univnet_mel(audio_resampled, do_normalization=True) + gen_mel = self.diffuser.p_sample_loop(self.model, umel.shape, + model_kwargs={'truth_mel': vmel, 'conditioning_input': None, 'disable_diversity': True}) gen_wav = self.local_modules['vocoder'].inference(denormalize_mel(gen_mel)) - real_dec = self.local_modules['vocoder'].inference(denormalize_mel(mel)) - return gen_wav.float(), real_dec, gen_mel, mel, SAMPLE_RATE + real_dec = self.local_modules['vocoder'].inference(denormalize_mel(umel)) + return gen_wav.float(), real_dec, gen_mel, umel, SAMPLE_RATE def load_projector(self): """ @@ -334,12 +335,12 @@ if __name__ == '__main__': if __name__ == '__main__': - diffusion = load_model_from_config('X:\\dlas\\experiments\\train_tts_diffusion_tfd11_quant\\train.yml', 'generator', + diffusion = load_model_from_config('X:\\dlas\\experiments\\train_tts_diffusion_tfd11_quant.yml', 'generator', also_load_savepoint=False, - load_path='X:\\dlas\\experiments\\train_tts_diffusion_tfd11_quant\\models\\14500_generator_ema.pth').cuda() - opt_eval = {'eval_tsv': 'Y:\\libritts\\test-clean\\transcribed-oco-realtext.tsv', 'diffusion_steps': 100, + load_path='X:\\dlas\\experiments\\train_tts_diffusion_tfd12_linear_dvae\\models\\12000_generator.pth').cuda() + opt_eval = {'eval_tsv': 'Y:\\libritts\\test-clean\\transcribed-brief-w2v.tsv', 'diffusion_steps': 50, 'conditioning_free': False, 'conditioning_free_k': 1, - 'diffusion_schedule': 'cosine', 'diffusion_type': 'tfd'} - env = {'rank': 0, 'base_path': 'D:\\tmp\\test_eval', 'step': 100, 'device': 'cuda', 'opt': {}} + 'diffusion_schedule': 'linear', 'diffusion_type': 'tfd'} + env = {'rank': 0, 'base_path': 'D:\\tmp\\test_eval', 'step': 101, 'device': 'cuda', 'opt': {}} eval = AudioDiffusionFid(diffusion, opt_eval, env) print(eval.perform_eval())