move speech utils

codes/models/gpt_voice/gpt_tts_hf.py

@@ -147,7 +147,7 @@ def register_gpt_tts_hf(opt_net, opt):
 
 
 if __name__ == '__main__':
-    gpt = GptTtsHf()
+    gpt = GptTtsHf(model_dim=1024, heads=16)
     l = gpt(torch.randint(high=len(symbols), size=(2,100)),
             torch.randn(2,2,80,800),
             torch.randint(high=8192, size=(2,200)),

codes/scripts/audio/gen/speech_synthesis_utils.py

@@ -54,7 +54,7 @@ def load_discrete_vocoder_diffuser(trained_diffusion_steps=4000, desired_diffusi
                            model_var_type='learned_range', loss_type='mse', betas=get_named_beta_schedule('linear', trained_diffusion_steps))
 
 
-def do_spectrogram_diffusion(diffusion_model, dvae_model, diffuser, mel_codes, conditioning_input, spectrogram_compression_factor=128, plt_spec=False, am=None):
+def do_spectrogram_diffusion(diffusion_model, dvae_model, diffuser, mel_codes, conditioning_input, spectrogram_compression_factor=128, plt_spec=False):
     """
     Uses the specified diffusion model and DVAE model to convert the provided MEL & conditioning inputs into an audio clip.
     """
@@ -62,11 +62,8 @@ def do_spectrogram_diffusion(diffusion_model, dvae_model, diffuser, mel_codes, c
     dvae_model.eval()
     with torch.no_grad():
         mel = dvae_model.decode(mel_codes)[0]
-
         if plt_spec:
             plot_spectrogram(mel[0].cpu())
-        m=mel[:,:,:am.shape[-1]]
-        print(torch.nn.MSELoss()(am,m))
 
         # Pad MEL to multiples of 4096//spectrogram_compression_factor
         msl = mel.shape[-1]

codes/scripts/audio/gen/use_discrete_vocoder.py

@@ -3,7 +3,7 @@ import argparse
 import torchaudio
 
 from data.audio.unsupervised_audio_dataset import load_audio
-from scripts.audio.speech_synthesis_utils import do_spectrogram_diffusion, \
+from scripts.audio.gen.speech_synthesis_utils import do_spectrogram_diffusion, \
     load_discrete_vocoder_diffuser, wav_to_mel, convert_mel_to_codes
 from utils.audio import plot_spectrogram
 from utils.util import load_model_from_config
@@ -19,17 +19,17 @@ def roundtrip_vocoding(dvae, vocoder, diffuser, clip, cond=None, plot_spec=False
     if plot_spec:
         plot_spectrogram(mel[0].cpu())
     codes = convert_mel_to_codes(dvae, mel)
-    return do_spectrogram_diffusion(vocoder, dvae, diffuser, codes, cond, spectrogram_compression_factor=128, plt_spec=plot_spec, am=mel)
+    return do_spectrogram_diffusion(vocoder, dvae, diffuser, codes, cond, spectrogram_compression_factor=128, plt_spec=plot_spec)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('-opt', type=str, help='Path to options YAML file used to train the diffusion model', default='X:\\dlas\\experiments\\train_diffusion_vocoder_with_cond_new_dvae.yml')
     parser.add_argument('-diffusion_model_name', type=str, help='Name of the diffusion model in opt.', default='generator')
-    parser.add_argument('-diffusion_model_path', type=str, help='Name of the diffusion model in opt.', default='X:\\dlas\\experiments\\train_diffusion_vocoder_with_cond_new_dvae_full\\models\\200_generator_ema.pth')
+    parser.add_argument('-diffusion_model_path', type=str, help='Name of the diffusion model in opt.', default='X:\\dlas\\experiments\\train_diffusion_vocoder_with_cond_new_dvae_full\\models\\6100_generator_ema.pth')
     parser.add_argument('-dvae_model_name', type=str, help='Name of the DVAE model in opt.', default='dvae')
     parser.add_argument('-input_file', type=str, help='Path to the input audio file.', default='Z:\\clips\\books1\\3_dchha04 Romancing The Tribes\\00036.wav')
-    parser.add_argument('-cond', type=str, help='Path to the conditioning input audio file.', default=None)
+    parser.add_argument('-cond', type=str, help='Path to the conditioning input audio file.', default='Z:\\clips\\books1\\3042_18_Holden__000000000\\00037.wav')
     args = parser.parse_args()
 
     print("Loading DVAE..")
@@ -40,7 +40,10 @@ if __name__ == '__main__':
     print("Loading data..")
     diffuser = load_discrete_vocoder_diffuser()
     inp = load_audio(args.input_file, 22050).cuda()
-    cond = None if args.cond is None else load_audio(args.cond, 22050).cuda()
+    cond = inp if args.cond is None else load_audio(args.cond, 22050)
+    if cond.shape[-1] > 44100+10000:
+        cond = cond[:,10000:54100]
+    cond = torchaudio.transforms.Resample(22050, 10025)(cond.cpu()).cuda()
 
     print("Performing inference..")
     roundtripped = roundtrip_vocoding(dvae, diffusion, diffuser, inp, cond).cpu()

codes/trainer/injectors/gaussian_diffusion_injector.py

@@ -61,12 +61,12 @@ class GaussianDiffusionInjector(Injector):
 
 
 def closest_multiple(inp, multiple):
-    div = inp / multiple
+    div = inp // multiple
     mod = inp % multiple
     if mod == 0:
         return inp
     else:
-        return (div+1)*multiple
+        return int((div+1)*multiple)
 
 
 # Performs inference using a network trained to predict a reverse diffusion process, which nets a image.