move speech utils

This commit is contained in:
James Betker 2021-12-16 20:47:37 -07:00
parent e7957e4897
commit 62c8ed9a29
5 changed files with 12 additions and 12 deletions

View File

@ -147,7 +147,7 @@ def register_gpt_tts_hf(opt_net, opt):
if __name__ == '__main__':
gpt = GptTtsHf()
gpt = GptTtsHf(model_dim=1024, heads=16)
l = gpt(torch.randint(high=len(symbols), size=(2,100)),
torch.randn(2,2,80,800),
torch.randint(high=8192, size=(2,200)),

View File

View File

@ -54,7 +54,7 @@ def load_discrete_vocoder_diffuser(trained_diffusion_steps=4000, desired_diffusi
model_var_type='learned_range', loss_type='mse', betas=get_named_beta_schedule('linear', trained_diffusion_steps))
def do_spectrogram_diffusion(diffusion_model, dvae_model, diffuser, mel_codes, conditioning_input, spectrogram_compression_factor=128, plt_spec=False, am=None):
def do_spectrogram_diffusion(diffusion_model, dvae_model, diffuser, mel_codes, conditioning_input, spectrogram_compression_factor=128, plt_spec=False):
"""
Uses the specified diffusion model and DVAE model to convert the provided MEL & conditioning inputs into an audio clip.
"""
@ -62,11 +62,8 @@ def do_spectrogram_diffusion(diffusion_model, dvae_model, diffuser, mel_codes, c
dvae_model.eval()
with torch.no_grad():
mel = dvae_model.decode(mel_codes)[0]
if plt_spec:
plot_spectrogram(mel[0].cpu())
m=mel[:,:,:am.shape[-1]]
print(torch.nn.MSELoss()(am,m))
# Pad MEL to multiples of 4096//spectrogram_compression_factor
msl = mel.shape[-1]

View File

@ -3,7 +3,7 @@ import argparse
import torchaudio
from data.audio.unsupervised_audio_dataset import load_audio
from scripts.audio.speech_synthesis_utils import do_spectrogram_diffusion, \
from scripts.audio.gen.speech_synthesis_utils import do_spectrogram_diffusion, \
load_discrete_vocoder_diffuser, wav_to_mel, convert_mel_to_codes
from utils.audio import plot_spectrogram
from utils.util import load_model_from_config
@ -19,17 +19,17 @@ def roundtrip_vocoding(dvae, vocoder, diffuser, clip, cond=None, plot_spec=False
if plot_spec:
plot_spectrogram(mel[0].cpu())
codes = convert_mel_to_codes(dvae, mel)
return do_spectrogram_diffusion(vocoder, dvae, diffuser, codes, cond, spectrogram_compression_factor=128, plt_spec=plot_spec, am=mel)
return do_spectrogram_diffusion(vocoder, dvae, diffuser, codes, cond, spectrogram_compression_factor=128, plt_spec=plot_spec)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-opt', type=str, help='Path to options YAML file used to train the diffusion model', default='X:\\dlas\\experiments\\train_diffusion_vocoder_with_cond_new_dvae.yml')
parser.add_argument('-diffusion_model_name', type=str, help='Name of the diffusion model in opt.', default='generator')
parser.add_argument('-diffusion_model_path', type=str, help='Name of the diffusion model in opt.', default='X:\\dlas\\experiments\\train_diffusion_vocoder_with_cond_new_dvae_full\\models\\200_generator_ema.pth')
parser.add_argument('-diffusion_model_path', type=str, help='Name of the diffusion model in opt.', default='X:\\dlas\\experiments\\train_diffusion_vocoder_with_cond_new_dvae_full\\models\\6100_generator_ema.pth')
parser.add_argument('-dvae_model_name', type=str, help='Name of the DVAE model in opt.', default='dvae')
parser.add_argument('-input_file', type=str, help='Path to the input audio file.', default='Z:\\clips\\books1\\3_dchha04 Romancing The Tribes\\00036.wav')
parser.add_argument('-cond', type=str, help='Path to the conditioning input audio file.', default=None)
parser.add_argument('-cond', type=str, help='Path to the conditioning input audio file.', default='Z:\\clips\\books1\\3042_18_Holden__000000000\\00037.wav')
args = parser.parse_args()
print("Loading DVAE..")
@ -40,7 +40,10 @@ if __name__ == '__main__':
print("Loading data..")
diffuser = load_discrete_vocoder_diffuser()
inp = load_audio(args.input_file, 22050).cuda()
cond = None if args.cond is None else load_audio(args.cond, 22050).cuda()
cond = inp if args.cond is None else load_audio(args.cond, 22050)
if cond.shape[-1] > 44100+10000:
cond = cond[:,10000:54100]
cond = torchaudio.transforms.Resample(22050, 10025)(cond.cpu()).cuda()
print("Performing inference..")
roundtripped = roundtrip_vocoding(dvae, diffusion, diffuser, inp, cond).cpu()

View File

@ -61,12 +61,12 @@ class GaussianDiffusionInjector(Injector):
def closest_multiple(inp, multiple):
div = inp / multiple
div = inp // multiple
mod = inp % multiple
if mod == 0:
return inp
else:
return (div+1)*multiple
return int((div+1)*multiple)
# Performs inference using a network trained to predict a reverse diffusion process, which nets a image.