From 03752c1cd62d65158224db8fbb1b8c4bb18a5fd8 Mon Sep 17 00:00:00 2001 From: James Betker Date: Tue, 22 Feb 2022 23:09:37 -0700 Subject: [PATCH] Report NaN --- codes/data/audio/paired_voice_audio_dataset.py | 2 +- codes/trainer/steps.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/codes/data/audio/paired_voice_audio_dataset.py b/codes/data/audio/paired_voice_audio_dataset.py index b7d40c95..4c8d6860 100644 --- a/codes/data/audio/paired_voice_audio_dataset.py +++ b/codes/data/audio/paired_voice_audio_dataset.py @@ -140,7 +140,7 @@ class TextWavLoader(torch.utils.data.Dataset): tseq, wav, text, path = self.get_wav_text_pair(self.audiopaths_and_text[index]) if text is None or len(text.strip()) == 0: raise ValueError - if wav is None or wav.shape[-1] < (.1 * self.sample_rate): + if wav is None or wav.shape[-1] < (.6 * self.sample_rate): # Ultra short clips are also useless (and can cause problems within some models). raise ValueError cond, cond_is_self = load_similar_clips(self.audiopaths_and_text[index][0], self.conditioning_length, self.sample_rate, diff --git a/codes/trainer/steps.py b/codes/trainer/steps.py index 086c2696..15e99aa1 100644 --- a/codes/trainer/steps.py +++ b/codes/trainer/steps.py @@ -332,6 +332,7 @@ class ConfigurableStep(Module): for pgn, pg in zip(opt._group_names, opt.param_groups): grad_norm = clip_grad_norm(pg['params'], pgn, self.clip_grad_eps) if torch.isnan(grad_norm): + print("NaN found in clip_grad; zeroing grad and trying again.") nan_found = True self.nan_counter += 1