Report NaN

codes/data/audio/paired_voice_audio_dataset.py

@@ -140,7 +140,7 @@ class TextWavLoader(torch.utils.data.Dataset):
             tseq, wav, text, path = self.get_wav_text_pair(self.audiopaths_and_text[index])
             if text is None or len(text.strip()) == 0:
                 raise ValueError
-            if wav is None or wav.shape[-1] < (.1 * self.sample_rate):
+            if wav is None or wav.shape[-1] < (.6 * self.sample_rate):
                 # Ultra short clips are also useless (and can cause problems within some models).
                 raise ValueError
             cond, cond_is_self = load_similar_clips(self.audiopaths_and_text[index][0], self.conditioning_length, self.sample_rate,

codes/trainer/steps.py

@@ -332,6 +332,7 @@ class ConfigurableStep(Module):
                 for pgn, pg in zip(opt._group_names, opt.param_groups):
                     grad_norm = clip_grad_norm(pg['params'], pgn, self.clip_grad_eps)
                     if torch.isnan(grad_norm):
+                        print("NaN found in clip_grad; zeroing grad and trying again.")
                         nan_found = True
                         self.nan_counter += 1