From c2a436d368acad3d51df95aaf82ac0af94f886da Mon Sep 17 00:00:00 2001 From: mrq Date: Sun, 2 Jun 2024 08:29:27 -0500 Subject: [PATCH] somehow between training sessions grad_norm = None even though it worked before --- vall_e/engines/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vall_e/engines/base.py b/vall_e/engines/base.py index 6d3a871..ae20ad8 100755 --- a/vall_e/engines/base.py +++ b/vall_e/engines/base.py @@ -493,10 +493,11 @@ class Engines(dict[str, Engine]): total_elapsed_time += elapsed_time grad_norm = engine.get_global_grad_norm() loss_scale = 1 - if hasattr(engine.optimizer, "loss_scale"): + if hasattr(engine.optimizer, "loss_scale") and engine.optimizer.loss_scale is not None: loss_scale = engine.optimizer.loss_scale - grad_norm /= loss_scale + if grad_norm is not None: + grad_norm /= loss_scale stats.update( flatten_dict(