From e045fb0ad728ff2b190cb8f093d2f3c6847764dc Mon Sep 17 00:00:00 2001 From: James Betker Date: Sun, 13 Mar 2022 16:28:23 -0600 Subject: [PATCH] fix clip grad norm with scaler --- codes/trainer/steps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codes/trainer/steps.py b/codes/trainer/steps.py index b092d093..12897f0b 100644 --- a/codes/trainer/steps.py +++ b/codes/trainer/steps.py @@ -304,6 +304,8 @@ class ConfigurableStep(Module): return self.grads_generated = False for opt in self.optimizers: + self.scaler.unscale_(opt) + # Optimizers can be opted out in the early stages of training. after = opt._config['after'] if 'after' in opt._config.keys() else 0 after_network = self.opt['networks'][opt._config['network']]['after'] if 'after' in self.opt['networks'][opt._config['network']].keys() else 0