From c3b43d24299a1384bb234507001c3a687b4becb0 Mon Sep 17 00:00:00 2001 From: mrq Date: Thu, 9 Mar 2023 19:42:31 +0000 Subject: [PATCH] today I learned adamw_zero actually negates ANY LR schemes --- src/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 191e677..f9a2b4c 100755 --- a/src/utils.py +++ b/src/utils.py @@ -1394,7 +1394,8 @@ def save_training_settings( **kwargs ): if settings['gpus'] > get_device_count(): settings['gpus'] = get_device_count() - settings['optimizer'] = 'adamw' if settings['gpus'] == 1 else 'adamw_zero' + # what an utter mistake this was + # settings['optimizer'] = 'adamw' if settings['gpus'] == 1 else 'adamw_zero' if 'learning_rate_scheme' not in settings or settings['learning_rate_scheme'] not in LEARNING_RATE_SCHEMES: settings['learning_rate_scheme'] = "Multistep"