From d89b7d60e0e69dd4ef4c4c8068597bea89a66fd2 Mon Sep 17 00:00:00 2001 From: mrq Date: Sun, 19 Feb 2023 07:05:11 +0000 Subject: [PATCH] forgot to divide checkpoint freq by iterations to get checkpoint counts --- src/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.py b/src/utils.py index 5a582f1..be0e304 100755 --- a/src/utils.py +++ b/src/utils.py @@ -465,7 +465,7 @@ def run_training(config_path, verbose=False, buffer_size=8, progress=gr.Progress its = config['train']['niter'] checkpoint = 0 - checkpoints = config['logger']['save_checkpoint_freq'] + checkpoints = config['logger']['save_checkpoint_freq'] / its buffer_size = 8 open_state = False @@ -497,7 +497,7 @@ def run_training(config_path, verbose=False, buffer_size=8, progress=gr.Progress progress(checkpoint / float(checkpoints), f'[{checkpoint}/{checkpoints}] Saving checkpoint...') print(f"[Training] [{datetime.now().isoformat()}] {line[:-1]}") - + if verbose: yield "".join(buffer[-buffer_size:])