From 8494628f3cb4578041457ea1c7a378663295403c Mon Sep 17 00:00:00 2001 From: mrq Date: Wed, 8 Mar 2023 05:27:20 +0000 Subject: [PATCH] normalize validation batch size because i oom'd without it getting scaled --- src/webui.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webui.py b/src/webui.py index 1ded6c6..c4e4a6e 100755 --- a/src/webui.py +++ b/src/webui.py @@ -318,7 +318,7 @@ def save_training_settings_proxy( epochs, learning_rate, text_ce_lr_weight, lear save_rate = int(save_rate * iterations / epochs) validation_rate = int(validation_rate * iterations / epochs) - validation_batch_size = batch_size + validation_batch_size = int(batch_size / gradient_accumulation_size) if iterations % save_rate != 0: adjustment = int(iterations / save_rate) * save_rate @@ -333,6 +333,7 @@ def save_training_settings_proxy( epochs, learning_rate, text_ce_lr_weight, lear with open(validation_path, 'r', encoding="utf-8") as f: validation_lines = len(f.readlines()) + if validation_lines < validation_batch_size: validation_batch_size = validation_lines messages.append(f"Batch size exceeds validation dataset size, clamping validation batch size to {validation_lines}")