From 58600274acd5876145fb44e8e836b453f6dc27f1 Mon Sep 17 00:00:00 2001 From: mrq Date: Thu, 23 Feb 2023 03:22:59 +0000 Subject: [PATCH] Disabling bitsandbytes optimization as default for now, in the off chance that it actually produces garbage (which shouldn't happen, there's no chance, if training at float16 from a model at float16 works fine, then this has to work) --- README.old.md => README.md | 11 +++++++++++ codes/torch_intermediary/__init__.py | 4 ++-- codes/train.py | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) rename README.old.md => README.md (91%) diff --git a/README.old.md b/README.md similarity index 91% rename from README.old.md rename to README.md index 059a5b0c..610c636b 100644 --- a/README.old.md +++ b/README.md @@ -1,3 +1,14 @@ +# (QoL improvements for) Deep Learning Art School + +This fork of [neonbjb/DL-Art-School](https://github.com/neonbjb/DL-Art-School/) contains a few fixes and QoL improvements, including but not limited to: +* sanity tidying, like: + - not outputing to `./DL-Art-School/experiments/` + - the custom module loader for networks/injectors getting fixed + - BitsAndBytes integration: + + working but output untested: Adam/AdamW + + toggles available in `./codes/torch_indermediary/__init__.py` + +--- # Deep Learning Art School Send your Pytorch model to art class! diff --git a/codes/torch_intermediary/__init__.py b/codes/torch_intermediary/__init__.py index bc8933a6..0fb0cbca 100644 --- a/codes/torch_intermediary/__init__.py +++ b/codes/torch_intermediary/__init__.py @@ -13,8 +13,8 @@ from torch.optim.adamw import AdamW OVERRIDE_LINEAR = False OVERRIDE_EMBEDDING = False -OVERRIDE_ADAM = True -OVERRIDE_ADAMW = True +OVERRIDE_ADAM = False # True +OVERRIDE_ADAMW = False # True USE_STABLE_EMBEDDING = True if OVERRIDE_LINEAR: diff --git a/codes/train.py b/codes/train.py index 88d1caac..077632ad 100644 --- a/codes/train.py +++ b/codes/train.py @@ -258,7 +258,7 @@ class Trainer: self.logger.info(message) #### save models and training states - if self.current_step % opt['logger']['save_checkpoint_freq'] == 0: + if self.current_step > 0 and self.current_step % opt['logger']['save_checkpoint_freq'] == 0: self.model.consolidate_state() if self.rank <= 0: self.logger.info('Saving models and training states.')