From 1b8af15f13cba2bfce249d9837660ea4f28d451e Mon Sep 17 00:00:00 2001 From: brkirch Date: Wed, 1 Feb 2023 09:28:16 -0500 Subject: [PATCH 01/10] Refactor Mac specific code to a separate file Move most Mac related code to a separate file, don't even load it unless web UI is run under macOS. --- modules/devices.py | 52 +++++--------------------------- modules/mac_specific.py | 56 +++++++++++++++++++++++++++++++++++ modules/sd_samplers_common.py | 16 ---------- modules/shared.py | 3 ++ 4 files changed, 66 insertions(+), 61 deletions(-) create mode 100644 modules/mac_specific.py diff --git a/modules/devices.py b/modules/devices.py index 919048d0..52c3e7cd 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -1,22 +1,17 @@ -import sys, os, shlex +import sys import contextlib import torch from modules import errors -from modules.sd_hijack_utils import CondFunc -from packaging import version + +if sys.platform == "darwin": + from modules import mac_specific -# has_mps is only available in nightly pytorch (for now) and macOS 12.3+. -# check `getattr` and try it for compatibility def has_mps() -> bool: - if not getattr(torch, 'has_mps', False): + if sys.platform != "darwin": return False - try: - torch.zeros(1).to(torch.device("mps")) - return True - except Exception: - return False - + else: + return mac_specific.has_mps def extract_device_id(args, name): for x in range(len(args)): @@ -155,36 +150,3 @@ def test_for_nans(x, where): message += " Use --disable-nan-check commandline argument to disable this check." raise NansException(message) - - -# MPS workaround for https://github.com/pytorch/pytorch/issues/89784 -def cumsum_fix(input, cumsum_func, *args, **kwargs): - if input.device.type == 'mps': - output_dtype = kwargs.get('dtype', input.dtype) - if output_dtype == torch.int64: - return cumsum_func(input.cpu(), *args, **kwargs).to(input.device) - elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16): - return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64) - return cumsum_func(input, *args, **kwargs) - - -if has_mps(): - if version.parse(torch.__version__) < version.parse("1.13"): - # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working - - # MPS workaround for https://github.com/pytorch/pytorch/issues/79383 - CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs), - lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')) - # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 - CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs), - lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps') - # MPS workaround for https://github.com/pytorch/pytorch/issues/90532 - CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad) - elif version.parse(torch.__version__) > version.parse("1.13.1"): - cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0)) - cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0)) - cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs) - CondFunc('torch.cumsum', cumsum_fix_func, None) - CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None) - CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None) - diff --git a/modules/mac_specific.py b/modules/mac_specific.py new file mode 100644 index 00000000..e39d670e --- /dev/null +++ b/modules/mac_specific.py @@ -0,0 +1,56 @@ +import torch +from modules import paths +from modules.sd_hijack_utils import CondFunc +from packaging import version + + +device = None + + +# has_mps is only available in nightly pytorch (for now) and macOS 12.3+. +# check `getattr` and try it for compatibility +def check_for_mps() -> bool: + if not getattr(torch, 'has_mps', False): + return False + try: + torch.zeros(1).to(torch.device("mps")) + return True + except Exception: + return False +has_mps = check_for_mps() + + +# MPS workaround for https://github.com/pytorch/pytorch/issues/89784 +def cumsum_fix(input, cumsum_func, *args, **kwargs): + if input.device.type == 'mps': + output_dtype = kwargs.get('dtype', input.dtype) + if output_dtype == torch.int64: + return cumsum_func(input.cpu(), *args, **kwargs).to(input.device) + elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16): + return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64) + return cumsum_func(input, *args, **kwargs) + + +if has_mps: + # MPS fix for randn in torchsde + CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'mps') + + if version.parse(torch.__version__) < version.parse("1.13"): + # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working + + # MPS workaround for https://github.com/pytorch/pytorch/issues/79383 + CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs), + lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')) + # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 + CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs), + lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps') + # MPS workaround for https://github.com/pytorch/pytorch/issues/90532 + CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad) + elif version.parse(torch.__version__) > version.parse("1.13.1"): + cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0)) + cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0)) + cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs) + CondFunc('torch.cumsum', cumsum_fix_func, None) + CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None) + CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None) + diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 3c03d442..a1aac7cf 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -2,7 +2,6 @@ from collections import namedtuple import numpy as np import torch from PIL import Image -import torchsde._brownian.brownian_interval from modules import devices, processing, images, sd_vae_approx from modules.shared import opts, state @@ -61,18 +60,3 @@ def store_latent(decoded): class InterruptedException(BaseException): pass - - -# MPS fix for randn in torchsde -# XXX move this to separate file for MPS -def torchsde_randn(size, dtype, device, seed): - if device.type == 'mps': - generator = torch.Generator(devices.cpu).manual_seed(int(seed)) - return torch.randn(size, dtype=dtype, device=devices.cpu, generator=generator).to(device) - else: - generator = torch.Generator(device).manual_seed(int(seed)) - return torch.randn(size, dtype=dtype, device=device, generator=generator) - - -torchsde._brownian.brownian_interval._randn = torchsde_randn - diff --git a/modules/shared.py b/modules/shared.py index 5600d480..59f12cd8 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -145,6 +145,9 @@ devices.device, devices.device_interrogate, devices.device_gfpgan, devices.devic (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer']) device = devices.device +if sys.platform == "darwin": + from modules import mac_specific + mac_specific.device = device weight_load_location = None if cmd_opts.lowram else "cpu" batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) From 92bae77b88fd90743eebec69ca7af1ee1c6e40f2 Mon Sep 17 00:00:00 2001 From: ctwrs <> Date: Wed, 1 Feb 2023 21:58:09 +0100 Subject: [PATCH 02/10] Add .jpg to allowed thumb formats --- modules/ui_extra_networks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 83367968..95b30f4a 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -29,8 +29,9 @@ def add_pages_to_demo(app): if not any([Path(x).resolve() in Path(filename).resolve().parents for x in allowed_dirs]): raise ValueError(f"File cannot be fetched: {filename}. Must be in one of directories registered by extra pages.") - if os.path.splitext(filename)[1].lower() != ".png": - raise ValueError(f"File cannot be fetched: {filename}. Only png.") + ext = os.path.splitext(filename)[1].lower() + if ext not in (".png", ".jpg"): + raise ValueError(f"File cannot be fetched: {filename}. Only png and jpg.") # would profit from returning 304 return FileResponse(filename, headers={"Accept-Ranges": "bytes"}) From fb97acef63ef50d1612566e47c5c0ba4823bd29f Mon Sep 17 00:00:00 2001 From: Cody Brownstein Date: Wed, 1 Feb 2023 14:46:13 -0800 Subject: [PATCH 03/10] Update error message WRT missing checkpoint file The Safetensors format is also supported. --- modules/sd_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 300387a9..45c8b0c2 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -158,7 +158,7 @@ def select_checkpoint(): print(f" - directory {model_path}", file=sys.stderr) if shared.cmd_opts.ckpt_dir is not None: print(f" - directory {os.path.abspath(shared.cmd_opts.ckpt_dir)}", file=sys.stderr) - print("Can't run without a checkpoint. Find and place a .ckpt file into any of those locations. The program will exit.", file=sys.stderr) + print("Can't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. The program will exit.", file=sys.stderr) exit(1) checkpoint_info = next(iter(checkpoints_list.values())) From 982295aee53dd4208fca1c13aa726f943806e790 Mon Sep 17 00:00:00 2001 From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com> Date: Sat, 4 Feb 2023 01:50:38 +0300 Subject: [PATCH 04/10] Fix img2imgalt after samplers separation --- scripts/img2imgalt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index cbdfc6b3..2572443f 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -6,7 +6,7 @@ from tqdm import trange import modules.scripts as scripts import gradio as gr -from modules import processing, shared, sd_samplers, prompt_parser +from modules import processing, shared, sd_samplers, prompt_parser, sd_samplers_common from modules.processing import Processed from modules.shared import opts, cmd_opts, state @@ -50,7 +50,7 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): x = x + d * dt - sd_samplers.store_latent(x) + sd_samplers_common.store_latent(x) # This shouldn't be necessary, but solved some VRAM issues del x_in, sigma_in, cond_in, c_out, c_in, t, @@ -104,7 +104,7 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): dt = sigmas[i] - sigmas[i - 1] x = x + d * dt - sd_samplers.store_latent(x) + sd_samplers_common.store_latent(x) # This shouldn't be necessary, but solved some VRAM issues del x_in, sigma_in, cond_in, c_out, c_in, t, From 4306659c4dab1a2ae611ac2a7487b87e1c513adf Mon Sep 17 00:00:00 2001 From: brkirch Date: Sat, 4 Feb 2023 01:22:06 -0500 Subject: [PATCH 05/10] Remove unused code --- modules/mac_specific.py | 3 --- modules/shared.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/modules/mac_specific.py b/modules/mac_specific.py index e39d670e..ddcea53b 100644 --- a/modules/mac_specific.py +++ b/modules/mac_specific.py @@ -4,9 +4,6 @@ from modules.sd_hijack_utils import CondFunc from packaging import version -device = None - - # has_mps is only available in nightly pytorch (for now) and macOS 12.3+. # check `getattr` and try it for compatibility def check_for_mps() -> bool: diff --git a/modules/shared.py b/modules/shared.py index 59f12cd8..5600d480 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -145,9 +145,6 @@ devices.device, devices.device_interrogate, devices.device_gfpgan, devices.devic (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer']) device = devices.device -if sys.platform == "darwin": - from modules import mac_specific - mac_specific.device = device weight_load_location = None if cmd_opts.lowram else "cpu" batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) From 72dd5785d9721b95e8d61210a56be8f6c6b1e97d Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 11:06:17 +0300 Subject: [PATCH 06/10] merge CFGDenoiserEdit and CFGDenoiser into single object --- modules/sd_samplers_kdiffusion.py | 133 +++++++++--------------------- 1 file changed, 37 insertions(+), 96 deletions(-) diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 6c57fdec..f076fc55 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -41,90 +41,6 @@ sampler_extra_params = { 'sample_dpm_2': ['s_churn', 's_tmin', 's_tmax', 's_noise'], } -class CFGDenoiserEdit(torch.nn.Module): - """ - Classifier free guidance denoiser. A wrapper for stable diffusion model (specifically for unet) - that can take a noisy picture and produce a noise-free picture using two guidances (prompts) - instead of one. Originally, the second prompt is just an empty string, but we use non-empty - negative prompt. - """ - - def __init__(self, model): - super().__init__() - self.inner_model = model - self.mask = None - self.nmask = None - self.init_latent = None - self.step = 0 - - def combine_denoised(self, x_out, conds_list, uncond, cond_scale, image_cfg_scale): - denoised_uncond = x_out[-uncond.shape[0]:] - denoised = torch.clone(denoised_uncond) - - for i, conds in enumerate(conds_list): - for cond_index, weight in conds: - out_cond, out_img_cond, out_uncond = x_out.chunk(3) - denoised[i] = out_uncond[cond_index] + cond_scale * (out_cond[cond_index] - out_img_cond[cond_index]) + image_cfg_scale * (out_img_cond[cond_index] - out_uncond[cond_index]) - - return denoised - - def forward(self, x, sigma, uncond, cond, cond_scale, image_cond, image_cfg_scale): - if state.interrupted or state.skipped: - raise sd_samplers_common.InterruptedException - - conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step) - uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step) - - batch_size = len(conds_list) - repeats = [len(conds_list[i]) for i in range(batch_size)] - - x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) - sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [torch.zeros_like(self.init_latent)]) - - denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) - cfg_denoiser_callback(denoiser_params) - x_in = denoiser_params.x - image_cond_in = denoiser_params.image_cond - sigma_in = denoiser_params.sigma - - if tensor.shape[1] == uncond.shape[1]: - cond_in = torch.cat([tensor, uncond, uncond]) - - if shared.batch_cond_uncond: - x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) - else: - x_out = torch.zeros_like(x_in) - for batch_offset in range(0, x_out.shape[0], batch_size): - a = batch_offset - b = a + batch_size - x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": [cond_in[a:b]], "c_concat": [image_cond_in[a:b]]}) - else: - x_out = torch.zeros_like(x_in) - batch_size = batch_size*2 if shared.batch_cond_uncond else batch_size - for batch_offset in range(0, tensor.shape[0], batch_size): - a = batch_offset - b = min(a + batch_size, tensor.shape[0]) - x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": torch.cat([tensor[a:b]], uncond) , "c_concat": [image_cond_in[a:b]]}) - - x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]}) - - devices.test_for_nans(x_out, "unet") - - if opts.live_preview_content == "Prompt": - sd_samplers_common.store_latent(x_out[0:uncond.shape[0]]) - elif opts.live_preview_content == "Negative prompt": - sd_samplers_common.store_latent(x_out[-uncond.shape[0]:]) - - denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale, image_cfg_scale) - - if self.mask is not None: - denoised = self.init_latent * self.mask + self.nmask * denoised - - self.step += 1 - - return denoised - class CFGDenoiser(torch.nn.Module): """ @@ -141,6 +57,7 @@ class CFGDenoiser(torch.nn.Module): self.nmask = None self.init_latent = None self.step = 0 + self.image_cfg_scale = None def combine_denoised(self, x_out, conds_list, uncond, cond_scale): denoised_uncond = x_out[-uncond.shape[0]:] @@ -152,19 +69,36 @@ class CFGDenoiser(torch.nn.Module): return denoised + def combine_denoised_for_edit_model(self, x_out, cond_scale): + out_cond, out_img_cond, out_uncond = x_out.chunk(3) + denoised = out_uncond + cond_scale * (out_cond - out_img_cond) + self.image_cfg_scale * (out_img_cond - out_uncond) + + return denoised + def forward(self, x, sigma, uncond, cond, cond_scale, image_cond): if state.interrupted or state.skipped: raise sd_samplers_common.InterruptedException + # at self.image_cfg_scale == 1.0 produced results for edit model are the same as with normal sampling, + # so is_edit_model is set to False to support AND composition. + is_edit_model = shared.sd_model.cond_stage_key == "edit" and self.image_cfg_scale is not None and self.image_cfg_scale != 1.0 + conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step) uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step) + assert not is_edit_model or all([len(conds) == 1 for conds in conds_list]), "AND is not supported for InstructPix2Pix checkpoint (unless using Image CFG scale = 1.0)" + batch_size = len(conds_list) repeats = [len(conds_list[i]) for i in range(batch_size)] - x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) - sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) + if not is_edit_model: + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) + else: + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [torch.zeros_like(self.init_latent)]) denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) cfg_denoiser_callback(denoiser_params) @@ -173,7 +107,10 @@ class CFGDenoiser(torch.nn.Module): sigma_in = denoiser_params.sigma if tensor.shape[1] == uncond.shape[1]: - cond_in = torch.cat([tensor, uncond]) + if not is_edit_model: + cond_in = torch.cat([tensor, uncond]) + else: + cond_in = torch.cat([tensor, uncond, uncond]) if shared.batch_cond_uncond: x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) @@ -189,7 +126,13 @@ class CFGDenoiser(torch.nn.Module): for batch_offset in range(0, tensor.shape[0], batch_size): a = batch_offset b = min(a + batch_size, tensor.shape[0]) - x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": [tensor[a:b]], "c_concat": [image_cond_in[a:b]]}) + + if not is_edit_model: + c_crossattn = [tensor[a:b]] + else: + c_crossattn = torch.cat([tensor[a:b]], uncond) + + x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": c_crossattn, "c_concat": [image_cond_in[a:b]]}) x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]}) @@ -200,7 +143,10 @@ class CFGDenoiser(torch.nn.Module): elif opts.live_preview_content == "Negative prompt": sd_samplers_common.store_latent(x_out[-uncond.shape[0]:]) - denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale) + if not is_edit_model: + denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale) + else: + denoised = self.combine_denoised_for_edit_model(x_out, cond_scale) if self.mask is not None: denoised = self.init_latent * self.mask + self.nmask * denoised @@ -280,12 +226,10 @@ class KDiffusionSampler: return p.steps def initialize(self, p): - if shared.sd_model.cond_stage_key == "edit" and getattr(p, 'image_cfg_scale', None) != 1: - self.model_wrap_cfg = CFGDenoiserEdit(self.model_wrap) - self.model_wrap_cfg.mask = p.mask if hasattr(p, 'mask') else None self.model_wrap_cfg.nmask = p.nmask if hasattr(p, 'nmask') else None self.model_wrap_cfg.step = 0 + self.model_wrap_cfg.image_cfg_scale = getattr(p, 'image_cfg_scale', None) self.eta = p.eta if p.eta is not None else opts.eta_ancestral k_diffusion.sampling.torch = TorchHijack(self.sampler_noises if self.sampler_noises is not None else []) @@ -355,9 +299,6 @@ class KDiffusionSampler: 'cond_scale': p.cfg_scale, } - if hasattr(p, 'image_cfg_scale') and p.image_cfg_scale != 1 and p.image_cfg_scale != None: - extra_args['image_cfg_scale'] = p.image_cfg_scale - samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args=extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs)) return samples From c4b9ed1a2791e411f95a96a6324b4986b8b85b84 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 11:18:44 +0300 Subject: [PATCH 07/10] make Image CFG Scale only show if instrutpix2pix model is loaded --- modules/ui.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/ui.py b/modules/ui.py index f2f7de8b..f5df1ffe 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -765,8 +765,9 @@ def create_ui(): elif category == "cfg": with FormGroup(): - cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=7.0, elem_id="img2img_cfg_scale") - image_cfg_scale = gr.Slider(minimum=0, maximum=3.0, step=0.05, label='Image CFG Scale (for instruct-pix2pix models only)', value=1.5, elem_id="img2img_image_cfg_scale") + with FormRow(): + cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=7.0, elem_id="img2img_cfg_scale") + image_cfg_scale = gr.Slider(minimum=0, maximum=3.0, step=0.05, label='Image CFG Scale', value=1.5, elem_id="img2img_image_cfg_scale", visible=shared.sd_model and shared.sd_model.cond_stage_key == "edit") denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength") elif category == "seed": @@ -1594,6 +1595,12 @@ def create_ui(): outputs=[component, text_settings], ) + text_settings.change( + fn=lambda: gr.update(visible=shared.sd_model and shared.sd_model.cond_stage_key == "edit"), + inputs=[], + outputs=[image_cfg_scale], + ) + button_set_checkpoint = gr.Button('Change checkpoint', elem_id='change_checkpoint', visible=False) button_set_checkpoint.click( fn=lambda value, _: run_settings_single(value, key='sd_model_checkpoint'), From 81823407d9b3c3daf2f9de59e0d75ef9a257f902 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 11:38:56 +0300 Subject: [PATCH 08/10] add --no-hashing --- modules/hashes.py | 4 ++++ modules/hypernetworks/hypernetwork.py | 2 +- modules/sd_models.py | 3 +++ modules/shared.py | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/hashes.py b/modules/hashes.py index 819362a3..83272a07 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -4,6 +4,7 @@ import os.path import filelock +from modules import shared from modules.paths import data_path @@ -68,6 +69,9 @@ def sha256(filename, title): if sha256_value is not None: return sha256_value + if shared.cmd_opts.no_hashing: + return None + print(f"Calculating sha256 for {filename}: ", end='') sha256_value = calculate_sha256(filename) print(f"{sha256_value}") diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 503534e2..825a93b2 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -307,7 +307,7 @@ class Hypernetwork: def shorthash(self): sha256 = hashes.sha256(self.filename, f'hypernet/{self.name}') - return sha256[0:10] + return sha256[0:10] if sha256 else None def list_hypernetworks(path): diff --git a/modules/sd_models.py b/modules/sd_models.py index 300387a9..6c6bb571 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -59,6 +59,9 @@ class CheckpointInfo: def calculate_shorthash(self): self.sha256 = hashes.sha256(self.filename, "checkpoint/" + self.name) + if self.sha256 is None: + return + self.shorthash = self.sha256[0:10] if self.shorthash not in self.ids: diff --git a/modules/shared.py b/modules/shared.py index 5600d480..79fbf724 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -106,7 +106,7 @@ parser.add_argument("--tls-certfile", type=str, help="Partially enables TLS, req parser.add_argument("--server-name", type=str, help="Sets hostname of server", default=None) parser.add_argument("--gradio-queue", action='store_true', help="Uses gradio queue; experimental option; breaks restart UI button") parser.add_argument("--skip-version-check", action='store_true', help="Do not check versions of torch and xformers") - +parser.add_argument("--no-hashing", action='store_true', help="disable sha256 hashing of checkpoints to help loading performance", default=False) script_loading.preload_extensions(extensions.extensions_dir, parser) From 40e51fd6efa9c09a82c5ab391dbbd2c806971582 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 13:28:53 +0300 Subject: [PATCH 09/10] add margin parameter to draw_grid_annotations --- modules/images.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/images.py b/modules/images.py index ae3cdaf4..4bdbd730 100644 --- a/modules/images.py +++ b/modules/images.py @@ -130,7 +130,7 @@ class GridAnnotation: self.size = None -def draw_grid_annotations(im, width, height, hor_texts, ver_texts): +def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0): def wrap(drawing, text, font, line_length): lines = [''] for word in text.split(): @@ -194,25 +194,28 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts): line.allowed_width = allowed_width hor_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in hor_texts] - ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in - ver_texts] + ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in ver_texts] pad_top = 0 if sum(hor_text_heights) == 0 else max(hor_text_heights) + line_spacing * 2 - result = Image.new("RGB", (im.width + pad_left, im.height + pad_top), "white") - result.paste(im, (pad_left, pad_top)) + result = Image.new("RGB", (im.width + pad_left + margin * (rows-1), im.height + pad_top + margin * (cols-1)), "white") + + for row in range(rows): + for col in range(cols): + cell = im.crop((width * col, height * row, width * (col+1), height * (row+1))) + result.paste(cell, (pad_left + (width + margin) * col, pad_top + (height + margin) * row)) d = ImageDraw.Draw(result) for col in range(cols): - x = pad_left + width * col + width / 2 + x = pad_left + (width + margin) * col + width / 2 y = pad_top / 2 - hor_text_heights[col] / 2 draw_texts(d, x, y, hor_texts[col], fnt, fontsize) for row in range(rows): x = pad_left / 2 - y = pad_top + height * row + height / 2 - ver_text_heights[row] / 2 + y = pad_top + (height + margin) * row + height / 2 - ver_text_heights[row] / 2 draw_texts(d, x, y, ver_texts[row], fnt, fontsize) From 3e0f9a75438fa815429b5530261bcf7d80f3f101 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 15:23:16 +0300 Subject: [PATCH 10/10] fix issue with switching back to checkpoint that had its checksum calculated during runtime mentioned in #7506 --- modules/sd_models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 0e61d323..af1731e5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -65,10 +65,11 @@ class CheckpointInfo: self.shorthash = self.sha256[0:10] if self.shorthash not in self.ids: - self.ids += [self.shorthash, self.sha256] - self.register() + self.ids += [self.shorthash, self.sha256, f'{self.name} [{self.shorthash}]'] + checkpoints_list.pop(self.title) self.title = f'{self.name} [{self.shorthash}]' + self.register() return self.shorthash