From 44ab954fabb9c1273366ebdca47f8da394d61aab Mon Sep 17 00:00:00 2001 From: random_thoughtss Date: Sat, 29 Oct 2022 10:02:56 -0700 Subject: [PATCH 1/5] Fix latent upscale highres fix #3888 --- modules/processing.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 548eec29..f18b7db2 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -653,6 +653,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): if opts.use_scale_latent_for_hires_fix: samples = torch.nn.functional.interpolate(samples, size=(self.height // opt_f, self.width // opt_f), mode="bilinear") + image_conditioning = self.txt2img_image_conditioning(samples) else: decoded_samples = decode_first_stage(self.sd_model, samples) @@ -674,6 +675,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples)) + image_conditioning = self.img2img_image_conditioning( + decoded_samples, + samples, + decoded_samples.new_ones(decoded_samples.shape[0], 1, decoded_samples.shape[2], decoded_samples.shape[3]) + ) + shared.state.nextjob() self.sampler = sd_samplers.create_sampler_with_index(sd_samplers.samplers, self.sampler_index, self.sd_model) @@ -684,11 +691,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): x = None devices.torch_gc() - image_conditioning = self.img2img_image_conditioning( - decoded_samples, - samples, - decoded_samples.new_ones(decoded_samples.shape[0], 1, decoded_samples.shape[2], decoded_samples.shape[3]) - ) samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.steps, image_conditioning=image_conditioning) return samples From 6e2ce4e735db64afcd0fe637327ca4ec78335706 Mon Sep 17 00:00:00 2001 From: random_thoughtss Date: Sat, 29 Oct 2022 10:35:51 -0700 Subject: [PATCH 2/5] Added image conditioning to latent upscale. Only comuted if the mask weight is not 1.0 to avoid extra memory. Also includes some code cleanup. --- modules/processing.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index f18b7db2..ee0e9e34 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -134,11 +134,7 @@ class StableDiffusionProcessing(): # Dummy zero conditioning if we're not using inpainting model. # Still takes up a bit of memory, but no encoder call. # Pretty sure we can just make this a 1x1 image since its not going to be used besides its batch size. - return torch.zeros( - x.shape[0], 5, 1, 1, - dtype=x.dtype, - device=x.device - ) + return x.new_zeros(x.shape[0], 5, 1, 1) height = height or self.height width = width or self.width @@ -156,11 +152,7 @@ class StableDiffusionProcessing(): def img2img_image_conditioning(self, source_image, latent_image, image_mask = None): if self.sampler.conditioning_key not in {'hybrid', 'concat'}: # Dummy zero conditioning if we're not using inpainting model. - return torch.zeros( - latent_image.shape[0], 5, 1, 1, - dtype=latent_image.dtype, - device=latent_image.device - ) + return latent_image.new_zeros(latent_image.shape[0], 5, 1, 1) # Handle the different mask inputs if image_mask is not None: @@ -174,11 +166,10 @@ class StableDiffusionProcessing(): # Inpainting model uses a discretized mask as input, so we round to either 1.0 or 0.0 conditioning_mask = torch.round(conditioning_mask) else: - conditioning_mask = torch.ones(1, 1, *source_image.shape[-2:]) + conditioning_mask = source_image.new_ones(1, 1, *source_image.shape[-2:]) # Create another latent image, this time with a masked version of the original input. # Smoothly interpolate between the masked and unmasked latent conditioning image using a parameter. - conditioning_mask = conditioning_mask.to(source_image.device) conditioning_image = torch.lerp( source_image, source_image * (1.0 - conditioning_mask), @@ -653,7 +644,13 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): if opts.use_scale_latent_for_hires_fix: samples = torch.nn.functional.interpolate(samples, size=(self.height // opt_f, self.width // opt_f), mode="bilinear") - image_conditioning = self.txt2img_image_conditioning(samples) + + # Avoid making the inpainting conditioning unless necessary as + # this does need some extra compute to decode / encode the image again. + if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0: + image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples), samples) + else: + image_conditioning = self.txt2img_image_conditioning(samples) else: decoded_samples = decode_first_stage(self.sd_model, samples) @@ -675,11 +672,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples)) - image_conditioning = self.img2img_image_conditioning( - decoded_samples, - samples, - decoded_samples.new_ones(decoded_samples.shape[0], 1, decoded_samples.shape[2], decoded_samples.shape[3]) - ) + image_conditioning = self.img2img_image_conditioning(decoded_samples, samples) shared.state.nextjob() From 39f55c3c35873bc7dd9792cb2155746a1c3d4292 Mon Sep 17 00:00:00 2001 From: random_thoughtss Date: Sat, 29 Oct 2022 14:13:02 -0700 Subject: [PATCH 3/5] Re-add explicit device move --- modules/processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/processing.py b/modules/processing.py index ee0e9e34..d07e3db9 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -170,6 +170,7 @@ class StableDiffusionProcessing(): # Create another latent image, this time with a masked version of the original input. # Smoothly interpolate between the masked and unmasked latent conditioning image using a parameter. + conditioning_mask = conditioning_mask.to(source_image.device).to(source_image.dtype) conditioning_image = torch.lerp( source_image, source_image * (1.0 - conditioning_mask), From 71571e3f055237d71ba2d47756846ad1d73be00c Mon Sep 17 00:00:00 2001 From: random_thoughtss Date: Sun, 30 Oct 2022 00:35:40 -0700 Subject: [PATCH 4/5] Replaced master branch fix with updated fix. --- modules/processing.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 3dd44d3a..512c484f 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -688,8 +688,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) - image_conditioning = self.txt2img_image_conditioning(x) - # GC now before running the next img2img to prevent running out of memory x = None devices.torch_gc() From d9e4e4d7a09d4aee8ce249a3c8e91ce165b10fa5 Mon Sep 17 00:00:00 2001 From: random_thoughtss Date: Sun, 30 Oct 2022 15:33:02 -0700 Subject: [PATCH 5/5] Fix non-square full resolution inpainting. --- modules/masking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/masking.py b/modules/masking.py index fd8d9241..a5c4d2da 100644 --- a/modules/masking.py +++ b/modules/masking.py @@ -49,7 +49,7 @@ def expand_crop_region(crop_region, processing_width, processing_height, image_w ratio_processing = processing_width / processing_height if ratio_crop_region > ratio_processing: - desired_height = (x2 - x1) * ratio_processing + desired_height = (x2 - x1) / ratio_processing desired_height_diff = int(desired_height - (y2-y1)) y1 -= desired_height_diff//2 y2 += desired_height_diff - desired_height_diff//2