From 44d36a0e0b957591d8fa3e42046801360d9dc293 Mon Sep 17 00:00:00 2001
From: Kevin Turner <83819+keturn@users.noreply.github.com>
Date: Wed, 1 Feb 2023 18:29:25 -0800
Subject: [PATCH 1/2] fix(img2img): do not attempt to do a zero-step img2img
when strength is low
---
ldm/invoke/generator/diffusers_pipeline.py | 26 ++++++++++++----------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/ldm/invoke/generator/diffusers_pipeline.py b/ldm/invoke/generator/diffusers_pipeline.py
index 1ef89913c5..a63159b118 100644
--- a/ldm/invoke/generator/diffusers_pipeline.py
+++ b/ldm/invoke/generator/diffusers_pipeline.py
@@ -17,10 +17,8 @@ import PIL.Image
import einops
import torch
import torchvision.transforms as T
-from diffusers.models import attention
from diffusers.utils.import_utils import is_xformers_available
-from ...models.diffusion import cross_attention_control
from ...models.diffusion.cross_attention_map_saving import AttentionMapSaver
from ...modules.prompt_to_embeddings_converter import WeightedPromptFragmentsToEmbeddingsConverter
@@ -506,11 +504,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
strength,
noise: torch.Tensor, run_id=None, callback=None
) -> InvokeAIStableDiffusionPipelineOutput:
- device = self.unet.device
- img2img_pipeline = StableDiffusionImg2ImgPipeline(**self.components)
- img2img_pipeline.scheduler.set_timesteps(num_inference_steps, device=device)
- timesteps, _ = img2img_pipeline.get_timesteps(num_inference_steps, strength, device=device)
-
+ timesteps, _ = self.get_img2img_timesteps(num_inference_steps, strength, self.unet.device)
result_latents, result_attention_maps = self.latents_from_embeddings(
initial_latents, num_inference_steps, conditioning_data,
timesteps=timesteps,
@@ -526,6 +520,18 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
output = InvokeAIStableDiffusionPipelineOutput(images=image, nsfw_content_detected=[], attention_map_saver=result_attention_maps)
return self.check_for_safety(output, dtype=conditioning_data.dtype)
+ def get_img2img_timesteps(self, num_inference_steps: int, strength: float, device) -> (torch.Tensor, int):
+ img2img_pipeline = StableDiffusionImg2ImgPipeline(**self.components)
+ assert img2img_pipeline.scheduler is self.scheduler
+ img2img_pipeline.scheduler.set_timesteps(num_inference_steps, device=device)
+ timesteps, adjusted_steps = img2img_pipeline.get_timesteps(num_inference_steps, strength, device=device)
+ # Workaround for low strength resulting in zero timesteps.
+ # TODO: submit upstream fix for zero-step img2img
+ if timesteps.numel() == 0:
+ timesteps = self.scheduler.timesteps[-1:]
+ adjusted_steps = timesteps.numel()
+ return timesteps, adjusted_steps
+
def inpaint_from_embeddings(
self,
init_image: torch.FloatTensor,
@@ -549,11 +555,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
if init_image.dim() == 3:
init_image = init_image.unsqueeze(0)
- img2img_pipeline = StableDiffusionImg2ImgPipeline(**self.components)
- img2img_pipeline.scheduler.set_timesteps(num_inference_steps, device=device)
- timesteps, _ = img2img_pipeline.get_timesteps(num_inference_steps, strength, device=device)
-
- assert img2img_pipeline.scheduler is self.scheduler
+ timesteps, _ = self.get_img2img_timesteps(num_inference_steps, strength, device=device)
# 6. Prepare latent variables
# can't quite use upstream StableDiffusionImg2ImgPipeline.prepare_latents
From 9e943ff7dcaab517148f6e642076992e113c6c5a Mon Sep 17 00:00:00 2001
From: noodlebox <ash@noodlebox.moe>
Date: Thu, 2 Feb 2023 05:26:15 -0500
Subject: [PATCH 2/2] Fix typo in xformers version, 0.16 -> 0.0.16
---
environments-and-requirements/requirements-lin-cuda.txt | 2 +-
environments-and-requirements/requirements-win-colab-cuda.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/environments-and-requirements/requirements-lin-cuda.txt b/environments-and-requirements/requirements-lin-cuda.txt
index f39aebea75..d0035da39f 100644
--- a/environments-and-requirements/requirements-lin-cuda.txt
+++ b/environments-and-requirements/requirements-lin-cuda.txt
@@ -2,5 +2,5 @@
-r environments-and-requirements/requirements-base.txt
torch>=1.13.1
torchvision>=0.14.1
-xformers~=0.16
+xformers~=0.0.16
-e .
diff --git a/environments-and-requirements/requirements-win-colab-cuda.txt b/environments-and-requirements/requirements-win-colab-cuda.txt
index a6375bd0ef..56b8e89822 100644
--- a/environments-and-requirements/requirements-win-colab-cuda.txt
+++ b/environments-and-requirements/requirements-win-colab-cuda.txt
@@ -3,5 +3,5 @@
--extra-index-url https://download.pytorch.org/whl/cu117 --trusted-host https://download.pytorch.org
torch==1.13.1
torchvision==0.14.1
-xformers~=0.16
+xformers~=0.0.16
-e .