add cross-attention support to im2img; prevent inpainting from crashing

ldm/generate.py

@@ -541,7 +541,8 @@ class Generate:
         image = Image.open(image_path)
 
         # used by multiple postfixers
-        uc, c = get_uc_and_c(
+        # todo: cross-attention
+        uc, c, _, _ = get_uc_and_c_and_ec(
             prompt, model =self.model,
             skip_normalize=opt.skip_normalize,
             log_tokens    =opt.log_tokenization

ldm/invoke/generator/img2img.py

@@ -32,7 +32,7 @@ class Img2Img(Generator):
             ) # move to latent space
 
         t_enc = int(strength * steps)
-        uc, c   = conditioning
+        uc, c, ec, edit_opcodes   = conditioning
 
         def make_image(x_T):
             # encode (scaled latent)
@@ -49,7 +49,10 @@ class Img2Img(Generator):
                 img_callback = step_callback,
                 unconditional_guidance_scale=cfg_scale,
                 unconditional_conditioning=uc,
-                init_latent = self.init_latent,  # changes how noising is performed in ksampler
+                init_latent = self.init_latent,
+                edited_conditioning = ec,
+                conditioning_edit_opcodes = edit_opcodes
+                # changes how noising is performed in ksampler
             )
 
             return self.sample_to_image(samples)

ldm/invoke/generator/inpaint.py

@@ -45,7 +45,8 @@ class Inpaint(Img2Img):
             ) # move to latent space
 
         t_enc   = int(strength * steps)
-        uc, c   = conditioning
+        # todo: support cross-attention control
+        uc, c, _, _ = conditioning
 
         print(f">> target t_enc is {t_enc} steps")
 

ldm/models/diffusion/ksampler.py

@@ -132,6 +132,7 @@ class KSampler(Sampler):
             use_original_steps=False,
             init_latent       = None,
             mask              = None,
+            **kwargs
     ):
         samples,_ = self.sample(
             batch_size = 1,
@@ -143,7 +144,8 @@ class KSampler(Sampler):
             unconditional_conditioning = unconditional_conditioning,
             img_callback = img_callback,
             x0           = init_latent,
-            mask         = mask
+            mask         = mask,
+            **kwargs
             )
         return samples
 
@@ -238,6 +240,8 @@ class KSampler(Sampler):
             index,
             unconditional_guidance_scale=1.0,
             unconditional_conditioning=None,
+            edited_conditioning=None,
+            conditioning_edit_opcodes=None,
             **kwargs,
     ):
         if self.model_wrap is None:
@@ -263,6 +267,7 @@ class KSampler(Sampler):
         # so the actual formula for indexing into sigmas:
         # sigma_index = (steps-index)
         s_index = t_enc - index - 1
+        self.model_wrap.prepare_to_sample(s_index, edited_conditioning=edited_conditioning, conditioning_edit_opcodes=conditioning_edit_opcodes)
         img =  K.sampling.__dict__[f'_{self.schedule}'](
             self.model_wrap,
             img,