Fix avoid storing extra conditioning info in two places.

invokeai/backend/stable_diffusion/diffusers_pipeline.py

@@ -420,10 +420,11 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
             return latents, attention_map_saver
 
         ip_adapter_unet_patcher = None
-        if conditioning_data.extra is not None and conditioning_data.extra.wants_cross_attention_control:
+        extra_conditioning_info = conditioning_data.text_embeddings.extra_conditioning
+        if extra_conditioning_info is not None and extra_conditioning_info.wants_cross_attention_control:
             attn_ctx = self.invokeai_diffuser.custom_attention_context(
                 self.invokeai_diffuser.model,
-                extra_conditioning_info=conditioning_data.extra,
+                extra_conditioning_info=extra_conditioning_info,
                 step_count=len(self.scheduler.timesteps),
             )
             self.use_ip_adapter = False

invokeai/backend/stable_diffusion/diffusion/conditioning_data.py

@@ -21,11 +21,7 @@ class ExtraConditioningInfo:
 @dataclass
 class BasicConditioningInfo:
     embeds: torch.Tensor
-    # TODO(ryand): Right now we awkwardly copy the extra conditioning info from here up to `ConditioningData`. This
-    # should only be stored in one place.
     extra_conditioning: Optional[ExtraConditioningInfo]
-    # weight: float
-    # mode: ConditioningAlgo
 
     def to(self, device, dtype=None):
         self.embeds = self.embeds.to(device=device, dtype=dtype)
@@ -78,7 +74,6 @@ class ConditioningData:
      ref [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf)
     """
     guidance_rescale_multiplier: float = 0
-    extra: Optional[ExtraConditioningInfo] = None
     scheduler_args: dict[str, Any] = field(default_factory=dict)
     """
     Additional arguments to pass to invokeai_diffuser.do_latent_postprocessing().