From 5d5157fc6512a3e264fa310c9d1030701ceef8b6 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Wed, 10 May 2023 18:08:33 -0400 Subject: [PATCH 1/7] make conditioning.py work with compel 1.1.5 --- invokeai/backend/prompting/conditioning.py | 100 ++++++++++----------- 1 file changed, 46 insertions(+), 54 deletions(-) diff --git a/invokeai/backend/prompting/conditioning.py b/invokeai/backend/prompting/conditioning.py index d9130ace04..f94f82ef72 100644 --- a/invokeai/backend/prompting/conditioning.py +++ b/invokeai/backend/prompting/conditioning.py @@ -16,6 +16,7 @@ from compel.prompt_parser import ( FlattenedPrompt, Fragment, PromptParser, + Conjunction, ) import invokeai.backend.util.logging as logger @@ -25,58 +26,51 @@ from ..stable_diffusion import InvokeAIDiffuserComponent from ..util import torch_dtype -def get_uc_and_c_and_ec( - prompt_string, model, log_tokens=False, skip_normalize_legacy_blend=False -): +def get_uc_and_c_and_ec(prompt_string, + model: InvokeAIDiffuserComponent, + log_tokens=False, skip_normalize_legacy_blend=False): # lazy-load any deferred textual inversions. # this might take a couple of seconds the first time a textual inversion is used. - model.textual_inversion_manager.create_deferred_token_ids_for_any_trigger_terms( - prompt_string - ) + model.textual_inversion_manager.create_deferred_token_ids_for_any_trigger_terms(prompt_string) - tokenizer = model.tokenizer - compel = Compel( - tokenizer=tokenizer, - text_encoder=model.text_encoder, - textual_inversion_manager=model.textual_inversion_manager, - dtype_for_device_getter=torch_dtype, - truncate_long_prompts=False - ) + compel = Compel(tokenizer=model.tokenizer, + text_encoder=model.text_encoder, + textual_inversion_manager=model.textual_inversion_manager, + dtype_for_device_getter=torch_dtype, + truncate_long_prompts=False, + ) # get rid of any newline characters prompt_string = prompt_string.replace("\n", " ") - ( - positive_prompt_string, - negative_prompt_string, - ) = split_prompt_to_positive_and_negative(prompt_string) - legacy_blend = try_parse_legacy_blend( - positive_prompt_string, skip_normalize_legacy_blend - ) - positive_prompt: Union[FlattenedPrompt, Blend] + positive_prompt_string, negative_prompt_string = split_prompt_to_positive_and_negative(prompt_string) + + legacy_blend = try_parse_legacy_blend(positive_prompt_string, skip_normalize_legacy_blend) + positive_conjunction: Conjunction if legacy_blend is not None: - positive_prompt = legacy_blend + positive_conjunction = legacy_blend else: - positive_prompt = Compel.parse_prompt_string(positive_prompt_string) - negative_prompt: Union[FlattenedPrompt, Blend] = Compel.parse_prompt_string( - negative_prompt_string - ) + positive_conjunction = Compel.parse_prompt_string(positive_prompt_string) + positive_prompt = positive_conjunction.prompts[0] + negative_conjunction = Compel.parse_prompt_string(negative_prompt_string) + negative_prompt: FlattenedPrompt | Blend = negative_conjunction.prompts[0] + + tokens_count = get_max_token_count(model.tokenizer, positive_prompt) if log_tokens or getattr(Globals, "log_tokenization", False): - log_tokenization(positive_prompt, negative_prompt, tokenizer=tokenizer) + log_tokenization(positive_prompt, negative_prompt, tokenizer=model.tokenizer) - c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) - uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) - [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) + with InvokeAIDiffuserComponent.custom_attention_context(model.unet, + extra_conditioning_info=None, + step_count=-1): + c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) + uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) - tokens_count = get_max_token_count(tokenizer, positive_prompt) - - ec = InvokeAIDiffuserComponent.ExtraConditioningInfo( - tokens_count_including_eos_bos=tokens_count, - cross_attention_control_args=options.get("cross_attention_control", None), - ) + # now build the "real" ec + ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(tokens_count_including_eos_bos=tokens_count, + cross_attention_control_args=options.get( + 'cross_attention_control', None)) return uc, c, ec - def get_prompt_structure( prompt_string, skip_normalize_legacy_blend: bool = False ) -> (Union[FlattenedPrompt, Blend], FlattenedPrompt): @@ -87,18 +81,17 @@ def get_prompt_structure( legacy_blend = try_parse_legacy_blend( positive_prompt_string, skip_normalize_legacy_blend ) - positive_prompt: Union[FlattenedPrompt, Blend] + positive_prompt: Conjunction if legacy_blend is not None: - positive_prompt = legacy_blend + positive_conjunction = legacy_blend else: - positive_prompt = Compel.parse_prompt_string(positive_prompt_string) - negative_prompt: Union[FlattenedPrompt, Blend] = Compel.parse_prompt_string( - negative_prompt_string - ) + positive_conjunction = Compel.parse_prompt_string(positive_prompt_string) + positive_prompt = positive_conjunction.prompts[0] + negative_conjunction = Compel.parse_prompt_string(negative_prompt_string) + negative_prompt: FlattenedPrompt|Blend = negative_conjunction.prompts[0] return positive_prompt, negative_prompt - def get_max_token_count( tokenizer, prompt: Union[FlattenedPrompt, Blend], truncate_if_too_long=False ) -> int: @@ -245,22 +238,21 @@ def log_tokenization_for_text(text, tokenizer, display_label=None, truncate_if_t logger.info(f"[TOKENLOG] Tokens Discarded ({totalTokens - usedTokens}):") logger.debug(f"{discarded}\x1b[0m") - -def try_parse_legacy_blend(text: str, skip_normalize: bool = False) -> Optional[Blend]: +def try_parse_legacy_blend(text: str, skip_normalize: bool = False) -> Optional[Conjunction]: weighted_subprompts = split_weighted_subprompts(text, skip_normalize=skip_normalize) if len(weighted_subprompts) <= 1: return None strings = [x[0] for x in weighted_subprompts] - weights = [x[1] for x in weighted_subprompts] pp = PromptParser() parsed_conjunctions = [pp.parse_conjunction(x) for x in strings] - flattened_prompts = [x.prompts[0] for x in parsed_conjunctions] - - return Blend( - prompts=flattened_prompts, weights=weights, normalize_weights=not skip_normalize - ) - + flattened_prompts = [] + weights = [] + for i, x in enumerate(parsed_conjunctions): + if len(x.prompts)>0: + flattened_prompts.append(x.prompts[0]) + weights.append(weighted_subprompts[i][1]) + return Conjunction([Blend(prompts=flattened_prompts, weights=weights, normalize_weights=not skip_normalize)]) def split_weighted_subprompts(text, skip_normalize=False) -> list: """ From aca47704814a2642dd2e58cba598abea25bbed19 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Wed, 10 May 2023 21:40:44 -0400 Subject: [PATCH 2/7] fixed compel.py as requested --- invokeai/app/invocations/compel.py | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/invokeai/app/invocations/compel.py b/invokeai/app/invocations/compel.py index 1fb7832031..329f1b6f08 100644 --- a/invokeai/app/invocations/compel.py +++ b/invokeai/app/invocations/compel.py @@ -100,7 +100,8 @@ class CompelInvocation(BaseInvocation): # TODO: support legacy blend? - prompt: Union[FlattenedPrompt, Blend] = Compel.parse_prompt_string(prompt_str) + conjunction = Compel.parse_prompt_string(prompt_str) + prompt: Union[FlattenedPrompt, Blend] = conjunction.prompts[0] if getattr(Globals, "log_tokenization", False): log_tokenization_for_prompt_object(prompt, tokenizer) diff --git a/pyproject.toml b/pyproject.toml index fd671fee23..2d685ffe02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "albumentations", "click", "clip_anytorch", # replacing "clip @ https://github.com/openai/CLIP/archive/eaa22acb90a5876642d0507623e859909230a52d.zip", - "compel~=1.1.5", + "compel~=1.1.5", "datasets", "diffusers[torch]~=0.16.1", "dnspython==2.2.1", From 9e594f90185de88b8e9c20f8ecaf6030d3ae7f17 Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 11 May 2023 00:34:12 -0400 Subject: [PATCH 3/7] pad conditioning tensors to same length fixes crash when prompt length is greater than 75 tokens --- invokeai/backend/prompting/conditioning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/invokeai/backend/prompting/conditioning.py b/invokeai/backend/prompting/conditioning.py index f94f82ef72..71e51f1103 100644 --- a/invokeai/backend/prompting/conditioning.py +++ b/invokeai/backend/prompting/conditioning.py @@ -64,6 +64,7 @@ def get_uc_and_c_and_ec(prompt_string, step_count=-1): c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) + [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) # now build the "real" ec ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(tokens_count_including_eos_bos=tokens_count, From 037078c8ad6d584949eb8671269aa8c0e7ec2e03 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 11 May 2023 21:13:18 -0400 Subject: [PATCH 4/7] make InvokeAIDiffuserComponent.custom_attention_control a classmethod --- .../stable_diffusion/diffusers_pipeline.py | 5 ++- .../diffusion/cross_attention_control.py | 37 ++++++------------ .../diffusion/shared_invokeai_diffusion.py | 38 ++++++++++++------- 3 files changed, 39 insertions(+), 41 deletions(-) diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index 94ec9da7e8..10e0ad4da3 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -545,8 +545,9 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): additional_guidance = [] extra_conditioning_info = conditioning_data.extra with self.invokeai_diffuser.custom_attention_context( - extra_conditioning_info=extra_conditioning_info, - step_count=len(self.scheduler.timesteps), + self.invokeai_diffuser.model, + extra_conditioning_info=extra_conditioning_info, + step_count=len(self.scheduler.timesteps), ): yield PipelineIntermediateState( run_id=run_id, diff --git a/invokeai/backend/stable_diffusion/diffusion/cross_attention_control.py b/invokeai/backend/stable_diffusion/diffusion/cross_attention_control.py index dfd19ea964..79a0982cfe 100644 --- a/invokeai/backend/stable_diffusion/diffusion/cross_attention_control.py +++ b/invokeai/backend/stable_diffusion/diffusion/cross_attention_control.py @@ -10,6 +10,7 @@ import diffusers import psutil import torch from compel.cross_attention_control import Arguments +from diffusers.models.unet_2d_condition import UNet2DConditionModel from diffusers.models.attention_processor import AttentionProcessor from torch import nn @@ -352,8 +353,7 @@ def restore_default_cross_attention( else: remove_attention_function(model) - -def override_cross_attention(model, context: Context, is_running_diffusers=False): +def setup_cross_attention_control_attention_processors(unet: UNet2DConditionModel, context: Context): """ Inject attention parameters and functions into the passed in model to enable cross attention editing. @@ -372,37 +372,22 @@ def override_cross_attention(model, context: Context, is_running_diffusers=False indices = torch.arange(max_length, dtype=torch.long) for name, a0, a1, b0, b1 in context.arguments.edit_opcodes: if b0 < max_length: - if name == "equal": # or (name == "replace" and a1 - a0 == b1 - b0): + if name == "equal":# or (name == "replace" and a1 - a0 == b1 - b0): # these tokens have not been edited indices[b0:b1] = indices_target[a0:a1] mask[b0:b1] = 1 context.cross_attention_mask = mask.to(device) context.cross_attention_index_map = indices.to(device) - if is_running_diffusers: - unet = model - old_attn_processors = unet.attn_processors - if torch.backends.mps.is_available(): - # see note in StableDiffusionGeneratorPipeline.__init__ about borked slicing on MPS - unet.set_attn_processor(SwapCrossAttnProcessor()) - else: - # try to re-use an existing slice size - default_slice_size = 4 - slice_size = next( - ( - p.slice_size - for p in old_attn_processors.values() - if type(p) is SlicedAttnProcessor - ), - default_slice_size, - ) - unet.set_attn_processor(SlicedSwapCrossAttnProcesser(slice_size=slice_size)) - return old_attn_processors + old_attn_processors = unet.attn_processors + if torch.backends.mps.is_available(): + # see note in StableDiffusionGeneratorPipeline.__init__ about borked slicing on MPS + unet.set_attn_processor(SwapCrossAttnProcessor()) else: - context.register_cross_attention_modules(model) - inject_attention_function(model, context) - return None - + # try to re-use an existing slice size + default_slice_size = 4 + slice_size = next((p.slice_size for p in old_attn_processors.values() if type(p) is SlicedAttnProcessor), default_slice_size) + unet.set_attn_processor(SlicedSwapCrossAttnProcesser(slice_size=slice_size)) def get_cross_attention_modules( model, which: CrossAttentionType diff --git a/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py b/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py index b0c85e9fd3..245317bcde 100644 --- a/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py +++ b/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, Optional, Union import numpy as np import torch +from diffusers import UNet2DConditionModel from diffusers.models.attention_processor import AttentionProcessor from typing_extensions import TypeAlias @@ -17,8 +18,8 @@ from .cross_attention_control import ( CrossAttentionType, SwapCrossAttnContext, get_cross_attention_modules, - override_cross_attention, restore_default_cross_attention, + setup_cross_attention_control_attention_processors, ) from .cross_attention_map_saving import AttentionMapSaver @@ -79,24 +80,35 @@ class InvokeAIDiffuserComponent: self.cross_attention_control_context = None self.sequential_guidance = Globals.sequential_guidance + @classmethod @contextmanager def custom_attention_context( - self, extra_conditioning_info: Optional[ExtraConditioningInfo], step_count: int + cls, + unet: UNet2DConditionModel, # note: also may futz with the text encoder depending on requested LoRAs + extra_conditioning_info: Optional[ExtraConditioningInfo], + step_count: int ): - do_swap = ( - extra_conditioning_info is not None - and extra_conditioning_info.wants_cross_attention_control - ) - old_attn_processor = None - if do_swap: - old_attn_processor = self.override_cross_attention( - extra_conditioning_info, step_count=step_count - ) + old_attn_processors = None + if extra_conditioning_info and ( + extra_conditioning_info.wants_cross_attention_control + ): + old_attn_processors = unet.attn_processors + # Load lora conditions into the model + if extra_conditioning_info.wants_cross_attention_control: + cross_attention_control_context = Context( + arguments=extra_conditioning_info.cross_attention_control_args, + step_count=step_count, + ) + setup_cross_attention_control_attention_processors( + unet, + cross_attention_control_context, + ) + try: yield None finally: - if old_attn_processor is not None: - self.restore_default_cross_attention(old_attn_processor) + if old_attn_processors is not None: + unet.set_attn_processor(old_attn_processors) # TODO resuscitate attention map saving # self.remove_attention_map_saving() From 8f8cd907878f00772e6c24185387d09dbd1167e7 Mon Sep 17 00:00:00 2001 From: Kent Keirsey <31807370+hipsterusername@users.noreply.github.com> Date: Fri, 12 May 2023 13:59:00 -0400 Subject: [PATCH 5/7] comment out customer_attention_context --- invokeai/backend/prompting/conditioning.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/invokeai/backend/prompting/conditioning.py b/invokeai/backend/prompting/conditioning.py index 71e51f1103..42b1736d00 100644 --- a/invokeai/backend/prompting/conditioning.py +++ b/invokeai/backend/prompting/conditioning.py @@ -59,12 +59,15 @@ def get_uc_and_c_and_ec(prompt_string, if log_tokens or getattr(Globals, "log_tokenization", False): log_tokenization(positive_prompt, negative_prompt, tokenizer=model.tokenizer) - with InvokeAIDiffuserComponent.custom_attention_context(model.unet, - extra_conditioning_info=None, - step_count=-1): - c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) - uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) - [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) + # The below has been commented out as it is an instance method used for cleanly loading LoRA models, but is not currently needed. + # TODO: Reimplement custom_attention for 3.0 support of LoRA. + + # with InvokeAIDiffuserComponent.custom_attention_context(model.unet, + # extra_conditioning_info=None, + # step_count=-1): + # c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) + # uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) + # [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) # now build the "real" ec ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(tokens_count_including_eos_bos=tokens_count, From b72c9787a931c9122e4953cfb3bbf01c3911a8a3 Mon Sep 17 00:00:00 2001 From: Eugene Date: Sun, 14 May 2023 00:37:55 -0400 Subject: [PATCH 6/7] Revert "comment out customer_attention_context" This reverts commit 8f8cd907878f00772e6c24185387d09dbd1167e7. Due to NameError: name 'options' is not defined --- invokeai/backend/prompting/conditioning.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/invokeai/backend/prompting/conditioning.py b/invokeai/backend/prompting/conditioning.py index 42b1736d00..71e51f1103 100644 --- a/invokeai/backend/prompting/conditioning.py +++ b/invokeai/backend/prompting/conditioning.py @@ -59,15 +59,12 @@ def get_uc_and_c_and_ec(prompt_string, if log_tokens or getattr(Globals, "log_tokenization", False): log_tokenization(positive_prompt, negative_prompt, tokenizer=model.tokenizer) - # The below has been commented out as it is an instance method used for cleanly loading LoRA models, but is not currently needed. - # TODO: Reimplement custom_attention for 3.0 support of LoRA. - - # with InvokeAIDiffuserComponent.custom_attention_context(model.unet, - # extra_conditioning_info=None, - # step_count=-1): - # c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) - # uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) - # [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) + with InvokeAIDiffuserComponent.custom_attention_context(model.unet, + extra_conditioning_info=None, + step_count=-1): + c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) + uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) + [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) # now build the "real" ec ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(tokens_count_including_eos_bos=tokens_count, From 050add58d2186c0df049f56bf08a624761bd1ce0 Mon Sep 17 00:00:00 2001 From: Damian Stewart Date: Sun, 14 May 2023 12:20:54 +0200 Subject: [PATCH 7/7] fix getting conditionings --- invokeai/backend/prompting/conditioning.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/invokeai/backend/prompting/conditioning.py b/invokeai/backend/prompting/conditioning.py index 71e51f1103..a6aa5b68f1 100644 --- a/invokeai/backend/prompting/conditioning.py +++ b/invokeai/backend/prompting/conditioning.py @@ -59,14 +59,10 @@ def get_uc_and_c_and_ec(prompt_string, if log_tokens or getattr(Globals, "log_tokenization", False): log_tokenization(positive_prompt, negative_prompt, tokenizer=model.tokenizer) - with InvokeAIDiffuserComponent.custom_attention_context(model.unet, - extra_conditioning_info=None, - step_count=-1): - c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) - uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) - [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) + c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) + uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) + [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) - # now build the "real" ec ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(tokens_count_including_eos_bos=tokens_count, cross_attention_control_args=options.get( 'cross_attention_control', None))