diff --git a/invokeai/app/invocations/ip_adapter.py b/invokeai/app/invocations/ip_adapter.py index 40cde8f3e9..b239e89bb6 100644 --- a/invokeai/app/invocations/ip_adapter.py +++ b/invokeai/app/invocations/ip_adapter.py @@ -4,20 +4,8 @@ from typing import List, Literal, Optional, Union from pydantic import BaseModel, Field, field_validator, model_validator from typing_extensions import Self -from invokeai.app.invocations.baseinvocation import ( - BaseInvocation, - BaseInvocationOutput, - invocation, - invocation_output, -) -from invokeai.app.invocations.fields import ( - FieldDescriptions, - Input, - InputField, - OutputField, - TensorField, - UIType, -) +from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output +from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, OutputField, TensorField, UIType from invokeai.app.invocations.model import ModelIdentifierField from invokeai.app.invocations.primitives import ImageField from invokeai.app.invocations.util import validate_begin_end_step, validate_weights @@ -36,6 +24,7 @@ class IPAdapterField(BaseModel): ip_adapter_model: ModelIdentifierField = Field(description="The IP-Adapter model to use.") image_encoder_model: ModelIdentifierField = Field(description="The name of the CLIP image encoder model.") weight: Union[float, List[float]] = Field(default=1, description="The weight given to the IP-Adapter.") + target_blocks: List[str] = Field(default=[], description="The IP Adapter blocks to apply") begin_step_percent: float = Field( default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)" ) @@ -90,6 +79,9 @@ class IPAdapterInvocation(BaseInvocation): weight: Union[float, List[float]] = InputField( default=1, description="The weight given to the IP-Adapter", title="Weight" ) + method: Literal["full", "style", "composition"] = InputField( + default="full", description="The method to apply the IP-Adapter" + ) begin_step_percent: float = InputField( default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)" ) @@ -124,12 +116,19 @@ class IPAdapterInvocation(BaseInvocation): image_encoder_model = self._get_image_encoder(context, image_encoder_model_name) + target_blocks = ["up_blocks.0.attentions.1", "down_blocks.2.attentions.1"] + if self.method == "style": + target_blocks = ["up_blocks.0.attentions.1"] + elif self.method == "composition": + target_blocks = ["down_blocks.2.attentions.1"] + return IPAdapterOutput( ip_adapter=IPAdapterField( image=self.image, ip_adapter_model=self.ip_adapter_model, image_encoder_model=ModelIdentifierField.from_config(image_encoder_model), weight=self.weight, + target_blocks=target_blocks, begin_step_percent=self.begin_step_percent, end_step_percent=self.end_step_percent, mask=self.mask, diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index ce63d568c6..ede26a823f 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -15,12 +15,10 @@ from diffusers import AutoencoderKL, AutoencoderTiny from diffusers.configuration_utils import ConfigMixin from diffusers.image_processor import VaeImageProcessor from diffusers.models.adapter import T2IAdapter -from diffusers.models.attention_processor import ( - AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, - XFormersAttnProcessor, -) +from diffusers.models.attention_processor import (AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor) from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel from diffusers.schedulers import DPMSolverSDEScheduler from diffusers.schedulers import SchedulerMixin as Scheduler @@ -29,22 +27,17 @@ from pydantic import field_validator from torchvision.transforms.functional import resize as tv_resize from transformers import CLIPVisionModelWithProjection -from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR, SCHEDULER_NAME_VALUES -from invokeai.app.invocations.fields import ( - ConditioningField, - DenoiseMaskField, - FieldDescriptions, - ImageField, - Input, - InputField, - LatentsField, - OutputField, - UIType, - WithBoard, - WithMetadata, -) +from invokeai.app.invocations.constants import (LATENT_SCALE_FACTOR, + SCHEDULER_NAME_VALUES) +from invokeai.app.invocations.fields import (ConditioningField, + DenoiseMaskField, + FieldDescriptions, ImageField, + Input, InputField, LatentsField, + OutputField, UIType, WithBoard, + WithMetadata) from invokeai.app.invocations.ip_adapter import IPAdapterField -from invokeai.app.invocations.primitives import DenoiseMaskOutput, ImageOutput, LatentsOutput +from invokeai.app.invocations.primitives import (DenoiseMaskOutput, + ImageOutput, LatentsOutput) from invokeai.app.invocations.t2i_adapter import T2IAdapterField from invokeai.app.services.shared.invocation_context import InvocationContext from invokeai.app.util.controlnet_utils import prepare_control_image @@ -52,28 +45,21 @@ from invokeai.backend.ip_adapter.ip_adapter import IPAdapter, IPAdapterPlus from invokeai.backend.lora import LoRAModelRaw from invokeai.backend.model_manager import BaseModelType, LoadedModel from invokeai.backend.model_patcher import ModelPatcher -from invokeai.backend.stable_diffusion import PipelineIntermediateState, set_seamless +from invokeai.backend.stable_diffusion import (PipelineIntermediateState, + set_seamless) from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ( - BasicConditioningInfo, - IPAdapterConditioningInfo, - IPAdapterData, - Range, - SDXLConditioningInfo, - TextConditioningData, - TextConditioningRegions, -) + BasicConditioningInfo, IPAdapterConditioningInfo, IPAdapterData, Range, + SDXLConditioningInfo, TextConditioningData, TextConditioningRegions) from invokeai.backend.util.mask import to_standard_float_mask from invokeai.backend.util.silence_warnings import SilenceWarnings from ...backend.stable_diffusion.diffusers_pipeline import ( - ControlNetData, - StableDiffusionGeneratorPipeline, - T2IAdapterData, - image_resized_to_grid_as_tensor, -) + ControlNetData, StableDiffusionGeneratorPipeline, T2IAdapterData, + image_resized_to_grid_as_tensor) from ...backend.stable_diffusion.schedulers import SCHEDULER_MAP from ...backend.util.devices import choose_precision, choose_torch_device -from .baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output +from .baseinvocation import (BaseInvocation, BaseInvocationOutput, invocation, + invocation_output) from .controlnet_image_processors import ControlField from .model import ModelIdentifierField, UNetField, VAEField @@ -682,6 +668,7 @@ class DenoiseLatentsInvocation(BaseInvocation): IPAdapterData( ip_adapter_model=ip_adapter_model, weight=single_ip_adapter.weight, + target_blocks=single_ip_adapter.target_blocks, begin_step_percent=single_ip_adapter.begin_step_percent, end_step_percent=single_ip_adapter.end_step_percent, ip_adapter_conditioning=IPAdapterConditioningInfo(image_prompt_embeds, uncond_image_prompt_embeds), diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index b4d1b3381c..befda72751 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -21,12 +21,9 @@ from pydantic import Field from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer from invokeai.app.services.config.config_default import get_config -from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ( - IPAdapterData, - TextConditioningData, -) +from invokeai.backend.stable_diffusion.diffusion.conditioning_data import IPAdapterData, TextConditioningData from invokeai.backend.stable_diffusion.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent -from invokeai.backend.stable_diffusion.diffusion.unet_attention_patcher import UNetAttentionPatcher +from invokeai.backend.stable_diffusion.diffusion.unet_attention_patcher import UNetAttentionPatcher, UNetIPAdapterData from invokeai.backend.util.attention import auto_detect_slice_size from invokeai.backend.util.devices import normalize_device @@ -394,8 +391,13 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): unet_attention_patcher = None self.use_ip_adapter = use_ip_adapter attn_ctx = nullcontext() + if use_ip_adapter or use_regional_prompting: - ip_adapters = [ipa.ip_adapter_model for ipa in ip_adapter_data] if use_ip_adapter else None + ip_adapters: Optional[List[UNetIPAdapterData]] = ( + [{"ip_adapter": ipa.ip_adapter_model, "target_blocks": ipa.target_blocks} for ipa in ip_adapter_data] + if use_ip_adapter + else None + ) unet_attention_patcher = UNetAttentionPatcher(ip_adapters) attn_ctx = unet_attention_patcher.apply_ip_adapter_attention(self.invokeai_diffuser.model) diff --git a/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py b/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py index 9b8ea0968a..85950a01df 100644 --- a/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py +++ b/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py @@ -53,6 +53,7 @@ class IPAdapterData: ip_adapter_model: IPAdapter ip_adapter_conditioning: IPAdapterConditioningInfo mask: torch.Tensor + target_blocks: List[str] # Either a single weight applied to all steps, or a list of weights for each step. weight: Union[float, List[float]] = 1.0 diff --git a/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py b/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py index ed706f6453..5a42d9572e 100644 --- a/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py +++ b/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional, TypedDict import torch import torch.nn.functional as F @@ -9,6 +9,11 @@ from invokeai.backend.stable_diffusion.diffusion.regional_ip_data import Regiona from invokeai.backend.stable_diffusion.diffusion.regional_prompt_data import RegionalPromptData +class IPAdapterAttentionWeights(TypedDict): + ip_adapter_weights: List[IPAttentionProcessorWeights] + skip: bool + + class CustomAttnProcessor2_0(AttnProcessor2_0): """A custom implementation of AttnProcessor2_0 that supports additional Invoke features. This implementation is based on @@ -20,7 +25,7 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): def __init__( self, - ip_adapter_weights: Optional[list[IPAttentionProcessorWeights]] = None, + ip_adapter_attention_weights: Optional[IPAdapterAttentionWeights] = None, ): """Initialize a CustomAttnProcessor2_0. Note: Arguments that are the same for all attention layers are passed to __call__(). Arguments that are @@ -30,10 +35,7 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): for the i'th IP-Adapter. """ super().__init__() - self._ip_adapter_weights = ip_adapter_weights - - def _is_ip_adapter_enabled(self) -> bool: - return self._ip_adapter_weights is not None + self._ip_adapter_attention_weights = ip_adapter_attention_weights def __call__( self, @@ -130,17 +132,17 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): # Apply IP-Adapter conditioning. if is_cross_attention: - if self._is_ip_adapter_enabled(): + if self._ip_adapter_attention_weights: assert regional_ip_data is not None ip_masks = regional_ip_data.get_masks(query_seq_len=query_seq_len) assert ( len(regional_ip_data.image_prompt_embeds) - == len(self._ip_adapter_weights) + == len(self._ip_adapter_attention_weights["ip_adapter_weights"]) == len(regional_ip_data.scales) == ip_masks.shape[1] ) for ipa_index, ipa_embed in enumerate(regional_ip_data.image_prompt_embeds): - ipa_weights = self._ip_adapter_weights[ipa_index] + ipa_weights = self._ip_adapter_attention_weights["ip_adapter_weights"][ipa_index] ipa_scale = regional_ip_data.scales[ipa_index] ip_mask = ip_masks[0, ipa_index, ...] @@ -153,29 +155,33 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): # Expected ip_hidden_state shape: (batch_size, num_ip_images, ip_seq_len, ip_image_embedding) - ip_key = ipa_weights.to_k_ip(ip_hidden_states) - ip_value = ipa_weights.to_v_ip(ip_hidden_states) + if self._ip_adapter_attention_weights["skip"]: - # Expected ip_key and ip_value shape: (batch_size, num_ip_images, ip_seq_len, head_dim * num_heads) + ip_key = ipa_weights.to_k_ip(ip_hidden_states) + ip_value = ipa_weights.to_v_ip(ip_hidden_states) - ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + # Expected ip_key and ip_value shape: (batch_size, num_ip_images, ip_seq_len, head_dim * num_heads) - # Expected ip_key and ip_value shape: (batch_size, num_heads, num_ip_images * ip_seq_len, head_dim) + ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - # TODO: add support for attn.scale when we move to Torch 2.1 - ip_hidden_states = F.scaled_dot_product_attention( - query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False - ) + # Expected ip_key and ip_value shape: (batch_size, num_heads, num_ip_images * ip_seq_len, head_dim) - # Expected ip_hidden_states shape: (batch_size, num_heads, query_seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + ip_hidden_states = F.scaled_dot_product_attention( + query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False + ) - ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) - ip_hidden_states = ip_hidden_states.to(query.dtype) + # Expected ip_hidden_states shape: (batch_size, num_heads, query_seq_len, head_dim) - # Expected ip_hidden_states shape: (batch_size, query_seq_len, num_heads * head_dim) + ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape( + batch_size, -1, attn.heads * head_dim + ) + ip_hidden_states = ip_hidden_states.to(query.dtype) - hidden_states = hidden_states + ipa_scale * ip_hidden_states * ip_mask + # Expected ip_hidden_states shape: (batch_size, query_seq_len, num_heads * head_dim) + + hidden_states = hidden_states + ipa_scale * ip_hidden_states * ip_mask else: # If IP-Adapter is not enabled, then regional_ip_data should not be passed in. assert regional_ip_data is None diff --git a/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py b/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py index 89a203f643..65992a62b5 100644 --- a/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py +++ b/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py @@ -1,17 +1,25 @@ from contextlib import contextmanager -from typing import Optional +from typing import List, Optional, TypedDict from diffusers.models import UNet2DConditionModel from invokeai.backend.ip_adapter.ip_adapter import IPAdapter -from invokeai.backend.stable_diffusion.diffusion.custom_atttention import CustomAttnProcessor2_0 +from invokeai.backend.stable_diffusion.diffusion.custom_atttention import ( + CustomAttnProcessor2_0, + IPAdapterAttentionWeights, +) + + +class UNetIPAdapterData(TypedDict): + ip_adapter: IPAdapter + target_blocks: List[str] class UNetAttentionPatcher: """A class for patching a UNet with CustomAttnProcessor2_0 attention layers.""" - def __init__(self, ip_adapters: Optional[list[IPAdapter]]): - self._ip_adapters = ip_adapters + def __init__(self, ip_adapter_data: Optional[List[UNetIPAdapterData]]): + self._ip_adapters = ip_adapter_data def _prepare_attention_processors(self, unet: UNet2DConditionModel): """Prepare a dict of attention processors that can be injected into a unet, and load the IP-Adapter attention @@ -25,10 +33,23 @@ class UNetAttentionPatcher: # "attn1" processors do not use IP-Adapters. attn_procs[name] = CustomAttnProcessor2_0() else: + + ip_adapter_attention_weights: IPAdapterAttentionWeights = {"ip_adapter_weights": [], "skip": False} + for ip_adapter in self._ip_adapters: + + ip_adapter_weight = ip_adapter["ip_adapter"].attn_weights.get_attention_processor_weights(idx) + skip = False + for block in ip_adapter["target_blocks"]: + if block in name: + skip = True + break + + ip_adapter_attention_weights.update({"ip_adapter_weights": [ip_adapter_weight], "skip": skip}) + # Collect the weights from each IP Adapter for the idx'th attention processor. - attn_procs[name] = CustomAttnProcessor2_0( - [ip_adapter.attn_weights.get_attention_processor_weights(idx) for ip_adapter in self._ip_adapters], - ) + + attn_procs[name] = CustomAttnProcessor2_0(ip_adapter_attention_weights) + return attn_procs @contextmanager diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts index ad563de468..ad530f7765 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts @@ -57,6 +57,7 @@ export const addIPAdapterToLinearGraph = async ( type: 'ip_adapter', is_intermediate: true, weight: weight, + method: 'composition', ip_adapter_model: model, clip_vision_model: clipVisionModel, begin_step_percent: beginStepPct, diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index cb222bd497..9028054a1f 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -1238,6 +1238,39 @@ export type components = { */ type: "boolean_output"; }; + /** + * BRIA AI Background Removal + * @description Uses the new Bria 1.4 model to remove backgrounds from images. + */ + BriaRemoveBackgroundInvocation: { + /** @description Optional metadata to be saved with the image */ + metadata?: components["schemas"]["MetadataField"] | null; + /** + * Id + * @description The id of this instance of an invocation. Must be unique among all instances of invocations. + */ + id: string; + /** + * Is Intermediate + * @description Whether or not this is an intermediate invocation. + * @default false + */ + is_intermediate?: boolean; + /** + * Use Cache + * @description Whether or not to use the cache + * @default true + */ + use_cache?: boolean; + /** @description The image to crop */ + image?: components["schemas"]["ImageField"]; + /** + * type + * @default bria_bg_remove + * @constant + */ + type: "bria_bg_remove"; + }; /** CLIPField */ CLIPField: { /** @description Info to load tokenizer submodel */ @@ -2070,6 +2103,8 @@ export type components = { * @description CLIP (tokenizer, text encoder, LoRAs) and skipped layer count */ clip?: components["schemas"]["CLIPField"]; + /** @description A mask defining the region that this conditioning prompt applies to. */ + mask?: components["schemas"]["TensorField"] | null; /** * type * @default compel @@ -2139,6 +2174,11 @@ export type components = { * @description The name of conditioning tensor */ conditioning_name: string; + /** + * @description The mask associated with this conditioning tensor. Excluded regions should be set to False, included regions should be set to True. + * @default null + */ + mask?: components["schemas"]["TensorField"] | null; }; /** * Conditioning Primitive @@ -3049,10 +3089,16 @@ export type components = { * @default true */ use_cache?: boolean; - /** @description Positive conditioning tensor */ - positive_conditioning?: components["schemas"]["ConditioningField"]; - /** @description Negative conditioning tensor */ - negative_conditioning?: components["schemas"]["ConditioningField"]; + /** + * Positive Conditioning + * @description Positive conditioning tensor + */ + positive_conditioning?: components["schemas"]["ConditioningField"] | components["schemas"]["ConditioningField"][]; + /** + * Negative Conditioning + * @description Negative conditioning tensor + */ + negative_conditioning?: components["schemas"]["ConditioningField"] | components["schemas"]["ConditioningField"][]; /** @description Noise tensor */ noise?: components["schemas"]["LatentsField"] | null; /** @@ -4112,7 +4158,7 @@ export type components = { * @description The nodes in this graph */ nodes: { - [key: string]: components["schemas"]["IntegerMathInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["LineartImageProcessorInvocation"]; + [key: string]: components["schemas"]["ImagePasteInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["BriaRemoveBackgroundInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["RectangleMaskInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["LineartImageProcessorInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["HandDepthMeshGraphormerProcessor"] | components["schemas"]["ContentShuffleImageProcessorInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["IntegerMathInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["FloatMathInvocation"]; }; /** * Edges @@ -4149,7 +4195,7 @@ export type components = { * @description The results of node executions */ results: { - [key: string]: components["schemas"]["FloatCollectionOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["String2Output"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["SchedulerOutput"] | components["schemas"]["CalculateImageTilesOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["LoRALoaderOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["SDXLRefinerModelLoaderOutput"]; + [key: string]: components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["CalculateImageTilesOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["MaskOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["SDXLRefinerModelLoaderOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["LoRALoaderOutput"] | components["schemas"]["SchedulerOutput"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["String2Output"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["FloatCollectionOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["HandDepthOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["LatentsCollectionOutput"]; }; /** * Errors @@ -4200,6 +4246,83 @@ export type components = { /** Detail */ detail?: components["schemas"]["ValidationError"][]; }; + /** + * Hand Depth w/ MeshGraphormer + * @description Generate hand depth maps to inpaint with using ControlNet + */ + HandDepthMeshGraphormerProcessor: { + /** @description Optional metadata to be saved with the image */ + metadata?: components["schemas"]["MetadataField"] | null; + /** + * Id + * @description The id of this instance of an invocation. Must be unique among all instances of invocations. + */ + id: string; + /** + * Is Intermediate + * @description Whether or not this is an intermediate invocation. + * @default false + */ + is_intermediate?: boolean; + /** + * Use Cache + * @description Whether or not to use the cache + * @default true + */ + use_cache?: boolean; + /** @description The image to process */ + image?: components["schemas"]["ImageField"]; + /** + * Resolution + * @description Pixel resolution for output image + * @default 512 + */ + resolution?: number; + /** + * Mask Padding + * @description Amount to pad the hand mask by + * @default 30 + */ + mask_padding?: number; + /** + * Offload + * @description Offload model after usage + * @default false + */ + offload?: boolean; + /** + * type + * @default hand_depth_mesh_graphormer_image_processor + * @constant + */ + type: "hand_depth_mesh_graphormer_image_processor"; + }; + /** + * HandDepthOutput + * @description Base class for to output Meshgraphormer results + */ + HandDepthOutput: { + /** @description Improved hands depth map */ + image: components["schemas"]["ImageField"]; + /** @description Hands area mask */ + mask: components["schemas"]["ImageField"]; + /** + * Width + * @description The width of the depth map in pixels + */ + width: number; + /** + * Height + * @description The height of the depth map in pixels + */ + height: number; + /** + * type + * @default meshgraphormer_output + * @constant + */ + type: "meshgraphormer_output"; + }; /** * HED (softedge) Processor * @description Applies HED edge detection to image @@ -4383,10 +4506,16 @@ export type components = { image_encoder_model: components["schemas"]["ModelIdentifierField"]; /** * Weight - * @description The weight given to the ControlNet + * @description The weight given to the IP-Adapter. * @default 1 */ weight?: number | number[]; + /** + * Target Blocks + * @description The IP Adapter blocks to apply + * @default [] + */ + target_blocks?: string[]; /** * Begin Step Percent * @description When the IP-Adapter is first applied (% of total steps) @@ -4399,6 +4528,11 @@ export type components = { * @default 1 */ end_step_percent?: number; + /** + * @description The bool mask associated with this IP-Adapter. Excluded regions should be set to False, included regions should be set to True. + * @default null + */ + mask?: components["schemas"]["TensorField"] | null; }; /** * IP-Adapter @@ -4445,6 +4579,13 @@ export type components = { * @default 1 */ weight?: number | number[]; + /** + * Method + * @description The method to apply the IP-Adapter + * @default full + * @enum {string} + */ + method?: "full" | "style" | "composition"; /** * Begin Step Percent * @description When the IP-Adapter is first applied (% of total steps) @@ -4457,6 +4598,8 @@ export type components = { * @default 1 */ end_step_percent?: number; + /** @description A mask defining the region that this IP-Adapter applies to. */ + mask?: components["schemas"]["TensorField"] | null; /** * type * @default ip_adapter @@ -7104,6 +7247,30 @@ export type components = { */ type: "mask_from_id"; }; + /** + * MaskOutput + * @description A torch mask tensor. + */ + MaskOutput: { + /** @description The mask. */ + mask: components["schemas"]["TensorField"]; + /** + * Width + * @description The width of the mask in pixels. + */ + width: number; + /** + * Height + * @description The height of the mask in pixels. + */ + height: number; + /** + * type + * @default mask_output + * @constant + */ + type: "mask_output"; + }; /** * Mediapipe Face Processor * @description Applies mediapipe face processing to image @@ -8364,6 +8531,67 @@ export type components = { */ type: "range_of_size"; }; + /** + * Create Rectangle Mask + * @description Create a rectangular mask. + */ + RectangleMaskInvocation: { + /** @description Optional metadata to be saved with the image */ + metadata?: components["schemas"]["MetadataField"] | null; + /** + * Id + * @description The id of this instance of an invocation. Must be unique among all instances of invocations. + */ + id: string; + /** + * Is Intermediate + * @description Whether or not this is an intermediate invocation. + * @default false + */ + is_intermediate?: boolean; + /** + * Use Cache + * @description Whether or not to use the cache + * @default true + */ + use_cache?: boolean; + /** + * Width + * @description The width of the entire mask. + */ + width?: number; + /** + * Height + * @description The height of the entire mask. + */ + height?: number; + /** + * X Left + * @description The left x-coordinate of the rectangular masked region (inclusive). + */ + x_left?: number; + /** + * Y Top + * @description The top y-coordinate of the rectangular masked region (inclusive). + */ + y_top?: number; + /** + * Rectangle Width + * @description The width of the rectangular masked region. + */ + rectangle_width?: number; + /** + * Rectangle Height + * @description The height of the rectangular masked region. + */ + rectangle_height?: number; + /** + * type + * @default rectangle_mask + * @constant + */ + type: "rectangle_mask"; + }; /** * RemoteModelFile * @description Information about a downloadable file that forms part of a model. @@ -8579,6 +8807,8 @@ export type components = { * @description CLIP (tokenizer, text encoder, LoRAs) and skipped layer count */ clip2?: components["schemas"]["CLIPField"]; + /** @description A mask defining the region that this conditioning prompt applies to. */ + mask?: components["schemas"]["TensorField"] | null; /** * type * @default sdxl_compel_prompt @@ -10115,6 +10345,17 @@ export type components = { /** Right */ right: number; }; + /** + * TensorField + * @description A tensor primitive field. + */ + TensorField: { + /** + * Tensor Name + * @description The name of a tensor. + */ + tensor_name: string; + }; /** * TextualInversionFileConfig * @description Model config for textual inversion embeddings.