diff --git a/invokeai/app/invocations/ip_adapter.py b/invokeai/app/invocations/ip_adapter.py index 40cde8f3e9..34a30628da 100644 --- a/invokeai/app/invocations/ip_adapter.py +++ b/invokeai/app/invocations/ip_adapter.py @@ -4,20 +4,8 @@ from typing import List, Literal, Optional, Union from pydantic import BaseModel, Field, field_validator, model_validator from typing_extensions import Self -from invokeai.app.invocations.baseinvocation import ( - BaseInvocation, - BaseInvocationOutput, - invocation, - invocation_output, -) -from invokeai.app.invocations.fields import ( - FieldDescriptions, - Input, - InputField, - OutputField, - TensorField, - UIType, -) +from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output +from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, OutputField, TensorField, UIType from invokeai.app.invocations.model import ModelIdentifierField from invokeai.app.invocations.primitives import ImageField from invokeai.app.invocations.util import validate_begin_end_step, validate_weights @@ -36,6 +24,7 @@ class IPAdapterField(BaseModel): ip_adapter_model: ModelIdentifierField = Field(description="The IP-Adapter model to use.") image_encoder_model: ModelIdentifierField = Field(description="The name of the CLIP image encoder model.") weight: Union[float, List[float]] = Field(default=1, description="The weight given to the IP-Adapter.") + target_blocks: List[str] = Field(default=[], description="The IP Adapter blocks to apply") begin_step_percent: float = Field( default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)" ) @@ -69,7 +58,7 @@ class IPAdapterOutput(BaseInvocationOutput): CLIP_VISION_MODEL_MAP = {"ViT-H": "ip_adapter_sd_image_encoder", "ViT-G": "ip_adapter_sdxl_image_encoder"} -@invocation("ip_adapter", title="IP-Adapter", tags=["ip_adapter", "control"], category="ip_adapter", version="1.3.0") +@invocation("ip_adapter", title="IP-Adapter", tags=["ip_adapter", "control"], category="ip_adapter", version="1.4.0") class IPAdapterInvocation(BaseInvocation): """Collects IP-Adapter info to pass to other nodes.""" @@ -90,6 +79,9 @@ class IPAdapterInvocation(BaseInvocation): weight: Union[float, List[float]] = InputField( default=1, description="The weight given to the IP-Adapter", title="Weight" ) + method: Literal["full", "style", "composition"] = InputField( + default="full", description="The method to apply the IP-Adapter" + ) begin_step_percent: float = InputField( default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)" ) @@ -124,12 +116,32 @@ class IPAdapterInvocation(BaseInvocation): image_encoder_model = self._get_image_encoder(context, image_encoder_model_name) + if self.method == "style": + if ip_adapter_info.base == "sd-1": + target_blocks = ["up_blocks.1"] + elif ip_adapter_info.base == "sdxl": + target_blocks = ["up_blocks.0.attentions.1"] + else: + raise ValueError(f"Unsupported IP-Adapter base type: '{ip_adapter_info.base}'.") + elif self.method == "composition": + if ip_adapter_info.base == "sd-1": + target_blocks = ["down_blocks.2", "mid_block"] + elif ip_adapter_info.base == "sdxl": + target_blocks = ["down_blocks.2.attentions.1"] + else: + raise ValueError(f"Unsupported IP-Adapter base type: '{ip_adapter_info.base}'.") + elif self.method == "full": + target_blocks = ["block"] + else: + raise ValueError(f"Unexpected IP-Adapter method: '{self.method}'.") + return IPAdapterOutput( ip_adapter=IPAdapterField( image=self.image, ip_adapter_model=self.ip_adapter_model, image_encoder_model=ModelIdentifierField.from_config(image_encoder_model), weight=self.weight, + target_blocks=target_blocks, begin_step_percent=self.begin_step_percent, end_step_percent=self.end_step_percent, mask=self.mask, diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index a8ead96f3a..4534df89c1 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -679,6 +679,7 @@ class DenoiseLatentsInvocation(BaseInvocation): IPAdapterData( ip_adapter_model=ip_adapter_model, weight=single_ip_adapter.weight, + target_blocks=single_ip_adapter.target_blocks, begin_step_percent=single_ip_adapter.begin_step_percent, end_step_percent=single_ip_adapter.end_step_percent, ip_adapter_conditioning=IPAdapterConditioningInfo(image_prompt_embeds, uncond_image_prompt_embeds), diff --git a/invokeai/app/invocations/metadata.py b/invokeai/app/invocations/metadata.py index 2da482c833..a02d0a57ef 100644 --- a/invokeai/app/invocations/metadata.py +++ b/invokeai/app/invocations/metadata.py @@ -36,6 +36,7 @@ class IPAdapterMetadataField(BaseModel): image: ImageField = Field(description="The IP-Adapter image prompt.") ip_adapter_model: ModelIdentifierField = Field(description="The IP-Adapter model.") clip_vision_model: Literal["ViT-H", "ViT-G"] = Field(description="The CLIP Vision model") + method: Literal["full", "style", "composition"] = Field(description="Method to apply IP Weights with") weight: Union[float, list[float]] = Field(description="The weight given to the IP-Adapter") begin_step_percent: float = Field(description="When the IP-Adapter is first applied (% of total steps)") end_step_percent: float = Field(description="When the IP-Adapter is last applied (% of total steps)") diff --git a/invokeai/app/services/model_install/model_install_default.py b/invokeai/app/services/model_install/model_install_default.py index 5aa0f199fc..6a3117bcb8 100644 --- a/invokeai/app/services/model_install/model_install_default.py +++ b/invokeai/app/services/model_install/model_install_default.py @@ -754,6 +754,8 @@ class ModelInstallService(ModelInstallServiceBase): self._download_cache[download_job.source] = install_job # matches a download job to an install job install_job.download_parts.add(download_job) + # only start the jobs once install_job.download_parts is fully populated + for download_job in install_job.download_parts: self._download_queue.submit_download_job( download_job, on_start=self._download_started_callback, @@ -762,6 +764,7 @@ class ModelInstallService(ModelInstallServiceBase): on_error=self._download_error_callback, on_cancelled=self._download_cancelled_callback, ) + return install_job def _stat_size(self, path: Path) -> int: diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index bd60b0b8c7..8b90c815ae 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -21,12 +21,9 @@ from pydantic import Field from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer from invokeai.app.services.config.config_default import get_config -from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ( - IPAdapterData, - TextConditioningData, -) +from invokeai.backend.stable_diffusion.diffusion.conditioning_data import IPAdapterData, TextConditioningData from invokeai.backend.stable_diffusion.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent -from invokeai.backend.stable_diffusion.diffusion.unet_attention_patcher import UNetAttentionPatcher +from invokeai.backend.stable_diffusion.diffusion.unet_attention_patcher import UNetAttentionPatcher, UNetIPAdapterData from invokeai.backend.util.attention import auto_detect_slice_size from invokeai.backend.util.devices import TorchDevice @@ -394,8 +391,13 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): unet_attention_patcher = None self.use_ip_adapter = use_ip_adapter attn_ctx = nullcontext() + if use_ip_adapter or use_regional_prompting: - ip_adapters = [ipa.ip_adapter_model for ipa in ip_adapter_data] if use_ip_adapter else None + ip_adapters: Optional[List[UNetIPAdapterData]] = ( + [{"ip_adapter": ipa.ip_adapter_model, "target_blocks": ipa.target_blocks} for ipa in ip_adapter_data] + if use_ip_adapter + else None + ) unet_attention_patcher = UNetAttentionPatcher(ip_adapters) attn_ctx = unet_attention_patcher.apply_ip_adapter_attention(self.invokeai_diffuser.model) diff --git a/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py b/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py index 9b8ea0968a..85950a01df 100644 --- a/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py +++ b/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py @@ -53,6 +53,7 @@ class IPAdapterData: ip_adapter_model: IPAdapter ip_adapter_conditioning: IPAdapterConditioningInfo mask: torch.Tensor + target_blocks: List[str] # Either a single weight applied to all steps, or a list of weights for each step. weight: Union[float, List[float]] = 1.0 diff --git a/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py b/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py index ed706f6453..1334313fe6 100644 --- a/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py +++ b/invokeai/backend/stable_diffusion/diffusion/custom_atttention.py @@ -1,4 +1,5 @@ -from typing import Optional +from dataclasses import dataclass +from typing import List, Optional, cast import torch import torch.nn.functional as F @@ -9,6 +10,12 @@ from invokeai.backend.stable_diffusion.diffusion.regional_ip_data import Regiona from invokeai.backend.stable_diffusion.diffusion.regional_prompt_data import RegionalPromptData +@dataclass +class IPAdapterAttentionWeights: + ip_adapter_weights: IPAttentionProcessorWeights + skip: bool + + class CustomAttnProcessor2_0(AttnProcessor2_0): """A custom implementation of AttnProcessor2_0 that supports additional Invoke features. This implementation is based on @@ -20,7 +27,7 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): def __init__( self, - ip_adapter_weights: Optional[list[IPAttentionProcessorWeights]] = None, + ip_adapter_attention_weights: Optional[List[IPAdapterAttentionWeights]] = None, ): """Initialize a CustomAttnProcessor2_0. Note: Arguments that are the same for all attention layers are passed to __call__(). Arguments that are @@ -30,23 +37,22 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): for the i'th IP-Adapter. """ super().__init__() - self._ip_adapter_weights = ip_adapter_weights - - def _is_ip_adapter_enabled(self) -> bool: - return self._ip_adapter_weights is not None + self._ip_adapter_attention_weights = ip_adapter_attention_weights def __call__( self, attn: Attention, - hidden_states: torch.FloatTensor, - encoder_hidden_states: Optional[torch.FloatTensor] = None, - attention_mask: Optional[torch.FloatTensor] = None, - temb: Optional[torch.FloatTensor] = None, - # For regional prompting: + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + temb: Optional[torch.Tensor] = None, + # For Regional Prompting: regional_prompt_data: Optional[RegionalPromptData] = None, - percent_through: Optional[torch.FloatTensor] = None, + percent_through: Optional[torch.Tensor] = None, # For IP-Adapter: regional_ip_data: Optional[RegionalIPData] = None, + *args, + **kwargs, ) -> torch.FloatTensor: """Apply attention. Args: @@ -130,17 +136,19 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): # Apply IP-Adapter conditioning. if is_cross_attention: - if self._is_ip_adapter_enabled(): + if self._ip_adapter_attention_weights: assert regional_ip_data is not None ip_masks = regional_ip_data.get_masks(query_seq_len=query_seq_len) + assert ( len(regional_ip_data.image_prompt_embeds) - == len(self._ip_adapter_weights) + == len(self._ip_adapter_attention_weights) == len(regional_ip_data.scales) == ip_masks.shape[1] ) + for ipa_index, ipa_embed in enumerate(regional_ip_data.image_prompt_embeds): - ipa_weights = self._ip_adapter_weights[ipa_index] + ipa_weights = self._ip_adapter_attention_weights[ipa_index].ip_adapter_weights ipa_scale = regional_ip_data.scales[ipa_index] ip_mask = ip_masks[0, ipa_index, ...] @@ -153,29 +161,33 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): # Expected ip_hidden_state shape: (batch_size, num_ip_images, ip_seq_len, ip_image_embedding) - ip_key = ipa_weights.to_k_ip(ip_hidden_states) - ip_value = ipa_weights.to_v_ip(ip_hidden_states) + if not self._ip_adapter_attention_weights[ipa_index].skip: + ip_key = ipa_weights.to_k_ip(ip_hidden_states) + ip_value = ipa_weights.to_v_ip(ip_hidden_states) - # Expected ip_key and ip_value shape: (batch_size, num_ip_images, ip_seq_len, head_dim * num_heads) + # Expected ip_key and ip_value shape: + # (batch_size, num_ip_images, ip_seq_len, head_dim * num_heads) - ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - # Expected ip_key and ip_value shape: (batch_size, num_heads, num_ip_images * ip_seq_len, head_dim) + # Expected ip_key and ip_value shape: + # (batch_size, num_heads, num_ip_images * ip_seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 - ip_hidden_states = F.scaled_dot_product_attention( - query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False - ) + # TODO: add support for attn.scale when we move to Torch 2.1 + ip_hidden_states = F.scaled_dot_product_attention( + query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False + ) - # Expected ip_hidden_states shape: (batch_size, num_heads, query_seq_len, head_dim) + # Expected ip_hidden_states shape: (batch_size, num_heads, query_seq_len, head_dim) + ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape( + batch_size, -1, attn.heads * head_dim + ) - ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) - ip_hidden_states = ip_hidden_states.to(query.dtype) + ip_hidden_states = ip_hidden_states.to(query.dtype) - # Expected ip_hidden_states shape: (batch_size, query_seq_len, num_heads * head_dim) - - hidden_states = hidden_states + ipa_scale * ip_hidden_states * ip_mask + # Expected ip_hidden_states shape: (batch_size, query_seq_len, num_heads * head_dim) + hidden_states = hidden_states + ipa_scale * ip_hidden_states * ip_mask else: # If IP-Adapter is not enabled, then regional_ip_data should not be passed in. assert regional_ip_data is None @@ -188,11 +200,15 @@ class CustomAttnProcessor2_0(AttnProcessor2_0): hidden_states = attn.to_out[1](hidden_states) if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) if attn.residual_connection: hidden_states = hidden_states + residual hidden_states = hidden_states / attn.rescale_output_factor + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # End of unmodified block from AttnProcessor2_0 - return hidden_states + # casting torch.Tensor to torch.FloatTensor to avoid type issues + return cast(torch.FloatTensor, hidden_states) diff --git a/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py b/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py index 89a203f643..ac00a8e06e 100644 --- a/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py +++ b/invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py @@ -1,17 +1,25 @@ from contextlib import contextmanager -from typing import Optional +from typing import List, Optional, TypedDict from diffusers.models import UNet2DConditionModel from invokeai.backend.ip_adapter.ip_adapter import IPAdapter -from invokeai.backend.stable_diffusion.diffusion.custom_atttention import CustomAttnProcessor2_0 +from invokeai.backend.stable_diffusion.diffusion.custom_atttention import ( + CustomAttnProcessor2_0, + IPAdapterAttentionWeights, +) + + +class UNetIPAdapterData(TypedDict): + ip_adapter: IPAdapter + target_blocks: List[str] class UNetAttentionPatcher: """A class for patching a UNet with CustomAttnProcessor2_0 attention layers.""" - def __init__(self, ip_adapters: Optional[list[IPAdapter]]): - self._ip_adapters = ip_adapters + def __init__(self, ip_adapter_data: Optional[List[UNetIPAdapterData]]): + self._ip_adapters = ip_adapter_data def _prepare_attention_processors(self, unet: UNet2DConditionModel): """Prepare a dict of attention processors that can be injected into a unet, and load the IP-Adapter attention @@ -26,9 +34,22 @@ class UNetAttentionPatcher: attn_procs[name] = CustomAttnProcessor2_0() else: # Collect the weights from each IP Adapter for the idx'th attention processor. - attn_procs[name] = CustomAttnProcessor2_0( - [ip_adapter.attn_weights.get_attention_processor_weights(idx) for ip_adapter in self._ip_adapters], - ) + ip_adapter_attention_weights_collection: list[IPAdapterAttentionWeights] = [] + + for ip_adapter in self._ip_adapters: + ip_adapter_weights = ip_adapter["ip_adapter"].attn_weights.get_attention_processor_weights(idx) + skip = True + for block in ip_adapter["target_blocks"]: + if block in name: + skip = False + break + ip_adapter_attention_weights: IPAdapterAttentionWeights = IPAdapterAttentionWeights( + ip_adapter_weights=ip_adapter_weights, skip=skip + ) + ip_adapter_attention_weights_collection.append(ip_adapter_attention_weights) + + attn_procs[name] = CustomAttnProcessor2_0(ip_adapter_attention_weights_collection) + return attn_procs @contextmanager diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index 1adac4c5dc..f69f09552a 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -213,6 +213,10 @@ "resize": "Resize", "resizeSimple": "Resize (Simple)", "resizeMode": "Resize Mode", + "ipAdapterMethod": "Method", + "full": "Full", + "style": "Style Only", + "composition": "Composition Only", "safe": "Safe", "saveControlImage": "Save Control Image", "scribble": "scribble", diff --git a/invokeai/frontend/web/src/features/controlAdapters/components/ControlAdapterConfig.tsx b/invokeai/frontend/web/src/features/controlAdapters/components/ControlAdapterConfig.tsx index 42499b015c..fcc816d75f 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/components/ControlAdapterConfig.tsx +++ b/invokeai/frontend/web/src/features/controlAdapters/components/ControlAdapterConfig.tsx @@ -21,6 +21,7 @@ import ControlAdapterShouldAutoConfig from './ControlAdapterShouldAutoConfig'; import ControlNetCanvasImageImports from './imports/ControlNetCanvasImageImports'; import { ParamControlAdapterBeginEnd } from './parameters/ParamControlAdapterBeginEnd'; import ParamControlAdapterControlMode from './parameters/ParamControlAdapterControlMode'; +import ParamControlAdapterIPMethod from './parameters/ParamControlAdapterIPMethod'; import ParamControlAdapterProcessorSelect from './parameters/ParamControlAdapterProcessorSelect'; import ParamControlAdapterResizeMode from './parameters/ParamControlAdapterResizeMode'; import ParamControlAdapterWeight from './parameters/ParamControlAdapterWeight'; @@ -111,7 +112,8 @@ const ControlAdapterConfig = (props: { id: string; number: number }) => { - + + diff --git a/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterIPMethod.tsx b/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterIPMethod.tsx new file mode 100644 index 0000000000..7385997804 --- /dev/null +++ b/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterIPMethod.tsx @@ -0,0 +1,63 @@ +import type { ComboboxOnChange } from '@invoke-ai/ui-library'; +import { Combobox, FormControl, FormLabel } from '@invoke-ai/ui-library'; +import { useAppDispatch } from 'app/store/storeHooks'; +import { InformationalPopover } from 'common/components/InformationalPopover/InformationalPopover'; +import { useControlAdapterIPMethod } from 'features/controlAdapters/hooks/useControlAdapterIPMethod'; +import { useControlAdapterIsEnabled } from 'features/controlAdapters/hooks/useControlAdapterIsEnabled'; +import { controlAdapterIPMethodChanged } from 'features/controlAdapters/store/controlAdaptersSlice'; +import type { IPMethod } from 'features/controlAdapters/store/types'; +import { isIPMethod } from 'features/controlAdapters/store/types'; +import { memo, useCallback, useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; + +type Props = { + id: string; +}; + +const ParamControlAdapterIPMethod = ({ id }: Props) => { + const isEnabled = useControlAdapterIsEnabled(id); + const method = useControlAdapterIPMethod(id); + const dispatch = useAppDispatch(); + const { t } = useTranslation(); + + const options: { label: string; value: IPMethod }[] = useMemo( + () => [ + { label: t('controlnet.full'), value: 'full' }, + { label: t('controlnet.style'), value: 'style' }, + { label: t('controlnet.composition'), value: 'composition' }, + ], + [t] + ); + + const handleIPMethodChanged = useCallback( + (v) => { + if (!isIPMethod(v?.value)) { + return; + } + dispatch( + controlAdapterIPMethodChanged({ + id, + method: v.value, + }) + ); + }, + [id, dispatch] + ); + + const value = useMemo(() => options.find((o) => o.value === method), [options, method]); + + if (!method) { + return null; + } + + return ( + + + {t('controlnet.ipAdapterMethod')} + + + + ); +}; + +export default memo(ParamControlAdapterIPMethod); diff --git a/invokeai/frontend/web/src/features/controlAdapters/hooks/useControlAdapterIPMethod.ts b/invokeai/frontend/web/src/features/controlAdapters/hooks/useControlAdapterIPMethod.ts new file mode 100644 index 0000000000..a179899396 --- /dev/null +++ b/invokeai/frontend/web/src/features/controlAdapters/hooks/useControlAdapterIPMethod.ts @@ -0,0 +1,24 @@ +import { createMemoizedSelector } from 'app/store/createMemoizedSelector'; +import { useAppSelector } from 'app/store/storeHooks'; +import { + selectControlAdapterById, + selectControlAdaptersSlice, +} from 'features/controlAdapters/store/controlAdaptersSlice'; +import { useMemo } from 'react'; + +export const useControlAdapterIPMethod = (id: string) => { + const selector = useMemo( + () => + createMemoizedSelector(selectControlAdaptersSlice, (controlAdapters) => { + const cn = selectControlAdapterById(controlAdapters, id); + if (cn && cn?.type === 'ip_adapter') { + return cn.method; + } + }), + [id] + ); + + const method = useAppSelector(selector); + + return method; +}; diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts b/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts index 100bb3f6ad..9a1ce5e984 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts @@ -21,6 +21,7 @@ import type { ControlAdapterType, ControlMode, ControlNetConfig, + IPMethod, RequiredControlAdapterProcessorNode, ResizeMode, T2IAdapterConfig, @@ -245,6 +246,10 @@ export const controlAdaptersSlice = createSlice({ } caAdapter.updateOne(state, { id, changes: { controlMode } }); }, + controlAdapterIPMethodChanged: (state, action: PayloadAction<{ id: string; method: IPMethod }>) => { + const { id, method } = action.payload; + caAdapter.updateOne(state, { id, changes: { method } }); + }, controlAdapterCLIPVisionModelChanged: ( state, action: PayloadAction<{ id: string; clipVisionModel: CLIPVisionModel }> @@ -390,6 +395,7 @@ export const { controlAdapterIsEnabledChanged, controlAdapterModelChanged, controlAdapterCLIPVisionModelChanged, + controlAdapterIPMethodChanged, controlAdapterWeightChanged, controlAdapterBeginStepPctChanged, controlAdapterEndStepPctChanged, diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/types.ts b/invokeai/frontend/web/src/features/controlAdapters/store/types.ts index 329c318759..7e2f18af5c 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/types.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/types.ts @@ -210,6 +210,10 @@ const zResizeMode = z.enum(['just_resize', 'crop_resize', 'fill_resize', 'just_r export type ResizeMode = z.infer; export const isResizeMode = (v: unknown): v is ResizeMode => zResizeMode.safeParse(v).success; +const zIPMethod = z.enum(['full', 'style', 'composition']); +export type IPMethod = z.infer; +export const isIPMethod = (v: unknown): v is IPMethod => zIPMethod.safeParse(v).success; + export type ControlNetConfig = { type: 'controlnet'; id: string; @@ -253,6 +257,7 @@ export type IPAdapterConfig = { model: ParameterIPAdapterModel | null; clipVisionModel: CLIPVisionModel; weight: number; + method: IPMethod; beginStepPct: number; endStepPct: number; }; diff --git a/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts b/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts index dc893ceb1c..ad7bdba363 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts @@ -46,6 +46,7 @@ export const initialIPAdapter: Omit = { isEnabled: true, controlImage: null, model: null, + method: 'full', clipVisionModel: 'ViT-H', weight: 1, beginStepPct: 0, diff --git a/invokeai/frontend/web/src/features/metadata/util/parsers.ts b/invokeai/frontend/web/src/features/metadata/util/parsers.ts index 2a3e3158e8..3decea6737 100644 --- a/invokeai/frontend/web/src/features/metadata/util/parsers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/parsers.ts @@ -386,6 +386,10 @@ const parseIPAdapter: MetadataParseFunc = async (metada .nullish() .catch(null) .parse(await getProperty(metadataItem, 'weight')); + const method = zIPAdapterField.shape.method + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'method')); const begin_step_percent = zIPAdapterField.shape.begin_step_percent .nullish() .catch(null) @@ -403,6 +407,7 @@ const parseIPAdapter: MetadataParseFunc = async (metada clipVisionModel: 'ViT-H', controlImage: image?.image_name ?? null, weight: weight ?? initialIPAdapter.weight, + method: method ?? initialIPAdapter.method, beginStepPct: begin_step_percent ?? initialIPAdapter.beginStepPct, endStepPct: end_step_percent ?? initialIPAdapter.endStepPct, }; diff --git a/invokeai/frontend/web/src/features/nodes/types/common.ts b/invokeai/frontend/web/src/features/nodes/types/common.ts index 06d5ecd5c7..e570054258 100644 --- a/invokeai/frontend/web/src/features/nodes/types/common.ts +++ b/invokeai/frontend/web/src/features/nodes/types/common.ts @@ -109,6 +109,7 @@ export const zIPAdapterField = z.object({ image: zImageField, ip_adapter_model: zModelIdentifierField, weight: z.number(), + method: z.enum(['full', 'style', 'composition']), begin_step_percent: z.number().optional(), end_step_percent: z.number().optional(), }); diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts index ad563de468..568b24ccfd 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts @@ -48,7 +48,7 @@ export const addIPAdapterToLinearGraph = async ( if (!ipAdapter.model) { return; } - const { id, weight, model, clipVisionModel, beginStepPct, endStepPct, controlImage } = ipAdapter; + const { id, weight, model, clipVisionModel, method, beginStepPct, endStepPct, controlImage } = ipAdapter; assert(controlImage, 'IP Adapter image is required'); @@ -57,6 +57,7 @@ export const addIPAdapterToLinearGraph = async ( type: 'ip_adapter', is_intermediate: true, weight: weight, + method: method, ip_adapter_model: model, clip_vision_model: clipVisionModel, begin_step_percent: beginStepPct, @@ -84,7 +85,7 @@ export const addIPAdapterToLinearGraph = async ( }; const buildIPAdapterMetadata = (ipAdapter: IPAdapterConfig): S['IPAdapterMetadataField'] => { - const { controlImage, beginStepPct, endStepPct, model, clipVisionModel, weight } = ipAdapter; + const { controlImage, beginStepPct, endStepPct, model, clipVisionModel, method, weight } = ipAdapter; assert(model, 'IP Adapter model is required'); @@ -102,6 +103,7 @@ const buildIPAdapterMetadata = (ipAdapter: IPAdapterConfig): S['IPAdapterMetadat ip_adapter_model: model, clip_vision_model: clipVisionModel, weight, + method, begin_step_percent: beginStepPct, end_step_percent: endStepPct, image, diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index cb222bd497..7157de227b 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -2070,6 +2070,8 @@ export type components = { * @description CLIP (tokenizer, text encoder, LoRAs) and skipped layer count */ clip?: components["schemas"]["CLIPField"]; + /** @description A mask defining the region that this conditioning prompt applies to. */ + mask?: components["schemas"]["TensorField"] | null; /** * type * @default compel @@ -2139,6 +2141,11 @@ export type components = { * @description The name of conditioning tensor */ conditioning_name: string; + /** + * @description The mask associated with this conditioning tensor. Excluded regions should be set to False, included regions should be set to True. + * @default null + */ + mask?: components["schemas"]["TensorField"] | null; }; /** * Conditioning Primitive @@ -3049,10 +3056,16 @@ export type components = { * @default true */ use_cache?: boolean; - /** @description Positive conditioning tensor */ - positive_conditioning?: components["schemas"]["ConditioningField"]; - /** @description Negative conditioning tensor */ - negative_conditioning?: components["schemas"]["ConditioningField"]; + /** + * Positive Conditioning + * @description Positive conditioning tensor + */ + positive_conditioning?: components["schemas"]["ConditioningField"] | components["schemas"]["ConditioningField"][]; + /** + * Negative Conditioning + * @description Negative conditioning tensor + */ + negative_conditioning?: components["schemas"]["ConditioningField"] | components["schemas"]["ConditioningField"][]; /** @description Noise tensor */ noise?: components["schemas"]["LatentsField"] | null; /** @@ -4112,7 +4125,7 @@ export type components = { * @description The nodes in this graph */ nodes: { - [key: string]: components["schemas"]["IntegerMathInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["LineartImageProcessorInvocation"]; + [key: string]: components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["LineartImageProcessorInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["RectangleMaskInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["IntegerMathInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"]; }; /** * Edges @@ -4149,7 +4162,7 @@ export type components = { * @description The results of node executions */ results: { - [key: string]: components["schemas"]["FloatCollectionOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["String2Output"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["SchedulerOutput"] | components["schemas"]["CalculateImageTilesOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["LoRALoaderOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["SDXLRefinerModelLoaderOutput"]; + [key: string]: components["schemas"]["ColorCollectionOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["String2Output"] | components["schemas"]["MaskOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["FloatCollectionOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["SchedulerOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["LoRALoaderOutput"] | components["schemas"]["SDXLRefinerModelLoaderOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["CalculateImageTilesOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["TileToPropertiesOutput"]; }; /** * Errors @@ -4383,10 +4396,16 @@ export type components = { image_encoder_model: components["schemas"]["ModelIdentifierField"]; /** * Weight - * @description The weight given to the ControlNet + * @description The weight given to the IP-Adapter. * @default 1 */ weight?: number | number[]; + /** + * Target Blocks + * @description The IP Adapter blocks to apply + * @default [] + */ + target_blocks?: string[]; /** * Begin Step Percent * @description When the IP-Adapter is first applied (% of total steps) @@ -4399,6 +4418,11 @@ export type components = { * @default 1 */ end_step_percent?: number; + /** + * @description The bool mask associated with this IP-Adapter. Excluded regions should be set to False, included regions should be set to True. + * @default null + */ + mask?: components["schemas"]["TensorField"] | null; }; /** * IP-Adapter @@ -4445,6 +4469,13 @@ export type components = { * @default 1 */ weight?: number | number[]; + /** + * Method + * @description The method to apply the IP-Adapter + * @default full + * @enum {string} + */ + method?: "full" | "style" | "composition"; /** * Begin Step Percent * @description When the IP-Adapter is first applied (% of total steps) @@ -4457,6 +4488,8 @@ export type components = { * @default 1 */ end_step_percent?: number; + /** @description A mask defining the region that this IP-Adapter applies to. */ + mask?: components["schemas"]["TensorField"] | null; /** * type * @default ip_adapter @@ -4542,6 +4575,12 @@ export type components = { * @enum {string} */ clip_vision_model: "ViT-H" | "ViT-G"; + /** + * Method + * @description Method to apply IP Weights with + * @enum {string} + */ + method: "full" | "style" | "composition"; /** * Weight * @description The weight given to the IP-Adapter @@ -7104,6 +7143,30 @@ export type components = { */ type: "mask_from_id"; }; + /** + * MaskOutput + * @description A torch mask tensor. + */ + MaskOutput: { + /** @description The mask. */ + mask: components["schemas"]["TensorField"]; + /** + * Width + * @description The width of the mask in pixels. + */ + width: number; + /** + * Height + * @description The height of the mask in pixels. + */ + height: number; + /** + * type + * @default mask_output + * @constant + */ + type: "mask_output"; + }; /** * Mediapipe Face Processor * @description Applies mediapipe face processing to image @@ -8364,6 +8427,67 @@ export type components = { */ type: "range_of_size"; }; + /** + * Create Rectangle Mask + * @description Create a rectangular mask. + */ + RectangleMaskInvocation: { + /** @description Optional metadata to be saved with the image */ + metadata?: components["schemas"]["MetadataField"] | null; + /** + * Id + * @description The id of this instance of an invocation. Must be unique among all instances of invocations. + */ + id: string; + /** + * Is Intermediate + * @description Whether or not this is an intermediate invocation. + * @default false + */ + is_intermediate?: boolean; + /** + * Use Cache + * @description Whether or not to use the cache + * @default true + */ + use_cache?: boolean; + /** + * Width + * @description The width of the entire mask. + */ + width?: number; + /** + * Height + * @description The height of the entire mask. + */ + height?: number; + /** + * X Left + * @description The left x-coordinate of the rectangular masked region (inclusive). + */ + x_left?: number; + /** + * Y Top + * @description The top y-coordinate of the rectangular masked region (inclusive). + */ + y_top?: number; + /** + * Rectangle Width + * @description The width of the rectangular masked region. + */ + rectangle_width?: number; + /** + * Rectangle Height + * @description The height of the rectangular masked region. + */ + rectangle_height?: number; + /** + * type + * @default rectangle_mask + * @constant + */ + type: "rectangle_mask"; + }; /** * RemoteModelFile * @description Information about a downloadable file that forms part of a model. @@ -8579,6 +8703,8 @@ export type components = { * @description CLIP (tokenizer, text encoder, LoRAs) and skipped layer count */ clip2?: components["schemas"]["CLIPField"]; + /** @description A mask defining the region that this conditioning prompt applies to. */ + mask?: components["schemas"]["TensorField"] | null; /** * type * @default sdxl_compel_prompt @@ -10115,6 +10241,17 @@ export type components = { /** Right */ right: number; }; + /** + * TensorField + * @description A tensor primitive field. + */ + TensorField: { + /** + * Tensor Name + * @description The name of a tensor. + */ + tensor_name: string; + }; /** * TextualInversionFileConfig * @description Model config for textual inversion embeddings.