Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added per-step setting of IP-Adapter weights (for param easing, etc.)

Browse Source
This commit is contained in:
user1 2023-09-16 12:36:16 -07:00
parent 29b4ddcc7f
commit c48e648cbb
2 changed files with 22 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
invokeai/app/invocations/ip_adapter.py
View File

@ -1,6 +1,8 @@
import os
from builtins import bool, float
from pydantic import BaseModel, Field
from typing import Dict, List, Literal, Optional, Union
from invokeai.app.invocations.baseinvocation import (
BaseInvocation,
@ -33,9 +35,14 @@ class IPAdapterField(BaseModel):
image: ImageField = Field(description="The IP-Adapter image prompt.")
ip_adapter_model: IPAdapterModelField = Field(description="The IP-Adapter model to use.")
image_encoder_model: CLIPVisionModelField = Field(description="The name of the CLIP image encoder model.")
weight: float = Field(default=1.0, ge=0, description="The weight of the IP-Adapter.")
begin_step_percent: float = Field(default=0.0, ge=0, le=1.0)
end_step_percent: float = Field(default=1.0, ge=0, le=1.0)
weight: Union[float, List[float]] = Field(default=1, description="The weight given to the ControlNet")
# weight: float = Field(default=1.0, ge=0, description="The weight of the IP-Adapter.")
begin_step_percent: float = Field(
default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)"
)
end_step_percent: float = Field(
default=1, ge=0, le=1, description="When the IP-Adapter is last applied (% of total steps)"
)
@invocation_output("ip_adapter_output")
@ -55,7 +62,11 @@ class IPAdapterInvocation(BaseInvocation):
title="IP-Adapter Model",
input=Input.Direct,
)
weight: float = InputField(default=1.0, description="The weight of the IP-Adapter.", ui_type=UIType.Float)
# weight: float = InputField(default=1.0, description="The weight of the IP-Adapter.", ui_type=UIType.Float)
weight: Union[float, List[float]] = InputField(default=1, ge=0, description="The weight given to the IP-Adapter",
ui_type=UIType.Float, title="Weight")
begin_step_percent: float = InputField(
default=0, ge=-1, le=2, description="When the IP-Adapter is first applied (% of total steps)"
)
@ -81,7 +92,6 @@ class IPAdapterInvocation(BaseInvocation):
model_name=image_encoder_model_name,
base_model=BaseModelType.Any,
)
return IPAdapterOutput(
ip_adapter=IPAdapterField(
image=self.image,

11
invokeai/backend/stable_diffusion/diffusers_pipeline.py
View File

@ -167,8 +167,8 @@ class ControlNetData:
class IPAdapterData:
ip_adapter_model: IPAdapter = Field(default=None)
# TODO: change to polymorphic so can do different weights per step (once implemented...)
# weight: Union[float, List[float]] = Field(default=1.0)
weight: float = Field(default=1.0)
weight: Union[float, List[float]] = Field(default=1.0)
# weight: float = Field(default=1.0)
begin_step_percent: float = Field(default=0.0)
end_step_percent: float = Field(default=1.0)
@ -421,8 +421,9 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
elif ip_adapter_data is not None:
# TODO(ryand): Should we raise an exception if both custom attention and IP-Adapter attention are active?
# As it is now, the IP-Adapter will silently be skipped.
weight = ip_adapter_data.weight[0] if isinstance(ip_adapter_data.weight, List) else ip_adapter_data.weight
attn_ctx = ip_adapter_data.ip_adapter_model.apply_ip_adapter_attention(
unet=self.invokeai_diffuser.model, scale=ip_adapter_data.weight
unet=self.invokeai_diffuser.model, scale=weight,
)
self.use_ip_adapter = True
else:
@ -512,9 +513,11 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
if self.use_ip_adapter and ip_adapter_data is not None: # somewhat redundant but logic is clearer
first_adapter_step = math.floor(ip_adapter_data.begin_step_percent * total_step_count)
last_adapter_step = math.ceil(ip_adapter_data.end_step_percent * total_step_count)
weight = ip_adapter_data.weight[step_index] if isinstance(ip_adapter_data.weight, List) else ip_adapter_data.weight
if step_index >= first_adapter_step and step_index <= last_adapter_step:
# only apply IP-Adapter if current step is within the IP-Adapter's begin/end step range
ip_adapter_data.ip_adapter_model.set_scale(ip_adapter_data.weight)
# ip_adapter_data.ip_adapter_model.set_scale(ip_adapter_data.weight)
ip_adapter_data.ip_adapter_model.set_scale(weight)
else:
# otherwise, set IP-Adapter scale to 0, so it has no effect
ip_adapter_data.ip_adapter_model.set_scale(0.0)