mirror of
https://github.com/invoke-ai/InvokeAI
synced 2024-08-30 20:32:17 +00:00
Compare commits
14 Commits
maryhipp/u
...
feat/contr
Author | SHA1 | Date | |
---|---|---|---|
e4a45341c8 | |||
4ca325e8e6 | |||
6b8e88ad7f | |||
6c53abc034 | |||
eb7047b21d | |||
43419ac761 | |||
5cd0e90816 | |||
cfd49e3921 | |||
a8e0490133 | |||
de3e6cdb02 | |||
8495764d45 | |||
8b7fac75ed | |||
9e0e26f4c4 | |||
fd715026a7 |
@ -1,7 +1,7 @@
|
||||
# InvokeAI nodes for ControlNet image preprocessors
|
||||
# initial implementation by Gregg Helt, 2023
|
||||
# heavily leverages controlnet_aux package: https://github.com/patrickvonplaten/controlnet_aux
|
||||
from builtins import float
|
||||
from builtins import float, bool
|
||||
|
||||
import numpy as np
|
||||
from typing import Literal, Optional, Union, List
|
||||
@ -94,6 +94,7 @@ CONTROLNET_DEFAULT_MODELS = [
|
||||
]
|
||||
|
||||
CONTROLNET_NAME_VALUES = Literal[tuple(CONTROLNET_DEFAULT_MODELS)]
|
||||
CONTROLNET_MODE_VALUES = Literal[tuple(["balanced", "more_prompt", "more_control", "unbalanced"])]
|
||||
|
||||
class ControlField(BaseModel):
|
||||
image: ImageField = Field(default=None, description="The control image")
|
||||
@ -104,6 +105,8 @@ class ControlField(BaseModel):
|
||||
description="When the ControlNet is first applied (% of total steps)")
|
||||
end_step_percent: float = Field(default=1, ge=0, le=1,
|
||||
description="When the ControlNet is last applied (% of total steps)")
|
||||
control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The contorl mode to use")
|
||||
|
||||
@validator("control_weight")
|
||||
def abs_le_one(cls, v):
|
||||
"""validate that all abs(values) are <=1"""
|
||||
@ -144,11 +147,11 @@ class ControlNetInvocation(BaseInvocation):
|
||||
control_model: CONTROLNET_NAME_VALUES = Field(default="lllyasviel/sd-controlnet-canny",
|
||||
description="control model used")
|
||||
control_weight: Union[float, List[float]] = Field(default=1.0, description="The weight given to the ControlNet")
|
||||
# TODO: add support in backend core for begin_step_percent, end_step_percent, guess_mode
|
||||
begin_step_percent: float = Field(default=0, ge=0, le=1,
|
||||
description="When the ControlNet is first applied (% of total steps)")
|
||||
end_step_percent: float = Field(default=1, ge=0, le=1,
|
||||
description="When the ControlNet is last applied (% of total steps)")
|
||||
control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The control mode used")
|
||||
# fmt: on
|
||||
|
||||
class Config(InvocationConfig):
|
||||
@ -166,7 +169,6 @@ class ControlNetInvocation(BaseInvocation):
|
||||
}
|
||||
|
||||
def invoke(self, context: InvocationContext) -> ControlOutput:
|
||||
|
||||
return ControlOutput(
|
||||
control=ControlField(
|
||||
image=self.image,
|
||||
@ -174,6 +176,7 @@ class ControlNetInvocation(BaseInvocation):
|
||||
control_weight=self.control_weight,
|
||||
begin_step_percent=self.begin_step_percent,
|
||||
end_step_percent=self.end_step_percent,
|
||||
control_mode=self.control_mode,
|
||||
),
|
||||
)
|
||||
|
||||
|
@ -282,19 +282,14 @@ class TextToLatentsInvocation(BaseInvocation):
|
||||
control_height_resize = latents_shape[2] * 8
|
||||
control_width_resize = latents_shape[3] * 8
|
||||
if control_input is None:
|
||||
# print("control input is None")
|
||||
control_list = None
|
||||
elif isinstance(control_input, list) and len(control_input) == 0:
|
||||
# print("control input is empty list")
|
||||
control_list = None
|
||||
elif isinstance(control_input, ControlField):
|
||||
# print("control input is ControlField")
|
||||
control_list = [control_input]
|
||||
elif isinstance(control_input, list) and len(control_input) > 0 and isinstance(control_input[0], ControlField):
|
||||
# print("control input is list[ControlField]")
|
||||
control_list = control_input
|
||||
else:
|
||||
# print("input control is unrecognized:", type(self.control))
|
||||
control_list = None
|
||||
if (control_list is None):
|
||||
control_data = None
|
||||
@ -336,12 +331,15 @@ class TextToLatentsInvocation(BaseInvocation):
|
||||
# num_images_per_prompt=num_images_per_prompt,
|
||||
device=control_model.device,
|
||||
dtype=control_model.dtype,
|
||||
control_mode=control_info.control_mode,
|
||||
)
|
||||
control_item = ControlNetData(model=control_model,
|
||||
image_tensor=control_image,
|
||||
weight=control_info.control_weight,
|
||||
begin_step_percent=control_info.begin_step_percent,
|
||||
end_step_percent=control_info.end_step_percent)
|
||||
end_step_percent=control_info.end_step_percent,
|
||||
control_mode=control_info.control_mode,
|
||||
)
|
||||
control_data.append(control_item)
|
||||
# MultiControlNetModel has been refactored out, just need list[ControlNetData]
|
||||
return control_data
|
||||
|
@ -46,6 +46,7 @@ from .diffusion import (
|
||||
AttentionMapSaver,
|
||||
InvokeAIDiffuserComponent,
|
||||
PostprocessingSettings,
|
||||
ControlNetData,
|
||||
)
|
||||
from .offloading import FullyLoadedModelGroup, LazilyLoadedModelGroup, ModelGroup
|
||||
from .textual_inversion_manager import TextualInversionManager
|
||||
@ -214,13 +215,6 @@ class GeneratorToCallbackinator(Generic[ParamType, ReturnType, CallbackType]):
|
||||
raise AssertionError("why was that an empty generator?")
|
||||
return result
|
||||
|
||||
@dataclass
|
||||
class ControlNetData:
|
||||
model: ControlNetModel = Field(default=None)
|
||||
image_tensor: torch.Tensor= Field(default=None)
|
||||
weight: Union[float, List[float]]= Field(default=1.0)
|
||||
begin_step_percent: float = Field(default=0.0)
|
||||
end_step_percent: float = Field(default=1.0)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConditioningData:
|
||||
@ -656,69 +650,18 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
|
||||
|
||||
# TODO: should this scaling happen here or inside self._unet_forward?
|
||||
# i.e. before or after passing it to InvokeAIDiffuserComponent
|
||||
latent_model_input = self.scheduler.scale_model_input(latents, timestep)
|
||||
|
||||
# default is no controlnet, so set controlnet processing output to None
|
||||
down_block_res_samples, mid_block_res_sample = None, None
|
||||
|
||||
if control_data is not None:
|
||||
# FIXME: make sure guidance_scale < 1.0 is handled correctly if doing per-step guidance setting
|
||||
# if conditioning_data.guidance_scale > 1.0:
|
||||
if conditioning_data.guidance_scale is not None:
|
||||
# expand the latents input to control model if doing classifier free guidance
|
||||
# (which I think for now is always true, there is conditional elsewhere that stops execution if
|
||||
# classifier_free_guidance is <= 1.0 ?)
|
||||
latent_control_input = torch.cat([latent_model_input] * 2)
|
||||
else:
|
||||
latent_control_input = latent_model_input
|
||||
# control_data should be type List[ControlNetData]
|
||||
# this loop covers both ControlNet (one ControlNetData in list)
|
||||
# and MultiControlNet (multiple ControlNetData in list)
|
||||
for i, control_datum in enumerate(control_data):
|
||||
# print("controlnet", i, "==>", type(control_datum))
|
||||
first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
|
||||
last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
|
||||
# only apply controlnet if current step is within the controlnet's begin/end step range
|
||||
if step_index >= first_control_step and step_index <= last_control_step:
|
||||
# print("running controlnet", i, "for step", step_index)
|
||||
if isinstance(control_datum.weight, list):
|
||||
# if controlnet has multiple weights, use the weight for the current step
|
||||
controlnet_weight = control_datum.weight[step_index]
|
||||
else:
|
||||
# if controlnet has a single weight, use it for all steps
|
||||
controlnet_weight = control_datum.weight
|
||||
down_samples, mid_sample = control_datum.model(
|
||||
sample=latent_control_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=torch.cat([conditioning_data.unconditioned_embeddings,
|
||||
conditioning_data.text_embeddings]),
|
||||
controlnet_cond=control_datum.image_tensor,
|
||||
conditioning_scale=controlnet_weight,
|
||||
# cross_attention_kwargs,
|
||||
guess_mode=False,
|
||||
return_dict=False,
|
||||
)
|
||||
if down_block_res_samples is None and mid_block_res_sample is None:
|
||||
down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
|
||||
else:
|
||||
# add controlnet outputs together if have multiple controlnets
|
||||
down_block_res_samples = [
|
||||
samples_prev + samples_curr
|
||||
for samples_prev, samples_curr in zip(down_block_res_samples, down_samples)
|
||||
]
|
||||
mid_block_res_sample += mid_sample
|
||||
unet_latent_input = self.scheduler.scale_model_input(latents, timestep)
|
||||
|
||||
# predict the noise residual
|
||||
noise_pred = self.invokeai_diffuser.do_diffusion_step(
|
||||
latent_model_input,
|
||||
t,
|
||||
conditioning_data.unconditioned_embeddings,
|
||||
conditioning_data.text_embeddings,
|
||||
conditioning_data.guidance_scale,
|
||||
x=unet_latent_input,
|
||||
sigma=t,
|
||||
unconditioning=conditioning_data.unconditioned_embeddings,
|
||||
conditioning=conditioning_data.text_embeddings,
|
||||
unconditional_guidance_scale=conditioning_data.guidance_scale,
|
||||
control_data=control_data,
|
||||
step_index=step_index,
|
||||
total_step_count=total_step_count,
|
||||
down_block_additional_residuals=down_block_res_samples, # from controlnet(s)
|
||||
mid_block_additional_residual=mid_block_res_sample, # from controlnet(s)
|
||||
)
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
@ -1038,6 +981,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
|
||||
device="cuda",
|
||||
dtype=torch.float16,
|
||||
do_classifier_free_guidance=True,
|
||||
control_mode="balanced"
|
||||
):
|
||||
|
||||
if not isinstance(image, torch.Tensor):
|
||||
@ -1068,6 +1012,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
|
||||
repeat_by = num_images_per_prompt
|
||||
image = image.repeat_interleave(repeat_by, dim=0)
|
||||
image = image.to(device=device, dtype=dtype)
|
||||
if do_classifier_free_guidance:
|
||||
image = torch.cat([image] * 2)
|
||||
#cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
|
||||
#if do_classifier_free_guidance and not cfg_injection:
|
||||
# image = torch.cat([image] * 2)
|
||||
return image
|
||||
|
@ -3,4 +3,4 @@ Initialization file for invokeai.models.diffusion
|
||||
"""
|
||||
from .cross_attention_control import InvokeAICrossAttentionMixin
|
||||
from .cross_attention_map_saving import AttentionMapSaver
|
||||
from .shared_invokeai_diffusion import InvokeAIDiffuserComponent, PostprocessingSettings
|
||||
from .shared_invokeai_diffusion import InvokeAIDiffuserComponent, PostprocessingSettings, ControlNetData
|
||||
|
@ -1,11 +1,14 @@
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pydantic import Field
|
||||
from math import ceil
|
||||
from typing import Any, Callable, Dict, Optional, Union, List
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import math
|
||||
from diffusers import UNet2DConditionModel
|
||||
from diffusers.models.controlnet import ControlNetModel
|
||||
from diffusers.models.attention_processor import AttentionProcessor
|
||||
from typing_extensions import TypeAlias
|
||||
|
||||
@ -40,6 +43,17 @@ class PostprocessingSettings:
|
||||
v_symmetry_time_pct: Optional[float]
|
||||
|
||||
|
||||
# TODO: pydantic Field work with dataclasses?
|
||||
@dataclass
|
||||
class ControlNetData:
|
||||
model: ControlNetModel = Field(default=None)
|
||||
image_tensor: torch.Tensor = Field(default=None)
|
||||
weight: Union[float, List[float]] = Field(default=1.0)
|
||||
begin_step_percent: float = Field(default=0.0)
|
||||
end_step_percent: float = Field(default=1.0)
|
||||
control_mode: str = Field(default="balanced")
|
||||
|
||||
|
||||
class InvokeAIDiffuserComponent:
|
||||
"""
|
||||
The aim of this component is to provide a single place for code that can be applied identically to
|
||||
@ -182,8 +196,9 @@ class InvokeAIDiffuserComponent:
|
||||
conditioning: Union[torch.Tensor, dict],
|
||||
# unconditional_guidance_scale: float,
|
||||
unconditional_guidance_scale: Union[float, List[float]],
|
||||
step_index: Optional[int] = None,
|
||||
total_step_count: Optional[int] = None,
|
||||
step_index: int,
|
||||
total_step_count: int,
|
||||
control_data: Optional[List[ControlNetData]],
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
@ -213,31 +228,56 @@ class InvokeAIDiffuserComponent:
|
||||
)
|
||||
)
|
||||
|
||||
if self.sequential_guidance:
|
||||
down_block_res_samples, mid_block_res_sample = self._run_controlnet_sequentially(
|
||||
unconditioning=unconditioning,
|
||||
conditioning=conditioning,
|
||||
control_data=control_data,
|
||||
sample=x,
|
||||
timestep=sigma,
|
||||
step_index=step_index,
|
||||
total_step_count=total_step_count,
|
||||
)
|
||||
else:
|
||||
down_block_res_samples, mid_block_res_sample = self._run_controlnet_normally(
|
||||
unconditioning=unconditioning,
|
||||
conditioning=conditioning,
|
||||
control_data=control_data,
|
||||
sample=x,
|
||||
timestep=sigma,
|
||||
step_index=step_index,
|
||||
total_step_count=total_step_count,
|
||||
)
|
||||
|
||||
wants_cross_attention_control = len(cross_attention_control_types_to_do) > 0
|
||||
wants_hybrid_conditioning = isinstance(conditioning, dict)
|
||||
|
||||
if wants_hybrid_conditioning:
|
||||
unconditioned_next_x, conditioned_next_x = self._apply_hybrid_conditioning(
|
||||
x, sigma, unconditioning, conditioning, **kwargs,
|
||||
x, sigma, unconditioning, conditioning,
|
||||
down_block_additional_residuals=down_block_res_samples, # from controlnet(s)
|
||||
mid_block_additional_residual=mid_block_res_sample, # from controlnet(s)
|
||||
**kwargs,
|
||||
)
|
||||
elif wants_cross_attention_control:
|
||||
(
|
||||
unconditioned_next_x,
|
||||
conditioned_next_x,
|
||||
) = self._apply_cross_attention_controlled_conditioning(
|
||||
x,
|
||||
sigma,
|
||||
unconditioning,
|
||||
conditioning,
|
||||
cross_attention_control_types_to_do,
|
||||
x, sigma, unconditioning, conditioning, cross_attention_control_types_to_do,
|
||||
down_block_additional_residuals=down_block_res_samples, # from controlnet(s)
|
||||
mid_block_additional_residual=mid_block_res_sample, # from controlnet(s)
|
||||
**kwargs,
|
||||
)
|
||||
elif self.sequential_guidance:
|
||||
elif True: #self.sequential_guidance:
|
||||
(
|
||||
unconditioned_next_x,
|
||||
conditioned_next_x,
|
||||
) = self._apply_standard_conditioning_sequentially(
|
||||
x, sigma, unconditioning, conditioning, **kwargs,
|
||||
x, sigma, unconditioning, conditioning,
|
||||
down_block_additional_residuals=down_block_res_samples, # from controlnet(s)
|
||||
mid_block_additional_residual=mid_block_res_sample, # from controlnet(s)
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
@ -245,7 +285,10 @@ class InvokeAIDiffuserComponent:
|
||||
unconditioned_next_x,
|
||||
conditioned_next_x,
|
||||
) = self._apply_standard_conditioning(
|
||||
x, sigma, unconditioning, conditioning, **kwargs,
|
||||
x, sigma, unconditioning, conditioning,
|
||||
down_block_additional_residuals=down_block_res_samples, # from controlnet(s)
|
||||
mid_block_additional_residual=mid_block_res_sample, # from controlnet(s)
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
combined_next_x = self._combine(
|
||||
@ -293,16 +336,160 @@ class InvokeAIDiffuserComponent:
|
||||
|
||||
# methods below are called from do_diffusion_step and should be considered private to this class.
|
||||
|
||||
def _apply_standard_conditioning(self, x, sigma, unconditioning, conditioning, **kwargs):
|
||||
def _run_controlnet_normally(
|
||||
self,
|
||||
unconditioning: torch.Tensor,
|
||||
conditioning: torch.Tensor,
|
||||
control_data: List[ControlNetData],
|
||||
sample: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
step_index: int,
|
||||
total_step_count: int,
|
||||
):
|
||||
if control_data is None:
|
||||
return (None, None)
|
||||
|
||||
down_block_res_samples, mid_block_res_sample = None, None
|
||||
|
||||
for i, control_datum in enumerate(control_data):
|
||||
control_mode = control_datum.control_mode
|
||||
soft_injection = (control_mode == "more_prompt" or control_mode == "more_control")
|
||||
cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
|
||||
|
||||
first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
|
||||
last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
|
||||
# only apply controlnet if current step is within the controlnet's begin/end step range
|
||||
if step_index >= first_control_step and step_index <= last_control_step:
|
||||
|
||||
if cfg_injection:
|
||||
control_sample = sample
|
||||
control_timestep = timestep
|
||||
control_image_tensor = control_datum.image_tensor
|
||||
encoder_hidden_states = conditioning # TODO: ask bug
|
||||
else:
|
||||
control_sample = torch.cat([sample] * 2)
|
||||
control_timestep = torch.cat([timestep] * 2)
|
||||
control_image_tensor = torch.cat([control_datum.image_tensor] * 2)
|
||||
encoder_hidden_states = torch.cat([unconditioning, conditioning])
|
||||
|
||||
if isinstance(control_datum.weight, list):
|
||||
weight = control_datum.weight[step_index]
|
||||
else:
|
||||
weight = control_datum.weight
|
||||
|
||||
# controlnet(s) inference
|
||||
down_samples, mid_sample = control_datum.model(
|
||||
sample=control_sample,
|
||||
timestep=control_timestep,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
controlnet_cond=control_image_tensor,
|
||||
conditioning_scale=weight, # controlnet specific, NOT the guidance scale
|
||||
guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
|
||||
return_dict=False,
|
||||
)
|
||||
|
||||
if cfg_injection:
|
||||
down_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_samples]
|
||||
mid_sample = torch.cat([torch.zeros_like(mid_sample), mid_sample])
|
||||
|
||||
if down_block_res_samples is None and mid_block_res_sample is None:
|
||||
down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
|
||||
else:
|
||||
down_block_res_samples = [
|
||||
samples_prev + samples_curr
|
||||
for samples_prev, samples_curr in zip(down_block_res_samples, down_samples)
|
||||
]
|
||||
mid_block_res_sample += mid_sample
|
||||
|
||||
return down_block_res_samples, mid_block_res_sample
|
||||
|
||||
def _run_controlnet_sequentially(
|
||||
self,
|
||||
unconditioning: torch.Tensor,
|
||||
conditioning: torch.Tensor,
|
||||
control_data: List[ControlNetData],
|
||||
sample: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
step_index: int,
|
||||
total_step_count: int,
|
||||
):
|
||||
if control_data is None:
|
||||
return (None, None)
|
||||
|
||||
down_block_res_samples, mid_block_res_sample = None, None
|
||||
|
||||
for i, control_datum in enumerate(control_data):
|
||||
control_mode = control_datum.control_mode
|
||||
soft_injection = (control_mode == "more_prompt" or control_mode == "more_control")
|
||||
cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
|
||||
|
||||
first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
|
||||
last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
|
||||
# only apply controlnet if current step is within the controlnet's begin/end step range
|
||||
if step_index >= first_control_step and step_index <= last_control_step:
|
||||
|
||||
if isinstance(control_datum.weight, list):
|
||||
weight = control_datum.weight[step_index]
|
||||
else:
|
||||
weight = control_datum.weight
|
||||
|
||||
# controlnet(s) inference
|
||||
cond_down_samples, cond_mid_sample = control_datum.model(
|
||||
sample=sample,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=conditioning, # TODO: ask bug
|
||||
controlnet_cond=control_datum.image_tensor,
|
||||
conditioning_scale=weight, # controlnet specific, NOT the guidance scale
|
||||
guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
|
||||
return_dict=False,
|
||||
)
|
||||
|
||||
if cfg_injection:
|
||||
uncond_down_samples = [torch.zeros_like(d) for d in cond_down_samples]
|
||||
uncond_mid_sample = torch.zeros_like(cond_mid_sample)
|
||||
|
||||
else:
|
||||
uncond_down_samples, uncond_mid_sample = control_datum.model(
|
||||
sample=sample,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=unconditioning,
|
||||
controlnet_cond=control_datum.image_tensor,
|
||||
conditioning_scale=weight, # controlnet specific, NOT the guidance scale
|
||||
guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
|
||||
return_dict=False,
|
||||
)
|
||||
|
||||
down_samples = [torch.cat([ud, cd]) for ud, cd in zip(uncond_down_samples, cond_down_samples)]
|
||||
mid_sample = torch.cat([uncond_mid_sample, cond_mid_sample])
|
||||
|
||||
if down_block_res_samples is None and mid_block_res_sample is None:
|
||||
down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
|
||||
else:
|
||||
down_block_res_samples = [
|
||||
samples_prev + samples_curr
|
||||
for samples_prev, samples_curr in zip(down_block_res_samples, down_samples)
|
||||
]
|
||||
mid_block_res_sample += mid_sample
|
||||
|
||||
return down_block_res_samples, mid_block_res_sample
|
||||
|
||||
def _apply_standard_conditioning(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
sigma: torch.Tensor,
|
||||
unconditioning: torch.Tensor,
|
||||
conditioning: torch.Tensor,
|
||||
**kwargs
|
||||
):
|
||||
# fast batched path
|
||||
x_twice = torch.cat([x] * 2)
|
||||
sigma_twice = torch.cat([sigma] * 2)
|
||||
both_conditionings = torch.cat([unconditioning, conditioning])
|
||||
both_results = self.model_forward_callback(
|
||||
x_twice, sigma_twice, both_conditionings, **kwargs,
|
||||
)
|
||||
|
||||
both_results = self.model_forward_callback(x_twice, sigma_twice, both_conditionings, **kwargs)
|
||||
unconditioned_next_x, conditioned_next_x = both_results.chunk(2)
|
||||
if conditioned_next_x.device.type == "mps":
|
||||
# TODO: check if this still present
|
||||
# prevent a result filled with zeros. seems to be a torch bug.
|
||||
conditioned_next_x = conditioned_next_x.clone()
|
||||
return unconditioned_next_x, conditioned_next_x
|
||||
@ -310,15 +497,43 @@ class InvokeAIDiffuserComponent:
|
||||
def _apply_standard_conditioning_sequentially(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
sigma,
|
||||
sigma: torch.Tensor,
|
||||
unconditioning: torch.Tensor,
|
||||
conditioning: torch.Tensor,
|
||||
down_block_additional_residuals, # from controlnet(s)
|
||||
mid_block_additional_residual, # from controlnet(s)
|
||||
**kwargs,
|
||||
):
|
||||
# split controlnet data to cond and uncond
|
||||
if down_block_additional_residuals is None:
|
||||
uncond_down_block_res_samples = None
|
||||
cond_down_block_res_samples = None
|
||||
uncond_mid_block_res_sample = None
|
||||
cond_mid_block_res_sample = None
|
||||
|
||||
else:
|
||||
uncond_down_block_res_samples = []
|
||||
cond_down_block_res_samples = []
|
||||
for d in down_block_additional_residuals:
|
||||
ud, cd = d.chunk(2)
|
||||
uncond_down_block_res_samples.append(ud)
|
||||
cond_down_block_res_samples.append(cd)
|
||||
|
||||
uncond_mid_block_res_sample, cond_mid_block_res_sample = mid_block_additional_residual.chunk(2)
|
||||
|
||||
# low-memory sequential path
|
||||
unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning, **kwargs)
|
||||
conditioned_next_x = self.model_forward_callback(x, sigma, conditioning, **kwargs)
|
||||
unconditioned_next_x = self.model_forward_callback(
|
||||
x, sigma, unconditioning, **kwargs,
|
||||
down_block_additional_residuals=uncond_down_block_res_samples,
|
||||
mid_block_additional_residual=uncond_mid_block_res_sample,
|
||||
)
|
||||
conditioned_next_x = self.model_forward_callback(
|
||||
x, sigma, conditioning, **kwargs,
|
||||
down_block_additional_residuals=cond_down_block_res_samples,
|
||||
mid_block_additional_residual=cond_mid_block_res_sample,
|
||||
)
|
||||
if conditioned_next_x.device.type == "mps":
|
||||
# TODO: check if still present
|
||||
# prevent a result filled with zeros. seems to be a torch bug.
|
||||
conditioned_next_x = conditioned_next_x.clone()
|
||||
return unconditioned_next_x, conditioned_next_x
|
||||
|
@ -524,7 +524,8 @@
|
||||
"initialImage": "Initial Image",
|
||||
"showOptionsPanel": "Show Options Panel",
|
||||
"hidePreview": "Hide Preview",
|
||||
"showPreview": "Show Preview"
|
||||
"showPreview": "Show Preview",
|
||||
"controlNetControlMode": "Control Mode"
|
||||
},
|
||||
"settings": {
|
||||
"models": "Models",
|
||||
|
@ -1,26 +1,27 @@
|
||||
import { Box, ChakraProps, Flex } from '@chakra-ui/react';
|
||||
import { useAppDispatch } from 'app/store/storeHooks';
|
||||
import { memo, useCallback } from 'react';
|
||||
import { FaCopy, FaTrash } from 'react-icons/fa';
|
||||
import {
|
||||
ControlNetConfig,
|
||||
controlNetAdded,
|
||||
controlNetRemoved,
|
||||
controlNetToggled,
|
||||
} from '../store/controlNetSlice';
|
||||
import { useAppDispatch } from 'app/store/storeHooks';
|
||||
import ParamControlNetModel from './parameters/ParamControlNetModel';
|
||||
import ParamControlNetWeight from './parameters/ParamControlNetWeight';
|
||||
import { Flex, Box, ChakraProps } from '@chakra-ui/react';
|
||||
import { FaCopy, FaTrash } from 'react-icons/fa';
|
||||
|
||||
import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
|
||||
import ControlNetImagePreview from './ControlNetImagePreview';
|
||||
import IAIIconButton from 'common/components/IAIIconButton';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { useToggle } from 'react-use';
|
||||
import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
|
||||
import ControlNetProcessorComponent from './ControlNetProcessorComponent';
|
||||
import IAISwitch from 'common/components/IAISwitch';
|
||||
import { ChevronUpIcon } from '@chakra-ui/icons';
|
||||
import IAIIconButton from 'common/components/IAIIconButton';
|
||||
import IAISwitch from 'common/components/IAISwitch';
|
||||
import { useToggle } from 'react-use';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import ControlNetImagePreview from './ControlNetImagePreview';
|
||||
import ControlNetProcessorComponent from './ControlNetProcessorComponent';
|
||||
import ParamControlNetShouldAutoConfig from './ParamControlNetShouldAutoConfig';
|
||||
import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
|
||||
import ParamControlNetControlMode from './parameters/ParamControlNetControlMode';
|
||||
import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
|
||||
|
||||
const expandedControlImageSx: ChakraProps['sx'] = { maxH: 96 };
|
||||
|
||||
@ -36,6 +37,7 @@ const ControlNet = (props: ControlNetProps) => {
|
||||
weight,
|
||||
beginStepPct,
|
||||
endStepPct,
|
||||
controlMode,
|
||||
controlImage,
|
||||
processedControlImage,
|
||||
processorNode,
|
||||
@ -137,45 +139,51 @@ const ControlNet = (props: ControlNetProps) => {
|
||||
</Flex>
|
||||
{isEnabled && (
|
||||
<>
|
||||
<Flex sx={{ gap: 4, w: 'full' }}>
|
||||
<Flex
|
||||
sx={{
|
||||
flexDir: 'column',
|
||||
gap: 2,
|
||||
w: 'full',
|
||||
h: isExpanded ? 28 : 24,
|
||||
paddingInlineStart: 1,
|
||||
paddingInlineEnd: isExpanded ? 1 : 0,
|
||||
pb: 2,
|
||||
justifyContent: 'space-between',
|
||||
}}
|
||||
>
|
||||
<ParamControlNetWeight
|
||||
controlNetId={controlNetId}
|
||||
weight={weight}
|
||||
mini={!isExpanded}
|
||||
/>
|
||||
<ParamControlNetBeginEnd
|
||||
controlNetId={controlNetId}
|
||||
beginStepPct={beginStepPct}
|
||||
endStepPct={endStepPct}
|
||||
mini={!isExpanded}
|
||||
/>
|
||||
</Flex>
|
||||
{!isExpanded && (
|
||||
<Flex sx={{ w: 'full', flexDirection: 'column' }}>
|
||||
<Flex sx={{ gap: 4, w: 'full' }}>
|
||||
<Flex
|
||||
sx={{
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
h: 24,
|
||||
w: 24,
|
||||
aspectRatio: '1/1',
|
||||
flexDir: 'column',
|
||||
gap: 3,
|
||||
w: 'full',
|
||||
paddingInlineStart: 1,
|
||||
paddingInlineEnd: isExpanded ? 1 : 0,
|
||||
pb: 2,
|
||||
justifyContent: 'space-between',
|
||||
}}
|
||||
>
|
||||
<ControlNetImagePreview controlNet={props.controlNet} />
|
||||
<ParamControlNetWeight
|
||||
controlNetId={controlNetId}
|
||||
weight={weight}
|
||||
mini={!isExpanded}
|
||||
/>
|
||||
<ParamControlNetBeginEnd
|
||||
controlNetId={controlNetId}
|
||||
beginStepPct={beginStepPct}
|
||||
endStepPct={endStepPct}
|
||||
mini={!isExpanded}
|
||||
/>
|
||||
</Flex>
|
||||
)}
|
||||
{!isExpanded && (
|
||||
<Flex
|
||||
sx={{
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
h: 24,
|
||||
w: 24,
|
||||
aspectRatio: '1/1',
|
||||
}}
|
||||
>
|
||||
<ControlNetImagePreview controlNet={props.controlNet} />
|
||||
</Flex>
|
||||
)}
|
||||
</Flex>
|
||||
<ParamControlNetControlMode
|
||||
controlNetId={controlNetId}
|
||||
controlMode={controlMode}
|
||||
/>
|
||||
</Flex>
|
||||
|
||||
{isExpanded && (
|
||||
<>
|
||||
<Box mt={2}>
|
||||
|
@ -0,0 +1,45 @@
|
||||
import { useAppDispatch } from 'app/store/storeHooks';
|
||||
import IAIMantineSelect from 'common/components/IAIMantineSelect';
|
||||
import {
|
||||
ControlModes,
|
||||
controlNetControlModeChanged,
|
||||
} from 'features/controlNet/store/controlNetSlice';
|
||||
import { useCallback } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
type ParamControlNetControlModeProps = {
|
||||
controlNetId: string;
|
||||
controlMode: string;
|
||||
};
|
||||
|
||||
const CONTROL_MODE_DATA = [
|
||||
{ label: 'Balanced', value: 'balanced' },
|
||||
{ label: 'Prompt', value: 'more_prompt' },
|
||||
{ label: 'Control', value: 'more_control' },
|
||||
{ label: 'Mega Control', value: 'unbalanced' },
|
||||
];
|
||||
|
||||
export default function ParamControlNetControlMode(
|
||||
props: ParamControlNetControlModeProps
|
||||
) {
|
||||
const { controlNetId, controlMode = false } = props;
|
||||
const dispatch = useAppDispatch();
|
||||
|
||||
const { t } = useTranslation();
|
||||
|
||||
const handleControlModeChange = useCallback(
|
||||
(controlMode: ControlModes) => {
|
||||
dispatch(controlNetControlModeChanged({ controlNetId, controlMode }));
|
||||
},
|
||||
[controlNetId, dispatch]
|
||||
);
|
||||
|
||||
return (
|
||||
<IAIMantineSelect
|
||||
label={t('parameters.controlNetControlMode')}
|
||||
data={CONTROL_MODE_DATA}
|
||||
value={String(controlMode)}
|
||||
onChange={handleControlModeChange}
|
||||
/>
|
||||
);
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
import {
|
||||
ControlNetProcessorType,
|
||||
RequiredCannyImageProcessorInvocation,
|
||||
RequiredControlNetProcessorNode,
|
||||
} from './types';
|
||||
|
||||
@ -23,7 +22,7 @@ type ControlNetProcessorsDict = Record<
|
||||
*
|
||||
* TODO: Generate from the OpenAPI schema
|
||||
*/
|
||||
export const CONTROLNET_PROCESSORS = {
|
||||
export const CONTROLNET_PROCESSORS: ControlNetProcessorsDict = {
|
||||
none: {
|
||||
type: 'none',
|
||||
label: 'none',
|
||||
@ -174,6 +173,8 @@ export const CONTROLNET_PROCESSORS = {
|
||||
},
|
||||
};
|
||||
|
||||
type ControlNetModelsDict = Record<string, ControlNetModel>;
|
||||
|
||||
type ControlNetModel = {
|
||||
type: string;
|
||||
label: string;
|
||||
@ -181,7 +182,7 @@ type ControlNetModel = {
|
||||
defaultProcessor?: ControlNetProcessorType;
|
||||
};
|
||||
|
||||
export const CONTROLNET_MODELS = {
|
||||
export const CONTROLNET_MODELS: ControlNetModelsDict = {
|
||||
'lllyasviel/control_v11p_sd15_canny': {
|
||||
type: 'lllyasviel/control_v11p_sd15_canny',
|
||||
label: 'Canny',
|
||||
@ -190,6 +191,7 @@ export const CONTROLNET_MODELS = {
|
||||
'lllyasviel/control_v11p_sd15_inpaint': {
|
||||
type: 'lllyasviel/control_v11p_sd15_inpaint',
|
||||
label: 'Inpaint',
|
||||
defaultProcessor: 'none',
|
||||
},
|
||||
'lllyasviel/control_v11p_sd15_mlsd': {
|
||||
type: 'lllyasviel/control_v11p_sd15_mlsd',
|
||||
@ -209,6 +211,7 @@ export const CONTROLNET_MODELS = {
|
||||
'lllyasviel/control_v11p_sd15_seg': {
|
||||
type: 'lllyasviel/control_v11p_sd15_seg',
|
||||
label: 'Segmentation',
|
||||
defaultProcessor: 'none',
|
||||
},
|
||||
'lllyasviel/control_v11p_sd15_lineart': {
|
||||
type: 'lllyasviel/control_v11p_sd15_lineart',
|
||||
@ -223,6 +226,7 @@ export const CONTROLNET_MODELS = {
|
||||
'lllyasviel/control_v11p_sd15_scribble': {
|
||||
type: 'lllyasviel/control_v11p_sd15_scribble',
|
||||
label: 'Scribble',
|
||||
defaultProcessor: 'none',
|
||||
},
|
||||
'lllyasviel/control_v11p_sd15_softedge': {
|
||||
type: 'lllyasviel/control_v11p_sd15_softedge',
|
||||
@ -242,10 +246,12 @@ export const CONTROLNET_MODELS = {
|
||||
'lllyasviel/control_v11f1e_sd15_tile': {
|
||||
type: 'lllyasviel/control_v11f1e_sd15_tile',
|
||||
label: 'Tile (experimental)',
|
||||
defaultProcessor: 'none',
|
||||
},
|
||||
'lllyasviel/control_v11e_sd15_ip2p': {
|
||||
type: 'lllyasviel/control_v11e_sd15_ip2p',
|
||||
label: 'Pix2Pix (experimental)',
|
||||
defaultProcessor: 'none',
|
||||
},
|
||||
'CrucibleAI/ControlNetMediaPipeFace': {
|
||||
type: 'CrucibleAI/ControlNetMediaPipeFace',
|
||||
|
@ -1,36 +1,27 @@
|
||||
import { PayloadAction } from '@reduxjs/toolkit';
|
||||
import { createSlice } from '@reduxjs/toolkit';
|
||||
import { PayloadAction, createSlice } from '@reduxjs/toolkit';
|
||||
import { RootState } from 'app/store/store';
|
||||
import { forEach } from 'lodash-es';
|
||||
import { ImageDTO } from 'services/api';
|
||||
import {
|
||||
ControlNetProcessorType,
|
||||
RequiredCannyImageProcessorInvocation,
|
||||
RequiredControlNetProcessorNode,
|
||||
} from './types';
|
||||
import { appSocketInvocationError } from 'services/events/actions';
|
||||
import { imageDeleted, imageUrlsReceived } from 'services/thunks/image';
|
||||
import { isAnySessionRejected } from 'services/thunks/session';
|
||||
import { controlNetImageProcessed } from './actions';
|
||||
import {
|
||||
CONTROLNET_MODELS,
|
||||
CONTROLNET_PROCESSORS,
|
||||
ControlNetModelName,
|
||||
} from './constants';
|
||||
import { controlNetImageProcessed } from './actions';
|
||||
import { imageDeleted, imageUrlsReceived } from 'services/thunks/image';
|
||||
import { forEach } from 'lodash-es';
|
||||
import { isAnySessionRejected } from 'services/thunks/session';
|
||||
import { appSocketInvocationError } from 'services/events/actions';
|
||||
import {
|
||||
ControlNetProcessorType,
|
||||
RequiredCannyImageProcessorInvocation,
|
||||
RequiredControlNetProcessorNode,
|
||||
} from './types';
|
||||
|
||||
export const initialControlNet: Omit<ControlNetConfig, 'controlNetId'> = {
|
||||
isEnabled: true,
|
||||
model: CONTROLNET_MODELS['lllyasviel/control_v11p_sd15_canny'].type,
|
||||
weight: 1,
|
||||
beginStepPct: 0,
|
||||
endStepPct: 1,
|
||||
controlImage: null,
|
||||
processedControlImage: null,
|
||||
processorType: 'canny_image_processor',
|
||||
processorNode: CONTROLNET_PROCESSORS.canny_image_processor
|
||||
.default as RequiredCannyImageProcessorInvocation,
|
||||
shouldAutoConfig: true,
|
||||
};
|
||||
export type ControlModes =
|
||||
| 'balanced'
|
||||
| 'more_prompt'
|
||||
| 'more_control'
|
||||
| 'unbalanced';
|
||||
|
||||
export type ControlNetConfig = {
|
||||
controlNetId: string;
|
||||
@ -39,6 +30,7 @@ export type ControlNetConfig = {
|
||||
weight: number;
|
||||
beginStepPct: number;
|
||||
endStepPct: number;
|
||||
controlMode: ControlModes;
|
||||
controlImage: ImageDTO | null;
|
||||
processedControlImage: ImageDTO | null;
|
||||
processorType: ControlNetProcessorType;
|
||||
@ -46,6 +38,21 @@ export type ControlNetConfig = {
|
||||
shouldAutoConfig: boolean;
|
||||
};
|
||||
|
||||
export const initialControlNet: Omit<ControlNetConfig, 'controlNetId'> = {
|
||||
isEnabled: true,
|
||||
model: CONTROLNET_MODELS['lllyasviel/control_v11p_sd15_canny'].type,
|
||||
weight: 1,
|
||||
beginStepPct: 0,
|
||||
endStepPct: 1,
|
||||
controlMode: 'balanced',
|
||||
controlImage: null,
|
||||
processedControlImage: null,
|
||||
processorType: 'canny_image_processor',
|
||||
processorNode: CONTROLNET_PROCESSORS.canny_image_processor
|
||||
.default as RequiredCannyImageProcessorInvocation,
|
||||
shouldAutoConfig: true,
|
||||
};
|
||||
|
||||
export type ControlNetState = {
|
||||
controlNets: Record<string, ControlNetConfig>;
|
||||
isEnabled: boolean;
|
||||
@ -147,11 +154,13 @@ export const controlNetSlice = createSlice({
|
||||
state.controlNets[controlNetId].processedControlImage = null;
|
||||
|
||||
if (state.controlNets[controlNetId].shouldAutoConfig) {
|
||||
const processorType = CONTROLNET_MODELS[model].defaultProcessor;
|
||||
const processorType =
|
||||
CONTROLNET_MODELS[model as keyof typeof CONTROLNET_MODELS]
|
||||
.defaultProcessor;
|
||||
if (processorType) {
|
||||
state.controlNets[controlNetId].processorType = processorType;
|
||||
state.controlNets[controlNetId].processorNode = CONTROLNET_PROCESSORS[
|
||||
processorType
|
||||
processorType as keyof typeof CONTROLNET_PROCESSORS
|
||||
].default as RequiredControlNetProcessorNode;
|
||||
} else {
|
||||
state.controlNets[controlNetId].processorType = 'none';
|
||||
@ -181,6 +190,13 @@ export const controlNetSlice = createSlice({
|
||||
const { controlNetId, endStepPct } = action.payload;
|
||||
state.controlNets[controlNetId].endStepPct = endStepPct;
|
||||
},
|
||||
controlNetControlModeChanged: (
|
||||
state,
|
||||
action: PayloadAction<{ controlNetId: string; controlMode: ControlModes }>
|
||||
) => {
|
||||
const { controlNetId, controlMode } = action.payload;
|
||||
state.controlNets[controlNetId].controlMode = controlMode;
|
||||
},
|
||||
controlNetProcessorParamsChanged: (
|
||||
state,
|
||||
action: PayloadAction<{
|
||||
@ -210,7 +226,7 @@ export const controlNetSlice = createSlice({
|
||||
state.controlNets[controlNetId].processedControlImage = null;
|
||||
state.controlNets[controlNetId].processorType = processorType;
|
||||
state.controlNets[controlNetId].processorNode = CONTROLNET_PROCESSORS[
|
||||
processorType
|
||||
processorType as keyof typeof CONTROLNET_PROCESSORS
|
||||
].default as RequiredControlNetProcessorNode;
|
||||
state.controlNets[controlNetId].shouldAutoConfig = false;
|
||||
},
|
||||
@ -227,12 +243,14 @@ export const controlNetSlice = createSlice({
|
||||
if (newShouldAutoConfig) {
|
||||
// manage the processor for the user
|
||||
const processorType =
|
||||
CONTROLNET_MODELS[state.controlNets[controlNetId].model]
|
||||
.defaultProcessor;
|
||||
CONTROLNET_MODELS[
|
||||
state.controlNets[controlNetId]
|
||||
.model as keyof typeof CONTROLNET_MODELS
|
||||
].defaultProcessor;
|
||||
if (processorType) {
|
||||
state.controlNets[controlNetId].processorType = processorType;
|
||||
state.controlNets[controlNetId].processorNode = CONTROLNET_PROCESSORS[
|
||||
processorType
|
||||
processorType as keyof typeof CONTROLNET_PROCESSORS
|
||||
].default as RequiredControlNetProcessorNode;
|
||||
} else {
|
||||
state.controlNets[controlNetId].processorType = 'none';
|
||||
@ -285,11 +303,11 @@ export const controlNetSlice = createSlice({
|
||||
});
|
||||
});
|
||||
|
||||
builder.addCase(appSocketInvocationError, (state, action) => {
|
||||
builder.addCase(appSocketInvocationError, (state) => {
|
||||
state.pendingControlImages = [];
|
||||
});
|
||||
|
||||
builder.addMatcher(isAnySessionRejected, (state, action) => {
|
||||
builder.addMatcher(isAnySessionRejected, (state) => {
|
||||
state.pendingControlImages = [];
|
||||
});
|
||||
},
|
||||
@ -307,6 +325,7 @@ export const {
|
||||
controlNetWeightChanged,
|
||||
controlNetBeginStepPctChanged,
|
||||
controlNetEndStepPctChanged,
|
||||
controlNetControlModeChanged,
|
||||
controlNetProcessorParamsChanged,
|
||||
controlNetProcessorTypeChanged,
|
||||
controlNetReset,
|
||||
|
@ -45,6 +45,7 @@ export const addControlNetToLinearGraph = (
|
||||
processedControlImage,
|
||||
beginStepPct,
|
||||
endStepPct,
|
||||
controlMode,
|
||||
model,
|
||||
processorType,
|
||||
weight,
|
||||
@ -60,6 +61,7 @@ export const addControlNetToLinearGraph = (
|
||||
type: 'controlnet',
|
||||
begin_step_percent: beginStepPct,
|
||||
end_step_percent: endStepPct,
|
||||
control_mode: controlMode,
|
||||
control_model: model as ControlNetInvocation['control_model'],
|
||||
control_weight: weight,
|
||||
};
|
||||
|
@ -24,4 +24,3 @@ export type AddInvocation = {
|
||||
*/
|
||||
'b'?: number;
|
||||
};
|
||||
|
||||
|
@ -5,4 +5,3 @@
|
||||
export type Body_upload_image = {
|
||||
file: Blob;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type CannyImageProcessorInvocation = {
|
||||
*/
|
||||
high_threshold?: number;
|
||||
};
|
||||
|
||||
|
@ -29,4 +29,3 @@ export type CkptModelInfo = {
|
||||
*/
|
||||
height?: number;
|
||||
};
|
||||
|
||||
|
@ -24,4 +24,3 @@ export type CollectInvocation = {
|
||||
*/
|
||||
collection?: Array<any>;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type CollectInvocationOutput = {
|
||||
*/
|
||||
collection: Array<any>;
|
||||
};
|
||||
|
||||
|
@ -20,4 +20,3 @@ export type ColorField = {
|
||||
*/
|
||||
'a': number;
|
||||
};
|
||||
|
||||
|
@ -24,4 +24,3 @@ export type CompelInvocation = {
|
||||
*/
|
||||
model?: string;
|
||||
};
|
||||
|
||||
|
@ -14,4 +14,3 @@ export type CompelOutput = {
|
||||
*/
|
||||
conditioning?: ConditioningField;
|
||||
};
|
||||
|
||||
|
@ -8,4 +8,3 @@ export type ConditioningField = {
|
||||
*/
|
||||
conditioning_name: string;
|
||||
};
|
||||
|
||||
|
@ -42,4 +42,3 @@ export type ContentShuffleImageProcessorInvocation = {
|
||||
*/
|
||||
'f'?: number;
|
||||
};
|
||||
|
||||
|
@ -25,5 +25,8 @@ export type ControlField = {
|
||||
* When the ControlNet is last applied (% of total steps)
|
||||
*/
|
||||
end_step_percent: number;
|
||||
/**
|
||||
* The contorl mode to use
|
||||
*/
|
||||
control_mode?: 'balanced' | 'more_prompt' | 'more_control' | 'unbalanced';
|
||||
};
|
||||
|
||||
|
@ -37,5 +37,8 @@ export type ControlNetInvocation = {
|
||||
* When the ControlNet is last applied (% of total steps)
|
||||
*/
|
||||
end_step_percent?: number;
|
||||
/**
|
||||
* The control mode used
|
||||
*/
|
||||
control_mode?: 'balanced' | 'more_prompt' | 'more_control' | 'unbalanced';
|
||||
};
|
||||
|
||||
|
@ -14,4 +14,3 @@ export type ControlOutput = {
|
||||
*/
|
||||
control?: ControlField;
|
||||
};
|
||||
|
||||
|
@ -15,4 +15,3 @@ export type CreateModelRequest = {
|
||||
*/
|
||||
info: (CkptModelInfo | DiffusersModelInfo);
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type CvInpaintInvocation = {
|
||||
*/
|
||||
mask?: ImageField;
|
||||
};
|
||||
|
||||
|
@ -23,4 +23,3 @@ export type DiffusersModelInfo = {
|
||||
*/
|
||||
path?: string;
|
||||
};
|
||||
|
||||
|
@ -24,4 +24,3 @@ export type DivideInvocation = {
|
||||
*/
|
||||
'b'?: number;
|
||||
};
|
||||
|
||||
|
@ -28,4 +28,3 @@ export type DynamicPromptInvocation = {
|
||||
*/
|
||||
combinatorial?: boolean;
|
||||
};
|
||||
|
||||
|
@ -14,4 +14,3 @@ export type Edge = {
|
||||
*/
|
||||
destination: EdgeConnection;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type EdgeConnection = {
|
||||
*/
|
||||
field: string;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type FloatCollectionOutput = {
|
||||
*/
|
||||
collection?: Array<number>;
|
||||
};
|
||||
|
||||
|
@ -28,4 +28,3 @@ export type FloatLinearRangeInvocation = {
|
||||
*/
|
||||
steps?: number;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type FloatOutput = {
|
||||
*/
|
||||
param?: number;
|
||||
};
|
||||
|
||||
|
@ -72,10 +72,9 @@ export type Graph = {
|
||||
/**
|
||||
* The nodes in this graph
|
||||
*/
|
||||
nodes?: Record<string, (LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | DynamicPromptInvocation | CompelInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | CvInpaintInvocation | RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | UpscaleInvocation | RestoreFaceInvocation | TextToImageInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | LatentsToLatentsInvocation | ImageToImageInvocation | InpaintInvocation)>;
|
||||
nodes?: Record<string, (RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | CompelInvocation | LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | CvInpaintInvocation | TextToImageInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | DynamicPromptInvocation | RestoreFaceInvocation | UpscaleInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | ImageToImageInvocation | LatentsToLatentsInvocation | InpaintInvocation)>;
|
||||
/**
|
||||
* The connections between nodes and their fields in this graph
|
||||
*/
|
||||
edges?: Array<Edge>;
|
||||
};
|
||||
|
||||
|
@ -46,7 +46,7 @@ export type GraphExecutionState = {
|
||||
/**
|
||||
* The results of node executions
|
||||
*/
|
||||
results: Record<string, (ImageOutput | MaskOutput | ControlOutput | PromptOutput | PromptCollectionOutput | CompelOutput | IntOutput | FloatOutput | LatentsOutput | NoiseOutput | IntCollectionOutput | FloatCollectionOutput | GraphInvocationOutput | IterateInvocationOutput | CollectInvocationOutput)>;
|
||||
results: Record<string, (IntCollectionOutput | FloatCollectionOutput | CompelOutput | ImageOutput | MaskOutput | ControlOutput | LatentsOutput | NoiseOutput | IntOutput | FloatOutput | PromptOutput | PromptCollectionOutput | GraphInvocationOutput | IterateInvocationOutput | CollectInvocationOutput)>;
|
||||
/**
|
||||
* Errors raised when executing nodes
|
||||
*/
|
||||
@ -60,4 +60,3 @@ export type GraphExecutionState = {
|
||||
*/
|
||||
source_prepared_mapping: Record<string, Array<string>>;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type GraphInvocation = {
|
||||
*/
|
||||
graph?: Graph;
|
||||
};
|
||||
|
||||
|
@ -8,4 +8,3 @@
|
||||
export type GraphInvocationOutput = {
|
||||
type: 'graph_output';
|
||||
};
|
||||
|
||||
|
@ -7,4 +7,3 @@ import type { ValidationError } from './ValidationError';
|
||||
export type HTTPValidationError = {
|
||||
detail?: Array<ValidationError>;
|
||||
};
|
||||
|
||||
|
@ -34,4 +34,3 @@ export type HedImageProcessorInvocation = {
|
||||
*/
|
||||
scribble?: boolean;
|
||||
};
|
||||
|
||||
|
@ -0,0 +1,36 @@
|
||||
/* istanbul ignore file */
|
||||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
|
||||
import type { ImageField } from './ImageField';
|
||||
|
||||
/**
|
||||
* Applies HED edge detection to image
|
||||
*/
|
||||
export type HedImageProcessorInvocation = {
|
||||
/**
|
||||
* The id of this node. Must be unique among all nodes.
|
||||
*/
|
||||
id: string;
|
||||
/**
|
||||
* Whether or not this node is an intermediate node.
|
||||
*/
|
||||
is_intermediate?: boolean;
|
||||
type?: 'hed_image_processor';
|
||||
/**
|
||||
* The image to process
|
||||
*/
|
||||
image?: ImageField;
|
||||
/**
|
||||
* The pixel resolution for detection
|
||||
*/
|
||||
detect_resolution?: number;
|
||||
/**
|
||||
* The pixel resolution for the output image
|
||||
*/
|
||||
image_resolution?: number;
|
||||
/**
|
||||
* Whether to use scribble mode
|
||||
*/
|
||||
scribble?: boolean;
|
||||
};
|
@ -30,4 +30,3 @@ export type ImageBlurInvocation = {
|
||||
*/
|
||||
blur_type?: 'gaussian' | 'box';
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type ImageChannelInvocation = {
|
||||
*/
|
||||
channel?: 'A' | 'R' | 'G' | 'B';
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type ImageConvertInvocation = {
|
||||
*/
|
||||
mode?: 'L' | 'RGB' | 'RGBA' | 'CMYK' | 'YCbCr' | 'LAB' | 'HSV' | 'I' | 'F';
|
||||
};
|
||||
|
||||
|
@ -38,4 +38,3 @@ export type ImageCropInvocation = {
|
||||
*/
|
||||
height?: number;
|
||||
};
|
||||
|
||||
|
@ -67,4 +67,3 @@ export type ImageDTO = {
|
||||
*/
|
||||
metadata?: ImageMetadata;
|
||||
};
|
||||
|
||||
|
@ -11,4 +11,3 @@ export type ImageField = {
|
||||
*/
|
||||
image_name: string;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type ImageInverseLerpInvocation = {
|
||||
*/
|
||||
max?: number;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type ImageLerpInvocation = {
|
||||
*/
|
||||
max?: number;
|
||||
};
|
||||
|
||||
|
@ -78,4 +78,3 @@ export type ImageMetadata = {
|
||||
*/
|
||||
extra?: string;
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type ImageMultiplyInvocation = {
|
||||
*/
|
||||
image2?: ImageField;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type ImageOutput = {
|
||||
*/
|
||||
height: number;
|
||||
};
|
||||
|
||||
|
@ -38,4 +38,3 @@ export type ImagePasteInvocation = {
|
||||
*/
|
||||
'y'?: number;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type ImageProcessorInvocation = {
|
||||
*/
|
||||
image?: ImageField;
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type ImageRecordChanges = {
|
||||
*/
|
||||
is_intermediate?: boolean;
|
||||
};
|
||||
|
||||
|
@ -34,4 +34,3 @@ export type ImageResizeInvocation = {
|
||||
*/
|
||||
resample_mode?: 'nearest' | 'box' | 'bilinear' | 'hamming' | 'bicubic' | 'lanczos';
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type ImageScaleInvocation = {
|
||||
*/
|
||||
resample_mode?: 'nearest' | 'box' | 'bilinear' | 'hamming' | 'bicubic' | 'lanczos';
|
||||
};
|
||||
|
||||
|
@ -74,4 +74,3 @@ export type ImageToImageInvocation = {
|
||||
*/
|
||||
fit?: boolean;
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type ImageToLatentsInvocation = {
|
||||
*/
|
||||
model?: string;
|
||||
};
|
||||
|
||||
|
@ -19,4 +19,3 @@ export type ImageUrlsDTO = {
|
||||
*/
|
||||
thumbnail_url: string;
|
||||
};
|
||||
|
||||
|
@ -27,4 +27,3 @@ export type InfillColorInvocation = {
|
||||
*/
|
||||
color?: ColorField;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type InfillPatchMatchInvocation = {
|
||||
*/
|
||||
image?: ImageField;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type InfillTileInvocation = {
|
||||
*/
|
||||
seed?: number;
|
||||
};
|
||||
|
||||
|
@ -119,4 +119,3 @@ export type InpaintInvocation = {
|
||||
*/
|
||||
inpaint_replace?: number;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type IntCollectionOutput = {
|
||||
*/
|
||||
collection?: Array<number>;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type IntOutput = {
|
||||
*/
|
||||
'a'?: number;
|
||||
};
|
||||
|
||||
|
@ -24,4 +24,3 @@ export type IterateInvocation = {
|
||||
*/
|
||||
index?: number;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type IterateInvocationOutput = {
|
||||
*/
|
||||
item: any;
|
||||
};
|
||||
|
||||
|
@ -11,4 +11,3 @@ export type LatentsField = {
|
||||
*/
|
||||
latents_name: string;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type LatentsOutput = {
|
||||
*/
|
||||
height: number;
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type LatentsToImageInvocation = {
|
||||
*/
|
||||
model?: string;
|
||||
};
|
||||
|
||||
|
@ -60,4 +60,3 @@ export type LatentsToLatentsInvocation = {
|
||||
*/
|
||||
strength?: number;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type LineartAnimeImageProcessorInvocation = {
|
||||
*/
|
||||
image_resolution?: number;
|
||||
};
|
||||
|
||||
|
@ -34,4 +34,3 @@ export type LineartImageProcessorInvocation = {
|
||||
*/
|
||||
coarse?: boolean;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type LoadImageInvocation = {
|
||||
*/
|
||||
image?: ImageField;
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type MaskFromAlphaInvocation = {
|
||||
*/
|
||||
invert?: boolean;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type MaskOutput = {
|
||||
*/
|
||||
height?: number;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type MediapipeFaceProcessorInvocation = {
|
||||
*/
|
||||
min_confidence?: number;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type MidasDepthImageProcessorInvocation = {
|
||||
*/
|
||||
bg_th?: number;
|
||||
};
|
||||
|
||||
|
@ -38,4 +38,3 @@ export type MlsdImageProcessorInvocation = {
|
||||
*/
|
||||
thr_d?: number;
|
||||
};
|
||||
|
||||
|
@ -8,4 +8,3 @@ import type { DiffusersModelInfo } from './DiffusersModelInfo';
|
||||
export type ModelsList = {
|
||||
models: Record<string, (CkptModelInfo | DiffusersModelInfo)>;
|
||||
};
|
||||
|
||||
|
@ -24,4 +24,3 @@ export type MultiplyInvocation = {
|
||||
*/
|
||||
'b'?: number;
|
||||
};
|
||||
|
||||
|
@ -28,4 +28,3 @@ export type NoiseInvocation = {
|
||||
*/
|
||||
height?: number;
|
||||
};
|
||||
|
||||
|
@ -22,4 +22,3 @@ export type NoiseOutput = {
|
||||
*/
|
||||
height: number;
|
||||
};
|
||||
|
||||
|
@ -30,4 +30,3 @@ export type NormalbaeImageProcessorInvocation = {
|
||||
*/
|
||||
image_resolution?: number;
|
||||
};
|
||||
|
||||
|
@ -25,4 +25,3 @@ export type OffsetPaginatedResults_ImageDTO_ = {
|
||||
*/
|
||||
total: number;
|
||||
};
|
||||
|
||||
|
@ -34,4 +34,3 @@ export type OpenposeImageProcessorInvocation = {
|
||||
*/
|
||||
image_resolution?: number;
|
||||
};
|
||||
|
||||
|
@ -29,4 +29,3 @@ export type PaginatedResults_GraphExecutionState_ = {
|
||||
*/
|
||||
total: number;
|
||||
};
|
||||
|
||||
|
@ -20,4 +20,3 @@ export type ParamFloatInvocation = {
|
||||
*/
|
||||
param?: number;
|
||||
};
|
||||
|
||||
|
@ -20,4 +20,3 @@ export type ParamIntInvocation = {
|
||||
*/
|
||||
'a'?: number;
|
||||
};
|
||||
|
||||
|
@ -38,4 +38,3 @@ export type PidiImageProcessorInvocation = {
|
||||
*/
|
||||
scribble?: boolean;
|
||||
};
|
||||
|
||||
|
@ -16,4 +16,3 @@ export type PromptCollectionOutput = {
|
||||
*/
|
||||
count: number;
|
||||
};
|
||||
|
||||
|
@ -12,4 +12,3 @@ export type PromptOutput = {
|
||||
*/
|
||||
prompt: string;
|
||||
};
|
||||
|
||||
|
@ -24,4 +24,3 @@ export type RandomIntInvocation = {
|
||||
*/
|
||||
high?: number;
|
||||
};
|
||||
|
||||
|
@ -32,4 +32,3 @@ export type RandomRangeInvocation = {
|
||||
*/
|
||||
seed?: number;
|
||||
};
|
||||
|
||||
|
@ -28,4 +28,3 @@ export type RangeInvocation = {
|
||||
*/
|
||||
step?: number;
|
||||
};
|
||||
|
||||
|
@ -28,4 +28,3 @@ export type RangeOfSizeInvocation = {
|
||||
*/
|
||||
step?: number;
|
||||
};
|
||||
|
||||
|
@ -38,4 +38,3 @@ export type ResizeLatentsInvocation = {
|
||||
*/
|
||||
antialias?: boolean;
|
||||
};
|
||||
|
||||
|
@ -26,4 +26,3 @@ export type RestoreFaceInvocation = {
|
||||
*/
|
||||
strength?: number;
|
||||
};
|
||||
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user