Controlnet implementation for sequential execution

invokeai/app/invocations/controlnet_image_processors.py

@@ -1,7 +1,7 @@
 # InvokeAI nodes for ControlNet image preprocessors
 # initial implementation by Gregg Helt, 2023
 # heavily leverages controlnet_aux package: https://github.com/patrickvonplaten/controlnet_aux
-from builtins import float
+from builtins import float, bool
 
 import numpy as np
 from typing import Literal, Optional, Union, List
@@ -94,6 +94,7 @@ CONTROLNET_DEFAULT_MODELS = [
 ]
 
 CONTROLNET_NAME_VALUES = Literal[tuple(CONTROLNET_DEFAULT_MODELS)]
+CONTROLNET_MODE_VALUES = Literal[tuple(["balanced", "more_prompt", "more_control", "unbalanced"])]
 
 class ControlField(BaseModel):
     image: ImageField = Field(default=None, description="The control image")
@@ -104,6 +105,8 @@ class ControlField(BaseModel):
                                       description="When the ControlNet is first applied (% of total steps)")
     end_step_percent: float = Field(default=1, ge=0, le=1,
                                     description="When the ControlNet is last applied (% of total steps)")
+    control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The contorl mode to use")
+
     @validator("control_weight")
     def abs_le_one(cls, v):
         """validate that all abs(values) are <=1"""
@@ -144,11 +147,11 @@ class ControlNetInvocation(BaseInvocation):
     control_model: CONTROLNET_NAME_VALUES = Field(default="lllyasviel/sd-controlnet-canny",
                                                   description="control model used")
     control_weight: Union[float, List[float]] = Field(default=1.0, description="The weight given to the ControlNet")
-    # TODO: add support in backend core for begin_step_percent, end_step_percent, guess_mode
     begin_step_percent: float = Field(default=0, ge=0, le=1,
                                       description="When the ControlNet is first applied (% of total steps)")
     end_step_percent: float = Field(default=1, ge=0, le=1,
                                     description="When the ControlNet is last applied (% of total steps)")
+    control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The control mode used")
     # fmt: on
 
     class Config(InvocationConfig):
@@ -166,7 +169,6 @@ class ControlNetInvocation(BaseInvocation):
         }
 
     def invoke(self, context: InvocationContext) -> ControlOutput:
-
         return ControlOutput(
             control=ControlField(
                 image=self.image,
@@ -174,6 +176,7 @@ class ControlNetInvocation(BaseInvocation):
                 control_weight=self.control_weight,
                 begin_step_percent=self.begin_step_percent,
                 end_step_percent=self.end_step_percent,
+                control_mode=self.control_mode,
             ),
         )
 

invokeai/app/invocations/latent.py

@@ -282,19 +282,14 @@ class TextToLatentsInvocation(BaseInvocation):
         control_height_resize = latents_shape[2] * 8
         control_width_resize = latents_shape[3] * 8
         if control_input is None:
-            # print("control input is None")
             control_list = None
         elif isinstance(control_input, list) and len(control_input) == 0:
-            # print("control input is empty list")
             control_list = None
         elif isinstance(control_input, ControlField):
-            # print("control input is ControlField")
             control_list = [control_input]
         elif isinstance(control_input, list) and len(control_input) > 0 and isinstance(control_input[0], ControlField):
-            # print("control input is list[ControlField]")
             control_list = control_input
         else:
-            # print("input control is unrecognized:", type(self.control))
             control_list = None
         if (control_list is None):
             control_data = None
@@ -336,12 +331,15 @@ class TextToLatentsInvocation(BaseInvocation):
                     # num_images_per_prompt=num_images_per_prompt,
                     device=control_model.device,
                     dtype=control_model.dtype,
+                    control_mode=control_info.control_mode,
                 )
                 control_item = ControlNetData(model=control_model,
                                               image_tensor=control_image,
                                               weight=control_info.control_weight,
                                               begin_step_percent=control_info.begin_step_percent,
-                                              end_step_percent=control_info.end_step_percent)
+                                              end_step_percent=control_info.end_step_percent,
+                                              control_mode=control_info.control_mode,
+                                              )
                 control_data.append(control_item)
                 # MultiControlNetModel has been refactored out, just need list[ControlNetData]
         return control_data

invokeai/backend/stable_diffusion/diffusers_pipeline.py

@@ -46,6 +46,7 @@ from .diffusion import (
     AttentionMapSaver,
     InvokeAIDiffuserComponent,
     PostprocessingSettings,
+    ControlNetData,
 )
 from .offloading import FullyLoadedModelGroup, LazilyLoadedModelGroup, ModelGroup
 from .textual_inversion_manager import TextualInversionManager
@@ -214,13 +215,6 @@ class GeneratorToCallbackinator(Generic[ParamType, ReturnType, CallbackType]):
             raise AssertionError("why was that an empty generator?")
         return result
 
-@dataclass
-class ControlNetData:
-    model: ControlNetModel = Field(default=None)
-    image_tensor: torch.Tensor= Field(default=None)
-    weight: Union[float, List[float]]= Field(default=1.0)
-    begin_step_percent: float = Field(default=0.0)
-    end_step_percent: float = Field(default=1.0)
 
 @dataclass(frozen=True)
 class ConditioningData:
@@ -656,69 +650,18 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
 
         # TODO: should this scaling happen here or inside self._unet_forward?
         #     i.e. before or after passing it to InvokeAIDiffuserComponent
-        latent_model_input = self.scheduler.scale_model_input(latents, timestep)
-
-        # default is no controlnet, so set controlnet processing output to None
-        down_block_res_samples, mid_block_res_sample = None, None
-
-        if control_data is not None:
-            # FIXME: make sure guidance_scale < 1.0 is handled correctly if doing per-step guidance setting
-            # if conditioning_data.guidance_scale > 1.0:
-            if conditioning_data.guidance_scale is not None:
-                # expand the latents input to control model if doing classifier free guidance
-                #    (which I think for now is always true, there is conditional elsewhere that stops execution if
-                #     classifier_free_guidance is <= 1.0 ?)
-                latent_control_input = torch.cat([latent_model_input] * 2)
-            else:
-                latent_control_input = latent_model_input
-            # control_data should be type List[ControlNetData]
-            # this loop covers both ControlNet (one ControlNetData in list)
-            #      and MultiControlNet (multiple ControlNetData in list)
-            for i, control_datum in enumerate(control_data):
-                # print("controlnet", i, "==>", type(control_datum))
-                first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
-                last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
-                # only apply controlnet if current step is within the controlnet's begin/end step range
-                if step_index >= first_control_step and step_index <= last_control_step:
-                    # print("running controlnet", i, "for step", step_index)
-                    if isinstance(control_datum.weight, list):
-                        # if controlnet has multiple weights, use the weight for the current step
-                        controlnet_weight = control_datum.weight[step_index]
-                    else:
-                        # if controlnet has a single weight, use it for all steps
-                        controlnet_weight = control_datum.weight
-                    down_samples, mid_sample = control_datum.model(
-                        sample=latent_control_input,
-                        timestep=timestep,
-                        encoder_hidden_states=torch.cat([conditioning_data.unconditioned_embeddings,
-                                                         conditioning_data.text_embeddings]),
-                        controlnet_cond=control_datum.image_tensor,
-                        conditioning_scale=controlnet_weight,
-                        # cross_attention_kwargs,
-                        guess_mode=False,
-                        return_dict=False,
-                    )
-                    if down_block_res_samples is None and mid_block_res_sample is None:
-                        down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
-                    else:
-                        # add controlnet outputs together if have multiple controlnets
-                        down_block_res_samples = [
-                            samples_prev + samples_curr
-                            for samples_prev, samples_curr in zip(down_block_res_samples, down_samples)
-                        ]
-                        mid_block_res_sample += mid_sample
+        unet_latent_input = self.scheduler.scale_model_input(latents, timestep)
 
         # predict the noise residual
         noise_pred = self.invokeai_diffuser.do_diffusion_step(
-            latent_model_input,
-            t,
-            conditioning_data.unconditioned_embeddings,
-            conditioning_data.text_embeddings,
-            conditioning_data.guidance_scale,
+            x=unet_latent_input,
+            sigma=t,
+            unconditioning=conditioning_data.unconditioned_embeddings,
+            conditioning=conditioning_data.text_embeddings,
+            unconditional_guidance_scale=conditioning_data.guidance_scale,
+            control_data=control_data,
             step_index=step_index,
             total_step_count=total_step_count,
-            down_block_additional_residuals=down_block_res_samples,  # from controlnet(s)
-            mid_block_additional_residual=mid_block_res_sample,      # from controlnet(s)
         )
 
         # compute the previous noisy sample x_t -> x_t-1
@@ -1038,6 +981,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
         device="cuda",
         dtype=torch.float16,
         do_classifier_free_guidance=True,
+        control_mode="balanced"
     ):
 
         if not isinstance(image, torch.Tensor):
@@ -1068,6 +1012,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
             repeat_by = num_images_per_prompt
         image = image.repeat_interleave(repeat_by, dim=0)
         image = image.to(device=device, dtype=dtype)
-        if do_classifier_free_guidance:
-            image = torch.cat([image] * 2)
+        #cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
+        #if do_classifier_free_guidance and not cfg_injection:
+        #    image = torch.cat([image] * 2)
         return image

invokeai/backend/stable_diffusion/diffusion/__init__.py

@@ -3,4 +3,4 @@ Initialization file for invokeai.models.diffusion
 """
 from .cross_attention_control import InvokeAICrossAttentionMixin
 from .cross_attention_map_saving import AttentionMapSaver
-from .shared_invokeai_diffusion import InvokeAIDiffuserComponent, PostprocessingSettings
+from .shared_invokeai_diffusion import InvokeAIDiffuserComponent, PostprocessingSettings, ControlNetData

invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py

@@ -1,11 +1,14 @@
 from contextlib import contextmanager
 from dataclasses import dataclass
+from pydantic import Field
 from math import ceil
 from typing import Any, Callable, Dict, Optional, Union, List
 
 import numpy as np
 import torch
+import math
 from diffusers import UNet2DConditionModel
+from diffusers.models.controlnet import ControlNetModel
 from diffusers.models.attention_processor import AttentionProcessor
 from typing_extensions import TypeAlias
 
@@ -40,6 +43,17 @@ class PostprocessingSettings:
     v_symmetry_time_pct: Optional[float]
 
 
+# TODO: pydantic Field work with dataclasses?
+@dataclass
+class ControlNetData:
+    model: ControlNetModel = Field(default=None)
+    image_tensor: torch.Tensor = Field(default=None)
+    weight: Union[float, List[float]] = Field(default=1.0)
+    begin_step_percent: float = Field(default=0.0)
+    end_step_percent: float = Field(default=1.0)
+    control_mode: str = Field(default="balanced")
+
+
 class InvokeAIDiffuserComponent:
     """
     The aim of this component is to provide a single place for code that can be applied identically to
@@ -182,8 +196,9 @@ class InvokeAIDiffuserComponent:
         conditioning: Union[torch.Tensor, dict],
         # unconditional_guidance_scale: float,
         unconditional_guidance_scale: Union[float, List[float]],
-        step_index: Optional[int] = None,
-        total_step_count: Optional[int] = None,
+        step_index: int,
+        total_step_count: int,
+        control_data: Optional[List[ControlNetData]],
         **kwargs,
     ):
         """
@@ -213,31 +228,56 @@ class InvokeAIDiffuserComponent:
                 )
             )
 
+        if self.sequential_guidance:
+            down_block_res_samples, mid_block_res_sample = self._run_controlnet_sequentially(
+                unconditioning=unconditioning,
+                conditioning=conditioning,
+                control_data=control_data,
+                sample=x,
+                timestep=sigma,
+                step_index=step_index,
+                total_step_count=total_step_count,
+            )
+        else:
+            down_block_res_samples, mid_block_res_sample = self._run_controlnet_normally(
+                unconditioning=unconditioning,
+                conditioning=conditioning,
+                control_data=control_data,
+                sample=x,
+                timestep=sigma,
+                step_index=step_index,
+                total_step_count=total_step_count,
+            )
+
         wants_cross_attention_control = len(cross_attention_control_types_to_do) > 0
         wants_hybrid_conditioning = isinstance(conditioning, dict)
 
         if wants_hybrid_conditioning:
             unconditioned_next_x, conditioned_next_x = self._apply_hybrid_conditioning(
-                x, sigma, unconditioning, conditioning, **kwargs,
+                x, sigma, unconditioning, conditioning,
+                down_block_additional_residuals=down_block_res_samples,  # from controlnet(s)
+                mid_block_additional_residual=mid_block_res_sample,      # from controlnet(s)
+                **kwargs,
             )
         elif wants_cross_attention_control:
             (
                 unconditioned_next_x,
                 conditioned_next_x,
             ) = self._apply_cross_attention_controlled_conditioning(
-                x,
-                sigma,
-                unconditioning,
-                conditioning,
-                cross_attention_control_types_to_do,
+                x, sigma, unconditioning, conditioning, cross_attention_control_types_to_do,
+                down_block_additional_residuals=down_block_res_samples,  # from controlnet(s)
+                mid_block_additional_residual=mid_block_res_sample,      # from controlnet(s)
                 **kwargs,
             )
-        elif self.sequential_guidance:
+        elif True: #self.sequential_guidance:
             (
                 unconditioned_next_x,
                 conditioned_next_x,
             ) = self._apply_standard_conditioning_sequentially(
-                x, sigma, unconditioning, conditioning, **kwargs,
+                x, sigma, unconditioning, conditioning,
+                down_block_additional_residuals=down_block_res_samples,  # from controlnet(s)
+                mid_block_additional_residual=mid_block_res_sample,      # from controlnet(s)
+                 **kwargs,
             )
 
         else:
@@ -245,7 +285,10 @@ class InvokeAIDiffuserComponent:
                 unconditioned_next_x,
                 conditioned_next_x,
             ) = self._apply_standard_conditioning(
-                x, sigma, unconditioning, conditioning, **kwargs,
+                x, sigma, unconditioning, conditioning,
+                down_block_additional_residuals=down_block_res_samples,  # from controlnet(s)
+                mid_block_additional_residual=mid_block_res_sample,      # from controlnet(s)
+                **kwargs,
             )
 
         combined_next_x = self._combine(
@@ -293,16 +336,160 @@ class InvokeAIDiffuserComponent:
 
     # methods below are called from do_diffusion_step and should be considered private to this class.
 
-    def _apply_standard_conditioning(self, x, sigma, unconditioning, conditioning, **kwargs):
+    def _run_controlnet_normally(
+        self,
+        unconditioning: torch.Tensor,
+        conditioning: torch.Tensor,
+        control_data: List[ControlNetData],
+        sample: torch.Tensor,
+        timestep: torch.Tensor,
+        step_index: int,
+        total_step_count: int,
+    ):
+        if control_data is None:
+            return (None, None)
+
+        down_block_res_samples, mid_block_res_sample = None, None
+
+        for i, control_datum in enumerate(control_data):
+            control_mode = control_datum.control_mode
+            soft_injection = (control_mode == "more_prompt" or control_mode == "more_control")
+            cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
+
+            first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
+            last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
+            # only apply controlnet if current step is within the controlnet's begin/end step range
+            if step_index >= first_control_step and step_index <= last_control_step:
+
+                if cfg_injection:
+                    control_sample = sample
+                    control_timestep = timestep
+                    control_image_tensor = control_datum.image_tensor
+                    encoder_hidden_states = conditioning  # TODO: ask bug
+                else:
+                    control_sample = torch.cat([sample] * 2)
+                    control_timestep = torch.cat([timestep] * 2)
+                    control_image_tensor = torch.cat([control_datum.image_tensor] * 2)
+                    encoder_hidden_states = torch.cat([unconditioning, conditioning])
+
+                if isinstance(control_datum.weight, list):
+                    weight = control_datum.weight[step_index]
+                else:
+                    weight = control_datum.weight
+
+                # controlnet(s) inference
+                down_samples, mid_sample = control_datum.model(
+                    sample=control_sample,
+                    timestep=control_timestep,
+                    encoder_hidden_states=encoder_hidden_states,
+                    controlnet_cond=control_image_tensor,
+                    conditioning_scale=weight, # controlnet specific, NOT the guidance scale
+                    guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
+                    return_dict=False,
+                )
+
+                if cfg_injection:
+                    down_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_samples]
+                    mid_sample = torch.cat([torch.zeros_like(mid_sample), mid_sample])
+
+                if down_block_res_samples is None and mid_block_res_sample is None:
+                    down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
+                else:
+                    down_block_res_samples = [
+                        samples_prev + samples_curr
+                        for samples_prev, samples_curr in zip(down_block_res_samples, down_samples)
+                    ]
+                    mid_block_res_sample += mid_sample
+
+        return down_block_res_samples, mid_block_res_sample
+
+    def _run_controlnet_sequentially(
+        self,
+        unconditioning: torch.Tensor,
+        conditioning: torch.Tensor,
+        control_data: List[ControlNetData],
+        sample: torch.Tensor,
+        timestep: torch.Tensor,
+        step_index: int,
+        total_step_count: int,
+    ):
+        if control_data is None:
+            return (None, None)
+
+        down_block_res_samples, mid_block_res_sample = None, None
+
+        for i, control_datum in enumerate(control_data):
+            control_mode = control_datum.control_mode
+            soft_injection = (control_mode == "more_prompt" or control_mode == "more_control")
+            cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
+
+            first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
+            last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
+            # only apply controlnet if current step is within the controlnet's begin/end step range
+            if step_index >= first_control_step and step_index <= last_control_step:
+
+                if isinstance(control_datum.weight, list):
+                    weight = control_datum.weight[step_index]
+                else:
+                    weight = control_datum.weight
+
+                # controlnet(s) inference
+                cond_down_samples, cond_mid_sample = control_datum.model(
+                    sample=sample,
+                    timestep=timestep,
+                    encoder_hidden_states=conditioning, # TODO: ask bug
+                    controlnet_cond=control_datum.image_tensor,
+                    conditioning_scale=weight, # controlnet specific, NOT the guidance scale
+                    guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
+                    return_dict=False,
+                )
+
+                if cfg_injection:
+                    uncond_down_samples = [torch.zeros_like(d) for d in cond_down_samples]
+                    uncond_mid_sample = torch.zeros_like(cond_mid_sample)
+
+                else:
+                    uncond_down_samples, uncond_mid_sample = control_datum.model(
+                        sample=sample,
+                        timestep=timestep,
+                        encoder_hidden_states=unconditioning,
+                        controlnet_cond=control_datum.image_tensor,
+                        conditioning_scale=weight, # controlnet specific, NOT the guidance scale
+                        guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
+                        return_dict=False,
+                    )
+
+                down_samples = [torch.cat([ud, cd]) for ud, cd in zip(uncond_down_samples, cond_down_samples)]
+                mid_sample = torch.cat([uncond_mid_sample, cond_mid_sample])
+
+                if down_block_res_samples is None and mid_block_res_sample is None:
+                    down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
+                else:
+                    down_block_res_samples = [
+                        samples_prev + samples_curr
+                        for samples_prev, samples_curr in zip(down_block_res_samples, down_samples)
+                    ]
+                    mid_block_res_sample += mid_sample
+
+        return down_block_res_samples, mid_block_res_sample
+
+    def _apply_standard_conditioning(
+        self,
+        x: torch.Tensor,
+        sigma: torch.Tensor,
+        unconditioning: torch.Tensor,
+        conditioning: torch.Tensor,
+        **kwargs
+    ):
         # fast batched path
         x_twice = torch.cat([x] * 2)
         sigma_twice = torch.cat([sigma] * 2)
         both_conditionings = torch.cat([unconditioning, conditioning])
-        both_results = self.model_forward_callback(
-            x_twice, sigma_twice, both_conditionings, **kwargs,
-        )
+
+        both_results = self.model_forward_callback(x_twice, sigma_twice, both_conditionings, **kwargs)
         unconditioned_next_x, conditioned_next_x = both_results.chunk(2)
         if conditioned_next_x.device.type == "mps":
+            # TODO: check if this still present
             # prevent a result filled with zeros. seems to be a torch bug.
             conditioned_next_x = conditioned_next_x.clone()
         return unconditioned_next_x, conditioned_next_x
@@ -310,15 +497,43 @@ class InvokeAIDiffuserComponent:
     def _apply_standard_conditioning_sequentially(
         self,
         x: torch.Tensor,
-        sigma,
+        sigma: torch.Tensor,
         unconditioning: torch.Tensor,
         conditioning: torch.Tensor,
+        down_block_additional_residuals,  # from controlnet(s)
+        mid_block_additional_residual,      # from controlnet(s)
         **kwargs,
     ):
+        # split controlnet data to cond and uncond
+        if down_block_additional_residuals is None:
+            uncond_down_block_res_samples = None
+            cond_down_block_res_samples = None
+            uncond_mid_block_res_sample = None
+            cond_mid_block_res_sample = None
+
+        else:
+            uncond_down_block_res_samples = []
+            cond_down_block_res_samples = []
+            for d in down_block_additional_residuals:
+                ud, cd = d.chunk(2)
+                uncond_down_block_res_samples.append(ud)
+                cond_down_block_res_samples.append(cd)
+
+            uncond_mid_block_res_sample, cond_mid_block_res_sample = mid_block_additional_residual.chunk(2)
+
         # low-memory sequential path
-        unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning, **kwargs)
-        conditioned_next_x = self.model_forward_callback(x, sigma, conditioning, **kwargs)
+        unconditioned_next_x = self.model_forward_callback(
+            x, sigma, unconditioning, **kwargs,
+            down_block_additional_residuals=uncond_down_block_res_samples,
+            mid_block_additional_residual=uncond_mid_block_res_sample,
+        )
+        conditioned_next_x = self.model_forward_callback(
+            x, sigma, conditioning, **kwargs,
+            down_block_additional_residuals=cond_down_block_res_samples,
+            mid_block_additional_residual=cond_mid_block_res_sample,
+        )
         if conditioned_next_x.device.type == "mps":
+            # TODO: check if still present
             # prevent a result filled with zeros. seems to be a torch bug.
             conditioned_next_x = conditioned_next_x.clone()
         return unconditioned_next_x, conditioned_next_x

invokeai/frontend/web/public/locales/en.json

@@ -524,7 +524,8 @@
         "initialImage": "Initial Image",
         "showOptionsPanel": "Show Options Panel",
         "hidePreview": "Hide Preview",
-        "showPreview": "Show Preview"
+        "showPreview": "Show Preview",
+        "controlNetControlMode": "Control Mode"
     },
     "settings": {
         "models": "Models",

invokeai/frontend/web/src/features/controlNet/components/ControlNet.tsx

@@ -1,26 +1,27 @@
+import { Box, ChakraProps, Flex } from '@chakra-ui/react';
+import { useAppDispatch } from 'app/store/storeHooks';
 import { memo, useCallback } from 'react';
+import { FaCopy, FaTrash } from 'react-icons/fa';
 import {
   ControlNetConfig,
   controlNetAdded,
   controlNetRemoved,
   controlNetToggled,
 } from '../store/controlNetSlice';
-import { useAppDispatch } from 'app/store/storeHooks';
 import ParamControlNetModel from './parameters/ParamControlNetModel';
 import ParamControlNetWeight from './parameters/ParamControlNetWeight';
-import { Flex, Box, ChakraProps } from '@chakra-ui/react';
-import { FaCopy, FaTrash } from 'react-icons/fa';
 
-import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
-import ControlNetImagePreview from './ControlNetImagePreview';
-import IAIIconButton from 'common/components/IAIIconButton';
-import { v4 as uuidv4 } from 'uuid';
-import { useToggle } from 'react-use';
-import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
-import ControlNetProcessorComponent from './ControlNetProcessorComponent';
-import IAISwitch from 'common/components/IAISwitch';
 import { ChevronUpIcon } from '@chakra-ui/icons';
+import IAIIconButton from 'common/components/IAIIconButton';
+import IAISwitch from 'common/components/IAISwitch';
+import { useToggle } from 'react-use';
+import { v4 as uuidv4 } from 'uuid';
+import ControlNetImagePreview from './ControlNetImagePreview';
+import ControlNetProcessorComponent from './ControlNetProcessorComponent';
 import ParamControlNetShouldAutoConfig from './ParamControlNetShouldAutoConfig';
+import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
+import ParamControlNetControlMode from './parameters/ParamControlNetControlMode';
+import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
 
 const expandedControlImageSx: ChakraProps['sx'] = { maxH: 96 };
 
@@ -36,6 +37,7 @@ const ControlNet = (props: ControlNetProps) => {
     weight,
     beginStepPct,
     endStepPct,
+    controlMode,
     controlImage,
     processedControlImage,
     processorNode,
@@ -137,45 +139,51 @@ const ControlNet = (props: ControlNetProps) => {
       </Flex>
       {isEnabled && (
         <>
-          <Flex sx={{ gap: 4, w: 'full' }}>
-            <Flex
-              sx={{
-                flexDir: 'column',
-                gap: 2,
-                w: 'full',
-                h: isExpanded ? 28 : 24,
-                paddingInlineStart: 1,
-                paddingInlineEnd: isExpanded ? 1 : 0,
-                pb: 2,
-                justifyContent: 'space-between',
-              }}
-            >
-              <ParamControlNetWeight
-                controlNetId={controlNetId}
-                weight={weight}
-                mini={!isExpanded}
-              />
-              <ParamControlNetBeginEnd
-                controlNetId={controlNetId}
-                beginStepPct={beginStepPct}
-                endStepPct={endStepPct}
-                mini={!isExpanded}
-              />
-            </Flex>
-            {!isExpanded && (
+          <Flex sx={{ w: 'full', flexDirection: 'column' }}>
+            <Flex sx={{ gap: 4, w: 'full' }}>
               <Flex
                 sx={{
-                  alignItems: 'center',
-                  justifyContent: 'center',
-                  h: 24,
-                  w: 24,
-                  aspectRatio: '1/1',
+                  flexDir: 'column',
+                  gap: 3,
+                  w: 'full',
+                  paddingInlineStart: 1,
+                  paddingInlineEnd: isExpanded ? 1 : 0,
+                  pb: 2,
+                  justifyContent: 'space-between',
                 }}
               >
-                <ControlNetImagePreview controlNet={props.controlNet} />
+                <ParamControlNetWeight
+                  controlNetId={controlNetId}
+                  weight={weight}
+                  mini={!isExpanded}
+                />
+                <ParamControlNetBeginEnd
+                  controlNetId={controlNetId}
+                  beginStepPct={beginStepPct}
+                  endStepPct={endStepPct}
+                  mini={!isExpanded}
+                />
               </Flex>
-            )}
+              {!isExpanded && (
+                <Flex
+                  sx={{
+                    alignItems: 'center',
+                    justifyContent: 'center',
+                    h: 24,
+                    w: 24,
+                    aspectRatio: '1/1',
+                  }}
+                >
+                  <ControlNetImagePreview controlNet={props.controlNet} />
+                </Flex>
+              )}
+            </Flex>
+            <ParamControlNetControlMode
+              controlNetId={controlNetId}
+              controlMode={controlMode}
+            />
           </Flex>
+
           {isExpanded && (
             <>
               <Box mt={2}>

invokeai/frontend/web/src/features/controlNet/components/parameters/ParamControlNetControlMode.tsx

@@ -0,0 +1,45 @@
+import { useAppDispatch } from 'app/store/storeHooks';
+import IAIMantineSelect from 'common/components/IAIMantineSelect';
+import {
+  ControlModes,
+  controlNetControlModeChanged,
+} from 'features/controlNet/store/controlNetSlice';
+import { useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+
+type ParamControlNetControlModeProps = {
+  controlNetId: string;
+  controlMode: string;
+};
+
+const CONTROL_MODE_DATA = [
+  { label: 'Balanced', value: 'balanced' },
+  { label: 'Prompt', value: 'more_prompt' },
+  { label: 'Control', value: 'more_control' },
+  { label: 'Mega Control', value: 'unbalanced' },
+];
+
+export default function ParamControlNetControlMode(
+  props: ParamControlNetControlModeProps
+) {
+  const { controlNetId, controlMode = false } = props;
+  const dispatch = useAppDispatch();
+
+  const { t } = useTranslation();
+
+  const handleControlModeChange = useCallback(
+    (controlMode: ControlModes) => {
+      dispatch(controlNetControlModeChanged({ controlNetId, controlMode }));
+    },
+    [controlNetId, dispatch]
+  );
+
+  return (
+    <IAIMantineSelect
+      label={t('parameters.controlNetControlMode')}
+      data={CONTROL_MODE_DATA}
+      value={String(controlMode)}
+      onChange={handleControlModeChange}
+    />
+  );
+}

invokeai/frontend/web/src/features/controlNet/store/constants.ts

@@ -1,6 +1,5 @@
 import {
   ControlNetProcessorType,
-  RequiredCannyImageProcessorInvocation,
   RequiredControlNetProcessorNode,
 } from './types';
 
@@ -23,7 +22,7 @@ type ControlNetProcessorsDict = Record<
  *
  * TODO: Generate from the OpenAPI schema
  */
-export const CONTROLNET_PROCESSORS = {
+export const CONTROLNET_PROCESSORS: ControlNetProcessorsDict = {
   none: {
     type: 'none',
     label: 'none',
@@ -174,6 +173,8 @@ export const CONTROLNET_PROCESSORS = {
   },
 };
 
+type ControlNetModelsDict = Record<string, ControlNetModel>;
+
 type ControlNetModel = {
   type: string;
   label: string;
@@ -181,7 +182,7 @@ type ControlNetModel = {
   defaultProcessor?: ControlNetProcessorType;
 };
 
-export const CONTROLNET_MODELS = {
+export const CONTROLNET_MODELS: ControlNetModelsDict = {
   'lllyasviel/control_v11p_sd15_canny': {
     type: 'lllyasviel/control_v11p_sd15_canny',
     label: 'Canny',
@@ -190,6 +191,7 @@ export const CONTROLNET_MODELS = {
   'lllyasviel/control_v11p_sd15_inpaint': {
     type: 'lllyasviel/control_v11p_sd15_inpaint',
     label: 'Inpaint',
+    defaultProcessor: 'none',
   },
   'lllyasviel/control_v11p_sd15_mlsd': {
     type: 'lllyasviel/control_v11p_sd15_mlsd',
@@ -209,6 +211,7 @@ export const CONTROLNET_MODELS = {
   'lllyasviel/control_v11p_sd15_seg': {
     type: 'lllyasviel/control_v11p_sd15_seg',
     label: 'Segmentation',
+    defaultProcessor: 'none',
   },
   'lllyasviel/control_v11p_sd15_lineart': {
     type: 'lllyasviel/control_v11p_sd15_lineart',
@@ -223,6 +226,7 @@ export const CONTROLNET_MODELS = {
   'lllyasviel/control_v11p_sd15_scribble': {
     type: 'lllyasviel/control_v11p_sd15_scribble',
     label: 'Scribble',
+    defaultProcessor: 'none',
   },
   'lllyasviel/control_v11p_sd15_softedge': {
     type: 'lllyasviel/control_v11p_sd15_softedge',
@@ -242,10 +246,12 @@ export const CONTROLNET_MODELS = {
   'lllyasviel/control_v11f1e_sd15_tile': {
     type: 'lllyasviel/control_v11f1e_sd15_tile',
     label: 'Tile (experimental)',
+    defaultProcessor: 'none',
   },
   'lllyasviel/control_v11e_sd15_ip2p': {
     type: 'lllyasviel/control_v11e_sd15_ip2p',
     label: 'Pix2Pix (experimental)',
+    defaultProcessor: 'none',
   },
   'CrucibleAI/ControlNetMediaPipeFace': {
     type: 'CrucibleAI/ControlNetMediaPipeFace',

invokeai/frontend/web/src/features/controlNet/store/controlNetSlice.ts

@@ -1,36 +1,27 @@
-import { PayloadAction } from '@reduxjs/toolkit';
-import { createSlice } from '@reduxjs/toolkit';
+import { PayloadAction, createSlice } from '@reduxjs/toolkit';
 import { RootState } from 'app/store/store';
+import { forEach } from 'lodash-es';
 import { ImageDTO } from 'services/api';
-import {
-  ControlNetProcessorType,
-  RequiredCannyImageProcessorInvocation,
-  RequiredControlNetProcessorNode,
-} from './types';
+import { appSocketInvocationError } from 'services/events/actions';
+import { imageDeleted, imageUrlsReceived } from 'services/thunks/image';
+import { isAnySessionRejected } from 'services/thunks/session';
+import { controlNetImageProcessed } from './actions';
 import {
   CONTROLNET_MODELS,
   CONTROLNET_PROCESSORS,
   ControlNetModelName,
 } from './constants';
-import { controlNetImageProcessed } from './actions';
-import { imageDeleted, imageUrlsReceived } from 'services/thunks/image';
-import { forEach } from 'lodash-es';
-import { isAnySessionRejected } from 'services/thunks/session';
-import { appSocketInvocationError } from 'services/events/actions';
+import {
+  ControlNetProcessorType,
+  RequiredCannyImageProcessorInvocation,
+  RequiredControlNetProcessorNode,
+} from './types';
 
-export const initialControlNet: Omit<ControlNetConfig, 'controlNetId'> = {
-  isEnabled: true,
-  model: CONTROLNET_MODELS['lllyasviel/control_v11p_sd15_canny'].type,
-  weight: 1,
-  beginStepPct: 0,
-  endStepPct: 1,
-  controlImage: null,
-  processedControlImage: null,
-  processorType: 'canny_image_processor',
-  processorNode: CONTROLNET_PROCESSORS.canny_image_processor
-    .default as RequiredCannyImageProcessorInvocation,
-  shouldAutoConfig: true,
-};
+export type ControlModes =
+  | 'balanced'
+  | 'more_prompt'
+  | 'more_control'
+  | 'unbalanced';
 
 export type ControlNetConfig = {
   controlNetId: string;
@@ -39,6 +30,7 @@ export type ControlNetConfig = {
   weight: number;
   beginStepPct: number;
   endStepPct: number;
+  controlMode: ControlModes;
   controlImage: ImageDTO | null;
   processedControlImage: ImageDTO | null;
   processorType: ControlNetProcessorType;
@@ -46,6 +38,21 @@ export type ControlNetConfig = {
   shouldAutoConfig: boolean;
 };
 
+export const initialControlNet: Omit<ControlNetConfig, 'controlNetId'> = {
+  isEnabled: true,
+  model: CONTROLNET_MODELS['lllyasviel/control_v11p_sd15_canny'].type,
+  weight: 1,
+  beginStepPct: 0,
+  endStepPct: 1,
+  controlMode: 'balanced',
+  controlImage: null,
+  processedControlImage: null,
+  processorType: 'canny_image_processor',
+  processorNode: CONTROLNET_PROCESSORS.canny_image_processor
+    .default as RequiredCannyImageProcessorInvocation,
+  shouldAutoConfig: true,
+};
+
 export type ControlNetState = {
   controlNets: Record<string, ControlNetConfig>;
   isEnabled: boolean;
@@ -147,11 +154,13 @@ export const controlNetSlice = createSlice({
       state.controlNets[controlNetId].processedControlImage = null;
 
       if (state.controlNets[controlNetId].shouldAutoConfig) {
-        const processorType = CONTROLNET_MODELS[model].defaultProcessor;
+        const processorType =
+          CONTROLNET_MODELS[model as keyof typeof CONTROLNET_MODELS]
+            .defaultProcessor;
         if (processorType) {
           state.controlNets[controlNetId].processorType = processorType;
           state.controlNets[controlNetId].processorNode = CONTROLNET_PROCESSORS[
-            processorType
+            processorType as keyof typeof CONTROLNET_PROCESSORS
           ].default as RequiredControlNetProcessorNode;
         } else {
           state.controlNets[controlNetId].processorType = 'none';
@@ -181,6 +190,13 @@ export const controlNetSlice = createSlice({
       const { controlNetId, endStepPct } = action.payload;
       state.controlNets[controlNetId].endStepPct = endStepPct;
     },
+    controlNetControlModeChanged: (
+      state,
+      action: PayloadAction<{ controlNetId: string; controlMode: ControlModes }>
+    ) => {
+      const { controlNetId, controlMode } = action.payload;
+      state.controlNets[controlNetId].controlMode = controlMode;
+    },
     controlNetProcessorParamsChanged: (
       state,
       action: PayloadAction<{
@@ -210,7 +226,7 @@ export const controlNetSlice = createSlice({
       state.controlNets[controlNetId].processedControlImage = null;
       state.controlNets[controlNetId].processorType = processorType;
       state.controlNets[controlNetId].processorNode = CONTROLNET_PROCESSORS[
-        processorType
+        processorType as keyof typeof CONTROLNET_PROCESSORS
       ].default as RequiredControlNetProcessorNode;
       state.controlNets[controlNetId].shouldAutoConfig = false;
     },
@@ -227,12 +243,14 @@ export const controlNetSlice = createSlice({
       if (newShouldAutoConfig) {
         // manage the processor for the user
         const processorType =
-          CONTROLNET_MODELS[state.controlNets[controlNetId].model]
-            .defaultProcessor;
+          CONTROLNET_MODELS[
+            state.controlNets[controlNetId]
+              .model as keyof typeof CONTROLNET_MODELS
+          ].defaultProcessor;
         if (processorType) {
           state.controlNets[controlNetId].processorType = processorType;
           state.controlNets[controlNetId].processorNode = CONTROLNET_PROCESSORS[
-            processorType
+            processorType as keyof typeof CONTROLNET_PROCESSORS
           ].default as RequiredControlNetProcessorNode;
         } else {
           state.controlNets[controlNetId].processorType = 'none';
@@ -285,11 +303,11 @@ export const controlNetSlice = createSlice({
       });
     });
 
-    builder.addCase(appSocketInvocationError, (state, action) => {
+    builder.addCase(appSocketInvocationError, (state) => {
       state.pendingControlImages = [];
     });
 
-    builder.addMatcher(isAnySessionRejected, (state, action) => {
+    builder.addMatcher(isAnySessionRejected, (state) => {
       state.pendingControlImages = [];
     });
   },
@@ -307,6 +325,7 @@ export const {
   controlNetWeightChanged,
   controlNetBeginStepPctChanged,
   controlNetEndStepPctChanged,
+  controlNetControlModeChanged,
   controlNetProcessorParamsChanged,
   controlNetProcessorTypeChanged,
   controlNetReset,

invokeai/frontend/web/src/features/nodes/util/addControlNetToLinearGraph.ts

@@ -45,6 +45,7 @@ export const addControlNetToLinearGraph = (
         processedControlImage,
         beginStepPct,
         endStepPct,
+        controlMode,
         model,
         processorType,
         weight,
@@ -60,6 +61,7 @@ export const addControlNetToLinearGraph = (
         type: 'controlnet',
         begin_step_percent: beginStepPct,
         end_step_percent: endStepPct,
+        control_mode: controlMode,
         control_model: model as ControlNetInvocation['control_model'],
         control_weight: weight,
       };

invokeai/frontend/web/src/services/api/models/AddInvocation.ts

@@ -24,4 +24,3 @@ export type AddInvocation = {
    */
   'b'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/Body_upload_image.ts

@@ -5,4 +5,3 @@
 export type Body_upload_image = {
   file: Blob;
 };
-

invokeai/frontend/web/src/services/api/models/CannyImageProcessorInvocation.ts

@@ -30,4 +30,3 @@ export type CannyImageProcessorInvocation = {
    */
   high_threshold?: number;
 };
-

invokeai/frontend/web/src/services/api/models/CkptModelInfo.ts

@@ -29,4 +29,3 @@ export type CkptModelInfo = {
    */
   height?: number;
 };
-

invokeai/frontend/web/src/services/api/models/CollectInvocation.ts

@@ -24,4 +24,3 @@ export type CollectInvocation = {
    */
   collection?: Array<any>;
 };
-

invokeai/frontend/web/src/services/api/models/CollectInvocationOutput.ts

@@ -12,4 +12,3 @@ export type CollectInvocationOutput = {
    */
   collection: Array<any>;
 };
-

invokeai/frontend/web/src/services/api/models/ColorField.ts

@@ -20,4 +20,3 @@ export type ColorField = {
    */
   'a': number;
 };
-

invokeai/frontend/web/src/services/api/models/CompelInvocation.ts

@@ -24,4 +24,3 @@ export type CompelInvocation = {
    */
   model?: string;
 };
-

invokeai/frontend/web/src/services/api/models/CompelOutput.ts

@@ -14,4 +14,3 @@ export type CompelOutput = {
    */
   conditioning?: ConditioningField;
 };
-

invokeai/frontend/web/src/services/api/models/ConditioningField.ts

@@ -8,4 +8,3 @@ export type ConditioningField = {
    */
   conditioning_name: string;
 };
-

invokeai/frontend/web/src/services/api/models/ContentShuffleImageProcessorInvocation.ts

@@ -42,4 +42,3 @@ export type ContentShuffleImageProcessorInvocation = {
    */
   'f'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ControlField.ts

@@ -25,5 +25,8 @@ export type ControlField = {
    * When the ControlNet is last applied (% of total steps)
    */
   end_step_percent: number;
+  /**
+   * The contorl mode to use
+   */
+  control_mode?: 'balanced' | 'more_prompt' | 'more_control' | 'unbalanced';
 };
-

invokeai/frontend/web/src/services/api/models/ControlNetInvocation.ts

@@ -37,5 +37,8 @@ export type ControlNetInvocation = {
    * When the ControlNet is last applied (% of total steps)
    */
   end_step_percent?: number;
+  /**
+   * The control mode used
+   */
+  control_mode?: 'balanced' | 'more_prompt' | 'more_control' | 'unbalanced';
 };
-

invokeai/frontend/web/src/services/api/models/ControlOutput.ts

@@ -14,4 +14,3 @@ export type ControlOutput = {
    */
   control?: ControlField;
 };
-

invokeai/frontend/web/src/services/api/models/CreateModelRequest.ts

@@ -15,4 +15,3 @@ export type CreateModelRequest = {
    */
   info: (CkptModelInfo | DiffusersModelInfo);
 };
-

invokeai/frontend/web/src/services/api/models/CvInpaintInvocation.ts

@@ -26,4 +26,3 @@ export type CvInpaintInvocation = {
    */
   mask?: ImageField;
 };
-

invokeai/frontend/web/src/services/api/models/DiffusersModelInfo.ts

@@ -23,4 +23,3 @@ export type DiffusersModelInfo = {
    */
   path?: string;
 };
-

invokeai/frontend/web/src/services/api/models/DivideInvocation.ts

@@ -24,4 +24,3 @@ export type DivideInvocation = {
    */
   'b'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/DynamicPromptInvocation.ts

@@ -28,4 +28,3 @@ export type DynamicPromptInvocation = {
    */
   combinatorial?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/Edge.ts

@@ -14,4 +14,3 @@ export type Edge = {
    */
   destination: EdgeConnection;
 };
-

invokeai/frontend/web/src/services/api/models/EdgeConnection.ts

@@ -12,4 +12,3 @@ export type EdgeConnection = {
    */
   field: string;
 };
-

invokeai/frontend/web/src/services/api/models/FloatCollectionOutput.ts

@@ -12,4 +12,3 @@ export type FloatCollectionOutput = {
    */
   collection?: Array<number>;
 };
-

invokeai/frontend/web/src/services/api/models/FloatLinearRangeInvocation.ts

@@ -28,4 +28,3 @@ export type FloatLinearRangeInvocation = {
    */
   steps?: number;
 };
-

invokeai/frontend/web/src/services/api/models/FloatOutput.ts

@@ -12,4 +12,3 @@ export type FloatOutput = {
    */
   param?: number;
 };
-

invokeai/frontend/web/src/services/api/models/Graph.ts

@@ -72,10 +72,9 @@ export type Graph = {
   /**
    * The nodes in this graph
    */
-  nodes?: Record<string, (LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | DynamicPromptInvocation | CompelInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | CvInpaintInvocation | RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | UpscaleInvocation | RestoreFaceInvocation | TextToImageInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | LatentsToLatentsInvocation | ImageToImageInvocation | InpaintInvocation)>;
+  nodes?: Record<string, (RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | CompelInvocation | LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | CvInpaintInvocation | TextToImageInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | DynamicPromptInvocation | RestoreFaceInvocation | UpscaleInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | ImageToImageInvocation | LatentsToLatentsInvocation | InpaintInvocation)>;
   /**
    * The connections between nodes and their fields in this graph
    */
   edges?: Array<Edge>;
 };
-

invokeai/frontend/web/src/services/api/models/GraphExecutionState.ts

@@ -46,7 +46,7 @@ export type GraphExecutionState = {
   /**
    * The results of node executions
    */
-  results: Record<string, (ImageOutput | MaskOutput | ControlOutput | PromptOutput | PromptCollectionOutput | CompelOutput | IntOutput | FloatOutput | LatentsOutput | NoiseOutput | IntCollectionOutput | FloatCollectionOutput | GraphInvocationOutput | IterateInvocationOutput | CollectInvocationOutput)>;
+  results: Record<string, (IntCollectionOutput | FloatCollectionOutput | CompelOutput | ImageOutput | MaskOutput | ControlOutput | LatentsOutput | NoiseOutput | IntOutput | FloatOutput | PromptOutput | PromptCollectionOutput | GraphInvocationOutput | IterateInvocationOutput | CollectInvocationOutput)>;
   /**
    * Errors raised when executing nodes
    */
@@ -60,4 +60,3 @@ export type GraphExecutionState = {
    */
   source_prepared_mapping: Record<string, Array<string>>;
 };
-

invokeai/frontend/web/src/services/api/models/GraphInvocation.ts

@@ -22,4 +22,3 @@ export type GraphInvocation = {
    */
   graph?: Graph;
 };
-

invokeai/frontend/web/src/services/api/models/GraphInvocationOutput.ts

@@ -8,4 +8,3 @@
 export type GraphInvocationOutput = {
   type: 'graph_output';
 };
-

invokeai/frontend/web/src/services/api/models/HTTPValidationError.ts

@@ -7,4 +7,3 @@ import type { ValidationError } from './ValidationError';
 export type HTTPValidationError = {
   detail?: Array<ValidationError>;
 };
-

invokeai/frontend/web/src/services/api/models/HedImageProcessorInvocation.ts

@@ -34,4 +34,3 @@ export type HedImageProcessorInvocation = {
    */
   scribble?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/HedImageprocessorInvocation.ts

@@ -0,0 +1,36 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { ImageField } from './ImageField';
+
+/**
+ * Applies HED edge detection to image
+ */
+export type HedImageProcessorInvocation = {
+  /**
+   * The id of this node. Must be unique among all nodes.
+   */
+  id: string;
+  /**
+   * Whether or not this node is an intermediate node.
+   */
+  is_intermediate?: boolean;
+  type?: 'hed_image_processor';
+  /**
+   * The image to process
+   */
+  image?: ImageField;
+  /**
+   * The pixel resolution for detection
+   */
+  detect_resolution?: number;
+  /**
+   * The pixel resolution for the output image
+   */
+  image_resolution?: number;
+  /**
+   * Whether to use scribble mode
+   */
+  scribble?: boolean;
+};

invokeai/frontend/web/src/services/api/models/ImageBlurInvocation.ts

@@ -30,4 +30,3 @@ export type ImageBlurInvocation = {
    */
   blur_type?: 'gaussian' | 'box';
 };
-

invokeai/frontend/web/src/services/api/models/ImageChannelInvocation.ts

@@ -26,4 +26,3 @@ export type ImageChannelInvocation = {
    */
   channel?: 'A' | 'R' | 'G' | 'B';
 };
-

invokeai/frontend/web/src/services/api/models/ImageConvertInvocation.ts

@@ -26,4 +26,3 @@ export type ImageConvertInvocation = {
    */
   mode?: 'L' | 'RGB' | 'RGBA' | 'CMYK' | 'YCbCr' | 'LAB' | 'HSV' | 'I' | 'F';
 };
-

invokeai/frontend/web/src/services/api/models/ImageCropInvocation.ts

@@ -38,4 +38,3 @@ export type ImageCropInvocation = {
    */
   height?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ImageDTO.ts

@@ -67,4 +67,3 @@ export type ImageDTO = {
    */
   metadata?: ImageMetadata;
 };
-

invokeai/frontend/web/src/services/api/models/ImageField.ts

@@ -11,4 +11,3 @@ export type ImageField = {
    */
   image_name: string;
 };
-

invokeai/frontend/web/src/services/api/models/ImageInverseLerpInvocation.ts

@@ -30,4 +30,3 @@ export type ImageInverseLerpInvocation = {
    */
   max?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ImageLerpInvocation.ts

@@ -30,4 +30,3 @@ export type ImageLerpInvocation = {
    */
   max?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ImageMetadata.ts

@@ -78,4 +78,3 @@ export type ImageMetadata = {
    */
   extra?: string;
 };
-

invokeai/frontend/web/src/services/api/models/ImageMultiplyInvocation.ts

@@ -26,4 +26,3 @@ export type ImageMultiplyInvocation = {
    */
   image2?: ImageField;
 };
-

invokeai/frontend/web/src/services/api/models/ImageOutput.ts

@@ -22,4 +22,3 @@ export type ImageOutput = {
    */
   height: number;
 };
-

invokeai/frontend/web/src/services/api/models/ImagePasteInvocation.ts

@@ -38,4 +38,3 @@ export type ImagePasteInvocation = {
    */
   'y'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ImageProcessorInvocation.ts

@@ -22,4 +22,3 @@ export type ImageProcessorInvocation = {
    */
   image?: ImageField;
 };
-

invokeai/frontend/web/src/services/api/models/ImageRecordChanges.ts

@@ -26,4 +26,3 @@ export type ImageRecordChanges = {
    */
   is_intermediate?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/ImageResizeInvocation.ts

@@ -34,4 +34,3 @@ export type ImageResizeInvocation = {
    */
   resample_mode?: 'nearest' | 'box' | 'bilinear' | 'hamming' | 'bicubic' | 'lanczos';
 };
-

invokeai/frontend/web/src/services/api/models/ImageScaleInvocation.ts

@@ -30,4 +30,3 @@ export type ImageScaleInvocation = {
    */
   resample_mode?: 'nearest' | 'box' | 'bilinear' | 'hamming' | 'bicubic' | 'lanczos';
 };
-

invokeai/frontend/web/src/services/api/models/ImageToImageInvocation.ts

@@ -74,4 +74,3 @@ export type ImageToImageInvocation = {
    */
   fit?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/ImageToLatentsInvocation.ts

@@ -26,4 +26,3 @@ export type ImageToLatentsInvocation = {
    */
   model?: string;
 };
-

invokeai/frontend/web/src/services/api/models/ImageUrlsDTO.ts

@@ -19,4 +19,3 @@ export type ImageUrlsDTO = {
    */
   thumbnail_url: string;
 };
-

invokeai/frontend/web/src/services/api/models/InfillColorInvocation.ts

@@ -27,4 +27,3 @@ export type InfillColorInvocation = {
    */
   color?: ColorField;
 };
-

invokeai/frontend/web/src/services/api/models/InfillPatchMatchInvocation.ts

@@ -22,4 +22,3 @@ export type InfillPatchMatchInvocation = {
    */
   image?: ImageField;
 };
-

invokeai/frontend/web/src/services/api/models/InfillTileInvocation.ts

@@ -30,4 +30,3 @@ export type InfillTileInvocation = {
    */
   seed?: number;
 };
-

invokeai/frontend/web/src/services/api/models/InpaintInvocation.ts

@@ -119,4 +119,3 @@ export type InpaintInvocation = {
    */
   inpaint_replace?: number;
 };
-

invokeai/frontend/web/src/services/api/models/IntCollectionOutput.ts

@@ -12,4 +12,3 @@ export type IntCollectionOutput = {
    */
   collection?: Array<number>;
 };
-

invokeai/frontend/web/src/services/api/models/IntOutput.ts

@@ -12,4 +12,3 @@ export type IntOutput = {
    */
   'a'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/IterateInvocation.ts

@@ -24,4 +24,3 @@ export type IterateInvocation = {
    */
   index?: number;
 };
-

invokeai/frontend/web/src/services/api/models/IterateInvocationOutput.ts

@@ -12,4 +12,3 @@ export type IterateInvocationOutput = {
    */
   item: any;
 };
-

invokeai/frontend/web/src/services/api/models/LatentsField.ts

@@ -11,4 +11,3 @@ export type LatentsField = {
    */
   latents_name: string;
 };
-

invokeai/frontend/web/src/services/api/models/LatentsOutput.ts

@@ -22,4 +22,3 @@ export type LatentsOutput = {
    */
   height: number;
 };
-

invokeai/frontend/web/src/services/api/models/LatentsToImageInvocation.ts

@@ -26,4 +26,3 @@ export type LatentsToImageInvocation = {
    */
   model?: string;
 };
-

invokeai/frontend/web/src/services/api/models/LatentsToLatentsInvocation.ts

@@ -60,4 +60,3 @@ export type LatentsToLatentsInvocation = {
    */
   strength?: number;
 };
-

invokeai/frontend/web/src/services/api/models/LineartAnimeImageProcessorInvocation.ts

@@ -30,4 +30,3 @@ export type LineartAnimeImageProcessorInvocation = {
    */
   image_resolution?: number;
 };
-

invokeai/frontend/web/src/services/api/models/LineartImageProcessorInvocation.ts

@@ -34,4 +34,3 @@ export type LineartImageProcessorInvocation = {
    */
   coarse?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/LoadImageInvocation.ts

@@ -22,4 +22,3 @@ export type LoadImageInvocation = {
    */
   image?: ImageField;
 };
-

invokeai/frontend/web/src/services/api/models/MaskFromAlphaInvocation.ts

@@ -26,4 +26,3 @@ export type MaskFromAlphaInvocation = {
    */
   invert?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/MaskOutput.ts

@@ -22,4 +22,3 @@ export type MaskOutput = {
    */
   height?: number;
 };
-

invokeai/frontend/web/src/services/api/models/MediapipeFaceProcessorInvocation.ts

@@ -30,4 +30,3 @@ export type MediapipeFaceProcessorInvocation = {
    */
   min_confidence?: number;
 };
-

invokeai/frontend/web/src/services/api/models/MidasDepthImageProcessorInvocation.ts

@@ -30,4 +30,3 @@ export type MidasDepthImageProcessorInvocation = {
    */
   bg_th?: number;
 };
-

invokeai/frontend/web/src/services/api/models/MlsdImageProcessorInvocation.ts

@@ -38,4 +38,3 @@ export type MlsdImageProcessorInvocation = {
    */
   thr_d?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ModelsList.ts

@@ -8,4 +8,3 @@ import type { DiffusersModelInfo } from './DiffusersModelInfo';
 export type ModelsList = {
   models: Record<string, (CkptModelInfo | DiffusersModelInfo)>;
 };
-

invokeai/frontend/web/src/services/api/models/MultiplyInvocation.ts

@@ -24,4 +24,3 @@ export type MultiplyInvocation = {
    */
   'b'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/NoiseInvocation.ts

@@ -28,4 +28,3 @@ export type NoiseInvocation = {
    */
   height?: number;
 };
-

invokeai/frontend/web/src/services/api/models/NoiseOutput.ts

@@ -22,4 +22,3 @@ export type NoiseOutput = {
    */
   height: number;
 };
-

invokeai/frontend/web/src/services/api/models/NormalbaeImageProcessorInvocation.ts

@@ -30,4 +30,3 @@ export type NormalbaeImageProcessorInvocation = {
    */
   image_resolution?: number;
 };
-

invokeai/frontend/web/src/services/api/models/OffsetPaginatedResults_ImageDTO_.ts

@@ -25,4 +25,3 @@ export type OffsetPaginatedResults_ImageDTO_ = {
    */
   total: number;
 };
-

invokeai/frontend/web/src/services/api/models/OpenposeImageProcessorInvocation.ts

@@ -34,4 +34,3 @@ export type OpenposeImageProcessorInvocation = {
    */
   image_resolution?: number;
 };
-

invokeai/frontend/web/src/services/api/models/PaginatedResults_GraphExecutionState_.ts

@@ -29,4 +29,3 @@ export type PaginatedResults_GraphExecutionState_ = {
    */
   total: number;
 };
-

invokeai/frontend/web/src/services/api/models/ParamFloatInvocation.ts

@@ -20,4 +20,3 @@ export type ParamFloatInvocation = {
    */
   param?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ParamIntInvocation.ts

@@ -20,4 +20,3 @@ export type ParamIntInvocation = {
    */
   'a'?: number;
 };
-

invokeai/frontend/web/src/services/api/models/PidiImageProcessorInvocation.ts

@@ -38,4 +38,3 @@ export type PidiImageProcessorInvocation = {
    */
   scribble?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/PromptCollectionOutput.ts

@@ -16,4 +16,3 @@ export type PromptCollectionOutput = {
    */
   count: number;
 };
-

invokeai/frontend/web/src/services/api/models/PromptOutput.ts

@@ -12,4 +12,3 @@ export type PromptOutput = {
    */
   prompt: string;
 };
-

invokeai/frontend/web/src/services/api/models/RandomIntInvocation.ts

@@ -24,4 +24,3 @@ export type RandomIntInvocation = {
    */
   high?: number;
 };
-

invokeai/frontend/web/src/services/api/models/RandomRangeInvocation.ts

@@ -32,4 +32,3 @@ export type RandomRangeInvocation = {
    */
   seed?: number;
 };
-

invokeai/frontend/web/src/services/api/models/RangeInvocation.ts

@@ -28,4 +28,3 @@ export type RangeInvocation = {
    */
   step?: number;
 };
-

invokeai/frontend/web/src/services/api/models/RangeOfSizeInvocation.ts

@@ -28,4 +28,3 @@ export type RangeOfSizeInvocation = {
    */
   step?: number;
 };
-

invokeai/frontend/web/src/services/api/models/ResizeLatentsInvocation.ts

@@ -38,4 +38,3 @@ export type ResizeLatentsInvocation = {
    */
   antialias?: boolean;
 };
-

invokeai/frontend/web/src/services/api/models/RestoreFaceInvocation.ts

@@ -26,4 +26,3 @@ export type RestoreFaceInvocation = {
    */
   strength?: number;
 };
-