Merge branch 'main' into lstein/installer-for-new-model-layout

This commit is contained in:
Eugene Brodsky 2023-06-26 01:24:30 -04:00 committed by GitHub
commit 7b97639961
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 1153 additions and 1609 deletions

View File

@ -1,7 +1,7 @@
# InvokeAI nodes for ControlNet image preprocessors
# initial implementation by Gregg Helt, 2023
# heavily leverages controlnet_aux package: https://github.com/patrickvonplaten/controlnet_aux
from builtins import float
from builtins import float, bool
import numpy as np
from typing import Literal, Optional, Union, List
@ -94,6 +94,7 @@ CONTROLNET_DEFAULT_MODELS = [
]
CONTROLNET_NAME_VALUES = Literal[tuple(CONTROLNET_DEFAULT_MODELS)]
CONTROLNET_MODE_VALUES = Literal[tuple(["balanced", "more_prompt", "more_control", "unbalanced"])]
class ControlField(BaseModel):
image: ImageField = Field(default=None, description="The control image")
@ -104,6 +105,8 @@ class ControlField(BaseModel):
description="When the ControlNet is first applied (% of total steps)")
end_step_percent: float = Field(default=1, ge=0, le=1,
description="When the ControlNet is last applied (% of total steps)")
control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The contorl mode to use")
@validator("control_weight")
def abs_le_one(cls, v):
"""validate that all abs(values) are <=1"""
@ -144,11 +147,11 @@ class ControlNetInvocation(BaseInvocation):
control_model: CONTROLNET_NAME_VALUES = Field(default="lllyasviel/sd-controlnet-canny",
description="control model used")
control_weight: Union[float, List[float]] = Field(default=1.0, description="The weight given to the ControlNet")
# TODO: add support in backend core for begin_step_percent, end_step_percent, guess_mode
begin_step_percent: float = Field(default=0, ge=0, le=1,
description="When the ControlNet is first applied (% of total steps)")
end_step_percent: float = Field(default=1, ge=0, le=1,
description="When the ControlNet is last applied (% of total steps)")
control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The control mode used")
# fmt: on
class Config(InvocationConfig):
@ -166,7 +169,6 @@ class ControlNetInvocation(BaseInvocation):
}
def invoke(self, context: InvocationContext) -> ControlOutput:
return ControlOutput(
control=ControlField(
image=self.image,
@ -174,6 +176,7 @@ class ControlNetInvocation(BaseInvocation):
control_weight=self.control_weight,
begin_step_percent=self.begin_step_percent,
end_step_percent=self.end_step_percent,
control_mode=self.control_mode,
),
)

View File

@ -287,19 +287,14 @@ class TextToLatentsInvocation(BaseInvocation):
control_height_resize = latents_shape[2] * 8
control_width_resize = latents_shape[3] * 8
if control_input is None:
# print("control input is None")
control_list = None
elif isinstance(control_input, list) and len(control_input) == 0:
# print("control input is empty list")
control_list = None
elif isinstance(control_input, ControlField):
# print("control input is ControlField")
control_list = [control_input]
elif isinstance(control_input, list) and len(control_input) > 0 and isinstance(control_input[0], ControlField):
# print("control input is list[ControlField]")
control_list = control_input
else:
# print("input control is unrecognized:", type(self.control))
control_list = None
if (control_list is None):
control_data = None
@ -341,12 +336,15 @@ class TextToLatentsInvocation(BaseInvocation):
# num_images_per_prompt=num_images_per_prompt,
device=control_model.device,
dtype=control_model.dtype,
control_mode=control_info.control_mode,
)
control_item = ControlNetData(model=control_model,
image_tensor=control_image,
weight=control_info.control_weight,
begin_step_percent=control_info.begin_step_percent,
end_step_percent=control_info.end_step_percent)
end_step_percent=control_info.end_step_percent,
control_mode=control_info.control_mode,
)
control_data.append(control_item)
# MultiControlNetModel has been refactored out, just need list[ControlNetData]
return control_data

View File

@ -215,10 +215,12 @@ class GeneratorToCallbackinator(Generic[ParamType, ReturnType, CallbackType]):
@dataclass
class ControlNetData:
model: ControlNetModel = Field(default=None)
image_tensor: torch.Tensor= Field(default=None)
weight: Union[float, List[float]]= Field(default=1.0)
image_tensor: torch.Tensor = Field(default=None)
weight: Union[float, List[float]] = Field(default=1.0)
begin_step_percent: float = Field(default=0.0)
end_step_percent: float = Field(default=1.0)
control_mode: str = Field(default="balanced")
@dataclass(frozen=True)
class ConditioningData:
@ -599,48 +601,68 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
# TODO: should this scaling happen here or inside self._unet_forward?
# i.e. before or after passing it to InvokeAIDiffuserComponent
latent_model_input = self.scheduler.scale_model_input(latents, timestep)
unet_latent_input = self.scheduler.scale_model_input(latents, timestep)
# default is no controlnet, so set controlnet processing output to None
down_block_res_samples, mid_block_res_sample = None, None
if control_data is not None:
# FIXME: make sure guidance_scale < 1.0 is handled correctly if doing per-step guidance setting
# if conditioning_data.guidance_scale > 1.0:
if conditioning_data.guidance_scale is not None:
# expand the latents input to control model if doing classifier free guidance
# (which I think for now is always true, there is conditional elsewhere that stops execution if
# classifier_free_guidance is <= 1.0 ?)
latent_control_input = torch.cat([latent_model_input] * 2)
else:
latent_control_input = latent_model_input
# control_data should be type List[ControlNetData]
# this loop covers both ControlNet (one ControlNetData in list)
# and MultiControlNet (multiple ControlNetData in list)
for i, control_datum in enumerate(control_data):
# print("controlnet", i, "==>", type(control_datum))
control_mode = control_datum.control_mode
# soft_injection and cfg_injection are the two ControlNet control_mode booleans
# that are combined at higher level to make control_mode enum
# soft_injection determines whether to do per-layer re-weighting adjustment (if True)
# or default weighting (if False)
soft_injection = (control_mode == "more_prompt" or control_mode == "more_control")
# cfg_injection = determines whether to apply ControlNet to only the conditional (if True)
# or the default both conditional and unconditional (if False)
cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
first_control_step = math.floor(control_datum.begin_step_percent * total_step_count)
last_control_step = math.ceil(control_datum.end_step_percent * total_step_count)
# only apply controlnet if current step is within the controlnet's begin/end step range
if step_index >= first_control_step and step_index <= last_control_step:
# print("running controlnet", i, "for step", step_index)
if cfg_injection:
control_latent_input = unet_latent_input
else:
# expand the latents input to control model if doing classifier free guidance
# (which I think for now is always true, there is conditional elsewhere that stops execution if
# classifier_free_guidance is <= 1.0 ?)
control_latent_input = torch.cat([unet_latent_input] * 2)
if cfg_injection: # only applying ControlNet to conditional instead of in unconditioned
encoder_hidden_states = torch.cat([conditioning_data.unconditioned_embeddings])
else:
encoder_hidden_states = torch.cat([conditioning_data.unconditioned_embeddings,
conditioning_data.text_embeddings])
if isinstance(control_datum.weight, list):
# if controlnet has multiple weights, use the weight for the current step
controlnet_weight = control_datum.weight[step_index]
else:
# if controlnet has a single weight, use it for all steps
controlnet_weight = control_datum.weight
# controlnet(s) inference
down_samples, mid_sample = control_datum.model(
sample=latent_control_input,
sample=control_latent_input,
timestep=timestep,
encoder_hidden_states=torch.cat([conditioning_data.unconditioned_embeddings,
conditioning_data.text_embeddings]),
encoder_hidden_states=encoder_hidden_states,
controlnet_cond=control_datum.image_tensor,
conditioning_scale=controlnet_weight,
# cross_attention_kwargs,
guess_mode=False,
conditioning_scale=controlnet_weight, # controlnet specific, NOT the guidance scale
guess_mode=soft_injection, # this is still called guess_mode in diffusers ControlNetModel
return_dict=False,
)
if cfg_injection:
# Inferred ControlNet only for the conditional batch.
# To apply the output of ControlNet to both the unconditional and conditional batches,
# add 0 to the unconditional batch to keep it unchanged.
down_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_samples]
mid_sample = torch.cat([torch.zeros_like(mid_sample), mid_sample])
if down_block_res_samples is None and mid_block_res_sample is None:
down_block_res_samples, mid_block_res_sample = down_samples, mid_sample
else:
@ -653,11 +675,11 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
# predict the noise residual
noise_pred = self.invokeai_diffuser.do_diffusion_step(
latent_model_input,
t,
conditioning_data.unconditioned_embeddings,
conditioning_data.text_embeddings,
conditioning_data.guidance_scale,
x=unet_latent_input,
sigma=t,
unconditioning=conditioning_data.unconditioned_embeddings,
conditioning=conditioning_data.text_embeddings,
unconditional_guidance_scale=conditioning_data.guidance_scale,
step_index=step_index,
total_step_count=total_step_count,
down_block_additional_residuals=down_block_res_samples, # from controlnet(s)
@ -962,6 +984,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
device="cuda",
dtype=torch.float16,
do_classifier_free_guidance=True,
control_mode="balanced"
):
if not isinstance(image, torch.Tensor):
@ -992,6 +1015,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
repeat_by = num_images_per_prompt
image = image.repeat_interleave(repeat_by, dim=0)
image = image.to(device=device, dtype=dtype)
if do_classifier_free_guidance:
cfg_injection = (control_mode == "more_control" or control_mode == "unbalanced")
if do_classifier_free_guidance and not cfg_injection:
image = torch.cat([image] * 2)
return image

View File

@ -23,7 +23,7 @@
"dev": "concurrently \"vite dev\" \"yarn run theme:watch\"",
"dev:host": "concurrently \"vite dev --host\" \"yarn run theme:watch\"",
"build": "yarn run lint && vite build",
"typegen": "npx openapi-typescript http://localhost:9090/openapi.json --output src/services/schema.d.ts -t",
"typegen": "npx openapi-typescript http://localhost:9090/openapi.json --output src/services/api/schema.d.ts -t",
"preview": "vite preview",
"lint:madge": "madge --circular src/main.tsx",
"lint:eslint": "eslint --max-warnings=0 .",
@ -55,34 +55,35 @@
"dependencies": {
"@chakra-ui/anatomy": "^2.1.1",
"@chakra-ui/icons": "^2.0.19",
"@chakra-ui/react": "^2.6.0",
"@chakra-ui/styled-system": "^2.9.0",
"@chakra-ui/theme-tools": "^2.0.16",
"@dagrejs/graphlib": "^2.1.12",
"@chakra-ui/react": "^2.7.1",
"@chakra-ui/styled-system": "^2.9.1",
"@chakra-ui/theme-tools": "^2.0.18",
"@dagrejs/graphlib": "^2.1.13",
"@dnd-kit/core": "^6.0.8",
"@dnd-kit/modifiers": "^6.0.1",
"@emotion/react": "^11.11.1",
"@emotion/styled": "^11.10.6",
"@floating-ui/react-dom": "^2.0.0",
"@fontsource/inter": "^4.5.15",
"@mantine/core": "^6.0.13",
"@mantine/hooks": "^6.0.13",
"@emotion/styled": "^11.11.0",
"@floating-ui/react-dom": "^2.0.1",
"@fontsource-variable/inter": "^5.0.3",
"@fontsource/inter": "^5.0.3",
"@mantine/core": "^6.0.14",
"@mantine/hooks": "^6.0.14",
"@reduxjs/toolkit": "^1.9.5",
"@roarr/browser-log-writer": "^1.1.5",
"chakra-ui-contextmenu": "^1.0.5",
"dateformat": "^5.0.3",
"downshift": "^7.6.0",
"formik": "^2.2.9",
"framer-motion": "^10.12.4",
"formik": "^2.4.2",
"framer-motion": "^10.12.17",
"fuse.js": "^6.6.2",
"i18next": "^22.4.15",
"i18next-browser-languagedetector": "^7.0.1",
"i18next-http-backend": "^2.2.0",
"konva": "^9.0.1",
"i18next": "^23.2.3",
"i18next-browser-languagedetector": "^7.0.2",
"i18next-http-backend": "^2.2.1",
"konva": "^9.2.0",
"lodash-es": "^4.17.21",
"nanostores": "^0.9.2",
"openapi-fetch": "^0.4.0",
"overlayscrollbars": "^2.1.1",
"overlayscrollbars": "^2.2.0",
"overlayscrollbars-react": "^0.5.0",
"patch-package": "^7.0.0",
"query-string": "^8.1.0",
@ -92,21 +93,21 @@
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-hotkeys-hook": "4.4.0",
"react-i18next": "^12.2.2",
"react-icons": "^4.9.0",
"react-konva": "^18.2.7",
"react-redux": "^8.0.5",
"react-resizable-panels": "^0.0.42",
"react-i18next": "^13.0.1",
"react-icons": "^4.10.1",
"react-konva": "^18.2.10",
"react-redux": "^8.1.1",
"react-resizable-panels": "^0.0.52",
"react-use": "^17.4.0",
"react-virtuoso": "^4.3.5",
"react-zoom-pan-pinch": "^3.0.7",
"reactflow": "^11.7.0",
"react-virtuoso": "^4.3.11",
"react-zoom-pan-pinch": "^3.0.8",
"reactflow": "^11.7.4",
"redux-dynamic-middlewares": "^2.2.0",
"redux-remember": "^3.3.1",
"roarr": "^7.15.0",
"serialize-error": "^11.0.0",
"socket.io-client": "^4.6.0",
"use-image": "^1.1.0",
"socket.io-client": "^4.7.0",
"use-image": "^1.1.1",
"uuid": "^9.0.0",
"zod": "^3.21.4"
},
@ -117,22 +118,22 @@
"ts-toolbelt": "^9.6.0"
},
"devDependencies": {
"@chakra-ui/cli": "^2.4.0",
"@chakra-ui/cli": "^2.4.1",
"@types/dateformat": "^5.0.0",
"@types/lodash-es": "^4.14.194",
"@types/node": "^18.16.2",
"@types/react": "^18.2.0",
"@types/react-dom": "^18.2.1",
"@types/node": "^20.3.1",
"@types/react": "^18.2.14",
"@types/react-dom": "^18.2.6",
"@types/react-redux": "^7.1.25",
"@types/react-transition-group": "^4.4.5",
"@types/uuid": "^9.0.0",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"@typescript-eslint/parser": "^5.59.1",
"@vitejs/plugin-react-swc": "^3.3.0",
"@types/react-transition-group": "^4.4.6",
"@types/uuid": "^9.0.2",
"@typescript-eslint/eslint-plugin": "^5.60.0",
"@typescript-eslint/parser": "^5.60.0",
"@vitejs/plugin-react-swc": "^3.3.2",
"axios": "^1.4.0",
"babel-plugin-transform-imports": "^2.0.0",
"concurrently": "^8.0.1",
"eslint": "^8.39.0",
"concurrently": "^8.2.0",
"eslint": "^8.43.0",
"eslint-config-prettier": "^8.8.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-react": "^7.32.2",
@ -140,16 +141,16 @@
"form-data": "^4.0.0",
"husky": "^8.0.3",
"lint-staged": "^13.2.2",
"madge": "^6.0.0",
"openapi-types": "^12.1.0",
"madge": "^6.1.0",
"openapi-types": "^12.1.3",
"openapi-typescript": "^6.2.8",
"openapi-typescript-codegen": "^0.24.0",
"postinstall-postinstall": "^2.1.0",
"prettier": "^2.8.8",
"rollup-plugin-visualizer": "^5.9.0",
"terser": "^5.17.1",
"rollup-plugin-visualizer": "^5.9.2",
"terser": "^5.18.1",
"ts-toolbelt": "^9.6.0",
"vite": "^4.3.3",
"vite": "^4.3.9",
"vite-plugin-css-injected-by-js": "^3.1.1",
"vite-plugin-dts": "^2.3.0",
"vite-plugin-eslint": "^1.8.1",

View File

@ -1,14 +0,0 @@
diff --git a/node_modules/@chakra-ui/cli/dist/scripts/read-theme-file.worker.js b/node_modules/@chakra-ui/cli/dist/scripts/read-theme-file.worker.js
index 937cf0d..7dcc0c0 100644
--- a/node_modules/@chakra-ui/cli/dist/scripts/read-theme-file.worker.js
+++ b/node_modules/@chakra-ui/cli/dist/scripts/read-theme-file.worker.js
@@ -50,7 +50,8 @@ async function readTheme(themeFilePath) {
project: tsConfig.configFileAbsolutePath,
compilerOptions: {
module: "CommonJS",
- esModuleInterop: true
+ esModuleInterop: true,
+ jsx: 'react'
},
transpileOnly: true,
swc: true

View File

@ -524,7 +524,8 @@
"initialImage": "Initial Image",
"showOptionsPanel": "Show Options Panel",
"hidePreview": "Hide Preview",
"showPreview": "Show Preview"
"showPreview": "Show Preview",
"controlNetControlMode": "Control Mode"
},
"settings": {
"models": "Models",

View File

@ -14,7 +14,7 @@ import { invokeAIThemeColors } from 'theme/colors/invokeAI';
import { lightThemeColors } from 'theme/colors/lightTheme';
import { oceanBlueColors } from 'theme/colors/oceanBlue';
import '@fontsource/inter/variable.css';
import '@fontsource-variable/inter';
import { MantineProvider } from '@mantine/core';
import { mantineTheme } from 'mantine-theme/theme';
import 'overlayscrollbars/overlayscrollbars.css';

View File

@ -1,26 +1,27 @@
import { Box, ChakraProps, Flex } from '@chakra-ui/react';
import { useAppDispatch } from 'app/store/storeHooks';
import { memo, useCallback } from 'react';
import { FaCopy, FaTrash } from 'react-icons/fa';
import {
ControlNetConfig,
controlNetAdded,
controlNetRemoved,
controlNetToggled,
} from '../store/controlNetSlice';
import { useAppDispatch } from 'app/store/storeHooks';
import ParamControlNetModel from './parameters/ParamControlNetModel';
import ParamControlNetWeight from './parameters/ParamControlNetWeight';
import { Flex, Box, ChakraProps } from '@chakra-ui/react';
import { FaCopy, FaTrash } from 'react-icons/fa';
import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
import ControlNetImagePreview from './ControlNetImagePreview';
import IAIIconButton from 'common/components/IAIIconButton';
import { v4 as uuidv4 } from 'uuid';
import { useToggle } from 'react-use';
import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
import ControlNetProcessorComponent from './ControlNetProcessorComponent';
import IAISwitch from 'common/components/IAISwitch';
import { ChevronUpIcon } from '@chakra-ui/icons';
import IAIIconButton from 'common/components/IAIIconButton';
import IAISwitch from 'common/components/IAISwitch';
import { useToggle } from 'react-use';
import { v4 as uuidv4 } from 'uuid';
import ControlNetImagePreview from './ControlNetImagePreview';
import ControlNetProcessorComponent from './ControlNetProcessorComponent';
import ParamControlNetShouldAutoConfig from './ParamControlNetShouldAutoConfig';
import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
import ParamControlNetControlMode from './parameters/ParamControlNetControlMode';
import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
const expandedControlImageSx: ChakraProps['sx'] = { maxH: 96 };
@ -36,6 +37,7 @@ const ControlNet = (props: ControlNetProps) => {
weight,
beginStepPct,
endStepPct,
controlMode,
controlImage,
processedControlImage,
processorNode,
@ -137,48 +139,51 @@ const ControlNet = (props: ControlNetProps) => {
</Flex>
{isEnabled && (
<>
<Flex sx={{ gap: 4, w: 'full' }}>
<Flex
sx={{
flexDir: 'column',
gap: 2,
w: 'full',
h: isExpanded ? 28 : 24,
paddingInlineStart: 1,
paddingInlineEnd: isExpanded ? 1 : 0,
pb: 2,
justifyContent: 'space-between',
}}
>
<ParamControlNetWeight
controlNetId={controlNetId}
weight={weight}
mini={!isExpanded}
/>
<ParamControlNetBeginEnd
controlNetId={controlNetId}
beginStepPct={beginStepPct}
endStepPct={endStepPct}
mini={!isExpanded}
/>
</Flex>
{!isExpanded && (
<Flex sx={{ w: 'full', flexDirection: 'column' }}>
<Flex sx={{ gap: 4, w: 'full' }}>
<Flex
sx={{
alignItems: 'center',
justifyContent: 'center',
h: 24,
w: 24,
aspectRatio: '1/1',
flexDir: 'column',
gap: 3,
w: 'full',
paddingInlineStart: 1,
paddingInlineEnd: isExpanded ? 1 : 0,
pb: 2,
justifyContent: 'space-between',
}}
>
<ControlNetImagePreview
controlNet={props.controlNet}
height={24}
<ParamControlNetWeight
controlNetId={controlNetId}
weight={weight}
mini={!isExpanded}
/>
<ParamControlNetBeginEnd
controlNetId={controlNetId}
beginStepPct={beginStepPct}
endStepPct={endStepPct}
mini={!isExpanded}
/>
</Flex>
)}
{!isExpanded && (
<Flex
sx={{
alignItems: 'center',
justifyContent: 'center',
h: 24,
w: 24,
aspectRatio: '1/1',
}}
>
<ControlNetImagePreview controlNet={props.controlNet} />
</Flex>
)}
</Flex>
<ParamControlNetControlMode
controlNetId={controlNetId}
controlMode={controlMode}
/>
</Flex>
{isExpanded && (
<>
<Box mt={2}>

View File

@ -0,0 +1,45 @@
import { useAppDispatch } from 'app/store/storeHooks';
import IAIMantineSelect from 'common/components/IAIMantineSelect';
import {
ControlModes,
controlNetControlModeChanged,
} from 'features/controlNet/store/controlNetSlice';
import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
type ParamControlNetControlModeProps = {
controlNetId: string;
controlMode: string;
};
const CONTROL_MODE_DATA = [
{ label: 'Balanced', value: 'balanced' },
{ label: 'Prompt', value: 'more_prompt' },
{ label: 'Control', value: 'more_control' },
{ label: 'Mega Control', value: 'unbalanced' },
];
export default function ParamControlNetControlMode(
props: ParamControlNetControlModeProps
) {
const { controlNetId, controlMode = false } = props;
const dispatch = useAppDispatch();
const { t } = useTranslation();
const handleControlModeChange = useCallback(
(controlMode: ControlModes) => {
dispatch(controlNetControlModeChanged({ controlNetId, controlMode }));
},
[controlNetId, dispatch]
);
return (
<IAIMantineSelect
label={t('parameters.controlNetControlMode')}
data={CONTROL_MODE_DATA}
value={String(controlMode)}
onChange={handleControlModeChange}
/>
);
}

View File

@ -1,6 +1,5 @@
import {
ControlNetProcessorType,
RequiredCannyImageProcessorInvocation,
RequiredControlNetProcessorNode,
} from './types';
@ -23,7 +22,7 @@ type ControlNetProcessorsDict = Record<
*
* TODO: Generate from the OpenAPI schema
*/
export const CONTROLNET_PROCESSORS = {
export const CONTROLNET_PROCESSORS: ControlNetProcessorsDict = {
none: {
type: 'none',
label: 'none',
@ -174,6 +173,8 @@ export const CONTROLNET_PROCESSORS = {
},
};
type ControlNetModelsDict = Record<string, ControlNetModel>;
type ControlNetModel = {
type: string;
label: string;
@ -181,7 +182,7 @@ type ControlNetModel = {
defaultProcessor?: ControlNetProcessorType;
};
export const CONTROLNET_MODELS = {
export const CONTROLNET_MODELS: ControlNetModelsDict = {
'lllyasviel/control_v11p_sd15_canny': {
type: 'lllyasviel/control_v11p_sd15_canny',
label: 'Canny',
@ -190,6 +191,7 @@ export const CONTROLNET_MODELS = {
'lllyasviel/control_v11p_sd15_inpaint': {
type: 'lllyasviel/control_v11p_sd15_inpaint',
label: 'Inpaint',
defaultProcessor: 'none',
},
'lllyasviel/control_v11p_sd15_mlsd': {
type: 'lllyasviel/control_v11p_sd15_mlsd',
@ -209,6 +211,7 @@ export const CONTROLNET_MODELS = {
'lllyasviel/control_v11p_sd15_seg': {
type: 'lllyasviel/control_v11p_sd15_seg',
label: 'Segmentation',
defaultProcessor: 'none',
},
'lllyasviel/control_v11p_sd15_lineart': {
type: 'lllyasviel/control_v11p_sd15_lineart',
@ -223,6 +226,7 @@ export const CONTROLNET_MODELS = {
'lllyasviel/control_v11p_sd15_scribble': {
type: 'lllyasviel/control_v11p_sd15_scribble',
label: 'Scribble',
defaultProcessor: 'none',
},
'lllyasviel/control_v11p_sd15_softedge': {
type: 'lllyasviel/control_v11p_sd15_softedge',
@ -242,10 +246,12 @@ export const CONTROLNET_MODELS = {
'lllyasviel/control_v11f1e_sd15_tile': {
type: 'lllyasviel/control_v11f1e_sd15_tile',
label: 'Tile (experimental)',
defaultProcessor: 'none',
},
'lllyasviel/control_v11e_sd15_ip2p': {
type: 'lllyasviel/control_v11e_sd15_ip2p',
label: 'Pix2Pix (experimental)',
defaultProcessor: 'none',
},
'CrucibleAI/ControlNetMediaPipeFace': {
type: 'CrucibleAI/ControlNetMediaPipeFace',

View File

@ -18,12 +18,19 @@ import { forEach } from 'lodash-es';
import { isAnySessionRejected } from 'services/api/thunks/session';
import { appSocketInvocationError } from 'services/events/actions';
export type ControlModes =
| 'balanced'
| 'more_prompt'
| 'more_control'
| 'unbalanced';
export const initialControlNet: Omit<ControlNetConfig, 'controlNetId'> = {
isEnabled: true,
model: CONTROLNET_MODELS['lllyasviel/control_v11p_sd15_canny'].type,
weight: 1,
beginStepPct: 0,
endStepPct: 1,
controlMode: 'balanced',
controlImage: null,
processedControlImage: null,
processorType: 'canny_image_processor',
@ -39,6 +46,7 @@ export type ControlNetConfig = {
weight: number;
beginStepPct: number;
endStepPct: number;
controlMode: ControlModes;
controlImage: string | null;
processedControlImage: string | null;
processorType: ControlNetProcessorType;
@ -181,6 +189,13 @@ export const controlNetSlice = createSlice({
const { controlNetId, endStepPct } = action.payload;
state.controlNets[controlNetId].endStepPct = endStepPct;
},
controlNetControlModeChanged: (
state,
action: PayloadAction<{ controlNetId: string; controlMode: ControlModes }>
) => {
const { controlNetId, controlMode } = action.payload;
state.controlNets[controlNetId].controlMode = controlMode;
},
controlNetProcessorParamsChanged: (
state,
action: PayloadAction<{
@ -307,6 +322,7 @@ export const {
controlNetWeightChanged,
controlNetBeginStepPctChanged,
controlNetEndStepPctChanged,
controlNetControlModeChanged,
controlNetProcessorParamsChanged,
controlNetProcessorTypeChanged,
controlNetReset,

View File

@ -44,6 +44,7 @@ export const addControlNetToLinearGraph = (
processedControlImage,
beginStepPct,
endStepPct,
controlMode,
model,
processorType,
weight,
@ -59,6 +60,7 @@ export const addControlNetToLinearGraph = (
type: 'controlnet',
begin_step_percent: beginStepPct,
end_step_percent: endStepPct,
control_mode: controlMode,
control_model: model as ControlNetInvocation['control_model'],
control_weight: weight,
};

View File

@ -17,7 +17,6 @@ import { setActiveTab, togglePanels } from 'features/ui/store/uiSlice';
import { memo, MouseEvent, ReactNode, useCallback, useMemo } from 'react';
import { useHotkeys } from 'react-hotkeys-hook';
import { MdDeviceHub, MdGridOn } from 'react-icons/md';
import { GoTextSize } from 'react-icons/go';
import {
activeTabIndexSelector,
activeTabNameSelector,
@ -33,7 +32,7 @@ import ImageGalleryContent from 'features/gallery/components/ImageGalleryContent
import TextToImageTab from './tabs/TextToImage/TextToImageTab';
import UnifiedCanvasTab from './tabs/UnifiedCanvas/UnifiedCanvasTab';
import NodesTab from './tabs/Nodes/NodesTab';
import { FaImage } from 'react-icons/fa';
import { FaFont, FaImage } from 'react-icons/fa';
import ResizeHandle from './tabs/ResizeHandle';
import ImageTab from './tabs/ImageToImage/ImageToImageTab';
import AuxiliaryProgressIndicator from 'app/components/AuxiliaryProgressIndicator';
@ -47,7 +46,7 @@ export interface InvokeTabInfo {
const tabs: InvokeTabInfo[] = [
{
id: 'txt2img',
icon: <Icon as={GoTextSize} sx={{ boxSize: 6, pointerEvents: 'none' }} />,
icon: <Icon as={FaFont} sx={{ boxSize: 6, pointerEvents: 'none' }} />,
content: <TextToImageTab />,
},
{

View File

@ -0,0 +1,36 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { ImageField } from './ImageField';
/**
* Applies HED edge detection to image
*/
export type HedImageProcessorInvocation = {
/**
* The id of this node. Must be unique among all nodes.
*/
id: string;
/**
* Whether or not this node is an intermediate node.
*/
is_intermediate?: boolean;
type?: 'hed_image_processor';
/**
* The image to process
*/
image?: ImageField;
/**
* The pixel resolution for detection
*/
detect_resolution?: number;
/**
* The pixel resolution for the output image
*/
image_resolution?: number;
/**
* Whether to use scribble mode
*/
scribble?: boolean;
};

View File

@ -648,6 +648,13 @@ export type components = {
* @default 1
*/
end_step_percent: number;
/**
* Control Mode
* @description The contorl mode to use
* @default balanced
* @enum {string}
*/
control_mode?: "balanced" | "more_prompt" | "more_control" | "unbalanced";
};
/**
* ControlNetInvocation
@ -701,6 +708,13 @@ export type components = {
* @default 1
*/
end_step_percent?: number;
/**
* Control Mode
* @description The control mode used
* @default balanced
* @enum {string}
*/
control_mode?: "balanced" | "more_prompt" | "more_control" | "unbalanced";
};
/** ControlNetModelConfig */
ControlNetModelConfig: {
@ -2903,7 +2917,7 @@ export type components = {
/** ModelsList */
ModelsList: {
/** Models */
models: (components["schemas"]["StableDiffusion1ModelCheckpointConfig"] | components["schemas"]["StableDiffusion1ModelDiffusersConfig"] | components["schemas"]["VaeModelConfig"] | components["schemas"]["LoRAModelConfig"] | components["schemas"]["ControlNetModelConfig"] | components["schemas"]["TextualInversionModelConfig"] | components["schemas"]["StableDiffusion2ModelDiffusersConfig"] | components["schemas"]["StableDiffusion2ModelCheckpointConfig"])[];
models: (components["schemas"]["StableDiffusion1ModelDiffusersConfig"] | components["schemas"]["StableDiffusion1ModelCheckpointConfig"] | components["schemas"]["VaeModelConfig"] | components["schemas"]["LoRAModelConfig"] | components["schemas"]["ControlNetModelConfig"] | components["schemas"]["TextualInversionModelConfig"] | components["schemas"]["StableDiffusion2ModelCheckpointConfig"] | components["schemas"]["StableDiffusion2ModelDiffusersConfig"])[];
};
/**
* MultiplyInvocation
@ -4163,18 +4177,18 @@ export type components = {
*/
image?: components["schemas"]["ImageField"];
};
/**
* StableDiffusion1ModelFormat
* @description An enumeration.
* @enum {string}
*/
StableDiffusion1ModelFormat: "checkpoint" | "diffusers";
/**
* StableDiffusion2ModelFormat
* @description An enumeration.
* @enum {string}
*/
StableDiffusion2ModelFormat: "checkpoint" | "diffusers";
/**
* StableDiffusion1ModelFormat
* @description An enumeration.
* @enum {string}
*/
StableDiffusion1ModelFormat: "checkpoint" | "diffusers";
};
responses: never;
parameters: never;

View File

@ -1,81 +1,78 @@
import { components } from './schema';
type schemas = components['schemas'];
/**
* Types from the API, re-exported from the types generated by `openapi-typescript`.
*/
// Images
export type ImageDTO = components['schemas']['ImageDTO'];
export type BoardDTO = components['schemas']['BoardDTO'];
export type BoardChanges = components['schemas']['BoardChanges'];
export type ImageChanges = components['schemas']['ImageRecordChanges'];
export type ImageCategory = components['schemas']['ImageCategory'];
export type ResourceOrigin = components['schemas']['ResourceOrigin'];
export type ImageField = components['schemas']['ImageField'];
export type ImageDTO = schemas['ImageDTO'];
export type BoardDTO = schemas['BoardDTO'];
export type BoardChanges = schemas['BoardChanges'];
export type ImageChanges = schemas['ImageRecordChanges'];
export type ImageCategory = schemas['ImageCategory'];
export type ResourceOrigin = schemas['ResourceOrigin'];
export type ImageField = schemas['ImageField'];
export type OffsetPaginatedResults_BoardDTO_ =
components['schemas']['OffsetPaginatedResults_BoardDTO_'];
schemas['OffsetPaginatedResults_BoardDTO_'];
export type OffsetPaginatedResults_ImageDTO_ =
components['schemas']['OffsetPaginatedResults_ImageDTO_'];
schemas['OffsetPaginatedResults_ImageDTO_'];
// Models
export type ModelType = components['schemas']['ModelType'];
export type BaseModelType = components['schemas']['BaseModelType'];
export type PipelineModelField = components['schemas']['PipelineModelField'];
export type ModelsList = components['schemas']['ModelsList'];
export type ModelType = schemas['ModelType'];
export type BaseModelType = schemas['BaseModelType'];
export type PipelineModelField = schemas['PipelineModelField'];
export type ModelsList = schemas['ModelsList'];
// Graphs
export type Graph = components['schemas']['Graph'];
export type Edge = components['schemas']['Edge'];
export type GraphExecutionState = components['schemas']['GraphExecutionState'];
export type Graph = schemas['Graph'];
export type Edge = schemas['Edge'];
export type GraphExecutionState = schemas['GraphExecutionState'];
// General nodes
export type CollectInvocation = components['schemas']['CollectInvocation'];
export type IterateInvocation = components['schemas']['IterateInvocation'];
export type RangeInvocation = components['schemas']['RangeInvocation'];
export type RandomRangeInvocation =
components['schemas']['RandomRangeInvocation'];
export type RangeOfSizeInvocation =
components['schemas']['RangeOfSizeInvocation'];
export type InpaintInvocation = components['schemas']['InpaintInvocation'];
export type ImageResizeInvocation =
components['schemas']['ImageResizeInvocation'];
export type RandomIntInvocation = components['schemas']['RandomIntInvocation'];
export type CompelInvocation = components['schemas']['CompelInvocation'];
export type CollectInvocation = schemas['CollectInvocation'];
export type IterateInvocation = schemas['IterateInvocation'];
export type RangeInvocation = schemas['RangeInvocation'];
export type RandomRangeInvocation = schemas['RandomRangeInvocation'];
export type RangeOfSizeInvocation = schemas['RangeOfSizeInvocation'];
export type InpaintInvocation = schemas['InpaintInvocation'];
export type ImageResizeInvocation = schemas['ImageResizeInvocation'];
export type RandomIntInvocation = schemas['RandomIntInvocation'];
export type CompelInvocation = schemas['CompelInvocation'];
// ControlNet Nodes
export type ControlNetInvocation = schemas['ControlNetInvocation'];
export type CannyImageProcessorInvocation =
components['schemas']['CannyImageProcessorInvocation'];
schemas['CannyImageProcessorInvocation'];
export type ContentShuffleImageProcessorInvocation =
components['schemas']['ContentShuffleImageProcessorInvocation'];
schemas['ContentShuffleImageProcessorInvocation'];
export type HedImageProcessorInvocation =
components['schemas']['HedImageProcessorInvocation'];
schemas['HedImageProcessorInvocation'];
export type LineartAnimeImageProcessorInvocation =
components['schemas']['LineartAnimeImageProcessorInvocation'];
schemas['LineartAnimeImageProcessorInvocation'];
export type LineartImageProcessorInvocation =
components['schemas']['LineartImageProcessorInvocation'];
schemas['LineartImageProcessorInvocation'];
export type MediapipeFaceProcessorInvocation =
components['schemas']['MediapipeFaceProcessorInvocation'];
schemas['MediapipeFaceProcessorInvocation'];
export type MidasDepthImageProcessorInvocation =
components['schemas']['MidasDepthImageProcessorInvocation'];
schemas['MidasDepthImageProcessorInvocation'];
export type MlsdImageProcessorInvocation =
components['schemas']['MlsdImageProcessorInvocation'];
schemas['MlsdImageProcessorInvocation'];
export type NormalbaeImageProcessorInvocation =
components['schemas']['NormalbaeImageProcessorInvocation'];
schemas['NormalbaeImageProcessorInvocation'];
export type OpenposeImageProcessorInvocation =
components['schemas']['OpenposeImageProcessorInvocation'];
schemas['OpenposeImageProcessorInvocation'];
export type PidiImageProcessorInvocation =
components['schemas']['PidiImageProcessorInvocation'];
schemas['PidiImageProcessorInvocation'];
export type ZoeDepthImageProcessorInvocation =
components['schemas']['ZoeDepthImageProcessorInvocation'];
schemas['ZoeDepthImageProcessorInvocation'];
// Node Outputs
export type ImageOutput = components['schemas']['ImageOutput'];
export type MaskOutput = components['schemas']['MaskOutput'];
export type PromptOutput = components['schemas']['PromptOutput'];
export type IterateInvocationOutput =
components['schemas']['IterateInvocationOutput'];
export type CollectInvocationOutput =
components['schemas']['CollectInvocationOutput'];
export type LatentsOutput = components['schemas']['LatentsOutput'];
export type GraphInvocationOutput =
components['schemas']['GraphInvocationOutput'];
export type ImageOutput = schemas['ImageOutput'];
export type MaskOutput = schemas['MaskOutput'];
export type PromptOutput = schemas['PromptOutput'];
export type IterateInvocationOutput = schemas['IterateInvocationOutput'];
export type CollectInvocationOutput = schemas['CollectInvocationOutput'];
export type LatentsOutput = schemas['LatentsOutput'];
export type GraphInvocationOutput = schemas['GraphInvocationOutput'];

File diff suppressed because it is too large Load Diff