diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index 0943b78bf8..1b65326f6e 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -546,11 +546,13 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): # Handle ControlNet(s) and T2I-Adapter(s) down_block_additional_residuals = None mid_block_additional_residual = None - if control_data is not None and t2i_adapter_data is not None: - # TODO(ryand): This is a limitation of the UNet2DConditionModel API, not a fundamental incompatibility - # between ControlNets and T2I-Adapters. We will try to fix this upstream in diffusers. - raise Exception("ControlNet(s) and T2I-Adapter(s) cannot be used simultaneously (yet).") - elif control_data is not None: + down_intrablock_additional_residuals = None + # if control_data is not None and t2i_adapter_data is not None: + # TODO(ryand): This is a limitation of the UNet2DConditionModel API, not a fundamental incompatibility + # between ControlNets and T2I-Adapters. We will try to fix this upstream in diffusers. + # raise Exception("ControlNet(s) and T2I-Adapter(s) cannot be used simultaneously (yet).") + # elif control_data is not None: + if control_data is not None: down_block_additional_residuals, mid_block_additional_residual = self.invokeai_diffuser.do_controlnet_step( control_data=control_data, sample=latent_model_input, @@ -559,7 +561,8 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): total_step_count=total_step_count, conditioning_data=conditioning_data, ) - elif t2i_adapter_data is not None: + # elif t2i_adapter_data is not None: + if t2i_adapter_data is not None: accum_adapter_state = None for single_t2i_adapter_data in t2i_adapter_data: # Determine the T2I-Adapter weights for the current denoising step. @@ -584,7 +587,8 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): for idx, value in enumerate(single_t2i_adapter_data.adapter_state): accum_adapter_state[idx] += value * t2i_adapter_weight - down_block_additional_residuals = accum_adapter_state + # down_block_additional_residuals = accum_adapter_state + down_intrablock_additional_residuals = accum_adapter_state uc_noise_pred, c_noise_pred = self.invokeai_diffuser.do_unet_step( sample=latent_model_input, @@ -593,8 +597,9 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): total_step_count=total_step_count, conditioning_data=conditioning_data, # extra: - down_block_additional_residuals=down_block_additional_residuals, - mid_block_additional_residual=mid_block_additional_residual, + down_block_additional_residuals=down_block_additional_residuals, # for ControlNet + mid_block_additional_residual=mid_block_additional_residual, # for ControlNet + down_intrablock_additional_residuals=down_intrablock_additional_residuals, # for T2I-Adapter ) guidance_scale = conditioning_data.guidance_scale diff --git a/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py b/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py index 943fe7b307..b7c0058fe9 100644 --- a/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py +++ b/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py @@ -260,7 +260,6 @@ class InvokeAIDiffuserComponent: conditioning_data, **kwargs, ) - else: ( unconditioned_next_x, @@ -410,6 +409,15 @@ class InvokeAIDiffuserComponent: uncond_down_block.append(_uncond_down) cond_down_block.append(_cond_down) + uncond_down_intrablock, cond_down_intrablock = None, None + down_intrablock_additional_residuals = kwargs.pop("down_intrablock_additional_residuals", None) + if down_intrablock_additional_residuals is not None: + uncond_down_intrablock, cond_down_intrablock = [], [] + for down_intrablock in down_intrablock_additional_residuals: + _uncond_down, _cond_down = down_intrablock.chunk(2) + uncond_down_intrablock.append(_uncond_down) + cond_down_intrablock.append(_cond_down) + uncond_mid_block, cond_mid_block = None, None mid_block_additional_residual = kwargs.pop("mid_block_additional_residual", None) if mid_block_additional_residual is not None: @@ -441,6 +449,7 @@ class InvokeAIDiffuserComponent: cross_attention_kwargs=cross_attention_kwargs, down_block_additional_residuals=uncond_down_block, mid_block_additional_residual=uncond_mid_block, + down_intrablock_additional_residuals=uncond_down_intrablock, added_cond_kwargs=added_cond_kwargs, **kwargs, ) @@ -470,6 +479,7 @@ class InvokeAIDiffuserComponent: cross_attention_kwargs=cross_attention_kwargs, down_block_additional_residuals=cond_down_block, mid_block_additional_residual=cond_mid_block, + down_intrablock_additional_residuals=cond_down_intrablock, added_cond_kwargs=added_cond_kwargs, **kwargs, ) @@ -494,6 +504,15 @@ class InvokeAIDiffuserComponent: uncond_down_block.append(_uncond_down) cond_down_block.append(_cond_down) + uncond_down_intrablock, cond_down_intrablock = None, None + down_intrablock_additional_residuals = kwargs.pop("down_intrablock_additional_residuals", None) + if down_intrablock_additional_residuals is not None: + uncond_down_intrablock, cond_down_intrablock = [], [] + for down_intrablock in down_intrablock_additional_residuals: + _uncond_down, _cond_down = down_intrablock.chunk(2) + uncond_down_intrablock.append(_uncond_down) + cond_down_intrablock.append(_cond_down) + uncond_mid_block, cond_mid_block = None, None mid_block_additional_residual = kwargs.pop("mid_block_additional_residual", None) if mid_block_additional_residual is not None: @@ -522,6 +541,7 @@ class InvokeAIDiffuserComponent: {"swap_cross_attn_context": cross_attn_processor_context}, down_block_additional_residuals=uncond_down_block, mid_block_additional_residual=uncond_mid_block, + down_intrablock_additional_residuals=uncond_down_intrablock, added_cond_kwargs=added_cond_kwargs, **kwargs, ) @@ -541,6 +561,7 @@ class InvokeAIDiffuserComponent: {"swap_cross_attn_context": cross_attn_processor_context}, down_block_additional_residuals=cond_down_block, mid_block_additional_residual=cond_mid_block, + down_intrablock_additional_residuals=cond_down_intrablock, added_cond_kwargs=added_cond_kwargs, **kwargs, ) diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts index cbc88966a7..772ea216c0 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts @@ -12,6 +12,7 @@ import { addFirstListImagesListener } from './listeners/addFirstListImagesListen import { addAnyEnqueuedListener } from './listeners/anyEnqueued'; import { addAppConfigReceivedListener } from './listeners/appConfigReceived'; import { addAppStartedListener } from './listeners/appStarted'; +import { addBatchEnqueuedListener } from './listeners/batchEnqueued'; import { addDeleteBoardAndImagesFulfilledListener } from './listeners/boardAndImagesDeleted'; import { addBoardIdSelectedListener } from './listeners/boardIdSelected'; import { addCanvasCopiedToClipboardListener } from './listeners/canvasCopiedToClipboard'; @@ -71,8 +72,6 @@ import { addStagingAreaImageSavedListener } from './listeners/stagingAreaImageSa import { addTabChangedListener } from './listeners/tabChanged'; import { addUpscaleRequestedListener } from './listeners/upscaleRequested'; import { addWorkflowLoadedListener } from './listeners/workflowLoaded'; -import { addBatchEnqueuedListener } from './listeners/batchEnqueued'; -import { addControlAdapterAddedOrEnabledListener } from './listeners/controlAdapterAddedOrEnabled'; export const listenerMiddleware = createListenerMiddleware(); @@ -200,7 +199,3 @@ addTabChangedListener(); // Dynamic prompts addDynamicPromptsListener(); - -// Display toast when controlnet or t2i adapter enabled -// TODO: Remove when they can both be enabled at same time -addControlAdapterAddedOrEnabledListener(); diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterAddedOrEnabled.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterAddedOrEnabled.ts deleted file mode 100644 index bc5387c1fb..0000000000 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterAddedOrEnabled.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { isAnyOf } from '@reduxjs/toolkit'; -import { - controlAdapterAdded, - controlAdapterAddedFromImage, - controlAdapterIsEnabledChanged, - controlAdapterRecalled, - selectControlAdapterAll, - selectControlAdapterById, -} from 'features/controlAdapters/store/controlAdaptersSlice'; -import { ControlAdapterType } from 'features/controlAdapters/store/types'; -import { addToast } from 'features/system/store/systemSlice'; -import i18n from 'i18n'; -import { startAppListening } from '..'; - -const isAnyControlAdapterAddedOrEnabled = isAnyOf( - controlAdapterAdded, - controlAdapterAddedFromImage, - controlAdapterRecalled, - controlAdapterIsEnabledChanged -); - -/** - * Until we can have both controlnet and t2i adapter enabled at once, they are mutually exclusive - * This displays a toast when one is enabled and the other is already enabled, or one is added - * with the other enabled - */ -export const addControlAdapterAddedOrEnabledListener = () => { - startAppListening({ - matcher: isAnyControlAdapterAddedOrEnabled, - effect: async (action, { dispatch, getOriginalState }) => { - const controlAdapters = getOriginalState().controlAdapters; - - const hasEnabledControlNets = selectControlAdapterAll( - controlAdapters - ).some((ca) => ca.isEnabled && ca.type === 'controlnet'); - - const hasEnabledT2IAdapters = selectControlAdapterAll( - controlAdapters - ).some((ca) => ca.isEnabled && ca.type === 't2i_adapter'); - - let caType: ControlAdapterType | null = null; - - if (controlAdapterAdded.match(action)) { - caType = action.payload.type; - } - - if (controlAdapterAddedFromImage.match(action)) { - caType = action.payload.type; - } - - if (controlAdapterRecalled.match(action)) { - caType = action.payload.type; - } - - if (controlAdapterIsEnabledChanged.match(action)) { - const _caType = selectControlAdapterById( - controlAdapters, - action.payload.id - )?.type; - if (!_caType) { - return; - } - caType = _caType; - } - - if ( - (caType === 'controlnet' && hasEnabledT2IAdapters) || - (caType === 't2i_adapter' && hasEnabledControlNets) - ) { - const title = - caType === 'controlnet' - ? i18n.t('controlnet.controlNetEnabledT2IDisabled') - : i18n.t('controlnet.t2iEnabledControlNetDisabled'); - - const description = i18n.t('controlnet.controlNetT2IMutexDesc'); - - dispatch( - addToast({ - title, - description, - status: 'warning', - }) - ); - } - }, - }); -}; diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts b/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts index a3645fad9d..9e293f1104 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts @@ -88,61 +88,6 @@ export const selectValidT2IAdapters = (controlAdapters: ControlAdaptersState) => (ca.processorType === 'none' && Boolean(ca.controlImage))) ); -// TODO: I think we can safely remove this? -// const disableAllIPAdapters = ( -// state: ControlAdaptersState, -// exclude?: string -// ) => { -// const updates: Update[] = selectAllIPAdapters(state) -// .filter((ca) => ca.id !== exclude) -// .map((ca) => ({ -// id: ca.id, -// changes: { isEnabled: false }, -// })); -// caAdapter.updateMany(state, updates); -// }; - -const disableAllControlNets = ( - state: ControlAdaptersState, - exclude?: string -) => { - const updates: Update[] = selectAllControlNets(state) - .filter((ca) => ca.id !== exclude) - .map((ca) => ({ - id: ca.id, - changes: { isEnabled: false }, - })); - caAdapter.updateMany(state, updates); -}; - -const disableAllT2IAdapters = ( - state: ControlAdaptersState, - exclude?: string -) => { - const updates: Update[] = selectAllT2IAdapters(state) - .filter((ca) => ca.id !== exclude) - .map((ca) => ({ - id: ca.id, - changes: { isEnabled: false }, - })); - caAdapter.updateMany(state, updates); -}; - -const disableIncompatibleControlAdapters = ( - state: ControlAdaptersState, - type: ControlAdapterType, - exclude?: string -) => { - if (type === 'controlnet') { - // we cannot do controlnet + t2i adapter, if we are enabled a controlnet, disable all t2is - disableAllT2IAdapters(state, exclude); - } - if (type === 't2i_adapter') { - // we cannot do controlnet + t2i adapter, if we are enabled a t2i, disable controlnets - disableAllControlNets(state, exclude); - } -}; - export const controlAdaptersSlice = createSlice({ name: 'controlAdapters', initialState: initialControlAdapterState, @@ -158,7 +103,6 @@ export const controlAdaptersSlice = createSlice({ ) => { const { id, type, overrides } = action.payload; caAdapter.addOne(state, buildControlAdapter(id, type, overrides)); - disableIncompatibleControlAdapters(state, type, id); }, prepare: ({ type, @@ -175,8 +119,6 @@ export const controlAdaptersSlice = createSlice({ action: PayloadAction ) => { caAdapter.addOne(state, action.payload); - const { type, id } = action.payload; - disableIncompatibleControlAdapters(state, type, id); }, controlAdapterDuplicated: { reducer: ( @@ -196,8 +138,6 @@ export const controlAdaptersSlice = createSlice({ isEnabled: true, }); caAdapter.addOne(state, newControlAdapter); - const { type } = newControlAdapter; - disableIncompatibleControlAdapters(state, type, newId); }, prepare: (id: string) => { return { payload: { id, newId: uuidv4() } }; @@ -217,7 +157,6 @@ export const controlAdaptersSlice = createSlice({ state, buildControlAdapter(id, type, { controlImage }) ); - disableIncompatibleControlAdapters(state, type, id); }, prepare: (payload: { type: ControlAdapterType; @@ -235,12 +174,6 @@ export const controlAdaptersSlice = createSlice({ ) => { const { id, isEnabled } = action.payload; caAdapter.updateOne(state, { id, changes: { isEnabled } }); - if (isEnabled) { - // we are enabling a control adapter. due to limitations in the current system, we may need to disable other adapters - // TODO: disable when multiple IP adapters are supported - const ca = selectControlAdapterById(state, id); - ca && disableIncompatibleControlAdapters(state, ca.type, id); - } }, controlAdapterImageChanged: ( state, diff --git a/invokeai/frontend/web/src/services/api/schema.d.ts b/invokeai/frontend/web/src/services/api/schema.d.ts index 6ff9d49bf9..8ac2e2dcb2 100644 --- a/invokeai/frontend/web/src/services/api/schema.d.ts +++ b/invokeai/frontend/web/src/services/api/schema.d.ts @@ -8808,11 +8808,11 @@ export type components = { ui_order: number | null; }; /** - * StableDiffusionOnnxModelFormat + * IPAdapterModelFormat * @description An enumeration. * @enum {string} */ - StableDiffusionOnnxModelFormat: "olive" | "onnx"; + IPAdapterModelFormat: "invokeai"; /** * IPAdapterModelFormat * @description An enumeration. @@ -8832,11 +8832,11 @@ export type components = { */ StableDiffusion2ModelFormat: "checkpoint" | "diffusers"; /** - * CLIPVisionModelFormat + * StableDiffusionOnnxModelFormat * @description An enumeration. * @enum {string} */ - CLIPVisionModelFormat: "diffusers"; + StableDiffusionOnnxModelFormat: "olive" | "onnx"; /** * StableDiffusion1ModelFormat * @description An enumeration.