diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 3d1439f7db..b3ac3973bf 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -586,13 +586,6 @@ class DenoiseLatentsInvocation(BaseInvocation): unet: UNet2DConditionModel, scheduler: Scheduler, ) -> StableDiffusionGeneratorPipeline: - # TODO: - # configure_model_padding( - # unet, - # self.seamless, - # self.seamless_axes, - # ) - class FakeVae: class FakeVaeConfig: def __init__(self) -> None: diff --git a/invokeai/backend/image_util/__init__.py b/invokeai/backend/image_util/__init__.py index dec2a92150..f45af9feb4 100644 --- a/invokeai/backend/image_util/__init__.py +++ b/invokeai/backend/image_util/__init__.py @@ -4,5 +4,4 @@ Initialization file for invokeai.backend.image_util methods. from .infill_methods.patchmatch import PatchMatch # noqa: F401 from .pngwriter import PngWriter, PromptFormatter, retrieve_metadata, write_metadata # noqa: F401 -from .seamless import configure_model_padding # noqa: F401 from .util import InitImageResizer, make_grid # noqa: F401 diff --git a/invokeai/backend/image_util/seamless.py b/invokeai/backend/image_util/seamless.py deleted file mode 100644 index 8a2580bfcc..0000000000 --- a/invokeai/backend/image_util/seamless.py +++ /dev/null @@ -1,52 +0,0 @@ -import torch.nn as nn - - -def _conv_forward_asymmetric(self, input, weight, bias): - """ - Patch for Conv2d._conv_forward that supports asymmetric padding - """ - working = nn.functional.pad(input, self.asymmetric_padding["x"], mode=self.asymmetric_padding_mode["x"]) - working = nn.functional.pad(working, self.asymmetric_padding["y"], mode=self.asymmetric_padding_mode["y"]) - return nn.functional.conv2d( - working, - weight, - bias, - self.stride, - nn.modules.utils._pair(0), - self.dilation, - self.groups, - ) - - -def configure_model_padding(model, seamless, seamless_axes): - """ - Modifies the 2D convolution layers to use a circular padding mode based on - the `seamless` and `seamless_axes` options. - """ - # TODO: get an explicit interface for this in diffusers: https://github.com/huggingface/diffusers/issues/556 - for m in model.modules(): - if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): - if seamless: - m.asymmetric_padding_mode = {} - m.asymmetric_padding = {} - m.asymmetric_padding_mode["x"] = "circular" if ("x" in seamless_axes) else "constant" - m.asymmetric_padding["x"] = ( - m._reversed_padding_repeated_twice[0], - m._reversed_padding_repeated_twice[1], - 0, - 0, - ) - m.asymmetric_padding_mode["y"] = "circular" if ("y" in seamless_axes) else "constant" - m.asymmetric_padding["y"] = ( - 0, - 0, - m._reversed_padding_repeated_twice[2], - m._reversed_padding_repeated_twice[3], - ) - m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) - else: - m._conv_forward = nn.Conv2d._conv_forward.__get__(m, nn.Conv2d) - if hasattr(m, "asymmetric_padding_mode"): - del m.asymmetric_padding_mode - if hasattr(m, "asymmetric_padding"): - del m.asymmetric_padding diff --git a/invokeai/backend/stable_diffusion/seamless.py b/invokeai/backend/stable_diffusion/seamless.py index 2e22c19d0e..23ed978c6d 100644 --- a/invokeai/backend/stable_diffusion/seamless.py +++ b/invokeai/backend/stable_diffusion/seamless.py @@ -1,89 +1,51 @@ -from __future__ import annotations - from contextlib import contextmanager -from typing import Callable, List, Union +from typing import Callable, List, Optional, Tuple, Union +import torch import torch.nn as nn from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL from diffusers.models.autoencoders.autoencoder_tiny import AutoencoderTiny +from diffusers.models.lora import LoRACompatibleConv from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel -def _conv_forward_asymmetric(self, input, weight, bias): - """ - Patch for Conv2d._conv_forward that supports asymmetric padding - """ - working = nn.functional.pad(input, self.asymmetric_padding["x"], mode=self.asymmetric_padding_mode["x"]) - working = nn.functional.pad(working, self.asymmetric_padding["y"], mode=self.asymmetric_padding_mode["y"]) - return nn.functional.conv2d( - working, - weight, - bias, - self.stride, - nn.modules.utils._pair(0), - self.dilation, - self.groups, - ) - - @contextmanager def set_seamless(model: Union[UNet2DConditionModel, AutoencoderKL, AutoencoderTiny], seamless_axes: List[str]): if not seamless_axes: yield return - # Callable: (input: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor - to_restore: list[tuple[nn.Conv2d | nn.ConvTranspose2d, Callable]] = [] + # override conv_forward + # https://github.com/huggingface/diffusers/issues/556#issuecomment-1993287019 + def _conv_forward_asymmetric(self, input: torch.Tensor, weight: torch.Tensor, bias: Optional[torch.Tensor] = None): + self.paddingX = (self._reversed_padding_repeated_twice[0], self._reversed_padding_repeated_twice[1], 0, 0) + self.paddingY = (0, 0, self._reversed_padding_repeated_twice[2], self._reversed_padding_repeated_twice[3]) + working = torch.nn.functional.pad(input, self.paddingX, mode=x_mode) + working = torch.nn.functional.pad(working, self.paddingY, mode=y_mode) + return torch.nn.functional.conv2d( + working, weight, bias, self.stride, torch.nn.modules.utils._pair(0), self.dilation, self.groups + ) + + original_layers: List[Tuple[nn.Conv2d, Callable]] = [] + try: - # Hard coded to skip down block layers, allowing for seamless tiling at the expense of prompt adherence - skipped_layers = 1 - for m_name, m in model.named_modules(): - if not isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): - continue + x_mode = "circular" if "x" in seamless_axes else "constant" + y_mode = "circular" if "y" in seamless_axes else "constant" - if isinstance(model, UNet2DConditionModel) and m_name.startswith("down_blocks.") and ".resnets." in m_name: - # down_blocks.1.resnets.1.conv1 - _, block_num, _, resnet_num, submodule_name = m_name.split(".") - block_num = int(block_num) - resnet_num = int(resnet_num) + conv_layers: List[torch.nn.Conv2d] = [] - if block_num >= len(model.down_blocks) - skipped_layers: - continue + for module in model.modules(): + if isinstance(module, torch.nn.Conv2d): + conv_layers.append(module) - # Skip the second resnet (could be configurable) - if resnet_num > 0: - continue - - # Skip Conv2d layers (could be configurable) - if submodule_name == "conv2": - continue - - m.asymmetric_padding_mode = {} - m.asymmetric_padding = {} - m.asymmetric_padding_mode["x"] = "circular" if ("x" in seamless_axes) else "constant" - m.asymmetric_padding["x"] = ( - m._reversed_padding_repeated_twice[0], - m._reversed_padding_repeated_twice[1], - 0, - 0, - ) - m.asymmetric_padding_mode["y"] = "circular" if ("y" in seamless_axes) else "constant" - m.asymmetric_padding["y"] = ( - 0, - 0, - m._reversed_padding_repeated_twice[2], - m._reversed_padding_repeated_twice[3], - ) - - to_restore.append((m, m._conv_forward)) - m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) + for layer in conv_layers: + if isinstance(layer, LoRACompatibleConv) and layer.lora_layer is None: + layer.lora_layer = lambda *x: 0 + original_layers.append((layer, layer._conv_forward)) + layer._conv_forward = _conv_forward_asymmetric.__get__(layer, torch.nn.Conv2d) yield finally: - for module, orig_conv_forward in to_restore: - module._conv_forward = orig_conv_forward - if hasattr(module, "asymmetric_padding_mode"): - del module.asymmetric_padding_mode - if hasattr(module, "asymmetric_padding"): - del module.asymmetric_padding + for layer, orig_conv_forward in original_layers: + layer._conv_forward = orig_conv_forward diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index b36fe0e26c..7aa4b03b8c 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -261,7 +261,6 @@ "queue": "Queue", "queueFront": "Add to Front of Queue", "queueBack": "Add to Queue", - "queueCountPrediction": "{{promptsCount}} prompts \u00d7 {{iterations}} iterations -> {{count}} generations", "queueEmpty": "Queue Empty", "enqueueing": "Queueing Batch", "resume": "Resume", @@ -314,7 +313,13 @@ "batchFailedToQueue": "Failed to Queue Batch", "graphQueued": "Graph queued", "graphFailedToQueue": "Failed to queue graph", - "openQueue": "Open Queue" + "openQueue": "Open Queue", + "prompts_one": "Prompt", + "prompts_other": "Prompts", + "iterations_one": "Iteration", + "iterations_other": "Iterations", + "generations_one": "Generation", + "generations_other": "Generations" }, "invocationCache": { "invocationCache": "Invocation Cache", @@ -934,7 +939,20 @@ "noModelSelected": "No model selected", "noPrompts": "No prompts generated", "noNodesInGraph": "No nodes in graph", - "systemDisconnected": "System disconnected" + "systemDisconnected": "System disconnected", + "layer": { + "initialImageNoImageSelected": "no initial image selected", + "controlAdapterNoModelSelected": "no Control Adapter model selected", + "controlAdapterIncompatibleBaseModel": "incompatible Control Adapter base model", + "controlAdapterNoImageSelected": "no Control Adapter image selected", + "controlAdapterImageNotProcessed": "Control Adapter image not processed", + "t2iAdapterIncompatibleDimensions": "T2I Adapter requires image dimension to be multiples of 64", + "ipAdapterNoModelSelected": "no IP adapter selected", + "ipAdapterIncompatibleBaseModel": "incompatible IP Adapter base model", + "ipAdapterNoImageSelected": "no IP Adapter image selected", + "rgNoPromptsOrIPAdapters": "no text prompts or IP Adapters", + "rgNoRegion": "no region selected" + } }, "maskBlur": "Mask Blur", "negativePromptPlaceholder": "Negative Prompt", @@ -945,8 +963,6 @@ "positivePromptPlaceholder": "Positive Prompt", "globalPositivePromptPlaceholder": "Global Positive Prompt", "iterations": "Iterations", - "iterationsWithCount_one": "{{count}} Iteration", - "iterationsWithCount_other": "{{count}} Iterations", "scale": "Scale", "scaleBeforeProcessing": "Scale Before Processing", "scaledHeight": "Scaled H", diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterPreprocessor.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterPreprocessor.ts index 57a7cccf6c..a1f7ebcca1 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterPreprocessor.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlAdapterPreprocessor.ts @@ -1,13 +1,14 @@ import { isAnyOf } from '@reduxjs/toolkit'; import { logger } from 'app/logging/logger'; import type { AppStartListening } from 'app/store/middleware/listenerMiddleware'; +import type { AppDispatch } from 'app/store/store'; import { parseify } from 'common/util/serialize'; import { caLayerImageChanged, - caLayerIsProcessingImageChanged, caLayerModelChanged, caLayerProcessedImageChanged, caLayerProcessorConfigChanged, + caLayerProcessorPendingBatchIdChanged, caLayerRecalled, isControlAdapterLayer, } from 'features/controlLayers/store/controlLayersSlice'; @@ -15,47 +16,39 @@ import { CA_PROCESSOR_DATA } from 'features/controlLayers/util/controlAdapters'; import { isImageOutput } from 'features/nodes/types/common'; import { addToast } from 'features/system/store/systemSlice'; import { t } from 'i18next'; -import { isEqual } from 'lodash-es'; -import { imagesApi } from 'services/api/endpoints/images'; +import { getImageDTO } from 'services/api/endpoints/images'; import { queueApi } from 'services/api/endpoints/queue'; -import type { BatchConfig, ImageDTO } from 'services/api/types'; +import type { BatchConfig } from 'services/api/types'; import { socketInvocationComplete } from 'services/events/actions'; +import { assert } from 'tsafe'; const matcher = isAnyOf(caLayerImageChanged, caLayerProcessorConfigChanged, caLayerModelChanged, caLayerRecalled); const DEBOUNCE_MS = 300; const log = logger('session'); +/** + * Simple helper to cancel a batch and reset the pending batch ID + */ +const cancelProcessorBatch = async (dispatch: AppDispatch, layerId: string, batchId: string) => { + const req = dispatch(queueApi.endpoints.cancelByBatchIds.initiate({ batch_ids: [batchId] })); + log.trace({ batchId }, 'Cancelling existing preprocessor batch'); + try { + await req.unwrap(); + } catch { + // no-op + } finally { + req.reset(); + // Always reset the pending batch ID - the cancel req could fail if the batch doesn't exist + dispatch(caLayerProcessorPendingBatchIdChanged({ layerId, batchId: null })); + } +}; + export const addControlAdapterPreprocessor = (startAppListening: AppStartListening) => { startAppListening({ matcher, - effect: async (action, { dispatch, getState, getOriginalState, cancelActiveListeners, delay, take }) => { + effect: async (action, { dispatch, getState, cancelActiveListeners, delay, take, signal }) => { const layerId = caLayerRecalled.match(action) ? action.payload.id : action.payload.layerId; - const precheckLayerOriginal = getOriginalState() - .controlLayers.present.layers.filter(isControlAdapterLayer) - .find((l) => l.id === layerId); - const precheckLayer = getState() - .controlLayers.present.layers.filter(isControlAdapterLayer) - .find((l) => l.id === layerId); - - // Conditions to bail - const layerDoesNotExist = !precheckLayer; - const layerHasNoImage = !precheckLayer?.controlAdapter.image; - const layerHasNoProcessorConfig = !precheckLayer?.controlAdapter.processorConfig; - const layerIsAlreadyProcessingImage = precheckLayer?.controlAdapter.isProcessingImage; - const areImageAndProcessorUnchanged = - isEqual(precheckLayer?.controlAdapter.image, precheckLayerOriginal?.controlAdapter.image) && - isEqual(precheckLayer?.controlAdapter.processorConfig, precheckLayerOriginal?.controlAdapter.processorConfig); - - if ( - layerDoesNotExist || - layerHasNoImage || - layerHasNoProcessorConfig || - areImageAndProcessorUnchanged || - layerIsAlreadyProcessingImage - ) { - return; - } // Cancel any in-progress instances of this listener cancelActiveListeners(); @@ -63,19 +56,31 @@ export const addControlAdapterPreprocessor = (startAppListening: AppStartListeni // Delay before starting actual work await delay(DEBOUNCE_MS); - dispatch(caLayerIsProcessingImageChanged({ layerId, isProcessingImage: true })); // Double-check that we are still eligible for processing const state = getState(); const layer = state.controlLayers.present.layers.filter(isControlAdapterLayer).find((l) => l.id === layerId); - const image = layer?.controlAdapter.image; - const config = layer?.controlAdapter.processorConfig; // If we have no image or there is no processor config, bail - if (!layer || !image || !config) { + if (!layer) { return; } + const image = layer.controlAdapter.image; + const config = layer.controlAdapter.processorConfig; + + if (!image || !config) { + // The user has reset the image or config, so we should clear the processed image + dispatch(caLayerProcessedImageChanged({ layerId, imageDTO: null })); + } + + // At this point, the user has stopped fiddling with the processor settings and there is a processor selected. + + // If there is a pending processor batch, cancel it. + if (layer.controlAdapter.processorPendingBatchId) { + cancelProcessorBatch(dispatch, layerId, layer.controlAdapter.processorPendingBatchId); + } + // @ts-expect-error: TS isn't able to narrow the typing of buildNode and `config` will error... const processorNode = CA_PROCESSOR_DATA[config.type].buildNode(image, config); const enqueueBatchArg: BatchConfig = { @@ -83,7 +88,11 @@ export const addControlAdapterPreprocessor = (startAppListening: AppStartListeni batch: { graph: { nodes: { - [processorNode.id]: { ...processorNode, is_intermediate: true }, + [processorNode.id]: { + ...processorNode, + // Control images are always intermediate - do not save to gallery + is_intermediate: true, + }, }, edges: [], }, @@ -91,16 +100,21 @@ export const addControlAdapterPreprocessor = (startAppListening: AppStartListeni }, }; + // Kick off the processor batch + const req = dispatch( + queueApi.endpoints.enqueueBatch.initiate(enqueueBatchArg, { + fixedCacheKey: 'enqueueBatch', + }) + ); + try { - const req = dispatch( - queueApi.endpoints.enqueueBatch.initiate(enqueueBatchArg, { - fixedCacheKey: 'enqueueBatch', - }) - ); const enqueueResult = await req.unwrap(); - req.reset(); + // TODO(psyche): Update the pydantic models, pretty sure we will _always_ have a batch_id here, but the model says it's optional + assert(enqueueResult.batch.batch_id, 'Batch ID not returned from queue'); + dispatch(caLayerProcessorPendingBatchIdChanged({ layerId, batchId: enqueueResult.batch.batch_id })); log.debug({ enqueueResult: parseify(enqueueResult) }, t('queue.graphQueued')); + // Wait for the processor node to complete const [invocationCompleteAction] = await take( (action): action is ReturnType => socketInvocationComplete.match(action) && @@ -109,47 +123,52 @@ export const addControlAdapterPreprocessor = (startAppListening: AppStartListeni ); // We still have to check the output type - if (isImageOutput(invocationCompleteAction.payload.data.result)) { - const { image_name } = invocationCompleteAction.payload.data.result.image; + assert( + isImageOutput(invocationCompleteAction.payload.data.result), + `Processor did not return an image output, got: ${invocationCompleteAction.payload.data.result}` + ); + const { image_name } = invocationCompleteAction.payload.data.result.image; - // Wait for the ImageDTO to be received - const [{ payload }] = await take( - (action) => - imagesApi.endpoints.getImageDTO.matchFulfilled(action) && action.payload.image_name === image_name - ); + const imageDTO = await getImageDTO(image_name); + assert(imageDTO, "Failed to fetch processor output's image DTO"); - const imageDTO = payload as ImageDTO; - - log.debug({ layerId, imageDTO }, 'ControlNet image processed'); - - // Update the processed image in the store - dispatch( - caLayerProcessedImageChanged({ - layerId, - imageDTO, - }) - ); - dispatch(caLayerIsProcessingImageChanged({ layerId, isProcessingImage: false })); - } + // Whew! We made it. Update the layer with the processed image + log.debug({ layerId, imageDTO }, 'ControlNet image processed'); + dispatch(caLayerProcessedImageChanged({ layerId, imageDTO })); + dispatch(caLayerProcessorPendingBatchIdChanged({ layerId, batchId: null })); } catch (error) { - log.error({ enqueueBatchArg: parseify(enqueueBatchArg) }, t('queue.graphFailedToQueue')); - dispatch(caLayerIsProcessingImageChanged({ layerId, isProcessingImage: false })); + if (signal.aborted) { + // The listener was canceled - we need to cancel the pending processor batch, if there is one (could have changed by now). + const pendingBatchId = getState() + .controlLayers.present.layers.filter(isControlAdapterLayer) + .find((l) => l.id === layerId)?.controlAdapter.processorPendingBatchId; + if (pendingBatchId) { + cancelProcessorBatch(dispatch, layerId, pendingBatchId); + } + log.trace('Control Adapter preprocessor cancelled'); + } else { + // Some other error condition... + console.log(error); + log.error({ enqueueBatchArg: parseify(enqueueBatchArg) }, t('queue.graphFailedToQueue')); - if (error instanceof Object) { - if ('data' in error && 'status' in error) { - if (error.status === 403) { - dispatch(caLayerImageChanged({ layerId, imageDTO: null })); - return; + if (error instanceof Object) { + if ('data' in error && 'status' in error) { + if (error.status === 403) { + dispatch(caLayerImageChanged({ layerId, imageDTO: null })); + return; + } } } - } - dispatch( - addToast({ - title: t('queue.graphFailedToQueue'), - status: 'error', - }) - ); + dispatch( + addToast({ + title: t('queue.graphFailedToQueue'), + status: 'error', + }) + ); + } + } finally { + req.reset(); } }, }); diff --git a/invokeai/frontend/web/src/common/hooks/useGroupedModelCombobox.ts b/invokeai/frontend/web/src/common/hooks/useGroupedModelCombobox.ts index 55887eb3be..5b57fcd2bb 100644 --- a/invokeai/frontend/web/src/common/hooks/useGroupedModelCombobox.ts +++ b/invokeai/frontend/web/src/common/hooks/useGroupedModelCombobox.ts @@ -13,6 +13,7 @@ type UseGroupedModelComboboxArg = { onChange: (value: T | null) => void; getIsDisabled?: (model: T) => boolean; isLoading?: boolean; + groupByType?: boolean; }; type UseGroupedModelComboboxReturn = { @@ -23,17 +24,21 @@ type UseGroupedModelComboboxReturn = { noOptionsMessage: () => string; }; +const groupByBaseFunc = (model: T) => model.base.toUpperCase(); +const groupByBaseAndTypeFunc = (model: T) => + `${model.base.toUpperCase()} / ${model.type.replaceAll('_', ' ').toUpperCase()}`; + export const useGroupedModelCombobox = ( arg: UseGroupedModelComboboxArg ): UseGroupedModelComboboxReturn => { const { t } = useTranslation(); const base_model = useAppSelector((s) => s.generation.model?.base ?? 'sdxl'); - const { modelConfigs, selectedModel, getIsDisabled, onChange, isLoading } = arg; + const { modelConfigs, selectedModel, getIsDisabled, onChange, isLoading, groupByType = false } = arg; const options = useMemo[]>(() => { if (!modelConfigs) { return []; } - const groupedModels = groupBy(modelConfigs, 'base'); + const groupedModels = groupBy(modelConfigs, groupByType ? groupByBaseAndTypeFunc : groupByBaseFunc); const _options = reduce( groupedModels, (acc, val, label) => { @@ -49,9 +54,9 @@ export const useGroupedModelCombobox = ( }, [] as GroupBase[] ); - _options.sort((a) => (a.label === base_model ? -1 : 1)); + _options.sort((a) => (a.label?.split('/')[0]?.toLowerCase().includes(base_model) ? -1 : 1)); return _options; - }, [getIsDisabled, modelConfigs, base_model]); + }, [modelConfigs, groupByType, getIsDisabled, base_model]); const value = useMemo( () => diff --git a/invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts b/invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts index 2aac5b8e72..3c863d0c93 100644 --- a/invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts +++ b/invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts @@ -6,6 +6,7 @@ import { } from 'features/controlAdapters/store/controlAdaptersSlice'; import { isControlNetOrT2IAdapter } from 'features/controlAdapters/store/types'; import { selectControlLayersSlice } from 'features/controlLayers/store/controlLayersSlice'; +import type { Layer } from 'features/controlLayers/store/types'; import { selectDynamicPromptsSlice } from 'features/dynamicPrompts/store/dynamicPromptsSlice'; import { getShouldProcessPrompt } from 'features/dynamicPrompts/util/getShouldProcessPrompt'; import { selectNodesSlice } from 'features/nodes/store/nodesSlice'; @@ -14,9 +15,16 @@ import { selectGenerationSlice } from 'features/parameters/store/generationSlice import { selectSystemSlice } from 'features/system/store/systemSlice'; import { activeTabNameSelector } from 'features/ui/store/uiSelectors'; import i18n from 'i18next'; -import { forEach } from 'lodash-es'; +import { forEach, upperFirst } from 'lodash-es'; import { getConnectedEdges } from 'reactflow'; +const LAYER_TYPE_TO_TKEY: Record = { + initial_image_layer: 'controlLayers.globalInitialImage', + control_adapter_layer: 'controlLayers.globalControlAdapter', + ip_adapter_layer: 'controlLayers.globalIPAdapter', + regional_guidance_layer: 'controlLayers.regionalGuidance', +}; + const selector = createMemoizedSelector( [ selectControlAdaptersSlice, @@ -29,21 +37,22 @@ const selector = createMemoizedSelector( ], (controlAdapters, generation, system, nodes, dynamicPrompts, controlLayers, activeTabName) => { const { model } = generation; + const { size } = controlLayers.present; const { positivePrompt } = controlLayers.present; const { isConnected } = system; - const reasons: string[] = []; + const reasons: { prefix?: string; content: string }[] = []; // Cannot generate if not connected if (!isConnected) { - reasons.push(i18n.t('parameters.invoke.systemDisconnected')); + reasons.push({ content: i18n.t('parameters.invoke.systemDisconnected') }); } if (activeTabName === 'workflows') { if (nodes.shouldValidateGraph) { if (!nodes.nodes.length) { - reasons.push(i18n.t('parameters.invoke.noNodesInGraph')); + reasons.push({ content: i18n.t('parameters.invoke.noNodesInGraph') }); } nodes.nodes.forEach((node) => { @@ -55,7 +64,7 @@ const selector = createMemoizedSelector( if (!nodeTemplate) { // Node type not found - reasons.push(i18n.t('parameters.invoke.missingNodeTemplate')); + reasons.push({ content: i18n.t('parameters.invoke.missingNodeTemplate') }); return; } @@ -68,17 +77,17 @@ const selector = createMemoizedSelector( ); if (!fieldTemplate) { - reasons.push(i18n.t('parameters.invoke.missingFieldTemplate')); + reasons.push({ content: i18n.t('parameters.invoke.missingFieldTemplate') }); return; } if (fieldTemplate.required && field.value === undefined && !hasConnection) { - reasons.push( - i18n.t('parameters.invoke.missingInputForField', { + reasons.push({ + content: i18n.t('parameters.invoke.missingInputForField', { nodeLabel: node.data.label || nodeTemplate.title, fieldLabel: field.label || fieldTemplate.title, - }) - ); + }), + }); return; } }); @@ -86,62 +95,94 @@ const selector = createMemoizedSelector( } } else { if (dynamicPrompts.prompts.length === 0 && getShouldProcessPrompt(positivePrompt)) { - reasons.push(i18n.t('parameters.invoke.noPrompts')); + reasons.push({ content: i18n.t('parameters.invoke.noPrompts') }); } if (!model) { - reasons.push(i18n.t('parameters.invoke.noModelSelected')); + reasons.push({ content: i18n.t('parameters.invoke.noModelSelected') }); } if (activeTabName === 'generation') { // Handling for generation tab controlLayers.present.layers .filter((l) => l.isEnabled) - .flatMap((l) => { + .forEach((l, i) => { + const layerLiteral = i18n.t('controlLayers.layers_one'); + const layerNumber = i + 1; + const layerType = i18n.t(LAYER_TYPE_TO_TKEY[l.type]); + const prefix = `${layerLiteral} #${layerNumber} (${layerType})`; + const problems: string[] = []; if (l.type === 'control_adapter_layer') { - return l.controlAdapter; - } else if (l.type === 'ip_adapter_layer') { - return l.ipAdapter; - } else if (l.type === 'regional_guidance_layer') { - return l.ipAdapters; + // Must have model + if (!l.controlAdapter.model) { + problems.push(i18n.t('parameters.invoke.layer.controlAdapterNoModelSelected')); + } + // Model base must match + if (l.controlAdapter.model?.base !== model?.base) { + problems.push(i18n.t('parameters.invoke.layer.controlAdapterIncompatibleBaseModel')); + } + // Must have a control image OR, if it has a processor, it must have a processed image + if (!l.controlAdapter.image) { + problems.push(i18n.t('parameters.invoke.layer.controlAdapterNoImageSelected')); + } else if (l.controlAdapter.processorConfig && !l.controlAdapter.processedImage) { + problems.push(i18n.t('parameters.invoke.layer.controlAdapterImageNotProcessed')); + } + // T2I Adapters require images have dimensions that are multiples of 64 + if (l.controlAdapter.type === 't2i_adapter' && (size.width % 64 !== 0 || size.height % 64 !== 0)) { + problems.push(i18n.t('parameters.invoke.layer.t2iAdapterIncompatibleDimensions')); + } } - return []; - }) - .forEach((ca, i) => { - const hasNoModel = !ca.model; - const mismatchedModelBase = ca.model?.base !== model?.base; - const hasNoImage = !ca.image; - const imageNotProcessed = - (ca.type === 'controlnet' || ca.type === 't2i_adapter') && !ca.processedImage && ca.processorConfig; - if (hasNoModel) { - reasons.push( - i18n.t('parameters.invoke.noModelForControlAdapter', { - number: i + 1, - }) - ); + if (l.type === 'ip_adapter_layer') { + // Must have model + if (!l.ipAdapter.model) { + problems.push(i18n.t('parameters.invoke.layer.ipAdapterNoModelSelected')); + } + // Model base must match + if (l.ipAdapter.model?.base !== model?.base) { + problems.push(i18n.t('parameters.invoke.layer.ipAdapterIncompatibleBaseModel')); + } + // Must have an image + if (!l.ipAdapter.image) { + problems.push(i18n.t('parameters.invoke.layer.ipAdapterNoImageSelected')); + } } - if (mismatchedModelBase) { - // This should never happen, just a sanity check - reasons.push( - i18n.t('parameters.invoke.incompatibleBaseModelForControlAdapter', { - number: i + 1, - }) - ); + + if (l.type === 'initial_image_layer') { + // Must have an image + if (!l.image) { + problems.push(i18n.t('parameters.invoke.layer.initialImageNoImageSelected')); + } } - if (hasNoImage) { - reasons.push( - i18n.t('parameters.invoke.noControlImageForControlAdapter', { - number: i + 1, - }) - ); + + if (l.type === 'regional_guidance_layer') { + // Must have a region + if (l.maskObjects.length === 0) { + problems.push(i18n.t('parameters.invoke.layer.rgNoRegion')); + } + // Must have at least 1 prompt or IP Adapter + if (l.positivePrompt === null && l.negativePrompt === null && l.ipAdapters.length === 0) { + problems.push(i18n.t('parameters.invoke.layer.rgNoPromptsOrIPAdapters')); + } + l.ipAdapters.forEach((ipAdapter) => { + // Must have model + if (!ipAdapter.model) { + problems.push(i18n.t('parameters.invoke.layer.ipAdapterNoModelSelected')); + } + // Model base must match + if (ipAdapter.model?.base !== model?.base) { + problems.push(i18n.t('parameters.invoke.layer.ipAdapterIncompatibleBaseModel')); + } + // Must have an image + if (!ipAdapter.image) { + problems.push(i18n.t('parameters.invoke.layer.ipAdapterNoImageSelected')); + } + }); } - if (imageNotProcessed) { - reasons.push( - i18n.t('parameters.invoke.imageNotProcessedForControlAdapter', { - number: i + 1, - }) - ); + + if (problems.length) { + const content = upperFirst(problems.join(', ')); + reasons.push({ prefix, content }); } }); } else { @@ -154,29 +195,19 @@ const selector = createMemoizedSelector( } if (!ca.model) { - reasons.push( - i18n.t('parameters.invoke.noModelForControlAdapter', { - number: i + 1, - }) - ); + reasons.push({ content: i18n.t('parameters.invoke.noModelForControlAdapter', { number: i + 1 }) }); } else if (ca.model.base !== model?.base) { // This should never happen, just a sanity check - reasons.push( - i18n.t('parameters.invoke.incompatibleBaseModelForControlAdapter', { - number: i + 1, - }) - ); + reasons.push({ + content: i18n.t('parameters.invoke.incompatibleBaseModelForControlAdapter', { number: i + 1 }), + }); } if ( !ca.controlImage || (isControlNetOrT2IAdapter(ca) && !ca.processedControlImage && ca.processorType !== 'none') ) { - reasons.push( - i18n.t('parameters.invoke.noControlImageForControlAdapter', { - number: i + 1, - }) - ); + reasons.push({ content: i18n.t('parameters.invoke.noControlImageForControlAdapter', { number: i + 1 }) }); } }); } @@ -187,6 +218,6 @@ const selector = createMemoizedSelector( ); export const useIsReadyToEnqueue = () => { - const { isReady, reasons } = useAppSelector(selector); - return { isReady, reasons }; + const value = useAppSelector(selector); + return value; }; diff --git a/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx b/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx index 686577b4a7..5ed5ffe573 100644 --- a/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx +++ b/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx @@ -21,7 +21,6 @@ import { setShouldShowBoundingBox, } from 'features/canvas/store/canvasSlice'; import type { CanvasLayer } from 'features/canvas/store/canvasTypes'; -import { LAYER_NAMES_DICT } from 'features/canvas/store/canvasTypes'; import { memo, useCallback, useMemo } from 'react'; import { useHotkeys } from 'react-hotkeys-hook'; import { useTranslation } from 'react-i18next'; @@ -216,13 +215,20 @@ const IAICanvasToolbar = () => { [dispatch, isMaskEnabled] ); - const value = useMemo(() => LAYER_NAMES_DICT.filter((o) => o.value === layer)[0], [layer]); + const layerOptions = useMemo<{ label: string; value: CanvasLayer }[]>( + () => [ + { label: t('unifiedCanvas.base'), value: 'base' }, + { label: t('unifiedCanvas.mask'), value: 'mask' }, + ], + [t] + ); + const layerValue = useMemo(() => layerOptions.filter((o) => o.value === layer)[0] ?? null, [layer, layerOptions]); return ( - + diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts index 2d30e18760..c41c6f329f 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts @@ -5,11 +5,6 @@ import { z } from 'zod'; export type CanvasLayer = 'base' | 'mask'; -export const LAYER_NAMES_DICT: { label: string; value: CanvasLayer }[] = [ - { label: 'Base', value: 'base' }, - { label: 'Mask', value: 'mask' }, -]; - const zBoundingBoxScaleMethod = z.enum(['none', 'auto', 'manual']); export type BoundingBoxScaleMethod = z.infer; export const isBoundingBoxScaleMethod = (v: unknown): v is BoundingBoxScaleMethod => diff --git a/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterImagePreview.tsx b/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterImagePreview.tsx index c1da425186..4d93eb12ec 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterImagePreview.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterImagePreview.tsx @@ -124,7 +124,7 @@ export const ControlAdapterImagePreview = memo( controlImage && processedControlImage && !isMouseOverImage && - !controlAdapter.isProcessingImage && + !controlAdapter.processorPendingBatchId && controlAdapter.processorConfig !== null; useEffect(() => { @@ -190,7 +190,7 @@ export const ControlAdapterImagePreview = memo( /> - {controlAdapter.isProcessingImage && ( + {controlAdapter.processorPendingBatchId !== null && ( ; -const DEFAULTS = CA_PROCESSOR_DATA['depth_anything_image_processor'].buildDefaults(); export const DepthAnythingProcessor = memo(({ onChange, config }: Props) => { const { t } = useTranslation(); @@ -38,12 +37,7 @@ export const DepthAnythingProcessor = memo(({ onChange, config }: Props) => { {t('controlnet.modelSize')} - + ); diff --git a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts index 425599561a..32e29918ae 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts @@ -27,7 +27,7 @@ import { modelChanged } from 'features/parameters/store/generationSlice'; import type { ParameterAutoNegative } from 'features/parameters/types/parameterSchemas'; import { getIsSizeOptimal, getOptimalDimension } from 'features/parameters/util/optimalDimension'; import type { IRect, Vector2d } from 'konva/lib/types'; -import { isEqual, partition } from 'lodash-es'; +import { isEqual, partition, unset } from 'lodash-es'; import { atom } from 'nanostores'; import type { RgbColor } from 'react-colorful'; import type { UndoableOptions } from 'redux-undo'; @@ -49,7 +49,7 @@ import type { } from './types'; export const initialControlLayersState: ControlLayersState = { - _version: 2, + _version: 3, selectedLayerId: null, brushSize: 100, layers: [], @@ -334,13 +334,13 @@ export const controlLayersSlice = createSlice({ const layer = selectCALayerOrThrow(state, layerId); layer.opacity = opacity; }, - caLayerIsProcessingImageChanged: ( + caLayerProcessorPendingBatchIdChanged: ( state, - action: PayloadAction<{ layerId: string; isProcessingImage: boolean }> + action: PayloadAction<{ layerId: string; batchId: string | null }> ) => { - const { layerId, isProcessingImage } = action.payload; + const { layerId, batchId } = action.payload; const layer = selectCALayerOrThrow(state, layerId); - layer.controlAdapter.isProcessingImage = isProcessingImage; + layer.controlAdapter.processorPendingBatchId = batchId; }, //#endregion @@ -800,7 +800,7 @@ export const { caLayerProcessorConfigChanged, caLayerIsFilterEnabledChanged, caLayerOpacityChanged, - caLayerIsProcessingImageChanged, + caLayerProcessorPendingBatchIdChanged, // IPA Layers ipaLayerAdded, ipaLayerRecalled, @@ -857,7 +857,16 @@ export const selectControlLayersSlice = (state: RootState) => state.controlLayer const migrateControlLayersState = (state: any): any => { if (state._version === 1) { // Reset state for users on v1 (e.g. beta users), some changes could cause - return deepClone(initialControlLayersState); + state = deepClone(initialControlLayersState); + } + if (state._version === 2) { + // The CA `isProcessingImage` flag was replaced with a `processorPendingBatchId` property, fix up CA layers + for (const layer of (state as ControlLayersState).layers) { + if (layer.type === 'control_adapter_layer') { + layer.controlAdapter.processorPendingBatchId = null; + unset(layer.controlAdapter, 'isProcessingImage'); + } + } } return state; }; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/types.ts b/invokeai/frontend/web/src/features/controlLayers/store/types.ts index 2d29b349cb..771e5060e1 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/types.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/types.ts @@ -113,7 +113,7 @@ export const zLayer = z.discriminatedUnion('type', [ export type Layer = z.infer; export type ControlLayersState = { - _version: 2; + _version: 3; selectedLayerId: string | null; layers: Layer[]; brushSize: number; diff --git a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts index 589c61b855..41f1cf438f 100644 --- a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts +++ b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts @@ -198,8 +198,8 @@ const zControlAdapterBase = z.object({ weight: z.number().gte(0).lte(1), image: zImageWithDims.nullable(), processedImage: zImageWithDims.nullable(), - isProcessingImage: z.boolean(), processorConfig: zProcessorConfig.nullable(), + processorPendingBatchId: z.string().nullable().default(null), beginEndStepPct: zBeginEndStepPct, }); @@ -521,8 +521,8 @@ export const initialControlNetV2: Omit = { controlMode: 'balanced', image: null, processedImage: null, - isProcessingImage: false, processorConfig: CA_PROCESSOR_DATA.canny_image_processor.buildDefaults(), + processorPendingBatchId: null, }; export const initialT2IAdapterV2: Omit = { @@ -532,8 +532,8 @@ export const initialT2IAdapterV2: Omit = { beginEndStepPct: [0, 1], image: null, processedImage: null, - isProcessingImage: false, processorConfig: CA_PROCESSOR_DATA.canny_image_processor.buildDefaults(), + processorPendingBatchId: null, }; export const initialIPAdapterV2: Omit = { diff --git a/invokeai/frontend/web/src/features/metadata/util/parsers.ts b/invokeai/frontend/web/src/features/metadata/util/parsers.ts index ac42142ea2..0757d2e8db 100644 --- a/invokeai/frontend/web/src/features/metadata/util/parsers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/parsers.ts @@ -587,7 +587,7 @@ const parseControlNetToControlAdapterLayer: MetadataParseFunc { return ( - + + + ); }); diff --git a/invokeai/frontend/web/src/features/queue/components/QueueButtonTooltip.tsx b/invokeai/frontend/web/src/features/queue/components/QueueButtonTooltip.tsx index f63e96c45f..498414d377 100644 --- a/invokeai/frontend/web/src/features/queue/components/QueueButtonTooltip.tsx +++ b/invokeai/frontend/web/src/features/queue/components/QueueButtonTooltip.tsx @@ -1,10 +1,11 @@ -import { Divider, Flex, ListItem, Text, UnorderedList } from '@invoke-ai/ui-library'; +import { Divider, Flex, ListItem, Text, Tooltip, UnorderedList } from '@invoke-ai/ui-library'; import { createSelector } from '@reduxjs/toolkit'; import { useAppSelector } from 'app/store/storeHooks'; import { useIsReadyToEnqueue } from 'common/hooks/useIsReadyToEnqueue'; import { selectControlLayersSlice } from 'features/controlLayers/store/controlLayersSlice'; import { selectDynamicPromptsSlice } from 'features/dynamicPrompts/store/dynamicPromptsSlice'; import { getShouldProcessPrompt } from 'features/dynamicPrompts/util/getShouldProcessPrompt'; +import type { PropsWithChildren } from 'react'; import { memo, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useEnqueueBatchMutation } from 'services/api/endpoints/queue'; @@ -21,17 +22,32 @@ type Props = { prepend?: boolean; }; -export const QueueButtonTooltip = memo(({ prepend = false }: Props) => { +export const QueueButtonTooltip = (props: PropsWithChildren) => { + return ( + } maxW={512}> + {props.children} + + ); +}; + +const TooltipContent = memo(({ prepend = false }: Props) => { const { t } = useTranslation(); const { isReady, reasons } = useIsReadyToEnqueue(); const isLoadingDynamicPrompts = useAppSelector((s) => s.dynamicPrompts.isLoading); const promptsCount = useAppSelector(selectPromptsCount); - const iterations = useAppSelector((s) => s.generation.iterations); + const iterationsCount = useAppSelector((s) => s.generation.iterations); const autoAddBoardId = useAppSelector((s) => s.gallery.autoAddBoardId); const autoAddBoardName = useBoardName(autoAddBoardId); const [_, { isLoading }] = useEnqueueBatchMutation({ fixedCacheKey: 'enqueueBatch', }); + const queueCountPredictionLabel = useMemo(() => { + const generationCount = Math.min(promptsCount * iterationsCount, 10000); + const prompts = t('queue.prompts', { count: promptsCount }); + const iterations = t('queue.iterations', { count: iterationsCount }); + const generations = t('queue.generations', { count: generationCount }); + return `${promptsCount} ${prompts} \u00d7 ${iterationsCount} ${iterations} -> ${generationCount} ${generations}`.toLowerCase(); + }, [iterationsCount, promptsCount, t]); const label = useMemo(() => { if (isLoading) { @@ -52,20 +68,21 @@ export const QueueButtonTooltip = memo(({ prepend = false }: Props) => { return ( {label} - - {t('queue.queueCountPrediction', { - promptsCount, - iterations, - count: Math.min(promptsCount * iterations, 10000), - })} - + {queueCountPredictionLabel} {reasons.length > 0 && ( <> {reasons.map((reason, i) => ( - - {reason} + + + {reason.prefix && ( + + {reason.prefix}:{' '} + + )} + {reason.content} + ))} @@ -82,4 +99,4 @@ export const QueueButtonTooltip = memo(({ prepend = false }: Props) => { ); }); -QueueButtonTooltip.displayName = 'QueueButtonTooltip'; +TooltipContent.displayName = 'QueueButtonTooltipContent'; diff --git a/invokeai/frontend/web/src/features/queue/components/QueueFrontButton.tsx b/invokeai/frontend/web/src/features/queue/components/QueueFrontButton.tsx index 07ad0f5b3c..eb0e72950f 100644 --- a/invokeai/frontend/web/src/features/queue/components/QueueFrontButton.tsx +++ b/invokeai/frontend/web/src/features/queue/components/QueueFrontButton.tsx @@ -10,15 +10,16 @@ const QueueFrontButton = () => { const { t } = useTranslation(); const { queueFront, isLoading, isDisabled } = useQueueFront(); return ( - } - icon={} - size="lg" - /> + + } + size="lg" + /> + ); }; diff --git a/invokeai/frontend/web/src/features/ui/components/FloatingParametersPanelButtons.tsx b/invokeai/frontend/web/src/features/ui/components/FloatingParametersPanelButtons.tsx index 32611b2354..5a8273b7fc 100644 --- a/invokeai/frontend/web/src/features/ui/components/FloatingParametersPanelButtons.tsx +++ b/invokeai/frontend/web/src/features/ui/components/FloatingParametersPanelButtons.tsx @@ -63,16 +63,17 @@ const FloatingSidePanelButtons = (props: Props) => { sx={floatingButtonStyles} icon={} /> - } - sx={floatingButtonStyles} - /> + + +