From 6c1fd584d29b5aefe45a4859214b8306d3640227 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Wed, 8 May 2024 19:06:04 +1000 Subject: [PATCH] feat(ui): pre-CL control adapter metadata recall --- .../controlLayers/store/controlLayersSlice.ts | 7 +- .../controlLayers/util/controlAdapters.ts | 6 +- .../ImageMetadataActions.tsx | 4 +- .../src/features/metadata/util/handlers.ts | 2 +- .../web/src/features/metadata/util/parsers.ts | 310 ++++++++++++++++-- 5 files changed, 300 insertions(+), 29 deletions(-) diff --git a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts index 27b7e4c3fd..fb890f7d45 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts @@ -662,7 +662,7 @@ export const controlLayersSlice = createSlice({ } } }, - prepare: (imageDTO: ImageDTO | null) => ({ payload: { layerId: 'initial_image_layer', imageDTO } }), + prepare: (imageDTO: ImageDTO | null) => ({ payload: { layerId: INITIAL_IMAGE_LAYER_ID, imageDTO } }), }, iiLayerRecalled: (state, action: PayloadAction) => { deselectAllLayers(state); @@ -914,6 +914,7 @@ export const RG_LAYER_NAME = 'regional_guidance_layer'; export const RG_LAYER_LINE_NAME = 'regional_guidance_layer.line'; export const RG_LAYER_OBJECT_GROUP_NAME = 'regional_guidance_layer.object_group'; export const RG_LAYER_RECT_NAME = 'regional_guidance_layer.rect'; +export const INITIAL_IMAGE_LAYER_ID = 'singleton_initial_image_layer'; export const INITIAL_IMAGE_LAYER_NAME = 'initial_image_layer'; export const INITIAL_IMAGE_LAYER_IMAGE_NAME = 'initial_image_layer.image'; export const LAYER_BBOX_NAME = 'layer.bbox'; @@ -925,10 +926,10 @@ const getRGLayerLineId = (layerId: string, lineId: string) => `${layerId}.line_$ const getRGLayerRectId = (layerId: string, lineId: string) => `${layerId}.rect_${lineId}`; export const getRGLayerObjectGroupId = (layerId: string, groupId: string) => `${layerId}.objectGroup_${groupId}`; export const getLayerBboxId = (layerId: string) => `${layerId}.bbox`; -const getCALayerId = (layerId: string) => `control_adapter_layer_${layerId}`; +export const getCALayerId = (layerId: string) => `control_adapter_layer_${layerId}`; export const getCALayerImageId = (layerId: string, imageName: string) => `${layerId}.image_${imageName}`; export const getIILayerImageId = (layerId: string, imageName: string) => `${layerId}.image_${imageName}`; -const getIPALayerId = (layerId: string) => `ip_adapter_layer_${layerId}`; +export const getIPALayerId = (layerId: string) => `ip_adapter_layer_${layerId}`; export const controlLayersPersistConfig: PersistConfig = { name: controlLayersSlice.name, diff --git a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts index 070ca64b32..589c61b855 100644 --- a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts +++ b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts @@ -513,7 +513,7 @@ export const CA_PROCESSOR_DATA: CAProcessorsData = { }, }; -const initialControlNetV2: Omit = { +export const initialControlNetV2: Omit = { type: 'controlnet', model: null, weight: 1, @@ -525,7 +525,7 @@ const initialControlNetV2: Omit = { processorConfig: CA_PROCESSOR_DATA.canny_image_processor.buildDefaults(), }; -const initialT2IAdapterV2: Omit = { +export const initialT2IAdapterV2: Omit = { type: 't2i_adapter', model: null, weight: 1, @@ -536,7 +536,7 @@ const initialT2IAdapterV2: Omit = { processorConfig: CA_PROCESSOR_DATA.canny_image_processor.buildDefaults(), }; -const initialIPAdapterV2: Omit = { +export const initialIPAdapterV2: Omit = { type: 'ip_adapter', image: null, model: null, diff --git a/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx b/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx index f8425182dd..a192ff4fbb 100644 --- a/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx +++ b/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx @@ -37,7 +37,7 @@ const ImageMetadataActions = (props: Props) => { - + {activeTabName !== 'generation' && } @@ -49,7 +49,7 @@ const ImageMetadataActions = (props: Props) => { - + {activeTabName === 'generation' && } {activeTabName !== 'generation' && } {activeTabName !== 'generation' && } {activeTabName !== 'generation' && } diff --git a/invokeai/frontend/web/src/features/metadata/util/handlers.ts b/invokeai/frontend/web/src/features/metadata/util/handlers.ts index bf5fa2eaec..b0d0e22688 100644 --- a/invokeai/frontend/web/src/features/metadata/util/handlers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/handlers.ts @@ -429,7 +429,7 @@ export const parseAndRecallImageDimensions = async (metadata: unknown) => { }; // These handlers should be omitted when recalling to control layers -const TO_CONTROL_LAYERS_SKIP_KEYS: (keyof typeof handlers)[] = ['controlNets', 'ipAdapters', 't2iAdapters']; +const TO_CONTROL_LAYERS_SKIP_KEYS: (keyof typeof handlers)[] = ['controlNets', 'ipAdapters', 't2iAdapters', 'strength']; // These handlers should be omitted when recalling to the rest of the app const NOT_TO_CONTROL_LAYERS_SKIP_KEYS: (keyof typeof handlers)[] = ['layers']; diff --git a/invokeai/frontend/web/src/features/metadata/util/parsers.ts b/invokeai/frontend/web/src/features/metadata/util/parsers.ts index 213a4666fe..1848152d2b 100644 --- a/invokeai/frontend/web/src/features/metadata/util/parsers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/parsers.ts @@ -1,12 +1,20 @@ -import { getStore } from 'app/store/nanostores/store'; import { initialControlNet, initialIPAdapter, initialT2IAdapter, } from 'features/controlAdapters/util/buildControlAdapter'; import { buildControlAdapterProcessor } from 'features/controlAdapters/util/buildControlAdapterProcessor'; -import type { Layer } from 'features/controlLayers/store/types'; +import { getCALayerId, getIPALayerId, INITIAL_IMAGE_LAYER_ID } from 'features/controlLayers/store/controlLayersSlice'; +import type { ControlAdapterLayer, InitialImageLayer, IPAdapterLayer, Layer } from 'features/controlLayers/store/types'; import { zLayer } from 'features/controlLayers/store/types'; +import { + CA_PROCESSOR_DATA, + imageDTOToImageWithDims, + initialControlNetV2, + initialIPAdapterV2, + initialT2IAdapterV2, + isProcessorTypeV2, +} from 'features/controlLayers/util/controlAdapters'; import type { LoRA } from 'features/lora/store/loraSlice'; import { defaultLoRAConfig } from 'features/lora/store/loraSlice'; import type { @@ -60,8 +68,7 @@ import { isParameterWidth, } from 'features/parameters/types/parameterSchemas'; import { get, isArray, isString } from 'lodash-es'; -import { imagesApi } from 'services/api/endpoints/images'; -import type { ImageDTO } from 'services/api/types'; +import { getImageDTO } from 'services/api/endpoints/images'; import { isControlNetModelConfig, isIPAdapterModelConfig, @@ -71,6 +78,7 @@ import { isT2IAdapterModelConfig, isVAEModelConfig, } from 'services/api/types'; +import { assert } from 'tsafe'; import { v4 as uuidv4 } from 'uuid'; export const MetadataParsePendingToken = Symbol('pending'); @@ -140,14 +148,6 @@ const parseCFGRescaleMultiplier: MetadataParseFunc = (metadata) => getProperty(metadata, 'scheduler', isParameterScheduler); -const parseInitialImage: MetadataParseFunc = async (metadata) => { - const imageName = await getProperty(metadata, 'init_image', isString); - const imageDTORequest = getStore().dispatch(imagesApi.endpoints.getImageDTO.initiate(imageName)); - const imageDTO = await imageDTORequest.unwrap(); - imageDTORequest.unsubscribe(); - return imageDTO; -}; - const parseWidth: MetadataParseFunc = (metadata) => getProperty(metadata, 'width', isParameterWidth); const parseHeight: MetadataParseFunc = (metadata) => @@ -300,7 +300,7 @@ const parseControlNet: MetadataParseFunc = async (meta const parseAllControlNets: MetadataParseFunc = async (metadata) => { try { - const controlNetsRaw = await getProperty(metadata, 'controlnets', isArray || undefined); + const controlNetsRaw = await getProperty(metadata, 'controlnets', isArray); const parseResults = await Promise.allSettled(controlNetsRaw.map((cn) => parseControlNet(cn))); const controlNets = parseResults .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') @@ -434,18 +434,286 @@ const parseAllIPAdapters: MetadataParseFunc = async ( const parseLayer: MetadataParseFunc = async (metadataItem) => zLayer.parseAsync(metadataItem); const parseLayers: MetadataParseFunc = async (metadata) => { + // We need to support recalling pre-Control Layers metadata into Control Layers. A separate set of parsers handles + // taking pre-CL metadata and parsing it into layers. It doesn't always map 1-to-1, so this is best-effort. For + // example, CL Control Adapters don't support resize mode, so we simply omit that property. + try { - const control_layers = await getProperty(metadata, 'control_layers'); - const layersRaw = await getProperty(control_layers, 'layers', isArray); - const parseResults = await Promise.allSettled(layersRaw.map(parseLayer)); - const layers = parseResults - .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') - .map((result) => result.value); + const layers: Layer[] = []; + + try { + const control_layers = await getProperty(metadata, 'control_layers'); + const controlLayersRaw = await getProperty(control_layers, 'layers', isArray); + const controlLayersParseResults = await Promise.allSettled(controlLayersRaw.map(parseLayer)); + const controlLayers = controlLayersParseResults + .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') + .map((result) => result.value); + layers.push(...controlLayers); + } catch { + // no-op + } + + try { + const controlNetsRaw = await getProperty(metadata, 'controlnets', isArray); + console.log('controlNetsRaw', controlNetsRaw); + const controlNetsParseResults = await Promise.allSettled( + controlNetsRaw.map(async (cn) => await parseControlNetToControlAdapterLayer(cn)) + ); + console.log('controlNetsParseResults', controlNetsParseResults); + const controlNetsAsLayers = controlNetsParseResults + .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') + .map((result) => result.value); + layers.push(...controlNetsAsLayers); + } catch { + // no-op + } + + try { + const t2iAdaptersRaw = await getProperty(metadata, 't2iAdapters', isArray); + console.log('t2iAdaptersRaw', t2iAdaptersRaw); + const t2iAdaptersParseResults = await Promise.allSettled( + t2iAdaptersRaw.map(async (cn) => await parseT2IAdapterToControlAdapterLayer(cn)) + ); + console.log('t2iAdaptersParseResults', t2iAdaptersParseResults); + const t2iAdaptersAsLayers = t2iAdaptersParseResults + .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') + .map((result) => result.value); + layers.push(...t2iAdaptersAsLayers); + } catch { + // no-op + } + + try { + const ipAdaptersRaw = await getProperty(metadata, 'ipAdapters', isArray); + console.log('ipAdaptersRaw', ipAdaptersRaw); + const ipAdaptersParseResults = await Promise.allSettled( + ipAdaptersRaw.map(async (cn) => await parseIPAdapterToIPAdapterLayer(cn)) + ); + console.log('ipAdaptersParseResults', ipAdaptersParseResults); + const ipAdaptersAsLayers = ipAdaptersParseResults + .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') + .map((result) => result.value); + layers.push(...ipAdaptersAsLayers); + } catch { + // no-op + } + + try { + const initialImageLayer = await parseInitialImageToInitialImageLayer(metadata); + layers.push(initialImageLayer); + } catch { + // no-op + } + return layers; } catch { return []; } }; + +const parseInitialImageToInitialImageLayer: MetadataParseFunc = async (metadata) => { + const denoisingStrength = await getProperty(metadata, 'strength', isParameterStrength); + const imageName = await getProperty(metadata, 'init_image', isString); + const imageDTO = await getImageDTO(imageName); + assert(imageDTO, 'ImageDTO is null'); + const layer: InitialImageLayer = { + id: INITIAL_IMAGE_LAYER_ID, + type: 'initial_image_layer', + bbox: null, + bboxNeedsUpdate: true, + x: 0, + y: 0, + isEnabled: true, + opacity: 1, + image: imageDTOToImageWithDims(imageDTO), + isSelected: true, + denoisingStrength, + }; + return layer; +}; + +const parseControlNetToControlAdapterLayer: MetadataParseFunc = async (metadataItem) => { + const control_model = await getProperty(metadataItem, 'control_model'); + const key = await getModelKey(control_model, 'controlnet'); + const controlNetModel = await fetchModelConfigWithTypeGuard(key, isControlNetModelConfig); + const image = zControlField.shape.image + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'image')); + const processedImage = zControlField.shape.image + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'processed_image')); + const control_weight = zControlField.shape.control_weight + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'control_weight')); + const begin_step_percent = zControlField.shape.begin_step_percent + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'begin_step_percent')); + const end_step_percent = zControlField.shape.end_step_percent + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'end_step_percent')); + const control_mode = zControlField.shape.control_mode + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'control_mode')); + + const defaultPreprocessor = controlNetModel.default_settings?.preprocessor; + const processorConfig = isProcessorTypeV2(defaultPreprocessor) + ? CA_PROCESSOR_DATA[defaultPreprocessor].buildDefaults() + : null; + const beginEndStepPct: [number, number] = [ + begin_step_percent ?? initialControlNetV2.beginEndStepPct[0], + end_step_percent ?? initialControlNetV2.beginEndStepPct[1], + ]; + const imageDTO = image ? await getImageDTO(image.image_name) : null; + const processedImageDTO = processedImage ? await getImageDTO(processedImage.image_name) : null; + + const layer: ControlAdapterLayer = { + id: getCALayerId(uuidv4()), + bbox: null, + bboxNeedsUpdate: true, + isEnabled: true, + isFilterEnabled: true, + isSelected: true, + opacity: 1, + type: 'control_adapter_layer', + x: 0, + y: 0, + controlAdapter: { + id: uuidv4(), + type: 'controlnet', + model: zModelIdentifierField.parse(controlNetModel), + weight: typeof control_weight === 'number' ? control_weight : initialControlNetV2.weight, + beginEndStepPct, + controlMode: control_mode ?? initialControlNetV2.controlMode, + image: imageDTO ? imageDTOToImageWithDims(imageDTO) : null, + processedImage: processedImageDTO ? imageDTOToImageWithDims(processedImageDTO) : null, + processorConfig, + isProcessingImage: false, + }, + }; + + return layer; +}; + +const parseT2IAdapterToControlAdapterLayer: MetadataParseFunc = async (metadataItem) => { + const t2i_adapter_model = await getProperty(metadataItem, 't2i_adapter_model'); + const key = await getModelKey(t2i_adapter_model, 't2i_adapter'); + const t2iAdapterModel = await fetchModelConfigWithTypeGuard(key, isT2IAdapterModelConfig); + + const image = zT2IAdapterField.shape.image + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'image')); + const processedImage = zT2IAdapterField.shape.image + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'processed_image')); + const weight = zT2IAdapterField.shape.weight + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'weight')); + const begin_step_percent = zT2IAdapterField.shape.begin_step_percent + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'begin_step_percent')); + const end_step_percent = zT2IAdapterField.shape.end_step_percent + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'end_step_percent')); + + const defaultPreprocessor = t2iAdapterModel.default_settings?.preprocessor; + const processorConfig = isProcessorTypeV2(defaultPreprocessor) + ? CA_PROCESSOR_DATA[defaultPreprocessor].buildDefaults() + : null; + const beginEndStepPct: [number, number] = [ + begin_step_percent ?? initialT2IAdapterV2.beginEndStepPct[0], + end_step_percent ?? initialT2IAdapterV2.beginEndStepPct[1], + ]; + const imageDTO = image ? await getImageDTO(image.image_name) : null; + const processedImageDTO = processedImage ? await getImageDTO(processedImage.image_name) : null; + + const layer: ControlAdapterLayer = { + id: getCALayerId(uuidv4()), + bbox: null, + bboxNeedsUpdate: true, + isEnabled: true, + isFilterEnabled: true, + isSelected: true, + opacity: 1, + type: 'control_adapter_layer', + x: 0, + y: 0, + controlAdapter: { + id: uuidv4(), + type: 't2i_adapter', + model: zModelIdentifierField.parse(t2iAdapterModel), + weight: typeof weight === 'number' ? weight : initialT2IAdapterV2.weight, + beginEndStepPct, + image: imageDTO ? imageDTOToImageWithDims(imageDTO) : null, + processedImage: processedImageDTO ? imageDTOToImageWithDims(processedImageDTO) : null, + processorConfig, + isProcessingImage: false, + }, + }; + + return layer; +}; + +const parseIPAdapterToIPAdapterLayer: MetadataParseFunc = async (metadataItem) => { + const ip_adapter_model = await getProperty(metadataItem, 'ip_adapter_model'); + const key = await getModelKey(ip_adapter_model, 'ip_adapter'); + const ipAdapterModel = await fetchModelConfigWithTypeGuard(key, isIPAdapterModelConfig); + + const image = zIPAdapterField.shape.image + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'image')); + const weight = zIPAdapterField.shape.weight + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'weight')); + const method = zIPAdapterField.shape.method + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'method')); + const begin_step_percent = zIPAdapterField.shape.begin_step_percent + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'begin_step_percent')); + const end_step_percent = zIPAdapterField.shape.end_step_percent + .nullish() + .catch(null) + .parse(await getProperty(metadataItem, 'end_step_percent')); + + const beginEndStepPct: [number, number] = [ + begin_step_percent ?? initialIPAdapterV2.beginEndStepPct[0], + end_step_percent ?? initialIPAdapterV2.beginEndStepPct[1], + ]; + const imageDTO = image ? await getImageDTO(image.image_name) : null; + + const layer: IPAdapterLayer = { + id: getIPALayerId(uuidv4()), + isEnabled: true, + type: 'ip_adapter_layer', + ipAdapter: { + id: uuidv4(), + type: 'ip_adapter', + model: zModelIdentifierField.parse(ipAdapterModel), + weight: typeof weight === 'number' ? weight : initialIPAdapterV2.weight, + beginEndStepPct, + image: imageDTO ? imageDTOToImageWithDims(imageDTO) : null, + clipVisionModel: initialIPAdapterV2.clipVisionModel, // TODO: This needs to be added to the zIPAdapterField... + method: method ?? initialIPAdapterV2.method, + }, + }; + + return layer; +}; //#endregion export const parsers = { @@ -459,7 +727,6 @@ export const parsers = { cfgScale: parseCFGScale, cfgRescaleMultiplier: parseCFGRescaleMultiplier, scheduler: parseScheduler, - initialImage: parseInitialImage, width: parseWidth, height: parseHeight, steps: parseSteps, @@ -484,6 +751,9 @@ export const parsers = { t2iAdapters: parseAllT2IAdapters, ipAdapter: parseIPAdapter, ipAdapters: parseAllIPAdapters, + controlNetToControlLayer: parseControlNetToControlAdapterLayer, + t2iAdapterToControlAdapterLayer: parseT2IAdapterToControlAdapterLayer, + ipAdapterToIPAdapterLayer: parseIPAdapterToIPAdapterLayer, layer: parseLayer, layers: parseLayers, } as const;