From b35cdc05a5699d76147f5c4b0f446d86499cee80 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Sun, 13 Aug 2023 20:17:23 +1200 Subject: [PATCH] feat: Scaled Processing to Inpainting & Outpainting / 1.x & SDXL --- .../listeners/userInvokedCanvas.ts | 1 - .../src/features/canvas/store/canvasSlice.ts | 2 +- .../src/features/canvas/store/canvasTypes.ts | 2 +- .../graphBuilders/buildCanvasInpaintGraph.ts | 194 +++++++++-- .../graphBuilders/buildCanvasOutpaintGraph.ts | 302 ++++++++++++++--- .../buildCanvasSDXLInpaintGraph.ts | 188 ++++++++++- .../buildCanvasSDXLOutpaintGraph.ts | 310 +++++++++++++++--- .../nodes/util/graphBuilders/constants.ts | 6 + .../frontend/web/src/services/api/types.ts | 9 + 9 files changed, 874 insertions(+), 140 deletions(-) diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts index dbcb87f3cf..cd6791cc0b 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts @@ -123,7 +123,6 @@ export const addUserInvokedCanvasListener = () => { log.debug({ graph: parseify(graph) }, `Canvas graph built`); // currently this action is just listened to for logging - console.log(canvasGraphBuilt(graph)); dispatch(canvasGraphBuilt(graph)); // Create the session, store the request id diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts b/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts index f63ab2fd67..11f829221a 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts @@ -47,7 +47,7 @@ export const initialCanvasState: CanvasState = { boundingBoxCoordinates: { x: 0, y: 0 }, boundingBoxDimensions: { width: 512, height: 512 }, boundingBoxPreviewFill: { r: 0, g: 0, b: 0, a: 0.5 }, - boundingBoxScaleMethod: 'auto', + boundingBoxScaleMethod: 'none', brushColor: { r: 90, g: 90, b: 255, a: 1 }, brushSize: 50, canvasContainerDimensions: { width: 0, height: 0 }, diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts index ba85a7e132..f2ba90b050 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts @@ -11,9 +11,9 @@ export const LAYER_NAMES = ['base', 'mask'] as const; export type CanvasLayer = (typeof LAYER_NAMES)[number]; export const BOUNDING_BOX_SCALES_DICT = [ + { label: 'None', value: 'none' }, { label: 'Auto', value: 'auto' }, { label: 'Manual', value: 'manual' }, - { label: 'None', value: 'none' }, ]; export const BOUNDING_BOX_SCALES = ['none', 'auto', 'manual'] as const; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts index c2b8b62e42..00af964350 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts @@ -2,7 +2,10 @@ import { logger } from 'app/logging/logger'; import { RootState } from 'app/store/store'; import { NonNullableGraph } from 'features/nodes/types/types'; import { + ImageBlurInvocation, ImageDTO, + ImageToLatentsInvocation, + NoiseInvocation, RandomIntInvocation, RangeOfSizeInvocation, } from 'services/api/types'; @@ -16,12 +19,16 @@ import { CANVAS_OUTPUT, CLIP_SKIP, COLOR_CORRECT, - INPAINT, + DENOISE_LATENTS, INPAINT_IMAGE, + INPAINT_IMAGE_RESIZE_DOWN, + INPAINT_IMAGE_RESIZE_UP, ITERATE, LATENTS_TO_IMAGE, MAIN_MODEL_LOADER, MASK_BLUR, + MASK_RESIZE_DOWN, + MASK_RESIZE_UP, NEGATIVE_CONDITIONING, NOISE, POSITIVE_CONDITIONING, @@ -108,7 +115,6 @@ export const buildCanvasInpaintGraph = ( type: 'img_blur', id: MASK_BLUR, is_intermediate: true, - image: canvasMaskImage, radius: maskBlur, blur_type: maskBlurMethod, }, @@ -117,19 +123,16 @@ export const buildCanvasInpaintGraph = ( id: INPAINT_IMAGE, is_intermediate: true, fp32: vaePrecision === 'fp32' ? true : false, - image: canvasInitImage, }, [NOISE]: { type: 'noise', id: NOISE, - width, - height, use_cpu, is_intermediate: true, }, - [INPAINT]: { + [DENOISE_LATENTS]: { type: 'denoise_latents', - id: INPAINT, + id: DENOISE_LATENTS, is_intermediate: true, steps: steps, cfg_scale: cfg_scale, @@ -152,7 +155,7 @@ export const buildCanvasInpaintGraph = ( [CANVAS_OUTPUT]: { type: 'img_paste', id: CANVAS_OUTPUT, - is_intermediate: true, + is_intermediate: !shouldAutoSave, base_image: canvasInitImage, }, [RANGE_OF_SIZE]: { @@ -178,7 +181,7 @@ export const buildCanvasInpaintGraph = ( field: 'unet', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'unet', }, }, @@ -220,7 +223,7 @@ export const buildCanvasInpaintGraph = ( field: 'conditioning', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'positive_conditioning', }, }, @@ -230,7 +233,7 @@ export const buildCanvasInpaintGraph = ( field: 'conditioning', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'negative_conditioning', }, }, @@ -240,7 +243,7 @@ export const buildCanvasInpaintGraph = ( field: 'noise', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'noise', }, }, @@ -250,7 +253,7 @@ export const buildCanvasInpaintGraph = ( field: 'latents', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'latents', }, }, @@ -260,7 +263,7 @@ export const buildCanvasInpaintGraph = ( field: 'image', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'mask', }, }, @@ -288,7 +291,7 @@ export const buildCanvasInpaintGraph = ( // Decode Inpainted Latents To Image { source: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'latents', }, destination: { @@ -296,6 +299,155 @@ export const buildCanvasInpaintGraph = ( field: 'latents', }, }, + ], + }; + + // Handle Scale Before Processing + if (['auto', 'manual'].includes(boundingBoxScaleMethod)) { + const scaledWidth: number = scaledBoundingBoxDimensions.width; + const scaledHeight: number = scaledBoundingBoxDimensions.height; + + // Add Scaling Nodes + graph.nodes[INPAINT_IMAGE_RESIZE_UP] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + image: canvasInitImage, + }; + graph.nodes[MASK_RESIZE_UP] = { + type: 'img_resize', + id: MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + image: canvasMaskImage, + }; + graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + graph.nodes[MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: scaledWidth, + height: scaledHeight, + }; + + // Connect Nodes + graph.edges.push( + // Scale Inpaint Image and Mask + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_IMAGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: MASK_BLUR, + field: 'image', + }, + }, + // Color Correct The Inpainted Result + { + source: { + node_id: LATENTS_TO_IMAGE, + field: 'image', + }, + destination: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'mask', + }, + }, + // Paste Back Onto Original Image + { + source: { + node_id: COLOR_CORRECT, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'mask', + }, + } + ); + } else { + // Add Images To Nodes + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: width, + height: height, + }; + graph.nodes[INPAINT_IMAGE] = { + ...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation), + image: canvasInitImage, + }; + graph.nodes[MASK_BLUR] = { + ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), + image: canvasMaskImage, + }; + + graph.edges.push( // Color Correct The Inpainted Result { source: { @@ -337,11 +489,11 @@ export const buildCanvasInpaintGraph = ( node_id: CANVAS_OUTPUT, field: 'mask', }, - }, - ], - }; + } + ); + } - // handle seed + // Handle Seed if (shouldRandomizeSeed) { // Random int node to generate the starting seed const randomIntNode: RandomIntInvocation = { @@ -365,10 +517,10 @@ export const buildCanvasInpaintGraph = ( addVAEToGraph(state, graph, MAIN_MODEL_LOADER); // add LoRA support - addLoRAsToGraph(state, graph, INPAINT, MAIN_MODEL_LOADER); + addLoRAsToGraph(state, graph, DENOISE_LATENTS, MAIN_MODEL_LOADER); // add controlnet, mutating `graph` - addControlNetToLinearGraph(state, graph, INPAINT); + addControlNetToLinearGraph(state, graph, DENOISE_LATENTS); // NSFW & watermark - must be last thing added to graph if (state.system.shouldUseNSFWChecker) { diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts index d434f3d7cd..2eb70884cf 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts @@ -2,9 +2,12 @@ import { logger } from 'app/logging/logger'; import { RootState } from 'app/store/store'; import { NonNullableGraph } from 'features/nodes/types/types'; import { + ImageBlurInvocation, ImageDTO, + ImageToLatentsInvocation, InfillPatchmatchInvocation, InfillTileInvocation, + NoiseInvocation, RandomIntInvocation, RangeOfSizeInvocation, } from 'services/api/types'; @@ -18,15 +21,20 @@ import { CANVAS_OUTPUT, CLIP_SKIP, COLOR_CORRECT, - INPAINT, + DENOISE_LATENTS, INPAINT_IMAGE, + INPAINT_IMAGE_RESIZE_DOWN, + INPAINT_IMAGE_RESIZE_UP, INPAINT_INFILL, + INPAINT_INFILL_RESIZE_DOWN, ITERATE, LATENTS_TO_IMAGE, MAIN_MODEL_LOADER, MASK_BLUR, MASK_COMBINE, MASK_FROM_ALPHA, + MASK_RESIZE_DOWN, + MASK_RESIZE_UP, NEGATIVE_CONDITIONING, NOISE, POSITIVE_CONDITIONING, @@ -84,23 +92,6 @@ export const buildCanvasOutpaintGraph = ( ? shouldUseCpuNoise : shouldUseCpuNoise; - let infillNode: InfillTileInvocation | InfillPatchmatchInvocation = { - type: 'infill_tile', - id: INPAINT_INFILL, - is_intermediate: true, - image: canvasInitImage, - tile_size: tileSize, - }; - - if (infillMethod === 'patchmatch') { - infillNode = { - type: 'infill_patchmatch', - id: INPAINT_INFILL, - is_intermediate: true, - image: canvasInitImage, - }; - } - const graph: NonNullableGraph = { id: CANVAS_OUTPAINT_GRAPH, nodes: { @@ -147,7 +138,12 @@ export const buildCanvasOutpaintGraph = ( radius: maskBlur, blur_type: maskBlurMethod, }, - [infillNode.id]: infillNode, + [INPAINT_INFILL]: { + type: 'infill_tile', + id: INPAINT_INFILL, + is_intermediate: true, + tile_size: tileSize, + }, [INPAINT_IMAGE]: { type: 'i2l', id: INPAINT_IMAGE, @@ -157,14 +153,12 @@ export const buildCanvasOutpaintGraph = ( [NOISE]: { type: 'noise', id: NOISE, - width, - height, use_cpu, is_intermediate: true, }, - [INPAINT]: { + [DENOISE_LATENTS]: { type: 'denoise_latents', - id: INPAINT, + id: DENOISE_LATENTS, is_intermediate: true, steps: steps, cfg_scale: cfg_scale, @@ -186,7 +180,7 @@ export const buildCanvasOutpaintGraph = ( [CANVAS_OUTPUT]: { type: 'img_paste', id: CANVAS_OUTPUT, - is_intermediate: true, + is_intermediate: !shouldAutoSave, }, [RANGE_OF_SIZE]: { type: 'range_of_size', @@ -211,7 +205,7 @@ export const buildCanvasOutpaintGraph = ( field: 'unet', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'unet', }, }, @@ -268,16 +262,6 @@ export const buildCanvasOutpaintGraph = ( field: 'mask1', }, }, - { - source: { - node_id: MASK_COMBINE, - field: 'image', - }, - destination: { - node_id: MASK_BLUR, - field: 'image', - }, - }, // Plug Everything Into Inpaint Node { source: { @@ -285,7 +269,7 @@ export const buildCanvasOutpaintGraph = ( field: 'conditioning', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'positive_conditioning', }, }, @@ -295,7 +279,7 @@ export const buildCanvasOutpaintGraph = ( field: 'conditioning', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'negative_conditioning', }, }, @@ -305,7 +289,7 @@ export const buildCanvasOutpaintGraph = ( field: 'noise', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'noise', }, }, @@ -315,7 +299,7 @@ export const buildCanvasOutpaintGraph = ( field: 'latents', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'latents', }, }, @@ -325,7 +309,7 @@ export const buildCanvasOutpaintGraph = ( field: 'image', }, destination: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'mask', }, }, @@ -353,7 +337,7 @@ export const buildCanvasOutpaintGraph = ( // Decode the result from Inpaint { source: { - node_id: INPAINT, + node_id: DENOISE_LATENTS, field: 'latents', }, destination: { @@ -361,6 +345,230 @@ export const buildCanvasOutpaintGraph = ( field: 'latents', }, }, + ], + }; + + // Add Infill Nodes + + if (infillMethod === 'patchmatch') { + graph.nodes[INPAINT_INFILL] = { + type: 'infill_patchmatch', + id: INPAINT_INFILL, + is_intermediate: true, + }; + } + + // Handle Scale Before Processing + if (['auto', 'manual'].includes(boundingBoxScaleMethod)) { + const scaledWidth: number = scaledBoundingBoxDimensions.width; + const scaledHeight: number = scaledBoundingBoxDimensions.height; + + // Add Scaling Nodes + graph.nodes[INPAINT_IMAGE_RESIZE_UP] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + image: canvasInitImage, + }; + graph.nodes[MASK_RESIZE_UP] = { + type: 'img_resize', + id: MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + }; + graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + graph.nodes[INPAINT_INFILL_RESIZE_DOWN] = { + type: 'img_resize', + id: INPAINT_INFILL_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + graph.nodes[MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: scaledWidth, + height: scaledHeight, + }; + + // Connect Nodes + graph.edges.push( + // Scale Inpaint Image + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_INFILL, + field: 'image', + }, + }, + // Take combined mask and resize and then blur + { + source: { + node_id: MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: MASK_RESIZE_UP, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: MASK_BLUR, + field: 'image', + }, + }, + // Resize Results Down + { + source: { + node_id: LATENTS_TO_IMAGE, + field: 'image', + }, + destination: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: INPAINT_INFILL, + field: 'image', + }, + destination: { + node_id: INPAINT_INFILL_RESIZE_DOWN, + field: 'image', + }, + }, + // Color Correct The Inpainted Result + { + source: { + node_id: INPAINT_INFILL_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'reference', + }, + }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'mask', + }, + }, + // Paste Everything Back + { + source: { + node_id: INPAINT_INFILL_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'base_image', + }, + }, + { + source: { + node_id: COLOR_CORRECT, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'mask', + }, + } + ); + } else { + // Add Images To Nodes + graph.nodes[INPAINT_INFILL] = { + ...(graph.nodes[INPAINT_INFILL] as + | InfillTileInvocation + | InfillPatchmatchInvocation), + image: canvasInitImage, + }; + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: width, + height: height, + }; + graph.nodes[INPAINT_IMAGE] = { + ...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation), + image: canvasInitImage, + }; + graph.nodes[MASK_BLUR] = { + ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), + image: canvasMaskImage, + }; + + graph.edges.push( + // Take combined mask and plug it to blur + { + source: { + node_id: MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: MASK_BLUR, + field: 'image', + }, + }, // Color Correct The Inpainted Result { source: { @@ -422,11 +630,11 @@ export const buildCanvasOutpaintGraph = ( node_id: CANVAS_OUTPUT, field: 'mask', }, - }, - ], - }; + } + ); + } - // handle seed + // Handle Seed if (shouldRandomizeSeed) { // Random int node to generate the starting seed const randomIntNode: RandomIntInvocation = { @@ -450,10 +658,10 @@ export const buildCanvasOutpaintGraph = ( addVAEToGraph(state, graph, MAIN_MODEL_LOADER); // add LoRA support - addLoRAsToGraph(state, graph, INPAINT, MAIN_MODEL_LOADER); + addLoRAsToGraph(state, graph, DENOISE_LATENTS, MAIN_MODEL_LOADER); // add controlnet, mutating `graph` - addControlNetToLinearGraph(state, graph, INPAINT); + addControlNetToLinearGraph(state, graph, DENOISE_LATENTS); // NSFW & watermark - must be last thing added to graph if (state.system.shouldUseNSFWChecker) { diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLInpaintGraph.ts index 7b43be497c..13ba385a66 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLInpaintGraph.ts @@ -2,7 +2,10 @@ import { logger } from 'app/logging/logger'; import { RootState } from 'app/store/store'; import { NonNullableGraph } from 'features/nodes/types/types'; import { + ImageBlurInvocation, ImageDTO, + ImageToLatentsInvocation, + NoiseInvocation, RandomIntInvocation, RangeOfSizeInvocation, } from 'services/api/types'; @@ -16,9 +19,13 @@ import { CANVAS_OUTPUT, COLOR_CORRECT, INPAINT_IMAGE, + INPAINT_IMAGE_RESIZE_DOWN, + INPAINT_IMAGE_RESIZE_UP, ITERATE, LATENTS_TO_IMAGE, MASK_BLUR, + MASK_RESIZE_DOWN, + MASK_RESIZE_UP, NEGATIVE_CONDITIONING, NOISE, POSITIVE_CONDITIONING, @@ -114,20 +121,16 @@ export const buildCanvasSDXLInpaintGraph = ( is_intermediate: true, radius: maskBlur, blur_type: maskBlurMethod, - image: canvasMaskImage, }, [INPAINT_IMAGE]: { type: 'i2l', id: INPAINT_IMAGE, is_intermediate: true, fp32: vaePrecision === 'fp32' ? true : false, - image: canvasInitImage, }, [NOISE]: { type: 'noise', id: NOISE, - width, - height, use_cpu, is_intermediate: true, }, @@ -156,7 +159,7 @@ export const buildCanvasSDXLInpaintGraph = ( [CANVAS_OUTPUT]: { type: 'img_paste', id: CANVAS_OUTPUT, - is_intermediate: true, + is_intermediate: !shouldAutoSave, base_image: canvasInitImage, }, [RANGE_OF_SIZE]: { @@ -309,7 +312,156 @@ export const buildCanvasSDXLInpaintGraph = ( field: 'latents', }, }, - // Color Correct Inpainted Result + ], + }; + + // Handle Scale Before Processing + if (['auto', 'manual'].includes(boundingBoxScaleMethod)) { + const scaledWidth: number = scaledBoundingBoxDimensions.width; + const scaledHeight: number = scaledBoundingBoxDimensions.height; + + // Add Scaling Nodes + graph.nodes[INPAINT_IMAGE_RESIZE_UP] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + image: canvasInitImage, + }; + graph.nodes[MASK_RESIZE_UP] = { + type: 'img_resize', + id: MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + image: canvasMaskImage, + }; + graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + graph.nodes[MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: scaledWidth, + height: scaledHeight, + }; + + // Connect Nodes + graph.edges.push( + // Scale Inpaint Image and Mask + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_IMAGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: MASK_BLUR, + field: 'image', + }, + }, + // Color Correct The Inpainted Result + { + source: { + node_id: LATENTS_TO_IMAGE, + field: 'image', + }, + destination: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'mask', + }, + }, + // Paste Back Onto Original Image + { + source: { + node_id: COLOR_CORRECT, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'mask', + }, + } + ); + } else { + // Add Images To Nodes + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: width, + height: height, + }; + graph.nodes[INPAINT_IMAGE] = { + ...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation), + image: canvasInitImage, + }; + graph.nodes[MASK_BLUR] = { + ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), + image: canvasMaskImage, + }; + + graph.edges.push( + // Color Correct The Inpainted Result { source: { node_id: LATENTS_TO_IMAGE, @@ -330,7 +482,7 @@ export const buildCanvasSDXLInpaintGraph = ( field: 'mask', }, }, - // Paste them back on original image + // Paste Back Onto Original Image { source: { node_id: COLOR_CORRECT, @@ -350,19 +502,11 @@ export const buildCanvasSDXLInpaintGraph = ( node_id: CANVAS_OUTPUT, field: 'mask', }, - }, - ], - }; - - // Add Refiner if enabled - if (shouldUseSDXLRefiner) { - addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS); + } + ); } - // optionally add custom VAE - addVAEToGraph(state, graph, SDXL_MODEL_LOADER); - - // handle seed + // Handle Seed if (shouldRandomizeSeed) { // Random int node to generate the starting seed const randomIntNode: RandomIntInvocation = { @@ -382,6 +526,14 @@ export const buildCanvasSDXLInpaintGraph = ( (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; } + // Add Refiner if enabled + if (shouldUseSDXLRefiner) { + addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS); + } + + // optionally add custom VAE + addVAEToGraph(state, graph, SDXL_MODEL_LOADER); + // add LoRA support addSDXLLoRAsToGraph(state, graph, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER); diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts index c75ef0f800..7dc46ababf 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts @@ -2,9 +2,12 @@ import { logger } from 'app/logging/logger'; import { RootState } from 'app/store/store'; import { NonNullableGraph } from 'features/nodes/types/types'; import { + ImageBlurInvocation, ImageDTO, + ImageToLatentsInvocation, InfillPatchmatchInvocation, InfillTileInvocation, + NoiseInvocation, RandomIntInvocation, RangeOfSizeInvocation, } from 'services/api/types'; @@ -18,12 +21,17 @@ import { CANVAS_OUTPUT, COLOR_CORRECT, INPAINT_IMAGE, + INPAINT_IMAGE_RESIZE_DOWN, + INPAINT_IMAGE_RESIZE_UP, INPAINT_INFILL, + INPAINT_INFILL_RESIZE_DOWN, ITERATE, LATENTS_TO_IMAGE, MASK_BLUR, MASK_COMBINE, MASK_FROM_ALPHA, + MASK_RESIZE_DOWN, + MASK_RESIZE_UP, NEGATIVE_CONDITIONING, NOISE, POSITIVE_CONDITIONING, @@ -91,23 +99,6 @@ export const buildCanvasSDXLOutpaintGraph = ( ? shouldUseCpuNoise : shouldUseCpuNoise; - let infillNode: InfillTileInvocation | InfillPatchmatchInvocation = { - type: 'infill_tile', - id: INPAINT_INFILL, - is_intermediate: true, - image: canvasInitImage, - tile_size: tileSize, - }; - - if (infillMethod === 'patchmatch') { - infillNode = { - type: 'infill_patchmatch', - id: INPAINT_INFILL, - is_intermediate: true, - image: canvasInitImage, - }; - } - const graph: NonNullableGraph = { id: SDXL_CANVAS_OUTPAINT_GRAPH, nodes: { @@ -132,13 +123,6 @@ export const buildCanvasSDXLOutpaintGraph = ( ? `${negativePrompt} ${negativeStylePrompt}` : negativeStylePrompt, }, - [infillNode.id]: infillNode, - [INPAINT_IMAGE]: { - type: 'i2l', - id: INPAINT_IMAGE, - is_intermediate: true, - fp32: vaePrecision === 'fp32' ? true : false, - }, [MASK_FROM_ALPHA]: { type: 'tomask', id: MASK_FROM_ALPHA, @@ -158,11 +142,21 @@ export const buildCanvasSDXLOutpaintGraph = ( radius: maskBlur, blur_type: maskBlurMethod, }, + [INPAINT_INFILL]: { + type: 'infill_tile', + id: INPAINT_INFILL, + is_intermediate: true, + tile_size: tileSize, + }, + [INPAINT_IMAGE]: { + type: 'i2l', + id: INPAINT_IMAGE, + is_intermediate: true, + fp32: vaePrecision === 'fp32' ? true : false, + }, [NOISE]: { type: 'noise', id: NOISE, - width, - height, use_cpu, is_intermediate: true, }, @@ -190,7 +184,7 @@ export const buildCanvasSDXLOutpaintGraph = ( [CANVAS_OUTPUT]: { type: 'img_paste', id: CANVAS_OUTPUT, - is_intermediate: true, + is_intermediate: !shouldAutoSave, }, [RANGE_OF_SIZE]: { type: 'range_of_size', @@ -259,7 +253,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'clip2', }, }, - // Infill The Image + // Connect Infill Result To Inpaint Image { source: { node_id: INPAINT_INFILL, @@ -270,7 +264,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, - // Create mask from image alpha & merge with user painted mask + // Combine Mask from Init Image with User Painted Mask { source: { node_id: MASK_FROM_ALPHA, @@ -281,16 +275,6 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'mask1', }, }, - { - source: { - node_id: MASK_COMBINE, - field: 'image', - }, - destination: { - node_id: MASK_BLUR, - field: 'image', - }, - }, // Connect Everything To Inpaint { source: { @@ -374,6 +358,230 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'latents', }, }, + ], + }; + + // Add Infill Nodes + + if (infillMethod === 'patchmatch') { + graph.nodes[INPAINT_INFILL] = { + type: 'infill_patchmatch', + id: INPAINT_INFILL, + is_intermediate: true, + }; + } + + // Handle Scale Before Processing + if (['auto', 'manual'].includes(boundingBoxScaleMethod)) { + const scaledWidth: number = scaledBoundingBoxDimensions.width; + const scaledHeight: number = scaledBoundingBoxDimensions.height; + + // Add Scaling Nodes + graph.nodes[INPAINT_IMAGE_RESIZE_UP] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + image: canvasInitImage, + }; + graph.nodes[MASK_RESIZE_UP] = { + type: 'img_resize', + id: MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + }; + graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { + type: 'img_resize', + id: INPAINT_IMAGE_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + graph.nodes[INPAINT_INFILL_RESIZE_DOWN] = { + type: 'img_resize', + id: INPAINT_INFILL_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + graph.nodes[MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; + + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: scaledWidth, + height: scaledHeight, + }; + + // Connect Nodes + graph.edges.push( + // Scale Inpaint Image + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_INFILL, + field: 'image', + }, + }, + // Take combined mask and resize and then blur + { + source: { + node_id: MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: MASK_RESIZE_UP, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: MASK_BLUR, + field: 'image', + }, + }, + // Resize Results Down + { + source: { + node_id: LATENTS_TO_IMAGE, + field: 'image', + }, + destination: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + }, + { + source: { + node_id: INPAINT_INFILL, + field: 'image', + }, + destination: { + node_id: INPAINT_INFILL_RESIZE_DOWN, + field: 'image', + }, + }, + // Color Correct The Inpainted Result + { + source: { + node_id: INPAINT_INFILL_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'reference', + }, + }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: COLOR_CORRECT, + field: 'mask', + }, + }, + // Paste Everything Back + { + source: { + node_id: INPAINT_INFILL_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'base_image', + }, + }, + { + source: { + node_id: COLOR_CORRECT, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'image', + }, + }, + { + source: { + node_id: MASK_RESIZE_DOWN, + field: 'image', + }, + destination: { + node_id: CANVAS_OUTPUT, + field: 'mask', + }, + } + ); + } else { + // Add Images To Nodes + graph.nodes[INPAINT_INFILL] = { + ...(graph.nodes[INPAINT_INFILL] as + | InfillTileInvocation + | InfillPatchmatchInvocation), + image: canvasInitImage, + }; + graph.nodes[NOISE] = { + ...(graph.nodes[NOISE] as NoiseInvocation), + width: width, + height: height, + }; + graph.nodes[INPAINT_IMAGE] = { + ...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation), + image: canvasInitImage, + }; + graph.nodes[MASK_BLUR] = { + ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), + image: canvasMaskImage, + }; + + graph.edges.push( + // Take combined mask and plug it to blur + { + source: { + node_id: MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: MASK_BLUR, + field: 'image', + }, + }, // Color Correct The Inpainted Result { source: { @@ -405,7 +613,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'mask', }, }, - // Paste Back Outpainted Image on Original + // Paste Everything Back { source: { node_id: INPAINT_INFILL, @@ -435,19 +643,11 @@ export const buildCanvasSDXLOutpaintGraph = ( node_id: CANVAS_OUTPUT, field: 'mask', }, - }, - ], - }; - - // Add Refiner if enabled - if (shouldUseSDXLRefiner) { - addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS); + } + ); } - // optionally add custom VAE - addVAEToGraph(state, graph, SDXL_MODEL_LOADER); - - // handle seed + // Handle seed if (shouldRandomizeSeed) { // Random int node to generate the starting seed const randomIntNode: RandomIntInvocation = { @@ -467,6 +667,14 @@ export const buildCanvasSDXLOutpaintGraph = ( (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; } + // Add Refiner if enabled + if (shouldUseSDXLRefiner) { + addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS); + } + + // optionally add custom VAE + addVAEToGraph(state, graph, SDXL_MODEL_LOADER); + // add LoRA support addSDXLLoRAsToGraph(state, graph, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER); diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts index f1654383dd..3e213120b3 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts @@ -21,12 +21,18 @@ export const CANVAS_OUTPUT = 'canvas_output'; export const INPAINT = 'inpaint'; export const INPAINT_SEAM_FIX = 'inpaint_seam_fix'; export const INPAINT_IMAGE = 'inpaint_image'; +export const SCALED_INPAINT_IMAGE = 'scaled_inpaint_image'; +export const INPAINT_IMAGE_RESIZE_UP = 'inpaint_image_resize_up'; +export const INPAINT_IMAGE_RESIZE_DOWN = 'inpaint_image_resize_down'; export const INPAINT_INFILL = 'inpaint_infill'; +export const INPAINT_INFILL_RESIZE_DOWN = 'inpaint_infill_resize_down'; export const INPAINT_FINAL_IMAGE = 'inpaint_final_image'; export const MASK_FROM_ALPHA = 'tomask'; export const MASK_EDGE = 'mask_edge'; export const MASK_BLUR = 'mask_blur'; export const MASK_COMBINE = 'mask_combine'; +export const MASK_RESIZE_UP = 'mask_resize_up'; +export const MASK_RESIZE_DOWN = 'mask_resize_down'; export const COLOR_CORRECT = 'color_correct'; export const PASTE_IMAGE = 'img_paste'; export const CONTROL_NET_COLLECT = 'control_net_collect'; diff --git a/invokeai/frontend/web/src/services/api/types.ts b/invokeai/frontend/web/src/services/api/types.ts index 471c995f4d..435b605489 100644 --- a/invokeai/frontend/web/src/services/api/types.ts +++ b/invokeai/frontend/web/src/services/api/types.ts @@ -184,6 +184,15 @@ export type ImageNSFWBlurInvocation = TypeReq< export type ImageWatermarkInvocation = TypeReq< components['schemas']['ImageWatermarkInvocation'] >; +export type ImageBlurInvocation = TypeReq< + components['schemas']['ImageBlurInvocation'] +>; +export type ColorCorrectInvocation = TypeReq< + components['schemas']['ColorCorrectInvocation'] +>; +export type ImagePasteInvocation = TypeReq< + components['schemas']['ImagePasteInvocation'] +>; // ControlNet Nodes export type ControlNetInvocation = TypeReq<