From 9cbaefaa8139352da24dc017e6d0b437126c4e66 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:46:48 +1200 Subject: [PATCH] feat: Add Seam Painting to SDXL --- .../buildCanvasSDXLOutpaintGraph.ts | 217 +++++++++++++++++- 1 file changed, 209 insertions(+), 8 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts index 4d098f959f..1cc268c03d 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts @@ -29,6 +29,7 @@ import { LATENTS_TO_IMAGE, MASK_BLUR, MASK_COMBINE, + MASK_EDGE, MASK_FROM_ALPHA, MASK_RESIZE_DOWN, MASK_RESIZE_UP, @@ -40,6 +41,10 @@ import { SDXL_CANVAS_OUTPAINT_GRAPH, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER, + SEAM_FIX_DENOISE_LATENTS, + SEAM_MASK_COMBINE, + SEAM_MASK_RESIZE_DOWN, + SEAM_MASK_RESIZE_UP, } from './constants'; import { craftSDXLStylePrompt } from './helpers/craftSDXLStylePrompt'; @@ -67,6 +72,12 @@ export const buildCanvasSDXLOutpaintGraph = ( shouldUseCpuNoise, maskBlur, maskBlurMethod, + seamSize, + seamBlur, + seamSteps, + seamStrength, + seamLowThreshold, + seamHighThreshold, tileSize, infillMethod, } = state.generation; @@ -133,6 +144,11 @@ export const buildCanvasSDXLOutpaintGraph = ( is_intermediate: true, mask2: canvasMaskImage, }, + [SEAM_MASK_COMBINE]: { + type: 'mask_combine', + id: MASK_COMBINE, + is_intermediate: true, + }, [MASK_BLUR]: { type: 'img_blur', id: MASK_BLUR, @@ -170,6 +186,25 @@ export const buildCanvasSDXLOutpaintGraph = ( : 1 - strength, denoising_end: shouldUseSDXLRefiner ? refinerStart : 1, }, + [MASK_EDGE]: { + type: 'mask_edge', + id: MASK_EDGE, + is_intermediate: true, + edge_size: seamSize, + edge_blur: seamBlur, + low_threshold: seamLowThreshold, + high_threshold: seamHighThreshold, + }, + [SEAM_FIX_DENOISE_LATENTS]: { + type: 'denoise_latents', + id: SEAM_FIX_DENOISE_LATENTS, + is_intermediate: true, + steps: seamSteps, + cfg_scale: cfg_scale, + scheduler: scheduler, + denoising_start: 1 - seamStrength, + denoising_end: 1, + }, [LATENTS_TO_IMAGE]: { type: 'l2i', id: LATENTS_TO_IMAGE, @@ -347,12 +382,63 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'seed', }, }, - // Decode inpainted latents to image + // Seam Paint + { + source: { + node_id: SDXL_MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'noise', + }, + }, { source: { node_id: SDXL_DENOISE_LATENTS, field: 'latents', }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'latents', + }, + }, + // Decode inpainted latents to image + { + source: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'latents', + }, destination: { node_id: LATENTS_TO_IMAGE, field: 'latents', @@ -392,6 +478,13 @@ export const buildCanvasSDXLOutpaintGraph = ( width: scaledWidth, height: scaledHeight, }; + graph.nodes[SEAM_MASK_RESIZE_UP] = { + type: 'img_resize', + id: SEAM_MASK_RESIZE_UP, + is_intermediate: true, + width: scaledWidth, + height: scaledHeight, + }; graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { type: 'img_resize', id: INPAINT_IMAGE_RESIZE_DOWN, @@ -413,6 +506,13 @@ export const buildCanvasSDXLOutpaintGraph = ( width: width, height: height, }; + graph.nodes[SEAM_MASK_RESIZE_DOWN] = { + type: 'img_resize', + id: SEAM_MASK_RESIZE_DOWN, + is_intermediate: true, + width: width, + height: height, + }; graph.nodes[NOISE] = { ...(graph.nodes[NOISE] as NoiseInvocation), @@ -454,6 +554,57 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, + // Seam Paint Mask + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: MASK_EDGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + }, + { + source: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'mask', + }, + }, + { + source: { + node_id: MASK_BLUR, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask1', + }, + }, + { + source: { + node_id: SEAM_MASK_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask2', + }, + }, // Resize Results Down { source: { @@ -467,7 +618,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: MASK_RESIZE_UP, field: 'image', }, destination: { @@ -475,6 +626,16 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, + { + source: { + node_id: SEAM_MASK_COMBINE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_RESIZE_DOWN, + field: 'image', + }, + }, { source: { node_id: INPAINT_INFILL, @@ -508,7 +669,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_RESIZE_DOWN, + node_id: SEAM_MASK_RESIZE_DOWN, field: 'image', }, destination: { @@ -539,7 +700,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_RESIZE_DOWN, + node_id: SEAM_MASK_RESIZE_DOWN, field: 'image', }, destination: { @@ -567,7 +728,6 @@ export const buildCanvasSDXLOutpaintGraph = ( }; graph.nodes[MASK_BLUR] = { ...(graph.nodes[MASK_BLUR] as ImageBlurInvocation), - image: canvasMaskImage, }; graph.edges.push( @@ -582,6 +742,47 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, + // Seam Paint Mask + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: MASK_EDGE, + field: 'image', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_FIX_DENOISE_LATENTS, + field: 'mask', + }, + }, + { + source: { + node_id: MASK_FROM_ALPHA, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask1', + }, + }, + { + source: { + node_id: MASK_EDGE, + field: 'image', + }, + destination: { + node_id: SEAM_MASK_COMBINE, + field: 'mask2', + }, + }, // Color Correct The Inpainted Result { source: { @@ -605,7 +806,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: SEAM_MASK_COMBINE, field: 'image', }, destination: { @@ -636,7 +837,7 @@ export const buildCanvasSDXLOutpaintGraph = ( }, { source: { - node_id: MASK_BLUR, + node_id: SEAM_MASK_COMBINE, field: 'image', }, destination: { @@ -669,7 +870,7 @@ export const buildCanvasSDXLOutpaintGraph = ( // Add Refiner if enabled if (shouldUseSDXLRefiner) { - addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS); + addSDXLRefinerToGraph(state, graph, SEAM_FIX_DENOISE_LATENTS); } // optionally add custom VAE