From 8277bfab5e15126504a511dd24ee24169f16576f Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Thu, 24 Aug 2023 04:59:18 +1200 Subject: [PATCH] feat: Add Refiner Pass to SDXL Outpainting Also fix Scale Before Processing --- .../graphBuilders/buildCanvasOutpaintGraph.ts | 18 -- .../buildCanvasSDXLOutpaintGraph.ts | 178 ++---------------- .../nodes/util/graphBuilders/constants.ts | 2 - 3 files changed, 18 insertions(+), 180 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts index ecdd5ce556..33217efbbf 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts @@ -40,7 +40,6 @@ import { POSITIVE_CONDITIONING, RANDOM_INT, RANGE_OF_SIZE, - SEAM_MASK_RESIZE_UP, } from './constants'; /** @@ -168,15 +167,6 @@ export const buildCanvasOutpaintGraph = ( denoising_start: 1 - strength, denoising_end: 1, }, - // [MASK_EDGE]: { - // type: 'mask_edge', - // id: MASK_EDGE, - // is_intermediate: true, - // edge_size: seamSize, - // edge_blur: seamBlur, - // low_threshold: seamLowThreshold, - // high_threshold: seamHighThreshold, - // }, [OUTPAINT_REFINE_DENOISE_LATENTS]: { type: 'denoise_latents', id: OUTPAINT_REFINE_DENOISE_LATENTS, @@ -361,7 +351,6 @@ export const buildCanvasOutpaintGraph = ( field: 'unet', }, }, - { source: { node_id: POSITIVE_CONDITIONING, @@ -446,13 +435,6 @@ export const buildCanvasOutpaintGraph = ( width: scaledWidth, height: scaledHeight, }; - graph.nodes[SEAM_MASK_RESIZE_UP] = { - type: 'img_resize', - id: SEAM_MASK_RESIZE_UP, - is_intermediate: true, - width: scaledWidth, - height: scaledHeight, - }; graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { type: 'img_resize', id: INPAINT_IMAGE_RESIZE_DOWN, diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts index f15f554dca..d90a2c6f58 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts @@ -19,7 +19,6 @@ import { addVAEToGraph } from './addVAEToGraph'; import { addWatermarkerToGraph } from './addWatermarkerToGraph'; import { CANVAS_OUTPUT, - COLOR_CORRECT, INPAINT_IMAGE, INPAINT_IMAGE_RESIZE_DOWN, INPAINT_IMAGE_RESIZE_UP, @@ -29,20 +28,18 @@ import { LATENTS_TO_IMAGE, MASK_BLUR, MASK_COMBINE, - MASK_EDGE, MASK_FROM_ALPHA, MASK_RESIZE_DOWN, MASK_RESIZE_UP, NEGATIVE_CONDITIONING, NOISE, + OUTPAINT_REFINE_DENOISE_LATENTS, POSITIVE_CONDITIONING, RANDOM_INT, RANGE_OF_SIZE, SDXL_CANVAS_OUTPAINT_GRAPH, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER, - SEAM_FIX_DENOISE_LATENTS, - SEAM_MASK_RESIZE_UP, } from './constants'; import { craftSDXLStylePrompt } from './helpers/craftSDXLStylePrompt'; @@ -70,12 +67,8 @@ export const buildCanvasSDXLOutpaintGraph = ( shouldUseCpuNoise, maskBlur, maskBlurMethod, - seamSize, - seamBlur, - seamSteps, - seamStrength, - seamLowThreshold, - seamHighThreshold, + refineSteps, + refineStrength, tileSize, infillMethod, } = state.generation; @@ -179,23 +172,14 @@ export const buildCanvasSDXLOutpaintGraph = ( : 1 - strength, denoising_end: shouldUseSDXLRefiner ? refinerStart : 1, }, - [MASK_EDGE]: { - type: 'mask_edge', - id: MASK_EDGE, - is_intermediate: true, - edge_size: seamSize, - edge_blur: seamBlur, - low_threshold: seamLowThreshold, - high_threshold: seamHighThreshold, - }, - [SEAM_FIX_DENOISE_LATENTS]: { + [OUTPAINT_REFINE_DENOISE_LATENTS]: { type: 'denoise_latents', - id: SEAM_FIX_DENOISE_LATENTS, + id: OUTPAINT_REFINE_DENOISE_LATENTS, is_intermediate: true, - steps: seamSteps, + steps: refineSteps, cfg_scale: cfg_scale, scheduler: scheduler, - denoising_start: 1 - seamStrength, + denoising_start: 1 - refineStrength, denoising_end: 1, }, [LATENTS_TO_IMAGE]: { @@ -204,13 +188,8 @@ export const buildCanvasSDXLOutpaintGraph = ( is_intermediate: true, fp32: vaePrecision === 'fp32' ? true : false, }, - [COLOR_CORRECT]: { - type: 'color_correct', - id: COLOR_CORRECT, - is_intermediate: true, - }, [CANVAS_OUTPUT]: { - type: 'img_paste', + type: 'color_correct', id: CANVAS_OUTPUT, is_intermediate: !shouldAutoSave, }, @@ -375,14 +354,14 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'seed', }, }, - // Seam Paint + // Image To Image Clean Up { source: { node_id: SDXL_MODEL_LOADER, field: 'unet', }, destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, + node_id: OUTPAINT_REFINE_DENOISE_LATENTS, field: 'unet', }, }, @@ -392,7 +371,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'conditioning', }, destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, + node_id: OUTPAINT_REFINE_DENOISE_LATENTS, field: 'positive_conditioning', }, }, @@ -402,7 +381,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'conditioning', }, destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, + node_id: OUTPAINT_REFINE_DENOISE_LATENTS, field: 'negative_conditioning', }, }, @@ -412,7 +391,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'noise', }, destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, + node_id: OUTPAINT_REFINE_DENOISE_LATENTS, field: 'noise', }, }, @@ -422,14 +401,14 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'latents', }, destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, + node_id: OUTPAINT_REFINE_DENOISE_LATENTS, field: 'latents', }, }, // Decode inpainted latents to image { source: { - node_id: SEAM_FIX_DENOISE_LATENTS, + node_id: OUTPAINT_REFINE_DENOISE_LATENTS, field: 'latents', }, destination: { @@ -471,13 +450,6 @@ export const buildCanvasSDXLOutpaintGraph = ( width: scaledWidth, height: scaledHeight, }; - graph.nodes[SEAM_MASK_RESIZE_UP] = { - type: 'img_resize', - id: SEAM_MASK_RESIZE_UP, - is_intermediate: true, - width: scaledWidth, - height: scaledHeight, - }; graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = { type: 'img_resize', id: INPAINT_IMAGE_RESIZE_DOWN, @@ -540,37 +512,6 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, - // Seam Paint Mask - { - source: { - node_id: MASK_FROM_ALPHA, - field: 'image', - }, - destination: { - node_id: MASK_EDGE, - field: 'image', - }, - }, - { - source: { - node_id: MASK_EDGE, - field: 'image', - }, - destination: { - node_id: SEAM_MASK_RESIZE_UP, - field: 'image', - }, - }, - { - source: { - node_id: SEAM_MASK_RESIZE_UP, - field: 'image', - }, - destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, - field: 'mask', - }, - }, // Resize Results Down { source: { @@ -609,7 +550,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, destination: { - node_id: COLOR_CORRECT, + node_id: CANVAS_OUTPUT, field: 'reference', }, }, @@ -618,37 +559,6 @@ export const buildCanvasSDXLOutpaintGraph = ( node_id: INPAINT_IMAGE_RESIZE_DOWN, field: 'image', }, - destination: { - node_id: COLOR_CORRECT, - field: 'image', - }, - }, - { - source: { - node_id: MASK_RESIZE_DOWN, - field: 'image', - }, - destination: { - node_id: COLOR_CORRECT, - field: 'mask', - }, - }, - // Paste Everything Back - { - source: { - node_id: INPAINT_INFILL_RESIZE_DOWN, - field: 'image', - }, - destination: { - node_id: CANVAS_OUTPUT, - field: 'base_image', - }, - }, - { - source: { - node_id: COLOR_CORRECT, - field: 'image', - }, destination: { node_id: CANVAS_OUTPUT, field: 'image', @@ -698,27 +608,6 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, }, - // Seam Paint Mask - { - source: { - node_id: MASK_FROM_ALPHA, - field: 'image', - }, - destination: { - node_id: MASK_EDGE, - field: 'image', - }, - }, - { - source: { - node_id: MASK_EDGE, - field: 'image', - }, - destination: { - node_id: SEAM_FIX_DENOISE_LATENTS, - field: 'mask', - }, - }, // Color Correct The Inpainted Result { source: { @@ -726,7 +615,7 @@ export const buildCanvasSDXLOutpaintGraph = ( field: 'image', }, destination: { - node_id: COLOR_CORRECT, + node_id: CANVAS_OUTPUT, field: 'reference', }, }, @@ -735,37 +624,6 @@ export const buildCanvasSDXLOutpaintGraph = ( node_id: LATENTS_TO_IMAGE, field: 'image', }, - destination: { - node_id: COLOR_CORRECT, - field: 'image', - }, - }, - { - source: { - node_id: MASK_BLUR, - field: 'image', - }, - destination: { - node_id: COLOR_CORRECT, - field: 'mask', - }, - }, - // Paste Everything Back - { - source: { - node_id: INPAINT_INFILL, - field: 'image', - }, - destination: { - node_id: CANVAS_OUTPUT, - field: 'base_image', - }, - }, - { - source: { - node_id: COLOR_CORRECT, - field: 'image', - }, destination: { node_id: CANVAS_OUTPUT, field: 'image', @@ -806,7 +664,7 @@ export const buildCanvasSDXLOutpaintGraph = ( // Add Refiner if enabled if (shouldUseSDXLRefiner) { - addSDXLRefinerToGraph(state, graph, SEAM_FIX_DENOISE_LATENTS); + addSDXLRefinerToGraph(state, graph, OUTPAINT_REFINE_DENOISE_LATENTS); } // optionally add custom VAE diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts index 00866fc633..6655001715 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts @@ -25,14 +25,12 @@ export const INPAINT_IMAGE_RESIZE_DOWN = 'inpaint_image_resize_down'; export const INPAINT_INFILL = 'inpaint_infill'; export const INPAINT_INFILL_RESIZE_DOWN = 'inpaint_infill_resize_down'; export const INPAINT_FINAL_IMAGE = 'inpaint_final_image'; -export const SEAM_FIX_DENOISE_LATENTS = 'seam_fix_denoise_latents'; export const OUTPAINT_REFINE_DENOISE_LATENTS = 'outpaint_refine_denoise_latents'; export const MASK_FROM_ALPHA = 'tomask'; export const MASK_EDGE = 'mask_edge'; export const MASK_BLUR = 'mask_blur'; export const MASK_COMBINE = 'mask_combine'; -export const SEAM_MASK_RESIZE_UP = 'seam_mask_resize_up'; export const MASK_RESIZE_UP = 'mask_resize_up'; export const MASK_RESIZE_DOWN = 'mask_resize_down'; export const COLOR_CORRECT = 'color_correct';