Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Scaled Processing to Inpainting & Outpainting / 1.x & SDXL

Browse Source
This commit is contained in:
blessedcoolant 2023-08-13 20:17:23 +12:00
parent c8864e475b
commit b35cdc05a5
9 changed files with 874 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts
View File

@ -123,7 +123,6 @@ export const addUserInvokedCanvasListener = () => {
log.debug({ graph: parseify(graph) }, `Canvas graph built`);
// currently this action is just listened to for logging
console.log(canvasGraphBuilt(graph));
dispatch(canvasGraphBuilt(graph));
// Create the session, store the request id

2
invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts
View File

@ -47,7 +47,7 @@ export const initialCanvasState: CanvasState = {
boundingBoxCoordinates: { x: 0, y: 0 },
boundingBoxDimensions: { width: 512, height: 512 },
boundingBoxPreviewFill: { r: 0, g: 0, b: 0, a: 0.5 },
boundingBoxScaleMethod: 'auto',
boundingBoxScaleMethod: 'none',
brushColor: { r: 90, g: 90, b: 255, a: 1 },
brushSize: 50,
canvasContainerDimensions: { width: 0, height: 0 },

2
invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts
View File

@ -11,9 +11,9 @@ export const LAYER_NAMES = ['base', 'mask'] as const;
export type CanvasLayer = (typeof LAYER_NAMES)[number];
export const BOUNDING_BOX_SCALES_DICT = [
{ label: 'None', value: 'none' },
{ label: 'Auto', value: 'auto' },
{ label: 'Manual', value: 'manual' },
{ label: 'None', value: 'none' },
];
export const BOUNDING_BOX_SCALES = ['none', 'auto', 'manual'] as const;

194
invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts
View File

@ -2,7 +2,10 @@ import { logger } from 'app/logging/logger';
import { RootState } from 'app/store/store';
import { NonNullableGraph } from 'features/nodes/types/types';
import {
ImageBlurInvocation,
ImageDTO,
ImageToLatentsInvocation,
NoiseInvocation,
RandomIntInvocation,
RangeOfSizeInvocation,
} from 'services/api/types';
@ -16,12 +19,16 @@ import {
CANVAS_OUTPUT,
CLIP_SKIP,
COLOR_CORRECT,
INPAINT,
DENOISE_LATENTS,
INPAINT_IMAGE,
INPAINT_IMAGE_RESIZE_DOWN,
INPAINT_IMAGE_RESIZE_UP,
ITERATE,
LATENTS_TO_IMAGE,
MAIN_MODEL_LOADER,
MASK_BLUR,
MASK_RESIZE_DOWN,
MASK_RESIZE_UP,
NEGATIVE_CONDITIONING,
NOISE,
POSITIVE_CONDITIONING,
@ -108,7 +115,6 @@ export const buildCanvasInpaintGraph = (
type: 'img_blur',
id: MASK_BLUR,
is_intermediate: true,
image: canvasMaskImage,
radius: maskBlur,
blur_type: maskBlurMethod,
},
@ -117,19 +123,16 @@ export const buildCanvasInpaintGraph = (
id: INPAINT_IMAGE,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
image: canvasInitImage,
},
[NOISE]: {
type: 'noise',
id: NOISE,
width,
height,
use_cpu,
is_intermediate: true,
},
[INPAINT]: {
[DENOISE_LATENTS]: {
type: 'denoise_latents',
id: INPAINT,
id: DENOISE_LATENTS,
is_intermediate: true,
steps: steps,
cfg_scale: cfg_scale,
@ -152,7 +155,7 @@ export const buildCanvasInpaintGraph = (
[CANVAS_OUTPUT]: {
type: 'img_paste',
id: CANVAS_OUTPUT,
is_intermediate: true,
is_intermediate: !shouldAutoSave,
base_image: canvasInitImage,
},
[RANGE_OF_SIZE]: {
@ -178,7 +181,7 @@ export const buildCanvasInpaintGraph = (
field: 'unet',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'unet',
},
},
@ -220,7 +223,7 @@ export const buildCanvasInpaintGraph = (
field: 'conditioning',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'positive_conditioning',
},
},
@ -230,7 +233,7 @@ export const buildCanvasInpaintGraph = (
field: 'conditioning',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'negative_conditioning',
},
},
@ -240,7 +243,7 @@ export const buildCanvasInpaintGraph = (
field: 'noise',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'noise',
},
},
@ -250,7 +253,7 @@ export const buildCanvasInpaintGraph = (
field: 'latents',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'latents',
},
},
@ -260,7 +263,7 @@ export const buildCanvasInpaintGraph = (
field: 'image',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'mask',
},
},
@ -288,7 +291,7 @@ export const buildCanvasInpaintGraph = (
// Decode Inpainted Latents To Image
{
source: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'latents',
},
destination: {
@ -296,6 +299,155 @@ export const buildCanvasInpaintGraph = (
field: 'latents',
},
},
],
};
// Handle Scale Before Processing
if (['auto', 'manual'].includes(boundingBoxScaleMethod)) {
const scaledWidth: number = scaledBoundingBoxDimensions.width;
const scaledHeight: number = scaledBoundingBoxDimensions.height;
// Add Scaling Nodes
graph.nodes[INPAINT_IMAGE_RESIZE_UP] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
image: canvasInitImage,
};
graph.nodes[MASK_RESIZE_UP] = {
type: 'img_resize',
id: MASK_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
image: canvasMaskImage,
};
graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[MASK_RESIZE_DOWN] = {
type: 'img_resize',
id: MASK_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: scaledWidth,
height: scaledHeight,
};
// Connect Nodes
graph.edges.push(
// Scale Inpaint Image and Mask
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: INPAINT_IMAGE,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'image',
},
},
{
source: {
node_id: MASK_BLUR,
field: 'image',
},
destination: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'mask',
},
},
// Paste Back Onto Original Image
{
source: {
node_id: COLOR_CORRECT,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'mask',
},
}
);
} else {
// Add Images To Nodes
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: width,
height: height,
};
graph.nodes[INPAINT_IMAGE] = {
...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation),
image: canvasInitImage,
};
graph.nodes[MASK_BLUR] = {
...(graph.nodes[MASK_BLUR] as ImageBlurInvocation),
image: canvasMaskImage,
};
graph.edges.push(
// Color Correct The Inpainted Result
{
source: {
@ -337,11 +489,11 @@ export const buildCanvasInpaintGraph = (
node_id: CANVAS_OUTPUT,
field: 'mask',
},
},
],
};
}
);
}
// handle seed
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed
const randomIntNode: RandomIntInvocation = {
@ -365,10 +517,10 @@ export const buildCanvasInpaintGraph = (
addVAEToGraph(state, graph, MAIN_MODEL_LOADER);
// add LoRA support
addLoRAsToGraph(state, graph, INPAINT, MAIN_MODEL_LOADER);
addLoRAsToGraph(state, graph, DENOISE_LATENTS, MAIN_MODEL_LOADER);
// add controlnet, mutating `graph`
addControlNetToLinearGraph(state, graph, INPAINT);
addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
// NSFW & watermark - must be last thing added to graph
if (state.system.shouldUseNSFWChecker) {

302
invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts
View File

@ -2,9 +2,12 @@ import { logger } from 'app/logging/logger';
import { RootState } from 'app/store/store';
import { NonNullableGraph } from 'features/nodes/types/types';
import {
ImageBlurInvocation,
ImageDTO,
ImageToLatentsInvocation,
InfillPatchmatchInvocation,
InfillTileInvocation,
NoiseInvocation,
RandomIntInvocation,
RangeOfSizeInvocation,
} from 'services/api/types';
@ -18,15 +21,20 @@ import {
CANVAS_OUTPUT,
CLIP_SKIP,
COLOR_CORRECT,
INPAINT,
DENOISE_LATENTS,
INPAINT_IMAGE,
INPAINT_IMAGE_RESIZE_DOWN,
INPAINT_IMAGE_RESIZE_UP,
INPAINT_INFILL,
INPAINT_INFILL_RESIZE_DOWN,
ITERATE,
LATENTS_TO_IMAGE,
MAIN_MODEL_LOADER,
MASK_BLUR,
MASK_COMBINE,
MASK_FROM_ALPHA,
MASK_RESIZE_DOWN,
MASK_RESIZE_UP,
NEGATIVE_CONDITIONING,
NOISE,
POSITIVE_CONDITIONING,
@ -84,23 +92,6 @@ export const buildCanvasOutpaintGraph = (
? shouldUseCpuNoise
: shouldUseCpuNoise;
let infillNode: InfillTileInvocation | InfillPatchmatchInvocation = {
type: 'infill_tile',
id: INPAINT_INFILL,
is_intermediate: true,
image: canvasInitImage,
tile_size: tileSize,
};
if (infillMethod === 'patchmatch') {
infillNode = {
type: 'infill_patchmatch',
id: INPAINT_INFILL,
is_intermediate: true,
image: canvasInitImage,
};
}
const graph: NonNullableGraph = {
id: CANVAS_OUTPAINT_GRAPH,
nodes: {
@ -147,7 +138,12 @@ export const buildCanvasOutpaintGraph = (
radius: maskBlur,
blur_type: maskBlurMethod,
},
[infillNode.id]: infillNode,
[INPAINT_INFILL]: {
type: 'infill_tile',
id: INPAINT_INFILL,
is_intermediate: true,
tile_size: tileSize,
},
[INPAINT_IMAGE]: {
type: 'i2l',
id: INPAINT_IMAGE,
@ -157,14 +153,12 @@ export const buildCanvasOutpaintGraph = (
[NOISE]: {
type: 'noise',
id: NOISE,
width,
height,
use_cpu,
is_intermediate: true,
},
[INPAINT]: {
[DENOISE_LATENTS]: {
type: 'denoise_latents',
id: INPAINT,
id: DENOISE_LATENTS,
is_intermediate: true,
steps: steps,
cfg_scale: cfg_scale,
@ -186,7 +180,7 @@ export const buildCanvasOutpaintGraph = (
[CANVAS_OUTPUT]: {
type: 'img_paste',
id: CANVAS_OUTPUT,
is_intermediate: true,
is_intermediate: !shouldAutoSave,
},
[RANGE_OF_SIZE]: {
type: 'range_of_size',
@ -211,7 +205,7 @@ export const buildCanvasOutpaintGraph = (
field: 'unet',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'unet',
},
},
@ -268,16 +262,6 @@ export const buildCanvasOutpaintGraph = (
field: 'mask1',
},
},
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Plug Everything Into Inpaint Node
{
source: {
@ -285,7 +269,7 @@ export const buildCanvasOutpaintGraph = (
field: 'conditioning',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'positive_conditioning',
},
},
@ -295,7 +279,7 @@ export const buildCanvasOutpaintGraph = (
field: 'conditioning',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'negative_conditioning',
},
},
@ -305,7 +289,7 @@ export const buildCanvasOutpaintGraph = (
field: 'noise',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'noise',
},
},
@ -315,7 +299,7 @@ export const buildCanvasOutpaintGraph = (
field: 'latents',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'latents',
},
},
@ -325,7 +309,7 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
destination: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'mask',
},
},
@ -353,7 +337,7 @@ export const buildCanvasOutpaintGraph = (
// Decode the result from Inpaint
{
source: {
node_id: INPAINT,
node_id: DENOISE_LATENTS,
field: 'latents',
},
destination: {
@ -361,6 +345,230 @@ export const buildCanvasOutpaintGraph = (
field: 'latents',
},
},
],
};
// Add Infill Nodes
if (infillMethod === 'patchmatch') {
graph.nodes[INPAINT_INFILL] = {
type: 'infill_patchmatch',
id: INPAINT_INFILL,
is_intermediate: true,
};
}
// Handle Scale Before Processing
if (['auto', 'manual'].includes(boundingBoxScaleMethod)) {
const scaledWidth: number = scaledBoundingBoxDimensions.width;
const scaledHeight: number = scaledBoundingBoxDimensions.height;
// Add Scaling Nodes
graph.nodes[INPAINT_IMAGE_RESIZE_UP] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
image: canvasInitImage,
};
graph.nodes[MASK_RESIZE_UP] = {
type: 'img_resize',
id: MASK_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
};
graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[INPAINT_INFILL_RESIZE_DOWN] = {
type: 'img_resize',
id: INPAINT_INFILL_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[MASK_RESIZE_DOWN] = {
type: 'img_resize',
id: MASK_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: scaledWidth,
height: scaledHeight,
};
// Connect Nodes
graph.edges.push(
// Scale Inpaint Image
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: INPAINT_INFILL,
field: 'image',
},
},
// Take combined mask and resize and then blur
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Resize Results Down
{
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: MASK_BLUR,
field: 'image',
},
destination: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'reference',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'mask',
},
},
// Paste Everything Back
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'base_image',
},
},
{
source: {
node_id: COLOR_CORRECT,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'mask',
},
}
);
} else {
// Add Images To Nodes
graph.nodes[INPAINT_INFILL] = {
...(graph.nodes[INPAINT_INFILL] as
| InfillTileInvocation
| InfillPatchmatchInvocation),
image: canvasInitImage,
};
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: width,
height: height,
};
graph.nodes[INPAINT_IMAGE] = {
...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation),
image: canvasInitImage,
};
graph.nodes[MASK_BLUR] = {
...(graph.nodes[MASK_BLUR] as ImageBlurInvocation),
image: canvasMaskImage,
};
graph.edges.push(
// Take combined mask and plug it to blur
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -422,11 +630,11 @@ export const buildCanvasOutpaintGraph = (
node_id: CANVAS_OUTPUT,
field: 'mask',
},
},
],
};
}
);
}
// handle seed
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed
const randomIntNode: RandomIntInvocation = {
@ -450,10 +658,10 @@ export const buildCanvasOutpaintGraph = (
addVAEToGraph(state, graph, MAIN_MODEL_LOADER);
// add LoRA support
addLoRAsToGraph(state, graph, INPAINT, MAIN_MODEL_LOADER);
addLoRAsToGraph(state, graph, DENOISE_LATENTS, MAIN_MODEL_LOADER);
// add controlnet, mutating `graph`
addControlNetToLinearGraph(state, graph, INPAINT);
addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
// NSFW & watermark - must be last thing added to graph
if (state.system.shouldUseNSFWChecker) {

188
invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLInpaintGraph.ts
View File

@ -2,7 +2,10 @@ import { logger } from 'app/logging/logger';
import { RootState } from 'app/store/store';
import { NonNullableGraph } from 'features/nodes/types/types';
import {
ImageBlurInvocation,
ImageDTO,
ImageToLatentsInvocation,
NoiseInvocation,
RandomIntInvocation,
RangeOfSizeInvocation,
} from 'services/api/types';
@ -16,9 +19,13 @@ import {
CANVAS_OUTPUT,
COLOR_CORRECT,
INPAINT_IMAGE,
INPAINT_IMAGE_RESIZE_DOWN,
INPAINT_IMAGE_RESIZE_UP,
ITERATE,
LATENTS_TO_IMAGE,
MASK_BLUR,
MASK_RESIZE_DOWN,
MASK_RESIZE_UP,
NEGATIVE_CONDITIONING,
NOISE,
POSITIVE_CONDITIONING,
@ -114,20 +121,16 @@ export const buildCanvasSDXLInpaintGraph = (
is_intermediate: true,
radius: maskBlur,
blur_type: maskBlurMethod,
image: canvasMaskImage,
},
[INPAINT_IMAGE]: {
type: 'i2l',
id: INPAINT_IMAGE,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
image: canvasInitImage,
},
[NOISE]: {
type: 'noise',
id: NOISE,
width,
height,
use_cpu,
is_intermediate: true,
},
@ -156,7 +159,7 @@ export const buildCanvasSDXLInpaintGraph = (
[CANVAS_OUTPUT]: {
type: 'img_paste',
id: CANVAS_OUTPUT,
is_intermediate: true,
is_intermediate: !shouldAutoSave,
base_image: canvasInitImage,
},
[RANGE_OF_SIZE]: {
@ -309,7 +312,156 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'latents',
},
},
// Color Correct Inpainted Result
],
};
// Handle Scale Before Processing
if (['auto', 'manual'].includes(boundingBoxScaleMethod)) {
const scaledWidth: number = scaledBoundingBoxDimensions.width;
const scaledHeight: number = scaledBoundingBoxDimensions.height;
// Add Scaling Nodes
graph.nodes[INPAINT_IMAGE_RESIZE_UP] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
image: canvasInitImage,
};
graph.nodes[MASK_RESIZE_UP] = {
type: 'img_resize',
id: MASK_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
image: canvasMaskImage,
};
graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[MASK_RESIZE_DOWN] = {
type: 'img_resize',
id: MASK_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: scaledWidth,
height: scaledHeight,
};
// Connect Nodes
graph.edges.push(
// Scale Inpaint Image and Mask
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: INPAINT_IMAGE,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'image',
},
},
{
source: {
node_id: MASK_BLUR,
field: 'image',
},
destination: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'mask',
},
},
// Paste Back Onto Original Image
{
source: {
node_id: COLOR_CORRECT,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'mask',
},
}
);
} else {
// Add Images To Nodes
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: width,
height: height,
};
graph.nodes[INPAINT_IMAGE] = {
...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation),
image: canvasInitImage,
};
graph.nodes[MASK_BLUR] = {
...(graph.nodes[MASK_BLUR] as ImageBlurInvocation),
image: canvasMaskImage,
};
graph.edges.push(
// Color Correct The Inpainted Result
{
source: {
node_id: LATENTS_TO_IMAGE,
@ -330,7 +482,7 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'mask',
},
},
// Paste them back on original image
// Paste Back Onto Original Image
{
source: {
node_id: COLOR_CORRECT,
@ -350,19 +502,11 @@ export const buildCanvasSDXLInpaintGraph = (
node_id: CANVAS_OUTPUT,
field: 'mask',
},
},
],
};
// Add Refiner if enabled
if (shouldUseSDXLRefiner) {
addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS);
}
);
}
// optionally add custom VAE
addVAEToGraph(state, graph, SDXL_MODEL_LOADER);
// handle seed
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed
const randomIntNode: RandomIntInvocation = {
@ -382,6 +526,14 @@ export const buildCanvasSDXLInpaintGraph = (
(graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed;
}
// Add Refiner if enabled
if (shouldUseSDXLRefiner) {
addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS);
}
// optionally add custom VAE
addVAEToGraph(state, graph, SDXL_MODEL_LOADER);
// add LoRA support
addSDXLLoRAsToGraph(state, graph, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER);

310
invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts
View File

@ -2,9 +2,12 @@ import { logger } from 'app/logging/logger';
import { RootState } from 'app/store/store';
import { NonNullableGraph } from 'features/nodes/types/types';
import {
ImageBlurInvocation,
ImageDTO,
ImageToLatentsInvocation,
InfillPatchmatchInvocation,
InfillTileInvocation,
NoiseInvocation,
RandomIntInvocation,
RangeOfSizeInvocation,
} from 'services/api/types';
@ -18,12 +21,17 @@ import {
CANVAS_OUTPUT,
COLOR_CORRECT,
INPAINT_IMAGE,
INPAINT_IMAGE_RESIZE_DOWN,
INPAINT_IMAGE_RESIZE_UP,
INPAINT_INFILL,
INPAINT_INFILL_RESIZE_DOWN,
ITERATE,
LATENTS_TO_IMAGE,
MASK_BLUR,
MASK_COMBINE,
MASK_FROM_ALPHA,
MASK_RESIZE_DOWN,
MASK_RESIZE_UP,
NEGATIVE_CONDITIONING,
NOISE,
POSITIVE_CONDITIONING,
@ -91,23 +99,6 @@ export const buildCanvasSDXLOutpaintGraph = (
? shouldUseCpuNoise
: shouldUseCpuNoise;
let infillNode: InfillTileInvocation | InfillPatchmatchInvocation = {
type: 'infill_tile',
id: INPAINT_INFILL,
is_intermediate: true,
image: canvasInitImage,
tile_size: tileSize,
};
if (infillMethod === 'patchmatch') {
infillNode = {
type: 'infill_patchmatch',
id: INPAINT_INFILL,
is_intermediate: true,
image: canvasInitImage,
};
}
const graph: NonNullableGraph = {
id: SDXL_CANVAS_OUTPAINT_GRAPH,
nodes: {
@ -132,13 +123,6 @@ export const buildCanvasSDXLOutpaintGraph = (
? `${negativePrompt} ${negativeStylePrompt}`
: negativeStylePrompt,
},
[infillNode.id]: infillNode,
[INPAINT_IMAGE]: {
type: 'i2l',
id: INPAINT_IMAGE,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[MASK_FROM_ALPHA]: {
type: 'tomask',
id: MASK_FROM_ALPHA,
@ -158,11 +142,21 @@ export const buildCanvasSDXLOutpaintGraph = (
radius: maskBlur,
blur_type: maskBlurMethod,
},
[INPAINT_INFILL]: {
type: 'infill_tile',
id: INPAINT_INFILL,
is_intermediate: true,
tile_size: tileSize,
},
[INPAINT_IMAGE]: {
type: 'i2l',
id: INPAINT_IMAGE,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[NOISE]: {
type: 'noise',
id: NOISE,
width,
height,
use_cpu,
is_intermediate: true,
},
@ -190,7 +184,7 @@ export const buildCanvasSDXLOutpaintGraph = (
[CANVAS_OUTPUT]: {
type: 'img_paste',
id: CANVAS_OUTPUT,
is_intermediate: true,
is_intermediate: !shouldAutoSave,
},
[RANGE_OF_SIZE]: {
type: 'range_of_size',
@ -259,7 +253,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'clip2',
},
},
// Infill The Image
// Connect Infill Result To Inpaint Image
{
source: {
node_id: INPAINT_INFILL,
@ -270,7 +264,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'image',
},
},
// Create mask from image alpha & merge with user painted mask
// Combine Mask from Init Image with User Painted Mask
{
source: {
node_id: MASK_FROM_ALPHA,
@ -281,16 +275,6 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'mask1',
},
},
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Connect Everything To Inpaint
{
source: {
@ -374,6 +358,230 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'latents',
},
},
],
};
// Add Infill Nodes
if (infillMethod === 'patchmatch') {
graph.nodes[INPAINT_INFILL] = {
type: 'infill_patchmatch',
id: INPAINT_INFILL,
is_intermediate: true,
};
}
// Handle Scale Before Processing
if (['auto', 'manual'].includes(boundingBoxScaleMethod)) {
const scaledWidth: number = scaledBoundingBoxDimensions.width;
const scaledHeight: number = scaledBoundingBoxDimensions.height;
// Add Scaling Nodes
graph.nodes[INPAINT_IMAGE_RESIZE_UP] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
image: canvasInitImage,
};
graph.nodes[MASK_RESIZE_UP] = {
type: 'img_resize',
id: MASK_RESIZE_UP,
is_intermediate: true,
width: scaledWidth,
height: scaledHeight,
};
graph.nodes[INPAINT_IMAGE_RESIZE_DOWN] = {
type: 'img_resize',
id: INPAINT_IMAGE_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[INPAINT_INFILL_RESIZE_DOWN] = {
type: 'img_resize',
id: INPAINT_INFILL_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[MASK_RESIZE_DOWN] = {
type: 'img_resize',
id: MASK_RESIZE_DOWN,
is_intermediate: true,
width: width,
height: height,
};
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: scaledWidth,
height: scaledHeight,
};
// Connect Nodes
graph.edges.push(
// Scale Inpaint Image
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: INPAINT_INFILL,
field: 'image',
},
},
// Take combined mask and resize and then blur
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Resize Results Down
{
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: MASK_BLUR,
field: 'image',
},
destination: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'reference',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'mask',
},
},
// Paste Everything Back
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'base_image',
},
},
{
source: {
node_id: COLOR_CORRECT,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'mask',
},
}
);
} else {
// Add Images To Nodes
graph.nodes[INPAINT_INFILL] = {
...(graph.nodes[INPAINT_INFILL] as
| InfillTileInvocation
| InfillPatchmatchInvocation),
image: canvasInitImage,
};
graph.nodes[NOISE] = {
...(graph.nodes[NOISE] as NoiseInvocation),
width: width,
height: height,
};
graph.nodes[INPAINT_IMAGE] = {
...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation),
image: canvasInitImage,
};
graph.nodes[MASK_BLUR] = {
...(graph.nodes[MASK_BLUR] as ImageBlurInvocation),
image: canvasMaskImage,
};
graph.edges.push(
// Take combined mask and plug it to blur
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: MASK_BLUR,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -405,7 +613,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'mask',
},
},
// Paste Back Outpainted Image on Original
// Paste Everything Back
{
source: {
node_id: INPAINT_INFILL,
@ -435,19 +643,11 @@ export const buildCanvasSDXLOutpaintGraph = (
node_id: CANVAS_OUTPUT,
field: 'mask',
},
},
],
};
// Add Refiner if enabled
if (shouldUseSDXLRefiner) {
addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS);
}
);
}
// optionally add custom VAE
addVAEToGraph(state, graph, SDXL_MODEL_LOADER);
// handle seed
// Handle seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed
const randomIntNode: RandomIntInvocation = {
@ -467,6 +667,14 @@ export const buildCanvasSDXLOutpaintGraph = (
(graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed;
}
// Add Refiner if enabled
if (shouldUseSDXLRefiner) {
addSDXLRefinerToGraph(state, graph, SDXL_DENOISE_LATENTS);
}
// optionally add custom VAE
addVAEToGraph(state, graph, SDXL_MODEL_LOADER);
// add LoRA support
addSDXLLoRAsToGraph(state, graph, SDXL_DENOISE_LATENTS, SDXL_MODEL_LOADER);

6
invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts
View File

@ -21,12 +21,18 @@ export const CANVAS_OUTPUT = 'canvas_output';
export const INPAINT = 'inpaint';
export const INPAINT_SEAM_FIX = 'inpaint_seam_fix';
export const INPAINT_IMAGE = 'inpaint_image';
export const SCALED_INPAINT_IMAGE = 'scaled_inpaint_image';
export const INPAINT_IMAGE_RESIZE_UP = 'inpaint_image_resize_up';
export const INPAINT_IMAGE_RESIZE_DOWN = 'inpaint_image_resize_down';
export const INPAINT_INFILL = 'inpaint_infill';
export const INPAINT_INFILL_RESIZE_DOWN = 'inpaint_infill_resize_down';
export const INPAINT_FINAL_IMAGE = 'inpaint_final_image';
export const MASK_FROM_ALPHA = 'tomask';
export const MASK_EDGE = 'mask_edge';
export const MASK_BLUR = 'mask_blur';
export const MASK_COMBINE = 'mask_combine';
export const MASK_RESIZE_UP = 'mask_resize_up';
export const MASK_RESIZE_DOWN = 'mask_resize_down';
export const COLOR_CORRECT = 'color_correct';
export const PASTE_IMAGE = 'img_paste';
export const CONTROL_NET_COLLECT = 'control_net_collect';

9
invokeai/frontend/web/src/services/api/types.ts
View File

@ -184,6 +184,15 @@ export type ImageNSFWBlurInvocation = TypeReq<
export type ImageWatermarkInvocation = TypeReq<
components['schemas']['ImageWatermarkInvocation']
>;
export type ImageBlurInvocation = TypeReq<
components['schemas']['ImageBlurInvocation']
>;
export type ColorCorrectInvocation = TypeReq<
components['schemas']['ColorCorrectInvocation']
>;
export type ImagePasteInvocation = TypeReq<
components['schemas']['ImagePasteInvocation']
>;
// ControlNet Nodes
export type ControlNetInvocation = TypeReq<