Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

canvas: improve paste back (or try to)

Browse Source
This commit is contained in:
blessedcoolant 2024-02-22 10:23:18 +05:30 committed by Brandon Rising
parent ba3cf1d873
commit 6df606ac2a
7 changed files with 208 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
invokeai/app/invocations/image.py
View File

@ -17,16 +17,12 @@ from invokeai.app.invocations.fields import (
WithMetadata,
)
from invokeai.app.invocations.primitives import ImageOutput
from invokeai.app.services.image_records.image_records_common import ImageCategory
from invokeai.app.services.image_records.image_records_common import ImageCategory, ResourceOrigin
from invokeai.app.services.shared.invocation_context import InvocationContext
from invokeai.backend.image_util.invisible_watermark import InvisibleWatermark
from invokeai.backend.image_util.safety_checker import SafetyChecker
from .baseinvocation import (
BaseInvocation,
Classification,
invocation,
)
from .baseinvocation import BaseInvocation, Classification, invocation
@invocation("show_image", title="Show Image", tags=["image"], category="image", version="1.0.1")
@ -934,3 +930,42 @@ class SaveImageInvocation(BaseInvocation, WithMetadata, WithBoard):
image_dto = context.images.save(image=image)
return ImageOutput.build(image_dto)
@invocation(
"iai_canvas_paste_back",
title="InvokeAI Canvas Paste Back",
tags=["image", "combine"],
category="image",
version="1.0.0",
)
class IAICanvasPasteBackInvocation(BaseInvocation, WithMetadata, WithBoard):
"""Combines two images by using the mask provided"""
source_image: ImageField = InputField(description="The source image")
target_image: ImageField = InputField(default=None, description="The target image")
mask: ImageField = InputField(
description="The mask to use when pasting",
)
mask_blur: int = InputField(default=0, ge=0, description="The amount to blur the mask by")
def _prepare_mask(self, mask: Image.Image):
mask_array = numpy.array(mask)
kernel = numpy.ones((self.mask_blur, self.mask_blur), numpy.uint8)
dilated_mask_array = cv2.erode(mask_array, kernel, iterations=3)
dilated_mask = Image.fromarray(dilated_mask_array)
if self.mask_blur > 0:
mask = dilated_mask.filter(ImageFilter.GaussianBlur(self.mask_blur))
return ImageOps.invert(mask.convert("L"))
def invoke(self, context: InvocationContext) -> ImageOutput:
source_image = context.images.get_pil(self.source_image.image_name)
target_image = context.images.get_pil(self.target_image.image_name)
mask = self._prepare_mask(context.images.get_pil(self.mask.image_name))
# Merge the bands back together
source_image.paste(target_image, (0, 0), mask)
image_dto = context.images.save(image=source_image)
return ImageOutput.build(image_dto)

87
invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts
View File

@ -1,7 +1,13 @@
import { logger } from 'app/logging/logger';
import type { RootState } from 'app/store/store';
import { getBoardField, getIsIntermediate } from 'features/nodes/util/graph/graphBuilderUtils';
import type { ImageDTO, ImageToLatentsInvocation, NoiseInvocation, NonNullableGraph } from 'services/api/types';
import type {
CreateGradientMaskInvocation,
IAICanvasPasteBackInvocation,
ImageDTO,
ImageToLatentsInvocation,
NoiseInvocation,
NonNullableGraph,
} from 'services/api/types';
import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
@ -29,6 +35,7 @@ import {
POSITIVE_CONDITIONING,
SEAMLESS,
} from './constants';
import { getBoardField, getIsIntermediate } from './graphBuilderUtils';
/**
* Builds the Canvas tab's Inpaint graph.
@ -61,8 +68,8 @@ export const buildCanvasInpaintGraph = (
} = state.generation;
if (!model) {
log.error('No Image found in state');
throw new Error('No Image found in state');
log.error('No model found in state');
throw new Error('No model found in state');
}
// The bounding box determines width and height, not the width and height params
@ -106,7 +113,6 @@ export const buildCanvasInpaintGraph = (
is_intermediate,
prompt: negativePrompt,
},
[INPAINT_IMAGE]: {
type: 'i2l',
id: INPAINT_IMAGE,
@ -146,12 +152,12 @@ export const buildCanvasInpaintGraph = (
fp32,
},
[CANVAS_OUTPUT]: {
type: 'color_correct',
type: 'iai_canvas_paste_back',
id: CANVAS_OUTPUT,
is_intermediate: getIsIntermediate(state),
board: getBoardField(state),
reference: canvasInitImage,
use_cache: false,
mask_blur: canvasCoherenceEdgeSize,
source_image: canvasInitImage,
},
},
edges: [
@ -325,7 +331,7 @@ export const buildCanvasInpaintGraph = (
field: 'mask',
},
},
// Color Correct The Inpainted Result
// Resize Down
{
source: {
node_id: LATENTS_TO_IMAGE,
@ -336,16 +342,6 @@ export const buildCanvasInpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
@ -356,6 +352,17 @@ export const buildCanvasInpaintGraph = (
field: 'image',
},
},
// Paste Back
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'target_image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
@ -377,29 +384,27 @@ export const buildCanvasInpaintGraph = (
image: canvasInitImage,
};
graph.edges.push(
// Color Correct The Inpainted Result
{
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
graph.nodes[INPAINT_CREATE_MASK] = {
...(graph.nodes[INPAINT_CREATE_MASK] as CreateGradientMaskInvocation),
mask: canvasMaskImage,
};
// Paste Back
graph.nodes[CANVAS_OUTPUT] = {
...(graph.nodes[CANVAS_OUTPUT] as IAICanvasPasteBackInvocation),
mask: canvasMaskImage,
};
graph.edges.push({
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'mask',
},
}
);
destination: {
node_id: CANVAS_OUTPUT,
field: 'target_image',
},
});
}
// Add Seamless To Graph

19
invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts
View File

@ -1,6 +1,5 @@
import { logger } from 'app/logging/logger';
import type { RootState } from 'app/store/store';
import { getBoardField, getIsIntermediate } from 'features/nodes/util/graph/graphBuilderUtils';
import type {
ImageDTO,
ImageToLatentsInvocation,
@ -40,6 +39,7 @@ import {
POSITIVE_CONDITIONING,
SEAMLESS,
} from './constants';
import { getBoardField, getIsIntermediate } from './graphBuilderUtils';
/**
* Builds the Canvas tab's Outpaint graph.
@ -149,8 +149,8 @@ export const buildCanvasOutpaintGraph = (
id: INPAINT_CREATE_MASK,
is_intermediate,
coherence_mode: canvasCoherenceMode,
minimum_denoise: canvasCoherenceMinDenoise,
edge_radius: canvasCoherenceEdgeSize,
minimum_denoise: canvasCoherenceMinDenoise,
},
[DENOISE_LATENTS]: {
type: 'denoise_latents',
@ -171,7 +171,7 @@ export const buildCanvasOutpaintGraph = (
fp32,
},
[CANVAS_OUTPUT]: {
type: 'color_correct',
type: 'iai_canvas_paste_back',
id: CANVAS_OUTPUT,
is_intermediate: getIsIntermediate(state),
board: getBoardField(state),
@ -455,7 +455,7 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
},
// Color Correct The Inpainted Result
// Paste Back
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
@ -463,17 +463,17 @@ export const buildCanvasOutpaintGraph = (
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'reference',
field: 'source_image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
field: 'target_image',
},
},
{
@ -503,7 +503,6 @@ export const buildCanvasOutpaintGraph = (
};
graph.edges.push(
// Color Correct The Inpainted Result
{
source: {
node_id: INPAINT_INFILL,
@ -511,7 +510,7 @@ export const buildCanvasOutpaintGraph = (
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'reference',
field: 'source_image',
},
},
{
@ -521,7 +520,7 @@ export const buildCanvasOutpaintGraph = (
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
field: 'target_image',
},
},
{

90
invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts
View File

@ -1,7 +1,8 @@
import { logger } from 'app/logging/logger';
import type { RootState } from 'app/store/store';
import type {
CreateDenoiseMaskInvocation,
CreateGradientMaskInvocation,
IAICanvasPasteBackInvocation,
ImageDTO,
ImageToLatentsInvocation,
NoiseInvocation,
@ -54,15 +55,15 @@ export const buildCanvasSDXLInpaintGraph = (
cfgRescaleMultiplier: cfg_rescale_multiplier,
scheduler,
steps,
img2imgStrength: strength,
seed,
vaePrecision,
shouldUseCpuNoise,
seamlessXAxis,
seamlessYAxis,
canvasCoherenceMode,
canvasCoherenceMinDenoise,
canvasCoherenceEdgeSize,
seamlessXAxis,
seamlessYAxis,
img2imgStrength: strength,
} = state.generation;
const { refinerModel, refinerStart } = state.sdxl;
@ -78,8 +79,9 @@ export const buildCanvasSDXLInpaintGraph = (
// We may need to set the inpaint width and height to scale the image
const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } = state.canvas;
const fp32 = vaePrecision === 'fp32';
const is_intermediate = true;
const fp32 = vaePrecision === 'fp32';
const isUsingScaledDimensions = ['auto', 'manual'].includes(boundingBoxScaleMethod);
let modelLoaderNodeId = SDXL_MODEL_LOADER;
@ -95,17 +97,20 @@ export const buildCanvasSDXLInpaintGraph = (
[modelLoaderNodeId]: {
type: 'sdxl_model_loader',
id: modelLoaderNodeId,
is_intermediate,
model,
},
[POSITIVE_CONDITIONING]: {
type: 'sdxl_compel_prompt',
id: POSITIVE_CONDITIONING,
is_intermediate,
prompt: positivePrompt,
style: positiveStylePrompt,
},
[NEGATIVE_CONDITIONING]: {
type: 'sdxl_compel_prompt',
id: NEGATIVE_CONDITIONING,
is_intermediate,
prompt: negativePrompt,
style: negativeStylePrompt,
},
@ -148,12 +153,12 @@ export const buildCanvasSDXLInpaintGraph = (
fp32,
},
[CANVAS_OUTPUT]: {
type: 'color_correct',
type: 'iai_canvas_paste_back',
id: CANVAS_OUTPUT,
is_intermediate: getIsIntermediate(state),
board: getBoardField(state),
reference: canvasInitImage,
use_cache: false,
mask_blur: canvasCoherenceEdgeSize,
source_image: canvasInitImage,
},
},
edges: [
@ -208,7 +213,7 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'clip2',
},
},
// Connect everything to Inpaint
// Connect Everything To Inpaint Node
{
source: {
node_id: POSITIVE_CONDITIONING,
@ -336,7 +341,7 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'mask',
},
},
// Color Correct The Inpainted Result
// Resize Down
{
source: {
node_id: LATENTS_TO_IMAGE,
@ -347,16 +352,6 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
@ -367,6 +362,17 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'image',
},
},
// Paste Back
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'target_image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
@ -387,34 +393,28 @@ export const buildCanvasSDXLInpaintGraph = (
...(graph.nodes[INPAINT_IMAGE] as ImageToLatentsInvocation),
image: canvasInitImage,
};
graph.nodes[INPAINT_CREATE_MASK] = {
...(graph.nodes[INPAINT_CREATE_MASK] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
...(graph.nodes[INPAINT_CREATE_MASK] as CreateGradientMaskInvocation),
mask: canvasMaskImage,
};
graph.edges.push(
// Color Correct The Inpainted Result
{
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
},
// Paste Back
graph.nodes[CANVAS_OUTPUT] = {
...(graph.nodes[CANVAS_OUTPUT] as IAICanvasPasteBackInvocation),
mask: canvasMaskImage,
};
graph.edges.push({
source: {
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'mask',
},
}
);
destination: {
node_id: CANVAS_OUTPUT,
field: 'target_image',
},
});
}
// Add Seamless To Graph
@ -431,7 +431,7 @@ export const buildCanvasSDXLInpaintGraph = (
}
}
// optionally add custom VAE
// Add VAE
addVAEToGraph(state, graph, modelLoaderNodeId);
// add LoRA support

36
invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts
View File

@ -59,18 +59,18 @@ export const buildCanvasSDXLOutpaintGraph = (
cfgRescaleMultiplier: cfg_rescale_multiplier,
scheduler,
steps,
img2imgStrength: strength,
seed,
vaePrecision,
shouldUseCpuNoise,
canvasCoherenceMode,
canvasCoherenceMinDenoise,
canvasCoherenceEdgeSize,
infillTileSize,
infillPatchmatchDownscaleSize,
infillMethod,
seamlessXAxis,
seamlessYAxis,
img2imgStrength: strength,
canvasCoherenceMode,
canvasCoherenceMinDenoise,
canvasCoherenceEdgeSize,
} = state.generation;
const { refinerModel, refinerStart } = state.sdxl;
@ -108,12 +108,14 @@ export const buildCanvasSDXLOutpaintGraph = (
[POSITIVE_CONDITIONING]: {
type: 'sdxl_compel_prompt',
id: POSITIVE_CONDITIONING,
is_intermediate,
prompt: positivePrompt,
style: positiveStylePrompt,
},
[NEGATIVE_CONDITIONING]: {
type: 'sdxl_compel_prompt',
id: NEGATIVE_CONDITIONING,
is_intermediate,
prompt: negativePrompt,
style: negativeStylePrompt,
},
@ -161,6 +163,7 @@ export const buildCanvasSDXLOutpaintGraph = (
denoising_start: refinerModel ? Math.min(refinerStart, 1 - strength) : 1 - strength,
denoising_end: refinerModel ? refinerStart : 1,
},
[LATENTS_TO_IMAGE]: {
type: 'l2i',
id: LATENTS_TO_IMAGE,
@ -168,7 +171,7 @@ export const buildCanvasSDXLOutpaintGraph = (
fp32,
},
[CANVAS_OUTPUT]: {
type: 'color_correct',
type: 'iai_canvas_paste_back',
id: CANVAS_OUTPUT,
is_intermediate: getIsIntermediate(state),
board: getBoardField(state),
@ -249,7 +252,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'mask1',
},
},
// Connect Everything To Inpaint
// Plug Everything Into Inpaint Node
{
source: {
node_id: POSITIVE_CONDITIONING,
@ -311,7 +314,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'denoise_mask',
},
},
// Decode inpainted latents to image
{
source: {
node_id: SDXL_DENOISE_LATENTS,
@ -419,8 +422,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'image',
},
},
// Take combined mask and resize and then blur
// Take combined mask and resize
{
source: {
node_id: MASK_COMBINE,
@ -431,7 +433,6 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'image',
},
},
// Resize Results Down
{
source: {
@ -463,7 +464,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'image',
},
},
// Color Correct The Inpainted Result
// Paste Back
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
@ -471,17 +472,17 @@ export const buildCanvasSDXLOutpaintGraph = (
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'reference',
field: 'source_image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_DOWN,
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
field: 'target_image',
},
},
{
@ -511,7 +512,6 @@ export const buildCanvasSDXLOutpaintGraph = (
};
graph.edges.push(
// Color Correct The Inpainted Result
{
source: {
node_id: INPAINT_INFILL,
@ -519,7 +519,7 @@ export const buildCanvasSDXLOutpaintGraph = (
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'reference',
field: 'source_image',
},
},
{
@ -529,7 +529,7 @@ export const buildCanvasSDXLOutpaintGraph = (
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
field: 'target_image',
},
},
{
@ -559,7 +559,7 @@ export const buildCanvasSDXLOutpaintGraph = (
}
}
// optionally add custom VAE
// Add VAE
addVAEToGraph(state, graph, modelLoaderNodeId);
// add LoRA support

49
invokeai/frontend/web/src/services/api/schema.ts
View File

File diff suppressed because one or more lines are too long

2
invokeai/frontend/web/src/services/api/types.ts
View File

@ -150,6 +150,8 @@ export type ImageScaleInvocation = S['ImageScaleInvocation'];
export type InfillPatchMatchInvocation = S['InfillPatchMatchInvocation'];
export type InfillTileInvocation = S['InfillTileInvocation'];
export type CreateDenoiseMaskInvocation = S['CreateDenoiseMaskInvocation'];
export type CreateGradientMaskInvocation = S['CreateGradientMaskInvocation'];
export type IAICanvasPasteBackInvocation = S['IAICanvasPasteBackInvocation'];
export type MaskEdgeInvocation = S['MaskEdgeInvocation'];
export type RandomIntInvocation = S['RandomIntInvocation'];
export type CompelInvocation = S['CompelInvocation'];