feat: Add Coherence Mode

This commit is contained in:
blessedcoolant
2023-09-03 00:34:37 +12:00
parent fded8bee39
commit b5f42bedce
10 changed files with 422 additions and 337 deletions

View File

@ -32,7 +32,7 @@ export const addVAEToGraph = (
graph: NonNullableGraph,
modelLoaderNodeId: string = MAIN_MODEL_LOADER
): void => {
const { vae } = state.generation;
const { vae, canvasCoherenceMode } = state.generation;
const { boundingBoxScaleMethod } = state.canvas;
const { shouldUseSDXLRefiner } = state.sdxl;
@ -136,16 +136,6 @@ export const addVAEToGraph = (
field: 'vae',
},
},
{
source: {
node_id: isAutoVae ? modelLoaderNodeId : VAE_LOADER,
field: isAutoVae && isOnnxModel ? 'vae_decoder' : 'vae',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'vae',
},
},
{
source: {
node_id: isAutoVae ? modelLoaderNodeId : VAE_LOADER,
@ -157,6 +147,20 @@ export const addVAEToGraph = (
},
}
);
// Handle Coherence Mode
if (canvasCoherenceMode === 'edge') {
graph.edges.push({
source: {
node_id: isAutoVae ? modelLoaderNodeId : VAE_LOADER,
field: isAutoVae && isOnnxModel ? 'vae_decoder' : 'vae',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'vae',
},
});
}
}
if (shouldUseSDXLRefiner) {

View File

@ -70,6 +70,7 @@ export const buildCanvasInpaintGraph = (
shouldUseCpuNoise,
maskBlur,
maskBlurMethod,
canvasCoherenceMode,
canvasCoherenceSteps,
canvasCoherenceStrength,
clipSkip,
@ -176,21 +177,6 @@ export const buildCanvasInpaintGraph = (
b: 1,
is_intermediate: true,
},
[CANVAS_COHERENCE_MASK_EDGE]: {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
},
[CANVAS_COHERENCE_INPAINT_CREATE_MASK]: {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[CANVAS_COHERENCE_DENOISE_LATENTS]: {
type: 'denoise_latents',
id: CANVAS_COHERENCE_DENOISE_LATENTS,
@ -333,27 +319,6 @@ export const buildCanvasInpaintGraph = (
field: 'denoise_mask',
},
},
// Create Coherence Mask
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
},
// Iterate
{
source: {
@ -537,26 +502,6 @@ export const buildCanvasInpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -618,16 +563,6 @@ export const buildCanvasInpaintGraph = (
...(graph.nodes[INPAINT_CREATE_MASK] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
};
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
...(graph.nodes[
CANVAS_COHERENCE_INPAINT_CREATE_MASK
] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
};
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
...(graph.nodes[CANVAS_COHERENCE_MASK_EDGE] as MaskEdgeInvocation),
image: canvasMaskImage,
};
graph.edges.push(
// Color Correct The Inpainted Result
@ -654,6 +589,84 @@ export const buildCanvasInpaintGraph = (
);
}
// Handle Coherence Mode
if (canvasCoherenceMode === 'edge') {
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
};
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
};
if (isUsingScaledDimensions) {
graph.edges.push(
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
}
);
} else {
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
...(graph.nodes[
CANVAS_COHERENCE_INPAINT_CREATE_MASK
] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
};
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
...(graph.nodes[CANVAS_COHERENCE_MASK_EDGE] as MaskEdgeInvocation),
image: canvasMaskImage,
};
}
graph.edges.push(
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
}
);
}
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed

View File

@ -71,6 +71,7 @@ export const buildCanvasOutpaintGraph = (
shouldUseNoiseSettings,
shouldUseCpuNoise,
maskBlur,
canvasCoherenceMode,
canvasCoherenceSteps,
canvasCoherenceStrength,
infillTileSize,
@ -185,21 +186,6 @@ export const buildCanvasOutpaintGraph = (
b: 1,
is_intermediate: true,
},
[CANVAS_COHERENCE_MASK_EDGE]: {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
},
[CANVAS_COHERENCE_INPAINT_CREATE_MASK]: {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[CANVAS_COHERENCE_DENOISE_LATENTS]: {
type: 'denoise_latents',
id: CANVAS_COHERENCE_DENOISE_LATENTS,
@ -363,27 +349,6 @@ export const buildCanvasOutpaintGraph = (
field: 'denoise_mask',
},
},
// Create Coherence Mask
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
},
// Iterate
{
source: {
@ -476,6 +441,16 @@ export const buildCanvasOutpaintGraph = (
field: 'latents',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_CREATE_MASK,
field: 'image',
},
},
// Decode the result from Inpaint
{
source: {
@ -588,26 +563,7 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
// Take combined mask and resize and then blur
{
source: {
@ -619,16 +575,7 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
// Resize Results Down
{
source: {
@ -712,37 +659,6 @@ export const buildCanvasOutpaintGraph = (
};
graph.edges.push(
// Take combined mask and plug it to blur
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -777,6 +693,82 @@ export const buildCanvasOutpaintGraph = (
);
}
// Handle Coherence Mode
if (canvasCoherenceMode === 'edge') {
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
};
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
};
if (isUsingScaledDimensions) {
graph.edges.push({
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
});
} else {
graph.edges.push({
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
});
}
graph.edges.push(
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
}
);
}
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed

View File

@ -71,6 +71,7 @@ export const buildCanvasSDXLInpaintGraph = (
shouldUseCpuNoise,
maskBlur,
maskBlurMethod,
canvasCoherenceMode,
canvasCoherenceSteps,
canvasCoherenceStrength,
seamlessXAxis,
@ -182,21 +183,6 @@ export const buildCanvasSDXLInpaintGraph = (
b: 1,
is_intermediate: true,
},
[CANVAS_COHERENCE_MASK_EDGE]: {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
},
[CANVAS_COHERENCE_INPAINT_CREATE_MASK]: {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[CANVAS_COHERENCE_DENOISE_LATENTS]: {
type: 'denoise_latents',
id: CANVAS_COHERENCE_DENOISE_LATENTS,
@ -348,27 +334,6 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'denoise_mask',
},
},
// Create Coherence Mask
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
},
// Iterate
{
source: {
@ -552,26 +517,6 @@ export const buildCanvasSDXLInpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -633,16 +578,6 @@ export const buildCanvasSDXLInpaintGraph = (
...(graph.nodes[INPAINT_CREATE_MASK] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
};
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
...(graph.nodes[
CANVAS_COHERENCE_INPAINT_CREATE_MASK
] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
};
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
...(graph.nodes[CANVAS_COHERENCE_MASK_EDGE] as MaskEdgeInvocation),
image: canvasMaskImage,
};
graph.edges.push(
// Color Correct The Inpainted Result
@ -669,6 +604,84 @@ export const buildCanvasSDXLInpaintGraph = (
);
}
// Handle Coherence Mode
if (canvasCoherenceMode === 'edge') {
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
};
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
};
if (isUsingScaledDimensions) {
graph.edges.push(
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
{
source: {
node_id: INPAINT_IMAGE_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
}
);
} else {
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
...(graph.nodes[
CANVAS_COHERENCE_INPAINT_CREATE_MASK
] as CreateDenoiseMaskInvocation),
image: canvasInitImage,
};
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
...(graph.nodes[CANVAS_COHERENCE_MASK_EDGE] as MaskEdgeInvocation),
image: canvasMaskImage,
};
}
graph.edges.push(
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
}
);
}
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed

View File

@ -72,9 +72,11 @@ export const buildCanvasSDXLOutpaintGraph = (
shouldUseNoiseSettings,
shouldUseCpuNoise,
maskBlur,
canvasCoherenceMode,
canvasCoherenceSteps,
canvasCoherenceStrength,
infillTileSize,
infillPatchmatchDownscaleSize,
infillMethod,
seamlessXAxis,
seamlessYAxis,
@ -190,21 +192,6 @@ export const buildCanvasSDXLOutpaintGraph = (
b: 1,
is_intermediate: true,
},
[CANVAS_COHERENCE_MASK_EDGE]: {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
},
[CANVAS_COHERENCE_INPAINT_CREATE_MASK]: {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[CANVAS_COHERENCE_DENOISE_LATENTS]: {
type: 'denoise_latents',
id: CANVAS_COHERENCE_DENOISE_LATENTS,
@ -377,27 +364,6 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'denoise_mask',
},
},
// Create Coherence Mask
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
},
// Iterate
{
source: {
@ -490,6 +456,16 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'latents',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_CREATE_MASK,
field: 'image',
},
},
// Decode inpainted latents to image
{
source: {
@ -602,26 +578,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
// Take combined mask and resize and then blur
{
source: {
@ -633,16 +590,7 @@ export const buildCanvasSDXLOutpaintGraph = (
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
// Resize Results Down
{
source: {
@ -726,37 +674,6 @@ export const buildCanvasSDXLOutpaintGraph = (
};
graph.edges.push(
// Take combined mask and plug it to blur
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
},
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -791,6 +708,82 @@ export const buildCanvasSDXLOutpaintGraph = (
);
}
// Handle Coherence Mode
if (canvasCoherenceMode === 'edge') {
graph.nodes[CANVAS_COHERENCE_MASK_EDGE] = {
type: 'mask_edge',
id: CANVAS_COHERENCE_MASK_EDGE,
is_intermediate: true,
edge_blur: maskBlur,
edge_size: maskBlur * 2,
low_threshold: 100,
high_threshold: 200,
};
graph.nodes[CANVAS_COHERENCE_INPAINT_CREATE_MASK] = {
type: 'create_denoise_mask',
id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
};
if (isUsingScaledDimensions) {
graph.edges.push({
source: {
node_id: MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
});
} else {
graph.edges.push({
source: {
node_id: MASK_COMBINE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
});
}
graph.edges.push(
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'image',
},
},
{
source: {
node_id: CANVAS_COHERENCE_MASK_EDGE,
field: 'image',
},
destination: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'mask',
},
},
{
source: {
node_id: CANVAS_COHERENCE_INPAINT_CREATE_MASK,
field: 'denoise_mask',
},
destination: {
node_id: CANVAS_COHERENCE_DENOISE_LATENTS,
field: 'denoise_mask',
},
}
);
}
// Handle Seed
if (shouldRandomizeSeed) {
// Random int node to generate the starting seed