wip: Add initial Inpaint Graph

invokeai/frontend/web/src/features/nodes/util/graphBuilders/addVAEToGraph.ts

@@ -4,8 +4,8 @@ import { MetadataAccumulatorInvocation } from 'services/api/types';
 import {
   IMAGE_TO_IMAGE_GRAPH,
   IMAGE_TO_LATENTS,
-  INPAINT,
   INPAINT_GRAPH,
+  INPAINT_IMAGE,
   LATENTS_TO_IMAGE,
   MAIN_MODEL_LOADER,
   METADATA_ACCUMULATOR,
@@ -35,7 +35,11 @@ export const addVAEToGraph = (
     };
   }
   const isOnnxModel = modelLoaderNodeId == ONNX_MODEL_LOADER;
-  if (graph.id === TEXT_TO_IMAGE_GRAPH || graph.id === IMAGE_TO_IMAGE_GRAPH) {
+  if (
+    graph.id === TEXT_TO_IMAGE_GRAPH ||
+    graph.id === IMAGE_TO_IMAGE_GRAPH ||
+    graph.id === INPAINT_GRAPH
+  ) {
     graph.edges.push({
       source: {
         node_id: isAutoVae ? modelLoaderNodeId : VAE_LOADER,
@@ -68,7 +72,7 @@ export const addVAEToGraph = (
         field: isAutoVae && isOnnxModel ? 'vae_decoder' : 'vae',
       },
       destination: {
-        node_id: INPAINT,
+        node_id: INPAINT_IMAGE,
         field: 'vae',
       },
     });

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts

@@ -3,10 +3,10 @@ import { RootState } from 'app/store/store';
 import { NonNullableGraph } from 'features/nodes/types/types';
 import {
   ImageDTO,
-  InpaintInvocation,
   RandomIntInvocation,
   RangeOfSizeInvocation,
 } from 'services/api/types';
+import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addVAEToGraph } from './addVAEToGraph';
@@ -15,9 +15,12 @@ import {
   CLIP_SKIP,
   INPAINT,
   INPAINT_GRAPH,
+  INPAINT_IMAGE,
   ITERATE,
+  LATENTS_TO_IMAGE,
   MAIN_MODEL_LOADER,
   NEGATIVE_CONDITIONING,
+  NOISE,
   POSITIVE_CONDITIONING,
   RANDOM_INT,
   RANGE_OF_SIZE,
@@ -44,6 +47,9 @@ export const buildCanvasInpaintGraph = (
     iterations,
     seed,
     shouldRandomizeSeed,
+    vaePrecision,
+    shouldUseNoiseSettings,
+    shouldUseCpuNoise,
     seamSize,
     seamBlur,
     seamSteps,
@@ -68,40 +74,38 @@ export const buildCanvasInpaintGraph = (
     shouldAutoSave,
   } = state.canvas;
 
+  const use_cpu = shouldUseNoiseSettings
+    ? shouldUseCpuNoise
+    : shouldUseCpuNoise;
+
   const graph: NonNullableGraph = {
     id: INPAINT_GRAPH,
     nodes: {
       [INPAINT]: {
-        is_intermediate: !shouldAutoSave,
-        type: 'inpaint',
+        type: 'denoise_latents',
         id: INPAINT,
-        steps,
+        is_intermediate: true,
+        steps: steps,
+        cfg_scale: cfg_scale,
+        scheduler: scheduler,
+        denoising_start: 1 - strength,
+        denoising_end: 1,
+        mask: canvasMaskImage,
+      },
+      [INPAINT_IMAGE]: {
+        type: 'i2l',
+        id: INPAINT_IMAGE,
+        is_intermediate: true,
+        image: canvasInitImage,
+        fp32: vaePrecision === 'fp32' ? true : false,
+      },
+      [NOISE]: {
+        type: 'noise',
+        id: NOISE,
         width,
         height,
-        cfg_scale,
-        scheduler,
-        image: {
-          image_name: canvasInitImage.image_name,
-        },
-        strength,
-        fit: shouldFitToWidthHeight,
-        mask: {
-          image_name: canvasMaskImage.image_name,
-        },
-        seam_size: seamSize,
-        seam_blur: seamBlur,
-        seam_strength: seamStrength,
-        seam_steps: seamSteps,
-        tile_size: infillMethod === 'tile' ? tileSize : undefined,
-        infill_method: infillMethod as InpaintInvocation['infill_method'],
-        inpaint_width:
-          boundingBoxScaleMethod !== 'none'
-            ? scaledBoundingBoxDimensions.width
-            : undefined,
-        inpaint_height:
-          boundingBoxScaleMethod !== 'none'
-            ? scaledBoundingBoxDimensions.height
-            : undefined,
+        use_cpu,
+        is_intermediate: true,
       },
       [POSITIVE_CONDITIONING]: {
         type: 'compel',
@@ -121,12 +125,19 @@ export const buildCanvasInpaintGraph = (
         is_intermediate: true,
         model,
       },
+      [LATENTS_TO_IMAGE]: {
+        type: 'l2i',
+        id: LATENTS_TO_IMAGE,
+        is_intermediate: true,
+        fp32: vaePrecision === 'fp32' ? true : false,
+      },
       [CLIP_SKIP]: {
         type: 'clip_skip',
         id: CLIP_SKIP,
         is_intermediate: true,
         skipped_layers: clipSkip,
       },
+
       [RANGE_OF_SIZE]: {
         type: 'range_of_size',
         id: RANGE_OF_SIZE,
@@ -203,6 +214,26 @@ export const buildCanvasInpaintGraph = (
           field: 'positive_conditioning',
         },
       },
+      {
+        source: {
+          node_id: NOISE,
+          field: 'noise',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'noise',
+        },
+      },
+      {
+        source: {
+          node_id: INPAINT_IMAGE,
+          field: 'latents',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'latents',
+        },
+      },
       {
         source: {
           node_id: RANGE_OF_SIZE,
@@ -219,17 +250,25 @@ export const buildCanvasInpaintGraph = (
           field: 'item',
         },
         destination: {
-          node_id: INPAINT,
+          node_id: NOISE,
           field: 'seed',
         },
       },
+      {
+        source: {
+          node_id: INPAINT,
+          field: 'latents',
+        },
+        destination: {
+          node_id: LATENTS_TO_IMAGE,
+          field: 'latents',
+        },
+      },
     ],
   };
 
-  addLoRAsToGraph(state, graph, INPAINT);
-
   // Add VAE
-  addVAEToGraph(state, graph);
+  addVAEToGraph(state, graph, MAIN_MODEL_LOADER);
 
   // handle seed
   if (shouldRandomizeSeed) {
@@ -251,6 +290,12 @@ export const buildCanvasInpaintGraph = (
     (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed;
   }
 
+  // add LoRA support
+  addLoRAsToGraph(state, graph, INPAINT, MAIN_MODEL_LOADER);
+
+  // add controlnet, mutating `graph`
+  addControlNetToLinearGraph(state, graph, INPAINT);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts

@@ -18,6 +18,15 @@ export const IMAGE_TO_LATENTS = 'image_to_latents';
 export const LATENTS_TO_LATENTS = 'latents_to_latents';
 export const RESIZE = 'resize_image';
 export const INPAINT = 'inpaint';
+export const INPAINT_SEAM_FIX = 'inpaint_seam_fix';
+export const INPAINT_IMAGE = 'inpaint_image';
+export const INFILL_TILE = 'infill_tile';
+export const INPAINT_FINAL_IMAGE = 'inpaint_final_image';
+export const MASK_FROM_ALPHA = 'tomask';
+export const MASK_EDGE = 'mask_edge';
+export const MASK_BLUR = 'mask_blur';
+export const COLOR_CORRECT = 'color_correct';
+export const PASTE_IMAGE = 'img_paste';
 export const CONTROL_NET_COLLECT = 'control_net_collect';
 export const DYNAMIC_PROMPT = 'dynamic_prompt';
 export const IMAGE_COLLECTION = 'image_collection';