Demotes inpainting to img2img when mask is empty

backend/invoke_ai_web_server.py

@@ -336,7 +336,8 @@ class InvokeAIWebServer:
 
                 seed = (
                     original_image["metadata"]["seed"]
-                    if "metadata" in original_image and "seed" in original_image["metadata"]
+                    if "metadata" in original_image
+                    and "seed" in original_image["metadata"]
                     else "unknown_seed"
                 )
 
@@ -561,20 +562,22 @@ class InvokeAIWebServer:
                 )
                 generation_parameters["init_img"] = cropped_init_image
 
-                # grab an Image of the mask
-                mask_image = Image.open(
-                    io.BytesIO(
-                        base64.decodebytes(
-                            bytes(generation_parameters["init_mask"], "utf-8")
+                if generation_parameters["is_mask_empty"]:
+                    generation_parameters["init_mask"] = None
+                else:
+                    # grab an Image of the mask
+                    mask_image = Image.open(
+                        io.BytesIO(
+                            base64.decodebytes(
+                                bytes(generation_parameters["init_mask"], "utf-8")
+                            )
                         )
                     )
-                )
-
-                # crop the mask image
-                cropped_mask_image = copy_image_from_bounding_box(
-                    mask_image, **generation_parameters["bounding_box"]
-                )
-                generation_parameters["init_mask"] = cropped_mask_image
+                    # crop the mask image
+                    cropped_mask_image = copy_image_from_bounding_box(
+                        mask_image, **generation_parameters["bounding_box"]
+                    )
+                    generation_parameters["init_mask"] = cropped_mask_image
 
             totalSteps = self.calculate_real_steps(
                 steps=generation_parameters["steps"],
@@ -750,7 +753,7 @@ class InvokeAIWebServer:
                     all_parameters["init_img"] = init_img_url
 
                 if "init_mask" in all_parameters:
-                    all_parameters["init_mask"] = ""  #
+                    all_parameters["init_mask"] = ""  # TODO: store the mask in metadata
 
                 metadata = self.parameters_to_generated_image_metadata(all_parameters)
 

frontend/src/common/util/parameterTranslation.ts

@@ -111,20 +111,26 @@ export const frontendToBackendParameters = (
       ...boundingBoxDimensions,
     };
 
-    if (shouldUseInpaintReplace) {
-      generationParameters.inpaint_replace = inpaintReplace;
-    }
-
     generationParameters.init_img = imageToProcessUrl;
     generationParameters.strength = img2imgStrength;
     generationParameters.fit = false;
 
-    const maskDataURL = generateMask(maskImageElement, lines);
+    const { maskDataURL, isMaskEmpty } = generateMask(
+      maskImageElement,
+      lines,
+      boundingBox
+    );
+
+    generationParameters.is_mask_empty = isMaskEmpty;
 
     generationParameters.init_mask = maskDataURL.split(
       'data:image/png;base64,'
     )[1];
 
+    if (shouldUseInpaintReplace) {
+      generationParameters.inpaint_replace = inpaintReplace;
+    }
+
     generationParameters.bounding_box = boundingBox;
   }
 

frontend/src/features/tabs/Inpainting/util/generateMask.ts

@@ -1,4 +1,5 @@
 import Konva from 'konva';
+import { IRect } from 'konva/lib/types';
 import { MaskLine } from '../inpaintingSlice';
 
 /**
@@ -11,15 +12,19 @@ import { MaskLine } from '../inpaintingSlice';
  * drawing the mask and compositing everything correctly to output a valid
  * mask image.
  */
-const generateMask = (image: HTMLImageElement, lines: MaskLine[]) => {
-  const { width, height } = image;
+const generateMask = (
+  image: HTMLImageElement,
+  lines: MaskLine[],
+  boundingBox: IRect
+) => {
+  const { x, y, width, height } = boundingBox;
 
   const offscreenContainer = document.createElement('div');
 
   const stage = new Konva.Stage({
     container: offscreenContainer,
-    width: width,
-    height: height,
+    width: image.width,
+    height: image.height,
   });
 
   const layer = new Konva.Layer();
@@ -42,13 +47,31 @@ const generateMask = (image: HTMLImageElement, lines: MaskLine[]) => {
     )
   );
 
+  // check if mask is empty
+  const pixelBuffer = new Uint32Array(
+    layer.getContext().getImageData(x, y, width, height).data.buffer
+  );
+
+  const isMaskEmpty = !pixelBuffer.some((color) => color !== 0);
+
+  if (isMaskEmpty) {
+    layer.add(
+      new Konva.Rect({
+        ...boundingBox,
+        fill: 'rgb(0,0,0)',
+      })
+    );
+  }
+
   layer.add(
     new Konva.Image({ image: image, globalCompositeOperation: 'source-out' })
   );
 
+  const maskDataURL = stage.toDataURL();
+
   offscreenContainer.remove();
 
-  return stage.toDataURL();
+  return { maskDataURL, isMaskEmpty };
 };
 
 export default generateMask;