Fixes empty canvas detection

frontend/src/features/tabs/Inpainting/util/generateMask.ts

@@ -5,19 +5,21 @@ import { MaskLine } from '../inpaintingSlice';
 /**
  * Re-draws the mask canvas onto a new Konva stage.
  */
-const generateMask = (
+export const generateMaskCanvas = (
   image: HTMLImageElement,
-  lines: MaskLine[],
-  boundingBox: IRect
-) => {
-  const { x, y, width, height } = boundingBox;
+  lines: MaskLine[]
+): {
+  stage: Konva.Stage;
+  layer: Konva.Layer;
+} => {
+  const { width, height } = image;
 
   const offscreenContainer = document.createElement('div');
 
   const stage = new Konva.Stage({
     container: offscreenContainer,
-    width: image.width,
-    height: image.height,
+    width: width,
+    height: height,
   });
 
   const layer = new Konva.Layer();
@@ -40,30 +42,67 @@ const generateMask = (
     )
   );
 
-  // check if mask is empty
-  const pixelBuffer = new Uint32Array(
-    layer.getContext().getImageData(x, y, width, height).data.buffer
-  );
+  layer.draw();
 
-  const isMaskEmpty = !pixelBuffer.some((color) => color !== 0);
+  offscreenContainer.remove();
 
-  if (isMaskEmpty) {
-    layer.add(
-      new Konva.Rect({
-        ...boundingBox,
-        fill: 'rgb(0,0,0)',
-      })
+  return { stage, layer };
+};
+
+/**
+ * Check if the bounding box region has only fully transparent pixels.
+ */
+export const checkIsRegionEmpty = (
+  stage: Konva.Stage,
+  boundingBox: IRect
+): boolean => {
+  const imageData = stage
+    .toCanvas()
+    .getContext('2d')
+    ?.getImageData(
+      boundingBox.x,
+      boundingBox.y,
+      boundingBox.width,
+      boundingBox.height
     );
+
+  if (!imageData) {
+    throw new Error('Unable to get image data from generated canvas');
   }
 
+  const pixelBuffer = new Uint32Array(imageData.data.buffer);
+
+  return !pixelBuffer.some((color) => color !== 0);
+};
+
+/**
+ * Generating a mask image from InpaintingCanvas.tsx is not as simple
+ * as calling toDataURL() on the canvas, because the mask may be represented
+ * by colored lines or transparency, or the user may have inverted the mask
+ * display.
+ *
+ * So we need to regenerate the mask image by creating an offscreen canvas,
+ * drawing the mask and compositing everything correctly to output a valid
+ * mask image.
+ */
+const generateMask = (
+  image: HTMLImageElement,
+  lines: MaskLine[],
+  boundingBox: IRect
+): { maskDataURL: string; isMaskEmpty: boolean } => {
+  // create an offscreen canvas and add the mask to it
+  const { stage, layer } = generateMaskCanvas(image, lines);
+
+  // check if the mask layer is empty
+  const isMaskEmpty = checkIsRegionEmpty(stage, boundingBox);
+
+  // composite the image onto the mask layer
   layer.add(
     new Konva.Image({ image: image, globalCompositeOperation: 'source-out' })
   );
 
   const maskDataURL = stage.toDataURL();
 
-  offscreenContainer.remove();
-
   return { maskDataURL, isMaskEmpty };
 };