feat(ui): check for transparency and clear masks if no pixel data

Mask vector data includes additive (brush, rect) shapes and subtractive (eraser) shapes. A different composite operation is used to draw a shape, depending on whether it is additive or subtractive.

This means that a mask may have vector objects, but once rendered, is _visually_ empty (fully transparent). The only way determine if a mask is visually empty is to render it and check every pixel.

When we generate and save layer metadata, these fully erased masks are still used. Generating with an empty mask is a no-op in the backend, so we want to avoid this and not pollute graphs/metadata.

Previously, we did that pixel-based when calculating the bbox, which we only did when using the move tool, and only for the selected layer.

This change introduces a simpler function to check if a mask is transparent, and if so, deletes all its objects to reset it. This allows us skip these no-op layers entirely.

This check is debounced to 300 ms, trailing edge only.

invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx

@@ -130,11 +130,11 @@ const useStageRenderer = (
   }, [stage, state.size.width, state.size.height, wrapper]);
 
   useLayoutEffect(() => {
-    log.trace('Rendering tool preview');
     if (asPreview) {
       // Preview should not display tool
       return;
     }
+    log.trace('Rendering tool preview');
     renderers.renderToolPreview(
       stage,
       tool,
@@ -182,11 +182,20 @@ const useStageRenderer = (
   }, [stage, asPreview, state.layers, tool, onBboxChanged, renderers]);
 
   useLayoutEffect(() => {
-    log.trace('Rendering background');
+    if (asPreview) {
+      // Preview should not check for transparency
+      return;
+    }
+    log.trace('Checking for transparency');
+    debouncedRenderers.checkForTransparency(stage, state.layers, onBboxChanged);
+  }, [stage, asPreview, state.layers, onBboxChanged]);
+
+  useLayoutEffect(() => {
     if (asPreview) {
       // The preview should not have a background
       return;
     }
+    log.trace('Rendering background');
     renderers.renderBackground(stage, state.size.width, state.size.height);
   }, [stage, asPreview, state.size.width, state.size.height, renderers]);
 
@@ -196,11 +205,11 @@ const useStageRenderer = (
   }, [stage, layerIds, renderers]);
 
   useLayoutEffect(() => {
-    log.trace('Rendering no layers message');
     if (asPreview) {
       // The preview should not display the no layers message
       return;
     }
+    log.trace('Rendering no layers message');
     renderers.renderNoLayersMessage(stage, layerCount, state.size.width, state.size.height);
   }, [stage, layerCount, renderers, asPreview, state.size.width, state.size.height]);
 

invokeai/frontend/web/src/features/controlLayers/util/bbox.ts

@@ -2,7 +2,6 @@ import openBase64ImageInTab from 'common/util/openBase64ImageInTab';
 import { imageDataToDataURL } from 'features/canvas/util/blobToDataURL';
 import { RG_LAYER_OBJECT_GROUP_NAME } from 'features/controlLayers/store/controlLayersSlice';
 import Konva from 'konva';
-import type { Layer as KonvaLayerType } from 'konva/lib/Layer';
 import type { IRect } from 'konva/lib/types';
 import { assert } from 'tsafe';
 
@@ -54,34 +53,49 @@ const getImageDataBbox = (imageData: ImageData): Extents | null => {
 };
 
 /**
- * Get the bounding box of a regional prompt konva layer. This function has special handling for regional prompt layers.
- * @param layer The konva layer to get the bounding box of.
- * @param preview Whether to open a new tab displaying the rendered layer, which is used to calculate the bbox.
+ * Check if an image is fully transparent.
+ * @param imageData The ImageData object to check for transparency.
+ * @returns Whether the image is fully transparent.
  */
-export const getLayerBboxPixels = (layer: KonvaLayerType, preview: boolean = false): IRect | null => {
-  // To calculate the layer's bounding box, we must first export it to a pixel array, then do some math.
-  //
-  // Though it is relatively fast, we can't use Konva's `getClientRect`. It programmatically determines the rect
-  // by calculating the extents of individual shapes from their "vector" shape data.
-  //
-  // This doesn't work when some shapes are drawn with composite operations that "erase" pixels, like eraser lines.
-  // These shapes' extents are still calculated as if they were solid, leading to a bounding box that is too large.
+const getIsFullyTransparent = (imageData: ImageData) => {
+  if (!imageData.height || !imageData.width || imageData.data.length === 0) {
+    return true;
+  }
+  const data = imageData.data;
+  const len = data.length / 4;
+  for (let i = 0; i < len; i++) {
+    if (data[i * 4 + 3] ?? 0 > 0) {
+      return false;
+    }
+  }
+  return true;
+};
+
+/**
+ * Clones a regional guidance konva layer onto an offscreen stage/canvas. This allows the pixel data for a given layer
+ * to be captured, manipulated or analyzed without interference from other layers.
+ * @param layer The konva layer to clone.
+ * @returns The cloned stage and layer.
+ */
+const getIsolatedRGLayerClone = (layer: Konva.Layer): { stageClone: Konva.Stage; layerClone: Konva.Layer } => {
   const stage = layer.getStage();
 
-  // Construct and offscreen canvas on which we will do the bbox calculations.
+  // Construct an offscreen canvas with the same dimensions as the layer's stage.
   const offscreenStageContainer = document.createElement('div');
-  const offscreenStage = new Konva.Stage({
+  const stageClone = new Konva.Stage({
     container: offscreenStageContainer,
+    x: stage.x(),
+    y: stage.y(),
     width: stage.width(),
     height: stage.height(),
   });
 
   // Clone the layer and filter out unwanted children.
   const layerClone = layer.clone();
-  offscreenStage.add(layerClone);
+  stageClone.add(layerClone);
 
   for (const child of layerClone.getChildren()) {
-    if (child.name() === RG_LAYER_OBJECT_GROUP_NAME) {
+    if (child.name() === RG_LAYER_OBJECT_GROUP_NAME && child.hasChildren()) {
       // We need to cache the group to ensure it composites out eraser strokes correctly
       child.opacity(1);
       child.cache();
@@ -91,11 +105,31 @@ export const getLayerBboxPixels = (layer: KonvaLayerType, preview: boolean = fal
     }
   }
 
+  return { stageClone, layerClone };
+};
+
+/**
+ * Get the bounding box of a regional prompt konva layer. This function has special handling for regional prompt layers.
+ * @param layer The konva layer to get the bounding box of.
+ * @param preview Whether to open a new tab displaying the rendered layer, which is used to calculate the bbox.
+ */
+export const getLayerBboxPixels = (layer: Konva.Layer, preview: boolean = false): IRect | null => {
+  // To calculate the layer's bounding box, we must first export it to a pixel array, then do some math.
+  //
+  // Though it is relatively fast, we can't use Konva's `getClientRect`. It programmatically determines the rect
+  // by calculating the extents of individual shapes from their "vector" shape data.
+  //
+  // This doesn't work when some shapes are drawn with composite operations that "erase" pixels, like eraser lines.
+  // These shapes' extents are still calculated as if they were solid, leading to a bounding box that is too large.
+  const { stageClone, layerClone } = getIsolatedRGLayerClone(layer);
+
   // Get a worst-case rect using the relatively fast `getClientRect`.
   const layerRect = layerClone.getClientRect();
-
+  if (layerRect.width === 0 || layerRect.height === 0) {
+    return null;
+  }
   // Capture the image data with the above rect.
-  const layerImageData = offscreenStage
+  const layerImageData = stageClone
     .toCanvas(layerRect)
     .getContext('2d')
     ?.getImageData(0, 0, layerRect.width, layerRect.height);
@@ -114,8 +148,8 @@ export const getLayerBboxPixels = (layer: KonvaLayerType, preview: boolean = fal
 
   // Correct the bounding box to be relative to the layer's position.
   const correctedLayerBbox = {
-    x: layerBbox.minX - Math.floor(stage.x()) + layerRect.x - Math.floor(layer.x()),
-    y: layerBbox.minY - Math.floor(stage.y()) + layerRect.y - Math.floor(layer.y()),
+    x: layerBbox.minX - Math.floor(stageClone.x()) + layerRect.x - Math.floor(layer.x()),
+    y: layerBbox.minY - Math.floor(stageClone.y()) + layerRect.y - Math.floor(layer.y()),
     width: layerBbox.maxX - layerBbox.minX,
     height: layerBbox.maxY - layerBbox.minY,
   };
@@ -123,7 +157,13 @@ export const getLayerBboxPixels = (layer: KonvaLayerType, preview: boolean = fal
   return correctedLayerBbox;
 };
 
-export const getLayerBboxFast = (layer: KonvaLayerType): IRect => {
+/**
+ * Get the bounding box of a konva layer. This function is faster than `getLayerBboxPixels` but less accurate. It
+ * should only be used when there are no eraser strokes or shapes in the layer.
+ * @param layer The konva layer to get the bounding box of.
+ * @returns The bounding box of the layer.
+ */
+export const getLayerBboxFast = (layer: Konva.Layer): IRect => {
   const bbox = layer.getClientRect(GET_CLIENT_RECT_CONFIG);
   return {
     x: Math.floor(bbox.x),
@@ -132,3 +172,22 @@ export const getLayerBboxFast = (layer: KonvaLayerType): IRect => {
     height: Math.floor(bbox.height),
   };
 };
+
+export const getIsLayerTransparent = (layer: Konva.Layer): boolean => {
+  const { stageClone, layerClone } = getIsolatedRGLayerClone(layer);
+
+  // Get a worst-case rect using the relatively fast `getClientRect`.
+  const layerRect = layerClone.getClientRect();
+  if (layerRect.width === 0 || layerRect.height === 0) {
+    return true;
+  }
+
+  // Capture the image data with the above rect.
+  const layerImageData = stageClone
+    .toCanvas(layerRect)
+    .getContext('2d')
+    ?.getImageData(0, 0, layerRect.width, layerRect.height);
+  assert(layerImageData, "Unable to get layer's image data");
+
+  return getIsFullyTransparent(layerImageData);
+};

invokeai/frontend/web/src/features/controlLayers/util/renderers.ts

@@ -40,7 +40,7 @@ import type {
   VectorMaskLine,
   VectorMaskRect,
 } from 'features/controlLayers/store/types';
-import { getLayerBboxFast, getLayerBboxPixels } from 'features/controlLayers/util/bbox';
+import { getIsLayerTransparent, getLayerBboxFast, getLayerBboxPixels } from 'features/controlLayers/util/bbox';
 import { t } from 'i18next';
 import Konva from 'konva';
 import type { IRect, Vector2d } from 'konva/lib/types';
@@ -890,6 +890,25 @@ const renderNoLayersMessage = (stage: Konva.Stage, layerCount: number, width: nu
   }
 };
 
+const checkForTransparency = (
+  stage: Konva.Stage,
+  reduxLayers: Layer[],
+  onBboxChanged: (layerId: string, bbox: IRect | null) => void
+) => {
+  for (const reduxLayer of reduxLayers.filter(isRegionalGuidanceLayer)) {
+    if (!reduxLayer.needsPixelBbox) {
+      continue;
+    }
+    const konvaLayer = stage.findOne<Konva.Layer>(`#${reduxLayer.id}`);
+    if (!konvaLayer) {
+      continue;
+    }
+    if (getIsLayerTransparent(konvaLayer)) {
+      onBboxChanged(reduxLayer.id, null);
+    }
+  }
+};
+
 export const renderers = {
   renderToolPreview,
   renderLayers,
@@ -897,6 +916,7 @@ export const renderers = {
   renderBackground,
   renderNoLayersMessage,
   arrangeLayers,
+  checkForTransparency,
 };
 
 const DEBOUNCE_MS = 300;
@@ -908,6 +928,7 @@ export const debouncedRenderers = {
   renderBackground: debounce(renderBackground, DEBOUNCE_MS),
   renderNoLayersMessage: debounce(renderNoLayersMessage, DEBOUNCE_MS),
   arrangeLayers: debounce(arrangeLayers, DEBOUNCE_MS),
+  checkForTransparency: debounce(checkForTransparency, DEBOUNCE_MS),
 };
 
 /**