feat(ui): wip canvas nodes migration 2

invokeai/frontend/web/src/common/util/arrayBuffer.ts

@@ -1,18 +1,3 @@
-export const getIsImageDataPartiallyTransparent = (imageData: ImageData) => {
-  let hasTransparency = false;
-  let isFullyTransparent = true;
-  const len = imageData.data.length;
-  let i = 3;
-  for (i; i < len; i += 4) {
-    if (imageData.data[i] !== 0) {
-      isFullyTransparent = false;
-    } else {
-      hasTransparency = true;
-    }
-  }
-  return { hasTransparency, isFullyTransparent };
-};
-
 export const getImageDataTransparency = (imageData: ImageData) => {
   let isFullyTransparent = true;
   let isPartiallyTransparent = false;
@@ -46,14 +31,3 @@ export const areAnyPixelsBlack = (imageData: ImageData) => {
   }
   return false;
 };
-
-export const getIsImageDataWhite = (imageData: ImageData) => {
-  const len = imageData.data.length;
-  let i = 0;
-  for (i; i < len; ) {
-    if (imageData.data[i++] !== 255) {
-      return false;
-    }
-  }
-  return true;
-};

invokeai/frontend/web/src/features/canvas/hooks/useGetCanvasNodeType.ts

@@ -0,0 +1,30 @@
+import { createSelector } from '@reduxjs/toolkit';
+import { useAppSelector } from 'app/store/storeHooks';
+import { canvasSelector } from '../store/canvasSelectors';
+import { useMemo } from 'react';
+import { getCanvasNodeType } from '../util/getCanvasNodeType';
+
+const selector = createSelector(canvasSelector, (canvas) => {
+  const {
+    layerState: { objects },
+    boundingBoxCoordinates,
+    boundingBoxDimensions,
+    stageScale,
+    isMaskEnabled,
+  } = canvas;
+  return {
+    objects,
+    boundingBoxCoordinates,
+    boundingBoxDimensions,
+    stageScale,
+    isMaskEnabled,
+  };
+});
+
+export const useGetCanvasNodeType = () => {
+  const data = useAppSelector(selector);
+
+  const nodeType = useMemo(() => getCanvasNodeType(data), [data]);
+
+  return nodeType;
+};

invokeai/frontend/web/src/features/canvas/util/generateMask.ts

@@ -1,5 +1,6 @@
 import { CanvasMaskLine } from 'features/canvas/store/canvasTypes';
 import Konva from 'konva';
+import { Stage } from 'konva/lib/Stage';
 import { IRect } from 'konva/lib/types';
 
 /**
@@ -12,10 +13,50 @@ import { IRect } from 'konva/lib/types';
  * drawing the mask and compositing everything correctly to output a valid
  * mask image.
  */
-const generateMask = (
+export const getStageDataURL = (stage: Stage, boundingBox: IRect): string => {
+  // create an offscreen canvas and add the mask to it
+  // const { stage, offscreenContainer } = buildMaskStage(lines, boundingBox);
+
+  const dataURL = stage.toDataURL({ ...boundingBox });
+
+  // const imageData = stage
+  //   .toCanvas()
+  //   .getContext('2d')
+  //   ?.getImageData(
+  //     boundingBox.x,
+  //     boundingBox.y,
+  //     boundingBox.width,
+  //     boundingBox.height
+  //   );
+
+  // offscreenContainer.remove();
+
+  // return { dataURL, imageData };
+
+  return dataURL;
+};
+
+export const getStageImageData = (
+  stage: Stage,
+  boundingBox: IRect
+): ImageData | undefined => {
+  const imageData = stage
+    .toCanvas()
+    .getContext('2d')
+    ?.getImageData(
+      boundingBox.x,
+      boundingBox.y,
+      boundingBox.width,
+      boundingBox.height
+    );
+
+  return imageData;
+};
+
+export const buildMaskStage = (
   lines: CanvasMaskLine[],
   boundingBox: IRect
-): { dataURL: string; imageData: ImageData } => {
+): { stage: Stage; offscreenContainer: HTMLDivElement } => {
   // create an offscreen canvas and add the mask to it
   const { width, height } = boundingBox;
 
@@ -57,20 +98,5 @@ const generateMask = (
   stage.add(baseLayer);
   stage.add(maskLayer);
 
-  const dataURL = stage.toDataURL({ ...boundingBox });
+  return { stage, offscreenContainer };
-  const imageData = stage
-    .toCanvas()
-    .getContext('2d')
-    ?.getImageData(
-      boundingBox.x,
-      boundingBox.y,
-      boundingBox.width,
-      boundingBox.height
-    );
-
-  offscreenContainer.remove();
-
-  return { dataURL, imageData };
 };
-
-export default generateMask;

invokeai/frontend/web/src/features/canvas/util/getCanvasDataURLs.ts

@@ -1,24 +1,27 @@
 import { RootState } from 'app/store/store';
 import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider';
 import { isCanvasMaskLine } from '../store/canvasTypes';
-import generateMask from './generateMask';
+import {
+  buildMaskStage,
+  getStageDataURL,
+  getStageImageData,
+} from './generateMask';
 import { log } from 'app/logging/useLogger';
 import {
   areAnyPixelsBlack,
   getImageDataTransparency,
-  getIsImageDataWhite,
 } from 'common/util/arrayBuffer';
 import openBase64ImageInTab from 'common/util/openBase64ImageInTab';
+import { masks } from 'dateformat';
+
+const moduleLog = log.child({ namespace: 'getCanvasDataURLs' });
 
 export const getCanvasDataURLs = (state: RootState) => {
   const canvasBaseLayer = getCanvasBaseLayer();
   const canvasStage = getCanvasStage();
 
   if (!canvasBaseLayer || !canvasStage) {
-    log.error(
+    moduleLog.error('Unable to find canvas / stage');
-      { namespace: 'getCanvasDataURLs' },
-      'Unable to find canvas / stage'
-    );
     return;
   }
 
@@ -55,30 +58,46 @@ export const getCanvasDataURLs = (state: RootState) => {
 
   const absPos = canvasBaseLayer.getAbsolutePosition();
 
-  const { dataURL: maskDataURL, imageData: maskImageData } = generateMask(
+  const offsetBoundingBox = {
-    isMaskEnabled ? objects.filter(isCanvasMaskLine) : [],
-    {
     x: boundingBox.x + absPos.x,
     y: boundingBox.y + absPos.y,
     width: boundingBox.width,
     height: boundingBox.height,
-    }
+  };
+
+  const { stage: maskStage, offscreenContainer } = buildMaskStage(
+    isMaskEnabled ? objects.filter(isCanvasMaskLine) : [],
+    offsetBoundingBox
   );
 
-  const baseDataURL = canvasBaseLayer.toDataURL({
+  const maskDataURL = maskStage.toDataURL(offsetBoundingBox);
-    x: boundingBox.x + absPos.x,
+
-    y: boundingBox.y + absPos.y,
+  const maskImageData = maskStage
-    width: boundingBox.width,
+    .toCanvas()
-    height: boundingBox.height,
+    .getContext('2d')
-  });
+    ?.getImageData(
+      offsetBoundingBox.x,
+      offsetBoundingBox.y,
+      offsetBoundingBox.width,
+      offsetBoundingBox.height
+    );
+
+  offscreenContainer.remove();
+
+  if (!maskImageData) {
+    moduleLog.error('Unable to get mask stage context');
+    return;
+  }
+
+  const baseDataURL = canvasBaseLayer.toDataURL(offsetBoundingBox);
 
   const ctx = canvasBaseLayer.getContext();
 
   const baseImageData = ctx.getImageData(
-    boundingBox.x + absPos.x,
+    offsetBoundingBox.x,
-    boundingBox.y + absPos.y,
+    offsetBoundingBox.y,
-    boundingBox.width,
+    offsetBoundingBox.width,
-    boundingBox.height
+    offsetBoundingBox.height
   );
 
   const {
@@ -86,6 +105,7 @@ export const getCanvasDataURLs = (state: RootState) => {
     isFullyTransparent: baseIsFullyTransparent,
   } = getImageDataTransparency(baseImageData);
 
+  // const doesMaskHaveBlackPixels = false;
   const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData);
 
   if (state.system.enableImageDebugging) {

invokeai/frontend/web/src/features/canvas/util/getCanvasNodeType.ts

@@ -0,0 +1,102 @@
+import { RootState } from 'app/store/store';
+import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider';
+import {
+  CanvasObject,
+  Dimensions,
+  isCanvasMaskLine,
+} from '../store/canvasTypes';
+import { buildMaskStage, getStageImageData } from './generateMask';
+import { log } from 'app/logging/useLogger';
+import {
+  areAnyPixelsBlack,
+  getImageDataTransparency,
+} from 'common/util/arrayBuffer';
+import { getNodeType } from 'features/nodes/util/getNodeType';
+import { Vector2d } from 'konva/lib/types';
+
+const moduleLog = log.child({ namespace: 'getCanvasNodeTypes' });
+
+export type GetCanvasNodeTypeArg = {
+  objects: CanvasObject[];
+  boundingBoxCoordinates: Vector2d;
+  boundingBoxDimensions: Dimensions;
+  stageScale: number;
+  isMaskEnabled: boolean;
+};
+
+export const getCanvasNodeType = (arg: GetCanvasNodeTypeArg) => {
+  const canvasBaseLayer = getCanvasBaseLayer();
+  const canvasStage = getCanvasStage();
+
+  if (!canvasBaseLayer || !canvasStage) {
+    moduleLog.error('Unable to find canvas / stage');
+    return;
+  }
+
+  const {
+    objects,
+    boundingBoxCoordinates,
+    boundingBoxDimensions,
+    stageScale,
+    isMaskEnabled,
+  } = arg;
+
+  const boundingBox = {
+    ...boundingBoxCoordinates,
+    ...boundingBoxDimensions,
+  };
+
+  const tempScale = canvasBaseLayer.scale();
+
+  canvasBaseLayer.scale({
+    x: 1 / stageScale,
+    y: 1 / stageScale,
+  });
+
+  const absPos = canvasBaseLayer.getAbsolutePosition();
+
+  const scaledBoundingBox = {
+    x: boundingBox.x + absPos.x,
+    y: boundingBox.y + absPos.y,
+    width: boundingBox.width,
+    height: boundingBox.height,
+  };
+
+  const { stage: maskStage, offscreenContainer } = buildMaskStage(
+    isMaskEnabled ? objects.filter(isCanvasMaskLine) : [],
+    scaledBoundingBox
+  );
+
+  const maskImageData = getStageImageData(maskStage, scaledBoundingBox);
+
+  offscreenContainer.remove();
+
+  if (!maskImageData) {
+    moduleLog.error('Unable to get mask stage context');
+    return;
+  }
+
+  const ctx = canvasBaseLayer.getContext();
+
+  const baseImageData = ctx.getImageData(
+    boundingBox.x + absPos.x,
+    boundingBox.y + absPos.y,
+    boundingBox.width,
+    boundingBox.height
+  );
+
+  canvasBaseLayer.scale(tempScale);
+
+  const {
+    isPartiallyTransparent: baseIsPartiallyTransparent,
+    isFullyTransparent: baseIsFullyTransparent,
+  } = getImageDataTransparency(baseImageData);
+
+  const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData);
+
+  return getNodeType(
+    baseIsPartiallyTransparent,
+    baseIsFullyTransparent,
+    doesMaskHaveBlackPixels
+  );
+};

invokeai/frontend/web/src/features/gallery/components/HoverableImage.tsx

@@ -134,6 +134,7 @@ const HoverableImage = memo((props: HoverableImageProps) => {
 
   const handleDragStart = useCallback(
     (e: DragEvent<HTMLDivElement>) => {
+      console.log('dragging');
       e.dataTransfer.setData('invokeai/imageName', image.name);
       e.dataTransfer.setData('invokeai/imageType', image.type);
       e.dataTransfer.effectAllowed = 'move';

invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasWorkarea.tsx

@@ -34,6 +34,7 @@ import ParametersSlide from '../../common/ParametersSlide';
 import UnifiedCanvasParameters from './UnifiedCanvasParameters';
 import UnifiedCanvasContentBeta from './UnifiedCanvasBeta/UnifiedCanvasContentBeta';
 import UnifiedCanvasContent from './UnifiedCanvasContent';
+import { useGetCanvasNodeType } from 'features/canvas/hooks/useGetCanvasNodeType';
 
 const CanvasWorkspace = () => {
   const shouldPinParametersPanel = useAppSelector(