feat(ui): wip canvas nodes migration

invokeai/frontend/web/src/features/canvas/hooks/usePrepareCanvasState.ts

@@ -0,0 +1,39 @@
+import { createSelector } from '@reduxjs/toolkit';
+import { useAppSelector } from 'app/store/storeHooks';
+import {
+  FrontendToBackendParametersConfig,
+  frontendToBackendParameters,
+} from 'common/util/parameterTranslation';
+import { generationSelector } from 'features/parameters/store/generationSelectors';
+import { postprocessingSelector } from 'features/parameters/store/postprocessingSelectors';
+import { systemSelector } from 'features/system/store/systemSelectors';
+import { canvasSelector } from '../store/canvasSelectors';
+import { useCallback, useMemo } from 'react';
+
+const selector = createSelector(
+  [generationSelector, postprocessingSelector, systemSelector, canvasSelector],
+  (generation, postprocessing, system, canvas) => {
+    const frontendToBackendParametersConfig: FrontendToBackendParametersConfig =
+      {
+        generationMode: 'unifiedCanvas',
+        generationState: generation,
+        postprocessingState: postprocessing,
+        canvasState: canvas,
+        systemState: system,
+      };
+
+    return frontendToBackendParametersConfig;
+  }
+);
+
+export const usePrepareCanvasState = () => {
+  const frontendToBackendParametersConfig = useAppSelector(selector);
+
+  const getGenerationParameters = useCallback(() => {
+    const { generationParameters, esrganParameters, facetoolParameters } =
+      frontendToBackendParameters(frontendToBackendParametersConfig);
+    console.log(generationParameters);
+  }, [frontendToBackendParametersConfig]);
+
+  return getGenerationParameters;
+};

invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts

@@ -156,22 +156,20 @@ export const canvasSlice = createSlice({
     setCursorPosition: (state, action: PayloadAction<Vector2d | null>) => {
       state.cursorPosition = action.payload;
     },
-    setInitialCanvasImage: (state, action: PayloadAction<InvokeAI._Image>) => {
+    setInitialCanvasImage: (state, action: PayloadAction<InvokeAI.Image>) => {
       const image = action.payload;
+      const { width, height } = image.metadata;
       const { stageDimensions } = state;
 
       const newBoundingBoxDimensions = {
-        width: roundDownToMultiple(clamp(image.width, 64, 512), 64),
-        height: roundDownToMultiple(clamp(image.height, 64, 512), 64),
+        width: roundDownToMultiple(clamp(width, 64, 512), 64),
+        height: roundDownToMultiple(clamp(height, 64, 512), 64),
       };
 
       const newBoundingBoxCoordinates = {
-        x: roundToMultiple(
-          image.width / 2 - newBoundingBoxDimensions.width / 2,
-          64
-        ),
+        x: roundToMultiple(width / 2 - newBoundingBoxDimensions.width / 2, 64),
         y: roundToMultiple(
-          image.height / 2 - newBoundingBoxDimensions.height / 2,
+          height / 2 - newBoundingBoxDimensions.height / 2,
           64
         ),
       };
@@ -196,8 +194,8 @@ export const canvasSlice = createSlice({
             layer: 'base',
             x: 0,
             y: 0,
-            width: image.width,
-            height: image.height,
+            width: width,
+            height: height,
             image: image,
           },
         ],
@@ -208,8 +206,8 @@ export const canvasSlice = createSlice({
       const newScale = calculateScale(
         stageDimensions.width,
         stageDimensions.height,
-        image.width,
-        image.height,
+        width,
+        height,
         STAGE_PADDING_PERCENTAGE
       );
 
@@ -218,8 +216,8 @@ export const canvasSlice = createSlice({
         stageDimensions.height,
         0,
         0,
-        image.width,
-        image.height,
+        width,
+        height,
         newScale
       );
       state.stageScale = newScale;

invokeai/frontend/web/src/features/canvas/util/generateMask.ts

@@ -12,7 +12,10 @@ import { IRect } from 'konva/lib/types';
  * drawing the mask and compositing everything correctly to output a valid
  * mask image.
  */
-const generateMask = (lines: CanvasMaskLine[], boundingBox: IRect): string => {
+const generateMask = (
+  lines: CanvasMaskLine[],
+  boundingBox: IRect
+): { dataURL: string; imageData: ImageData } => {
   // create an offscreen canvas and add the mask to it
   const { width, height } = boundingBox;
 
@@ -55,10 +58,19 @@ const generateMask = (lines: CanvasMaskLine[], boundingBox: IRect): string => {
   stage.add(maskLayer);
 
   const dataURL = stage.toDataURL({ ...boundingBox });
+  const imageData = stage
+    .toCanvas()
+    .getContext('2d')
+    ?.getImageData(
+      boundingBox.x,
+      boundingBox.y,
+      boundingBox.width,
+      boundingBox.height
+    );
 
   offscreenContainer.remove();
 
-  return dataURL;
+  return { dataURL, imageData };
 };
 
 export default generateMask;

invokeai/frontend/web/src/features/canvas/util/getCanvasDataURLs.ts

@@ -0,0 +1,123 @@
+import { RootState } from 'app/store/store';
+import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider';
+import { isCanvasMaskLine } from '../store/canvasTypes';
+import generateMask from './generateMask';
+import { log } from 'app/logging/useLogger';
+import {
+  areAnyPixelsBlack,
+  getImageDataTransparency,
+  getIsImageDataWhite,
+} from 'common/util/arrayBuffer';
+import openBase64ImageInTab from 'common/util/openBase64ImageInTab';
+
+export const getCanvasDataURLs = (state: RootState) => {
+  const canvasBaseLayer = getCanvasBaseLayer();
+  const canvasStage = getCanvasStage();
+
+  if (!canvasBaseLayer || !canvasStage) {
+    log.error(
+      { namespace: 'getCanvasDataURLs' },
+      'Unable to find canvas / stage'
+    );
+    return;
+  }
+
+  const {
+    layerState: { objects },
+    boundingBoxCoordinates,
+    boundingBoxDimensions,
+    stageScale,
+    isMaskEnabled,
+    shouldPreserveMaskedArea,
+    boundingBoxScaleMethod: boundingBoxScale,
+    scaledBoundingBoxDimensions,
+  } = state.canvas;
+
+  const boundingBox = {
+    ...boundingBoxCoordinates,
+    ...boundingBoxDimensions,
+  };
+
+  // generationParameters.fit = false;
+
+  // generationParameters.strength = img2imgStrength;
+
+  // generationParameters.invert_mask = shouldPreserveMaskedArea;
+
+  // generationParameters.bounding_box = boundingBox;
+
+  const tempScale = canvasBaseLayer.scale();
+
+  canvasBaseLayer.scale({
+    x: 1 / stageScale,
+    y: 1 / stageScale,
+  });
+
+  const absPos = canvasBaseLayer.getAbsolutePosition();
+
+  const { dataURL: maskDataURL, imageData: maskImageData } = generateMask(
+    isMaskEnabled ? objects.filter(isCanvasMaskLine) : [],
+    {
+      x: boundingBox.x + absPos.x,
+      y: boundingBox.y + absPos.y,
+      width: boundingBox.width,
+      height: boundingBox.height,
+    }
+  );
+
+  const baseDataURL = canvasBaseLayer.toDataURL({
+    x: boundingBox.x + absPos.x,
+    y: boundingBox.y + absPos.y,
+    width: boundingBox.width,
+    height: boundingBox.height,
+  });
+
+  const ctx = canvasBaseLayer.getContext();
+
+  const baseImageData = ctx.getImageData(
+    boundingBox.x + absPos.x,
+    boundingBox.y + absPos.y,
+    boundingBox.width,
+    boundingBox.height
+  );
+
+  const {
+    isPartiallyTransparent: baseIsPartiallyTransparent,
+    isFullyTransparent: baseIsFullyTransparent,
+  } = getImageDataTransparency(baseImageData);
+
+  const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData);
+
+  if (state.system.enableImageDebugging) {
+    openBase64ImageInTab([
+      { base64: maskDataURL, caption: 'mask sent as init_mask' },
+      { base64: baseDataURL, caption: 'image sent as init_img' },
+    ]);
+  }
+
+  canvasBaseLayer.scale(tempScale);
+
+  // generationParameters.init_img = imageDataURL;
+  // generationParameters.progress_images = false;
+
+  // if (boundingBoxScale !== 'none') {
+  //   generationParameters.inpaint_width = scaledBoundingBoxDimensions.width;
+  //   generationParameters.inpaint_height = scaledBoundingBoxDimensions.height;
+  // }
+
+  // generationParameters.seam_size = seamSize;
+  // generationParameters.seam_blur = seamBlur;
+  // generationParameters.seam_strength = seamStrength;
+  // generationParameters.seam_steps = seamSteps;
+  // generationParameters.tile_size = tileSize;
+  // generationParameters.infill_method = infillMethod;
+  // generationParameters.force_outpaint = false;
+
+  return {
+    baseDataURL,
+    maskDataURL,
+    baseIsPartiallyTransparent,
+    baseIsFullyTransparent,
+    doesMaskHaveBlackPixels,
+  };
+};