Initial unification of canvas

backend/invoke_ai_web_server.py

@@ -8,6 +8,7 @@ import math
 import io
 import base64
 import os
+import json
 
 from werkzeug.utils import secure_filename
 from flask import Flask, redirect, send_from_directory, request, make_response
@@ -107,6 +108,7 @@ class InvokeAIWebServer:
         @self.app.route("/upload", methods=["POST"])
         def upload():
             try:
+                data = json.loads(request.form["data"])
                 filename = ""
                 # check if the post request has the file part
                 if "file" in request.files:
@@ -116,15 +118,15 @@ class InvokeAIWebServer:
                     if file.filename == "":
                         return make_response("No file selected", 400)
                     filename = file.filename
-                elif "dataURL" in request.form:
+                elif "dataURL" in data:
-                    file = dataURL_to_bytes(request.form["dataURL"])
+                    file = dataURL_to_bytes(data["dataURL"])
-                    if "filename" not in request.form or request.form["filename"] == "":
+                    if "filename" not in data or data["filename"] == "":
                         return make_response("No filename provided", 400)
-                    filename = request.form["filename"]
+                    filename = data["filename"]
                 else:
                     return make_response("No file or dataURL", 400)
 
-                kind = request.form["kind"]
+                kind = data["kind"]
 
                 if kind == "init":
                     path = self.init_image_path
@@ -153,22 +155,32 @@ class InvokeAIWebServer:
 
                 file_path = os.path.join(path, name)
 
-                if "dataURL" in request.form:
+                if "dataURL" in data:
                     with open(file_path, "wb") as f:
                         f.write(file)
                 else:
                     file.save(file_path)
 
                 mtime = os.path.getmtime(file_path)
-                (width, height) = Image.open(file_path).size
+
+                pil_image = Image.open(file_path)
+
+                # visible_image_bbox = pil_image.getbbox()
+                # pil_image = pil_image.crop(visible_image_bbox)
+                # pil_image.save(file_path)
+                # if "cropVisible" in data and data["cropVisible"] == True:
+                #     visible_image_bbox = pil_image.getbbox()
+                #     pil_image = pil_image.crop(visible_image_bbox)
+                #     pil_image.save(file_path)
+
+                (width, height) = pil_image.size
 
                 response = {
-                    "image": {
                     "url": self.get_url_from_image_path(file_path),
                     "mtime": mtime,
+                    # "bbox": visible_image_bbox,
                     "width": width,
                     "height": height,
-                    },
                 }
 
                 return make_response(response, 200)

frontend/src/app/invokeai.d.ts

@@ -179,7 +179,12 @@ export declare type ImageResultResponse = Omit<Image, 'uuid'> & {
 };
 
 export declare type ImageUploadResponse = {
-  image: Omit<Image, 'uuid' | 'metadata' | 'category'>;
+  // image: Omit<Image, 'uuid' | 'metadata' | 'category'>;
+  url: string;
+  mtime: number;
+  width: number;
+  height: number;
+  // bbox: [number, number, number, number];
 };
 
 export declare type ErrorResponse = {

frontend/src/app/store.ts

@@ -82,8 +82,6 @@ const rootPersistConfig = getPersistConfig({
   debounce: 300,
 });
 
-// console.log(rootPersistConfig)
-
 const persistedReducer = persistReducer(rootPersistConfig, rootReducer);
 
 // Continue with store setup
@@ -95,6 +93,7 @@ export const store = configureStore({
       serializableCheck: false,
     }).concat(socketioMiddleware()),
   devTools: {
+    // Uncommenting these very rapidly called actions makes the redux dev tools output much more readable
     actionsDenylist: [
       // 'canvas/setCursorPosition',
       // 'canvas/setStageCoordinates',

frontend/src/features/canvas/IAICanvas.tsx

@@ -1,16 +1,19 @@
 // lib
-import { MutableRefObject, useRef } from 'react';
+import { MutableRefObject, useCallback, useRef } from 'react';
 import Konva from 'konva';
 import { Layer, Stage } from 'react-konva';
 import { Stage as StageType } from 'konva/lib/Stage';
 
 // app
-import { useAppSelector } from 'app/store';
+import { RootState, useAppDispatch, useAppSelector } from 'app/store';
 import {
   baseCanvasImageSelector,
   currentCanvasSelector,
   isStagingSelector,
   outpaintingCanvasSelector,
+  setStageCoordinates,
+  setStageScale,
+  shouldLockToInitialImageSelector,
 } from 'features/canvas/canvasSlice';
 
 // component
@@ -35,15 +38,31 @@ import IAICanvasGrid from './IAICanvasGrid';
 import IAICanvasIntermediateImage from './IAICanvasIntermediateImage';
 import IAICanvasStatusText from './IAICanvasStatusText';
 import IAICanvasStagingArea from './IAICanvasStagingArea';
+import IAICanvasStagingAreaToolbar from './IAICanvasStagingAreaToolbar';
+import { KonvaEventObject } from 'konva/lib/Node';
+import {
+  CANVAS_SCALE_BY,
+  MAX_CANVAS_SCALE,
+  MIN_CANVAS_SCALE,
+} from './util/constants';
 
 const canvasSelector = createSelector(
   [
+    shouldLockToInitialImageSelector,
     currentCanvasSelector,
     outpaintingCanvasSelector,
     isStagingSelector,
     activeTabNameSelector,
+    baseCanvasImageSelector,
   ],
-  (currentCanvas, outpaintingCanvas, isStaging, activeTabName) => {
+  (
+    shouldLockToInitialImage,
+    currentCanvas,
+    outpaintingCanvas,
+    isStaging,
+    activeTabName,
+    baseCanvasImage
+  ) => {
     const {
       isMaskEnabled,
       stageScale,
@@ -56,6 +75,7 @@ const canvasSelector = createSelector(
       tool,
       isMovingStage,
       shouldShowIntermediates,
+      minimumStageScale,
     } = currentCanvas;
 
     const { shouldShowGrid } = outpaintingCanvas;
@@ -89,6 +109,10 @@ const canvasSelector = createSelector(
       isOnOutpaintingTab: activeTabName === 'outpainting',
       isStaging,
       shouldShowIntermediates,
+      shouldLockToInitialImage,
+      activeTabName,
+      minimumStageScale,
+      baseCanvasImage,
     };
   },
   {
@@ -116,8 +140,12 @@ const IAICanvas = () => {
     isOnOutpaintingTab,
     isStaging,
     shouldShowIntermediates,
+    shouldLockToInitialImage,
+    activeTabName,
+    minimumStageScale,
+    baseCanvasImage,
   } = useAppSelector(canvasSelector);
-
+  const dispatch = useAppDispatch();
   useCanvasHotkeys();
 
   // set the closure'd refs
@@ -142,6 +170,34 @@ const IAICanvas = () => {
   const { handleDragStart, handleDragMove, handleDragEnd } =
     useCanvasDragMove();
 
+  const dragBoundFunc = useCallback(
+    (newCoordinates: Vector2d) => {
+      if (shouldLockToInitialImage && baseCanvasImage) {
+        newCoordinates.x = _.clamp(
+          newCoordinates.x,
+          stageDimensions.width -
+            Math.floor(baseCanvasImage.width * stageScale),
+          0
+        );
+        newCoordinates.y = _.clamp(
+          newCoordinates.y,
+          stageDimensions.height -
+            Math.floor(baseCanvasImage.height * stageScale),
+          0
+        );
+      }
+
+      return newCoordinates;
+    },
+    [
+      baseCanvasImage,
+      shouldLockToInitialImage,
+      stageDimensions.height,
+      stageDimensions.width,
+      stageScale,
+    ]
+  );
+
   return (
     <div className="inpainting-canvas-container">
       <div className="inpainting-canvas-wrapper">
@@ -157,6 +213,7 @@ const IAICanvas = () => {
           width={stageDimensions.width}
           height={stageDimensions.height}
           scale={{ x: stageScale, y: stageScale }}
+          dragBoundFunc={dragBoundFunc}
           onMouseDown={handleMouseDown}
           onMouseEnter={handleMouseEnter}
           onMouseLeave={handleMouseOut}
@@ -205,6 +262,7 @@ const IAICanvas = () => {
           </Layer>
         </Stage>
         {isOnOutpaintingTab && <IAICanvasStatusText />}
+        <IAICanvasStagingAreaToolbar />
       </div>
     </div>
   );

frontend/src/features/canvas/IAICanvasBoundingBox.tsx

@@ -6,8 +6,11 @@ import { Vector2d } from 'konva/lib/types';
 import _ from 'lodash';
 import { useCallback, useEffect, useRef } from 'react';
 import { Group, Rect, Transformer } from 'react-konva';
-import { useAppDispatch, useAppSelector } from 'app/store';
+import { RootState, useAppDispatch, useAppSelector } from 'app/store';
-import { roundToMultiple } from 'common/util/roundDownToMultiple';
+import {
+  roundDownToMultiple,
+  roundToMultiple,
+} from 'common/util/roundDownToMultiple';
 import {
   baseCanvasImageSelector,
   currentCanvasSelector,
@@ -17,16 +20,24 @@ import {
   setIsMouseOverBoundingBox,
   setIsMovingBoundingBox,
   setIsTransformingBoundingBox,
+  shouldLockToInitialImageSelector,
 } from 'features/canvas/canvasSlice';
 import { GroupConfig } from 'konva/lib/Group';
 import { activeTabNameSelector } from 'features/options/optionsSelectors';
 
 const boundingBoxPreviewSelector = createSelector(
+  shouldLockToInitialImageSelector,
   currentCanvasSelector,
   outpaintingCanvasSelector,
   baseCanvasImageSelector,
   activeTabNameSelector,
-  (currentCanvas, outpaintingCanvas, baseCanvasImage, activeTabName) => {
+  (
+    shouldLockToInitialImage,
+    currentCanvas,
+    outpaintingCanvas,
+    baseCanvasImage,
+    activeTabName
+  ) => {
     const {
       boundingBoxCoordinates,
       boundingBoxDimensions,
@@ -40,6 +51,7 @@ const boundingBoxPreviewSelector = createSelector(
       tool,
       stageCoordinates,
     } = currentCanvas;
+
     const { shouldSnapToGrid } = outpaintingCanvas;
 
     return {
@@ -59,6 +71,7 @@ const boundingBoxPreviewSelector = createSelector(
       stageCoordinates,
       boundingBoxStrokeWidth: (isMouseOverBoundingBox ? 8 : 1) / stageScale,
       hitStrokeWidth: 20 / stageScale,
+      shouldLockToInitialImage,
     };
   },
   {
@@ -91,6 +104,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
     tool,
     boundingBoxStrokeWidth,
     hitStrokeWidth,
+    shouldLockToInitialImage,
   } = useAppSelector(boundingBoxPreviewSelector);
 
   const transformerRef = useRef<Konva.Transformer>(null);
@@ -106,7 +120,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
 
   const handleOnDragMove = useCallback(
     (e: KonvaEventObject<DragEvent>) => {
-      if (activeTabName === 'inpainting' || !shouldSnapToGrid) {
+      if (!shouldSnapToGrid) {
         dispatch(
           setBoundingBoxCoordinates({
             x: Math.floor(e.target.x()),
@@ -132,20 +146,27 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
         })
       );
     },
-    [activeTabName, dispatch, shouldSnapToGrid]
+    [dispatch, shouldSnapToGrid]
   );
 
   const dragBoundFunc = useCallback(
     (position: Vector2d) => {
-      if (!baseCanvasImage && activeTabName !== 'outpainting')
+      /**
-        return boundingBoxCoordinates;
+       * This limits the bounding box's drag coordinates.
+       */
+      if (!shouldLockToInitialImage) return boundingBoxCoordinates;
 
       const { x, y } = position;
 
       const maxX =
-        stageDimensions.width - boundingBoxDimensions.width * stageScale;
+        stageDimensions.width -
+        boundingBoxDimensions.width -
+        (stageDimensions.width % 64);
+
       const maxY =
-        stageDimensions.height - boundingBoxDimensions.height * stageScale;
+        stageDimensions.height -
+        boundingBoxDimensions.height -
+        (stageDimensions.height % 64);
 
       const clampedX = Math.floor(_.clamp(x, 0, maxX));
       const clampedY = Math.floor(_.clamp(y, 0, maxY));
@@ -153,14 +174,12 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
       return { x: clampedX, y: clampedY };
     },
     [
-      baseCanvasImage,
+      shouldLockToInitialImage,
-      activeTabName,
       boundingBoxCoordinates,
       stageDimensions.width,
       stageDimensions.height,
       boundingBoxDimensions.width,
       boundingBoxDimensions.height,
-      stageScale,
     ]
   );
 
@@ -203,7 +222,6 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
     rect.scaleY(1);
   }, [dispatch]);
 
-  // OK
   const anchorDragBoundFunc = useCallback(
     (
       oldPos: Vector2d, // old absolute position of anchor point
@@ -215,49 +233,14 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
        * Konva does not transform with width or height. It transforms the anchor point
        * and scale factor. This is then sent to the shape's onTransform listeners.
        *
-       * We need to snap the new width to steps of 64 without also snapping the
+       * We need to snap the new dimensions to steps of 64. But because the whole
-       * coordinates of the bounding box to steps of 64. But because the whole
        * stage is scaled, our actual desired step is actually 64 * the stage scale.
        */
 
-      // Difference of the old coords from the nearest multiple the scaled step
+      return {
-      const offsetX = oldPos.x % scaledStep;
+        x: roundToMultiple(newPos.x, scaledStep),
-      const offsetY = oldPos.y % scaledStep;
+        y: roundToMultiple(newPos.y, scaledStep),
-
-      // Round new position to the nearest multiple of the scaled step
-      const closestX = roundToMultiple(newPos.x, scaledStep) + offsetX;
-      const closestY = roundToMultiple(newPos.y, scaledStep) + offsetY;
-
-      // the difference between the old coord and new
-      const diffX = Math.abs(newPos.x - closestX);
-      const diffY = Math.abs(newPos.y - closestY);
-
-      // if the difference is less than the scaled step, we want to snap
-      const didSnapX = diffX < scaledStep;
-      const didSnapY = diffY < scaledStep;
-
-      // We may not change anything, stash the old position
-      let newCoordinate = { ...oldPos };
-
-      // Set the new coords based on what snapped
-      if (didSnapX && !didSnapY) {
-        newCoordinate = {
-          x: closestX,
-          y: oldPos.y,
       };
-      } else if (!didSnapX && didSnapY) {
-        newCoordinate = {
-          x: oldPos.x,
-          y: closestY,
-        };
-      } else if (didSnapX && didSnapY) {
-        newCoordinate = {
-          x: closestX,
-          y: closestY,
-        };
-      }
-
-      return newCoordinate;
     },
     [scaledStep]
   );
@@ -272,7 +255,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
 
       // On the Inpainting canvas, the bounding box needs to stay in the stage
       if (
-        activeTabName === 'inpainting' &&
+        shouldLockToInitialImage &&
         (newBoundBox.width + newBoundBox.x > stageDimensions.width ||
           newBoundBox.height + newBoundBox.y > stageDimensions.height ||
           newBoundBox.x < 0 ||
@@ -283,7 +266,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
 
       return newBoundBox;
     },
-    [activeTabName, stageDimensions.height, stageDimensions.width]
+    [shouldLockToInitialImage, stageDimensions.height, stageDimensions.width]
   );
 
   const handleStartedTransforming = () => {
@@ -337,7 +320,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
         globalCompositeOperation={'destination-out'}
       />
       <Rect
-        {...(activeTabName === 'inpainting' ? { dragBoundFunc } : {})}
+        {...(shouldLockToInitialImage ? { dragBoundFunc } : {})}
         listening={!isDrawing && tool === 'move'}
         draggable={true}
         fillEnabled={false}

frontend/src/features/canvas/IAICanvasMaskLines.tsx

@@ -2,17 +2,13 @@ import { GroupConfig } from 'konva/lib/Group';
 import { Group, Line } from 'react-konva';
 import { useAppSelector } from 'app/store';
 import { createSelector } from '@reduxjs/toolkit';
-import {
+import { currentCanvasSelector, isCanvasMaskLine } from './canvasSlice';
-  canvasClipSelector,
-  currentCanvasSelector,
-  isCanvasMaskLine,
-} from './canvasSlice';
 import _ from 'lodash';
 
 export const canvasLinesSelector = createSelector(
-  [currentCanvasSelector, canvasClipSelector],
+  [currentCanvasSelector],
-  (currentCanvas, canvasClip) => {
+  (currentCanvas) => {
-    return { objects: currentCanvas.layerState.objects, canvasClip };
+    return { objects: currentCanvas.layerState.objects };
   },
   {
     memoizeOptions: {
@@ -30,10 +26,10 @@ type InpaintingCanvasLinesProps = GroupConfig;
  */
 const IAICanvasLines = (props: InpaintingCanvasLinesProps) => {
   const { ...rest } = props;
-  const { objects, canvasClip } = useAppSelector(canvasLinesSelector);
+  const { objects } = useAppSelector(canvasLinesSelector);
 
   return (
-    <Group listening={false} {...rest} {...canvasClip}>
+    <Group listening={false} {...rest}>
       {objects.filter(isCanvasMaskLine).map((line, i) => (
         <Line
           key={i}

frontend/src/features/canvas/IAICanvasObjectRenderer.tsx

@@ -4,8 +4,8 @@ import _ from 'lodash';
 import { Group, Line } from 'react-konva';
 import {
   baseCanvasImageSelector,
-  canvasClipSelector,
+  // canvasClipSelector,
-  canvasModeSelector,
+  // canvasModeSelector,
   currentCanvasSelector,
   isCanvasBaseImage,
   isCanvasBaseLine,
@@ -14,13 +14,12 @@ import IAICanvasImage from './IAICanvasImage';
 import { rgbaColorToString } from './util/colorToString';
 
 const selector = createSelector(
-  [currentCanvasSelector, canvasClipSelector],
+  [currentCanvasSelector],
-  (currentCanvas, canvasClip) => {
+  (currentCanvas) => {
     const { objects } = currentCanvas.layerState;
 
     return {
       objects,
-      canvasClip,
     };
   },
   {
@@ -31,12 +30,12 @@ const selector = createSelector(
 );
 
 const IAICanvasObjectRenderer = () => {
-  const { objects, canvasClip } = useAppSelector(selector);
+  const { objects } = useAppSelector(selector);
 
   if (!objects) return null;
 
   return (
-    <Group name="outpainting-objects" listening={false} {...canvasClip}>
+    <Group name="outpainting-objects" listening={false}>
       {objects.map((obj, i) => {
         if (isCanvasBaseImage(obj)) {
           return (

frontend/src/features/canvas/IAICanvasOutpaintingControls.tsx

@@ -2,10 +2,11 @@ import { ButtonGroup } from '@chakra-ui/react';
 import { createSelector } from '@reduxjs/toolkit';
 import {
   currentCanvasSelector,
+  resizeAndScaleCanvas,
   isStagingSelector,
   resetCanvas,
   resetCanvasView,
-  setCanvasMode,
+  setShouldLockToInitialImage,
   setTool,
 } from './canvasSlice';
 import { RootState, useAppDispatch, useAppSelector } from 'app/store';
@@ -30,6 +31,7 @@ import IAICanvasBrushButtonPopover from './IAICanvasBrushButtonPopover';
 import IAICanvasMaskButtonPopover from './IAICanvasMaskButtonPopover';
 import { mergeAndUploadCanvas } from './util/mergeAndUploadCanvas';
 import IAICheckbox from 'common/components/IAICheckbox';
+import { ChangeEvent } from 'react';
 
 export const canvasControlsSelector = createSelector(
   [
@@ -38,12 +40,12 @@ export const canvasControlsSelector = createSelector(
     isStagingSelector,
   ],
   (canvas, currentCanvas, isStaging) => {
+    const { shouldLockToInitialImage } = canvas;
     const { tool } = currentCanvas;
-    const { mode } = canvas;
     return {
       tool,
       isStaging,
-      mode,
+      shouldLockToInitialImage,
     };
   },
   {
@@ -55,7 +57,16 @@ export const canvasControlsSelector = createSelector(
 
 const IAICanvasOutpaintingControls = () => {
   const dispatch = useAppDispatch();
-  const { tool, isStaging, mode } = useAppSelector(canvasControlsSelector);
+  const { tool, isStaging, shouldLockToInitialImage } = useAppSelector(
+    canvasControlsSelector
+  );
+
+  const handleToggleShouldLockToInitialImage = (
+    e: ChangeEvent<HTMLInputElement>
+  ) => {
+    dispatch(setShouldLockToInitialImage(e.target.checked));
+    dispatch(resizeAndScaleCanvas());
+  };
 
   return (
     <div className="inpainting-settings">
@@ -125,10 +136,12 @@ const IAICanvasOutpaintingControls = () => {
           icon={<FaCrosshairs />}
           onClick={() => {
             if (!stageRef.current || !canvasImageLayerRef.current) return;
-            const clientRect = canvasImageLayerRef.current.getClientRect({skipTransform: true});
+            const clientRect = canvasImageLayerRef.current.getClientRect({
+              skipTransform: true,
+            });
             dispatch(
               resetCanvasView({
-                clientRect,
+                contentRect: clientRect,
               })
             );
           }}
@@ -141,13 +154,9 @@ const IAICanvasOutpaintingControls = () => {
         />
       </ButtonGroup>
       <IAICheckbox
-        label={'inpainting'}
+        label={'Lock Canvas to Initial Image'}
-        isChecked={mode === 'inpainting'}
+        isChecked={shouldLockToInitialImage}
-        onChange={(e) =>
+        onChange={handleToggleShouldLockToInitialImage}
-          dispatch(
-            setCanvasMode(e.target.checked ? 'inpainting' : 'outpainting')
-          )
-        }
       />
     </div>
   );

frontend/src/features/canvas/IAICanvasResizer.tsx

@@ -5,8 +5,9 @@ import { activeTabNameSelector } from 'features/options/optionsSelectors';
 import {
   baseCanvasImageSelector,
   currentCanvasSelector,
-  initializeCanvas,
+  resizeAndScaleCanvas,
   resizeCanvas,
+  setCanvasContainerDimensions,
   setDoesCanvasNeedScaling,
 } from 'features/canvas/canvasSlice';
 import { createSelector } from '@reduxjs/toolkit';
@@ -17,10 +18,14 @@ const canvasResizerSelector = createSelector(
   baseCanvasImageSelector,
   activeTabNameSelector,
   (canvas, currentCanvas, baseCanvasImage, activeTabName) => {
-    const { doesCanvasNeedScaling, mode, isCanvasInitialized } = canvas;
+    const {
+      doesCanvasNeedScaling,
+      shouldLockToInitialImage,
+      isCanvasInitialized,
+    } = canvas;
     return {
       doesCanvasNeedScaling,
-      mode,
+      shouldLockToInitialImage,
       activeTabName,
       baseCanvasImage,
       isCanvasInitialized,
@@ -32,7 +37,7 @@ const IAICanvasResizer = () => {
   const dispatch = useAppDispatch();
   const {
     doesCanvasNeedScaling,
-    mode,
+    shouldLockToInitialImage,
     activeTabName,
     baseCanvasImage,
     isCanvasInitialized,
@@ -50,22 +55,17 @@ const IAICanvasResizer = () => {
 
       const { width: imageWidth, height: imageHeight } = baseCanvasImage.image;
 
-      if (!isCanvasInitialized) {
       dispatch(
-          initializeCanvas({
+        setCanvasContainerDimensions({
-            clientWidth,
+          width: clientWidth,
-            clientHeight,
+          height: clientHeight,
-            imageWidth,
-            imageHeight,
         })
       );
+
+      if (!isCanvasInitialized || shouldLockToInitialImage) {
+        dispatch(resizeAndScaleCanvas());
       } else {
-        dispatch(
+        dispatch(resizeCanvas());
-          resizeCanvas({
-            clientWidth,
-            clientHeight,
-          })
-        );
       }
 
       dispatch(setDoesCanvasNeedScaling(false));

frontend/src/features/canvas/IAICanvasStagingArea.tsx

@@ -106,7 +106,7 @@ const IAICanvasStagingArea = (props: Props) => {
           />
         </Group>
       )}
-      <Html>
+      {/* <Html>
         <CacheProvider value={emotionCache}>
           <ChakraProvider>
             <div
@@ -172,7 +172,7 @@ const IAICanvasStagingArea = (props: Props) => {
             </div>
           </ChakraProvider>
         </CacheProvider>
-      </Html>
+      </Html> */}
     </Group>
   );
 };

frontend/src/features/canvas/IAICanvasStagingAreaToolbar.tsx

@@ -0,0 +1,140 @@
+import {
+  background,
+  ButtonGroup,
+  ChakraProvider,
+  Flex,
+} from '@chakra-ui/react';
+import { CacheProvider } from '@emotion/react';
+import { createSelector } from '@reduxjs/toolkit';
+import { useAppDispatch, useAppSelector } from 'app/store';
+import IAIButton from 'common/components/IAIButton';
+import IAIIconButton from 'common/components/IAIIconButton';
+import { GroupConfig } from 'konva/lib/Group';
+import _ from 'lodash';
+import { emotionCache } from 'main';
+import { useCallback, useState } from 'react';
+import {
+  FaArrowLeft,
+  FaArrowRight,
+  FaCheck,
+  FaEye,
+  FaEyeSlash,
+  FaTrash,
+} from 'react-icons/fa';
+import { Group, Rect } from 'react-konva';
+import { Html } from 'react-konva-utils';
+import {
+  commitStagingAreaImage,
+  currentCanvasSelector,
+  discardStagedImages,
+  nextStagingAreaImage,
+  prevStagingAreaImage,
+} from './canvasSlice';
+import IAICanvasImage from './IAICanvasImage';
+
+const selector = createSelector(
+  [currentCanvasSelector],
+  (currentCanvas) => {
+    const {
+      layerState: {
+        stagingArea: { images, selectedImageIndex },
+      },
+    } = currentCanvas;
+
+    return {
+      currentStagingAreaImage:
+        images.length > 0 ? images[selectedImageIndex] : undefined,
+      isOnFirstImage: selectedImageIndex === 0,
+      isOnLastImage: selectedImageIndex === images.length - 1,
+    };
+  },
+  {
+    memoizeOptions: {
+      resultEqualityCheck: _.isEqual,
+    },
+  }
+);
+
+const IAICanvasStagingAreaToolbar = () => {
+  const dispatch = useAppDispatch();
+  const { isOnFirstImage, isOnLastImage, currentStagingAreaImage } =
+    useAppSelector(selector);
+
+  const [shouldShowStagedImage, setShouldShowStagedImage] =
+    useState<boolean>(true);
+
+  const [shouldShowStagingAreaOutline, setShouldShowStagingAreaOutline] =
+    useState<boolean>(true);
+
+  const handleMouseOver = useCallback(() => {
+    setShouldShowStagingAreaOutline(false);
+  }, []);
+
+  const handleMouseOut = useCallback(() => {
+    setShouldShowStagingAreaOutline(true);
+  }, []);
+
+  if (!currentStagingAreaImage) return null;
+
+  return (
+    <Flex
+      pos={'absolute'}
+      bottom={'1rem'}
+      w={'100%'}
+      align={'center'}
+      justify={'center'}
+    >
+      <ButtonGroup isAttached>
+        <IAIIconButton
+          tooltip="Previous"
+          tooltipProps={{ placement: 'bottom' }}
+          aria-label="Previous"
+          icon={<FaArrowLeft />}
+          onClick={() => dispatch(prevStagingAreaImage())}
+          onMouseOver={handleMouseOver}
+          onMouseOut={handleMouseOut}
+          data-selected={true}
+          isDisabled={isOnFirstImage}
+        />
+        <IAIIconButton
+          tooltip="Next"
+          tooltipProps={{ placement: 'bottom' }}
+          aria-label="Next"
+          icon={<FaArrowRight />}
+          onClick={() => dispatch(nextStagingAreaImage())}
+          onMouseOver={handleMouseOver}
+          onMouseOut={handleMouseOut}
+          data-selected={true}
+          isDisabled={isOnLastImage}
+        />
+        <IAIIconButton
+          tooltip="Accept"
+          tooltipProps={{ placement: 'bottom' }}
+          aria-label="Accept"
+          icon={<FaCheck />}
+          onClick={() => dispatch(commitStagingAreaImage())}
+          data-selected={true}
+        />
+        <IAIIconButton
+          tooltip="Show/Hide"
+          tooltipProps={{ placement: 'bottom' }}
+          aria-label="Show/Hide"
+          data-alert={!shouldShowStagedImage}
+          icon={shouldShowStagedImage ? <FaEye /> : <FaEyeSlash />}
+          onClick={() => setShouldShowStagedImage(!shouldShowStagedImage)}
+          data-selected={true}
+        />
+        <IAIIconButton
+          tooltip="Discard All"
+          tooltipProps={{ placement: 'bottom' }}
+          aria-label="Discard All"
+          icon={<FaTrash />}
+          onClick={() => dispatch(discardStagedImages())}
+          data-selected={true}
+        />
+      </ButtonGroup>
+    </Flex>
+  );
+};
+
+export default IAICanvasStagingAreaToolbar;

frontend/src/features/canvas/canvasSlice.ts

@@ -21,6 +21,7 @@ import { uploadImage } from 'features/gallery/util/uploadImage';
 import { setInitialCanvasImage } from './canvasReducers';
 import calculateScale from './util/calculateScale';
 import calculateCoordinates from './util/calculateCoordinates';
+import floorCoordinates from './util/floorCoordinates';
 
 export interface GenericCanvasState {
   boundingBoxCoordinates: Vector2d;
@@ -58,6 +59,7 @@ export interface GenericCanvasState {
   stageDimensions: Dimensions;
   stageScale: number;
   tool: CanvasTool;
+  minimumStageScale: number;
 }
 
 export type CanvasMode = 'inpainting' | 'outpainting';
@@ -147,8 +149,10 @@ export interface CanvasState {
   currentCanvas: ValidCanvasName;
   inpainting: InpaintingCanvasState;
   outpainting: OutpaintingCanvasState;
-  mode: CanvasMode;
+  // mode: CanvasMode;
+  shouldLockToInitialImage: boolean;
   isCanvasInitialized: boolean;
+  canvasContainerDimensions: Dimensions;
 }
 
 export const initialLayerState: CanvasLayerState = {
@@ -197,14 +201,17 @@ const initialGenericCanvasState: GenericCanvasState = {
   stageCoordinates: { x: 0, y: 0 },
   stageDimensions: { width: 0, height: 0 },
   stageScale: 1,
+  minimumStageScale: 1,
   tool: 'brush',
 };
 
 const initialCanvasState: CanvasState = {
   currentCanvas: 'inpainting',
   doesCanvasNeedScaling: false,
-  mode: 'outpainting',
+  shouldLockToInitialImage: false,
+  // mode: 'outpainting',
   isCanvasInitialized: false,
+  canvasContainerDimensions: { width: 0, height: 0 },
   inpainting: {
     layer: 'mask',
     ...initialGenericCanvasState,
@@ -335,80 +342,15 @@ export const canvasSlice = createSlice({
       };
     },
     setBoundingBoxDimensions: (state, action: PayloadAction<Dimensions>) => {
-      const currentCanvas = state[state.currentCanvas];
+      state[state.currentCanvas].boundingBoxDimensions = action.payload;
-      currentCanvas.boundingBoxDimensions = action.payload;
-
-      const { width: boundingBoxWidth, height: boundingBoxHeight } =
-        action.payload;
-      const { x: boundingBoxX, y: boundingBoxY } =
-        currentCanvas.boundingBoxCoordinates;
-      const { width: canvasWidth, height: canvasHeight } =
-        currentCanvas.stageDimensions;
-
-      const scaledCanvasWidth = canvasWidth / currentCanvas.stageScale;
-      const scaledCanvasHeight = canvasHeight / currentCanvas.stageScale;
-
-      const roundedCanvasWidth = roundDownToMultiple(scaledCanvasWidth, 64);
-      const roundedCanvasHeight = roundDownToMultiple(scaledCanvasHeight, 64);
-
-      const roundedBoundingBoxWidth = roundDownToMultiple(boundingBoxWidth, 64);
-      const roundedBoundingBoxHeight = roundDownToMultiple(
-        boundingBoxHeight,
-        64
-      );
-
-      const overflowX = boundingBoxX + boundingBoxWidth - scaledCanvasWidth;
-      const overflowY = boundingBoxY + boundingBoxHeight - scaledCanvasHeight;
-
-      const newBoundingBoxWidth = _.clamp(
-        roundedBoundingBoxWidth,
-        64,
-        roundedCanvasWidth
-      );
-
-      const newBoundingBoxHeight = _.clamp(
-        roundedBoundingBoxHeight,
-        64,
-        roundedCanvasHeight
-      );
-
-      const overflowCorrectedX =
-        overflowX > 0 ? boundingBoxX - overflowX : boundingBoxX;
-
-      const overflowCorrectedY =
-        overflowY > 0 ? boundingBoxY - overflowY : boundingBoxY;
-
-      const clampedX = _.clamp(
-        overflowCorrectedX,
-        currentCanvas.stageCoordinates.x,
-        roundedCanvasWidth - newBoundingBoxWidth
-      );
-
-      const clampedY = _.clamp(
-        overflowCorrectedY,
-        currentCanvas.stageCoordinates.y,
-        roundedCanvasHeight - newBoundingBoxHeight
-      );
-
-      currentCanvas.boundingBoxDimensions = {
-        width: newBoundingBoxWidth,
-        height: newBoundingBoxHeight,
-      };
-
-      currentCanvas.boundingBoxCoordinates = {
-        x: clampedX,
-        y: clampedY,
-      };
     },
     setBoundingBoxCoordinates: (state, action: PayloadAction<Vector2d>) => {
-      const { x, y } = action.payload;
+      state[state.currentCanvas].boundingBoxCoordinates = floorCoordinates(
-      state[state.currentCanvas].boundingBoxCoordinates = {
+        action.payload
-        x: Math.floor(x),
+      );
-        y: Math.floor(y),
-      };
     },
     setStageCoordinates: (state, action: PayloadAction<Vector2d>) => {
-      state[state.currentCanvas].stageCoordinates = action.payload;
+      state.outpainting.stageCoordinates = floorCoordinates(action.payload);
     },
     setBoundingBoxPreviewFill: (state, action: PayloadAction<RgbaColor>) => {
       state[state.currentCanvas].boundingBoxPreviewFill = action.payload;
@@ -595,30 +537,52 @@ export const canvasSlice = createSlice({
       state[state.currentCanvas].layerState = initialLayerState;
       state[state.currentCanvas].futureLayerStates = [];
     },
-    initializeCanvas: (
+    setCanvasContainerDimensions: (
       state,
-      action: PayloadAction<{
+      action: PayloadAction<Dimensions>
-        clientWidth: number;
-        clientHeight: number;
-        imageWidth: number;
-        imageHeight: number;
-      }>
     ) => {
-      const { clientWidth, clientHeight, imageWidth, imageHeight } =
+      state.canvasContainerDimensions = action.payload;
-        action.payload;
+    },
+    resizeAndScaleCanvas: (state) => {
+      const { width: containerWidth, height: containerHeight } =
+        state.canvasContainerDimensions;
+
+      const initialCanvasImage =
+        state.outpainting.layerState.objects.find(isCanvasBaseImage);
+
+      if (!initialCanvasImage) return;
+
+      const { width: imageWidth, height: imageHeight } = initialCanvasImage;
+
+      // const { clientWidth, clientHeight, imageWidth, imageHeight } =
+      //   action.payload;
+
+      const { shouldLockToInitialImage } = state;
 
       const currentCanvas = state[state.currentCanvas];
 
+      const padding = shouldLockToInitialImage ? 1 : 0.95;
+
       const newScale = calculateScale(
-        clientWidth,
+        containerWidth,
-        clientHeight,
+        containerHeight,
         imageWidth,
-        imageHeight
+        imageHeight,
+        padding
       );
 
+      const newDimensions = {
+        width: shouldLockToInitialImage
+          ? Math.floor(imageWidth * newScale)
+          : Math.floor(containerWidth),
+        height: shouldLockToInitialImage
+          ? Math.floor(imageHeight * newScale)
+          : Math.floor(containerHeight),
+      };
+
       const newCoordinates = calculateCoordinates(
-        clientWidth,
+        newDimensions.width,
-        clientHeight,
+        newDimensions.height,
         0,
         0,
         imageWidth,
@@ -627,50 +591,53 @@ export const canvasSlice = createSlice({
       );
 
       currentCanvas.stageScale = newScale;
+      currentCanvas.minimumStageScale = newScale;
       currentCanvas.stageCoordinates = newCoordinates;
 
-      currentCanvas.stageDimensions = {
+      currentCanvas.stageDimensions = newDimensions;
-        width: Math.floor(clientWidth),
-        height: Math.floor(clientHeight),
-      };
       state.isCanvasInitialized = true;
     },
-    resizeCanvas: (
+    resizeCanvas: (state) => {
-      state,
+      const { width: containerWidth, height: containerHeight } =
-      action: PayloadAction<{
+        state.canvasContainerDimensions;
-        clientWidth: number;
-        clientHeight: number;
-      }>
-    ) => {
-      const { clientWidth, clientHeight } = action.payload;
 
       const currentCanvas = state[state.currentCanvas];
 
       currentCanvas.stageDimensions = {
-        width: Math.floor(clientWidth),
+        width: Math.floor(containerWidth),
-        height: Math.floor(clientHeight),
+        height: Math.floor(containerHeight),
       };
     },
     resetCanvasView: (
       state,
       action: PayloadAction<{
-        clientRect: IRect;
+        contentRect: IRect;
       }>
     ) => {
-      const { clientRect } = action.payload;
+      const { contentRect } = action.payload;
+
       const currentCanvas = state[state.currentCanvas];
+
       const baseCanvasImage =
         currentCanvas.layerState.objects.find(isCanvasBaseImage);
-
+      const { shouldLockToInitialImage } = state;
       if (!baseCanvasImage) return;
 
       const {
         stageDimensions: { width: stageWidth, height: stageHeight },
       } = currentCanvas;
 
-      const { x, y, width, height } = clientRect;
+      const { x, y, width, height } = contentRect;
+
+      const padding = shouldLockToInitialImage ? 1 : 0.95;
+      const newScale = calculateScale(
+        stageWidth,
+        stageHeight,
+        width,
+        height,
+        padding
+      );
 
-      const newScale = calculateScale(stageWidth, stageHeight, width, height);
       const newCoordinates = calculateCoordinates(
         stageWidth,
         stageHeight,
@@ -683,10 +650,7 @@ export const canvasSlice = createSlice({
 
       currentCanvas.stageScale = newScale;
 
-      currentCanvas.stageCoordinates = {
+      currentCanvas.stageCoordinates = newCoordinates;
-        x: stageWidth / 2 - (x + width / 2) * newScale,
-        y: stageHeight / 2 - (y + height / 2) * newScale,
-      };
     },
     nextStagingAreaImage: (state) => {
       const currentIndex =
@@ -728,14 +692,17 @@ export const canvasSlice = createSlice({
 
       currentCanvas.futureLayerStates = [];
     },
-    setCanvasMode: (state, action: PayloadAction<CanvasMode>) => {
+    setShouldLockToInitialImage: (state, action: PayloadAction<boolean>) => {
-      state.mode = action.payload;
+      state.shouldLockToInitialImage = action.payload;
     },
+    // setCanvasMode: (state, action: PayloadAction<CanvasMode>) => {
+    //   state.mode = action.payload;
+    // },
   },
   extraReducers: (builder) => {
     builder.addCase(mergeAndUploadCanvas.fulfilled, (state, action) => {
       if (!action.payload) return;
-      const { image, kind, boundingBox } = action.payload;
+      const { image, kind, originalBoundingBox } = action.payload;
 
       if (kind === 'temp_merged_canvas') {
         state.outpainting.pastLayerStates.push({
@@ -748,7 +715,7 @@ export const canvasSlice = createSlice({
           {
             kind: 'image',
             layer: 'base',
-            ...boundingBox,
+            ...originalBoundingBox,
             image,
           },
         ];
@@ -824,10 +791,11 @@ export const {
   prevStagingAreaImage,
   commitStagingAreaImage,
   discardStagedImages,
-  setCanvasMode,
+  setShouldLockToInitialImage,
-  initializeCanvas,
+  resizeAndScaleCanvas,
   resizeCanvas,
   resetCanvasView,
+  setCanvasContainerDimensions,
 } = canvasSlice.actions;
 
 export default canvasSlice.reducer;
@@ -847,8 +815,8 @@ export const inpaintingCanvasSelector = (
   state: RootState
 ): InpaintingCanvasState => state.canvas.inpainting;
 
-export const canvasModeSelector = (state: RootState): CanvasMode =>
+export const shouldLockToInitialImageSelector = (state: RootState): boolean =>
-  state.canvas.mode;
+  state.canvas.shouldLockToInitialImage;
 
 export const baseCanvasImageSelector = createSelector(
   [currentCanvasSelector],
@@ -857,16 +825,16 @@ export const baseCanvasImageSelector = createSelector(
   }
 );
 
-export const canvasClipSelector = createSelector(
+// export const canvasClipSelector = createSelector(
-  [canvasModeSelector, baseCanvasImageSelector],
+//   [canvasModeSelector, baseCanvasImageSelector],
-  (canvasMode, baseCanvasImage) => {
+//   (canvasMode, baseCanvasImage) => {
-    return canvasMode === 'inpainting'
+//     return canvasMode === 'inpainting'
-      ? {
+//       ? {
-          clipX: 0,
+//           clipX: 0,
-          clipY: 0,
+//           clipY: 0,
-          clipWidth: baseCanvasImage?.width,
+//           clipWidth: baseCanvasImage?.width,
-          clipHeight: baseCanvasImage?.height,
+//           clipHeight: baseCanvasImage?.height,
-        }
+//         }
-      : {};
+//       : {};
-  }
+//   }
-);
+// );

frontend/src/features/canvas/hooks/useCanvasDragMove.ts

@@ -5,20 +5,21 @@ import { KonvaEventObject } from 'konva/lib/Node';
 import _ from 'lodash';
 import { useCallback } from 'react';
 import {
+  baseCanvasImageSelector,
   currentCanvasSelector,
   isStagingSelector,
   setIsMovingStage,
   setStageCoordinates,
+  shouldLockToInitialImageSelector,
 } from '../canvasSlice';
 
 const selector = createSelector(
-  [currentCanvasSelector, isStagingSelector, activeTabNameSelector],
+  [currentCanvasSelector, isStagingSelector],
-  (canvas, isStaging, activeTabName) => {
+  (canvas, isStaging) => {
     const { tool } = canvas;
     return {
       tool,
       isStaging,
-      activeTabName,
     };
   },
   { memoizeOptions: { resultEqualityCheck: _.isEqual } }
@@ -26,7 +27,7 @@ const selector = createSelector(
 
 const useCanvasDrag = () => {
   const dispatch = useAppDispatch();
-  const { tool, activeTabName, isStaging } = useAppSelector(selector);
+  const { tool, isStaging } = useAppSelector(selector);
 
   return {
     handleDragStart: useCallback(() => {
@@ -37,7 +38,10 @@ const useCanvasDrag = () => {
     handleDragMove: useCallback(
       (e: KonvaEventObject<MouseEvent>) => {
         if (!(tool === 'move' || isStaging)) return;
-        dispatch(setStageCoordinates(e.target.getPosition()));
+
+        const newCoordinates = { x: e.target.x(), y: e.target.y() };
+
+        dispatch(setStageCoordinates(newCoordinates));
       },
       [dispatch, isStaging, tool]
     ),

frontend/src/features/canvas/hooks/useCanvasZoom.ts

@@ -1,15 +1,17 @@
 import { createSelector } from '@reduxjs/toolkit';
-import { useAppDispatch, useAppSelector } from 'app/store';
+import { RootState, useAppDispatch, useAppSelector } from 'app/store';
 import { activeTabNameSelector } from 'features/options/optionsSelectors';
 import Konva from 'konva';
 import { KonvaEventObject } from 'konva/lib/Node';
 import _ from 'lodash';
 import { MutableRefObject, useCallback } from 'react';
 import {
+  baseCanvasImageSelector,
   currentCanvasSelector,
   GenericCanvasState,
   setStageCoordinates,
   setStageScale,
+  shouldLockToInitialImageSelector,
 } from '../canvasSlice';
 import {
   CANVAS_SCALE_BY,
@@ -18,13 +20,34 @@ import {
 } from '../util/constants';
 
 const selector = createSelector(
-  [activeTabNameSelector, currentCanvasSelector],
+  [
-  (activeTabName, canvas: GenericCanvasState) => {
+    (state: RootState) => state.canvas,
-    const { isMoveStageKeyHeld, stageScale } = canvas;
+    activeTabNameSelector,
+    currentCanvasSelector,
+    baseCanvasImageSelector,
+    shouldLockToInitialImageSelector,
+  ],
+  (
+    canvas,
+    activeTabName,
+    currentCanvas,
+    baseCanvasImage,
+    shouldLockToInitialImage
+  ) => {
+    const {
+      isMoveStageKeyHeld,
+      stageScale,
+      stageDimensions,
+      minimumStageScale,
+    } = currentCanvas;
     return {
       isMoveStageKeyHeld,
       stageScale,
       activeTabName,
+      baseCanvasImage,
+      shouldLockToInitialImage,
+      stageDimensions,
+      minimumStageScale,
     };
   },
   { memoizeOptions: { resultEqualityCheck: _.isEqual } }
@@ -32,19 +55,29 @@ const selector = createSelector(
 
 const useCanvasWheel = (stageRef: MutableRefObject<Konva.Stage | null>) => {
   const dispatch = useAppDispatch();
-  const { isMoveStageKeyHeld, stageScale, activeTabName } =
+  const {
-    useAppSelector(selector);
+    isMoveStageKeyHeld,
+    stageScale,
+    activeTabName,
+    baseCanvasImage,
+    shouldLockToInitialImage,
+    stageDimensions,
+    minimumStageScale,
+  } = useAppSelector(selector);
 
   return useCallback(
     (e: KonvaEventObject<WheelEvent>) => {
       // stop default scrolling
-      if (activeTabName !== 'outpainting') return;
+      if (
+        activeTabName !== 'outpainting' ||
+        !stageRef.current ||
+        isMoveStageKeyHeld ||
+        !baseCanvasImage
+      )
+        return;
 
       e.evt.preventDefault();
 
-      // const oldScale = stageRef.current.scaleX();
-      if (!stageRef.current || isMoveStageKeyHeld) return;
-
       const cursorPos = stageRef.current.getPointerPosition();
 
       if (!cursorPos) return;
@@ -64,19 +97,44 @@ const useCanvasWheel = (stageRef: MutableRefObject<Konva.Stage | null>) => {
 
       const newScale = _.clamp(
         stageScale * CANVAS_SCALE_BY ** delta,
-        MIN_CANVAS_SCALE,
+        shouldLockToInitialImage ? minimumStageScale : MIN_CANVAS_SCALE,
         MAX_CANVAS_SCALE
       );
 
-      const newPos = {
+      const newCoordinates = {
         x: cursorPos.x - mousePointTo.x * newScale,
         y: cursorPos.y - mousePointTo.y * newScale,
       };
 
+      if (shouldLockToInitialImage) {
+        newCoordinates.x = _.clamp(
+          newCoordinates.x,
+          stageDimensions.width - Math.floor(baseCanvasImage.width * newScale),
+          0
+        );
+        newCoordinates.y = _.clamp(
+          newCoordinates.y,
+          stageDimensions.height -
+            Math.floor(baseCanvasImage.height * newScale),
+          0
+        );
+      }
+
       dispatch(setStageScale(newScale));
-      dispatch(setStageCoordinates(newPos));
+      dispatch(setStageCoordinates(newCoordinates));
     },
-    [activeTabName, dispatch, isMoveStageKeyHeld, stageRef, stageScale]
+    [
+      activeTabName,
+      stageRef,
+      isMoveStageKeyHeld,
+      baseCanvasImage,
+      stageScale,
+      shouldLockToInitialImage,
+      minimumStageScale,
+      dispatch,
+      stageDimensions.width,
+      stageDimensions.height,
+    ]
   );
 };
 

frontend/src/features/canvas/util/floorCoordinates.ts

@@ -0,0 +1,10 @@
+import { Vector2d } from 'konva/lib/types';
+
+const floorCoordinates = (coord: Vector2d): Vector2d => {
+  return {
+    x: Math.floor(coord.x),
+    y: Math.floor(coord.y),
+  };
+};
+
+export default floorCoordinates;

frontend/src/features/canvas/util/layerToDataURL.ts

@@ -3,7 +3,7 @@ import Konva from 'konva';
 const layerToDataURL = (layer: Konva.Layer, stageScale: number) => {
   const tempScale = layer.scale();
 
-  const { x: relativeX, y: relativeY } = layer.getClientRect({
+  const relativeClientRect = layer.getClientRect({
     relativeTo: layer.getParent(),
   });
 
@@ -13,14 +13,27 @@ const layerToDataURL = (layer: Konva.Layer, stageScale: number) => {
     y: 1 / stageScale,
   });
 
-  const clientRect = layer.getClientRect();
+  const { x, y, width, height } = layer.getClientRect();
 
-  const dataURL = layer.toDataURL(clientRect);
+  const dataURL = layer.toDataURL({
+    x: Math.round(x),
+    y: Math.round(y),
+    width: Math.round(width),
+    height: Math.round(height),
+  });
 
   // Unscale the canvas
   layer.scale(tempScale);
 
-  return { dataURL, relativeX, relativeY };
+  return {
+    dataURL,
+    boundingBox: {
+      x: Math.round(relativeClientRect.x),
+      y: Math.round(relativeClientRect.y),
+      width: Math.round(width),
+      height: Math.round(height),
+    },
+  };
 };
 
 export default layerToDataURL;

frontend/src/features/canvas/util/mergeAndUploadCanvas.ts

@@ -25,7 +25,7 @@ export const mergeAndUploadCanvas = createAsyncThunk(
 
     if (!canvasImageLayerRef.current) return;
 
-    const { dataURL, relativeX, relativeY } = layerToDataURL(
+    const { dataURL, boundingBox: originalBoundingBox } = layerToDataURL(
       canvasImageLayerRef.current,
       stageScale
     );
@@ -34,32 +34,45 @@ export const mergeAndUploadCanvas = createAsyncThunk(
 
     const formData = new FormData();
 
-    formData.append('dataURL', dataURL);
+    formData.append(
-    formData.append('filename', 'merged_canvas.png');
+      'data',
-    formData.append('kind', saveToGallery ? 'result' : 'temp');
+      JSON.stringify({
+        dataURL,
+        filename: 'merged_canvas.png',
+        kind: saveToGallery ? 'result' : 'temp',
+        cropVisible: saveToGallery,
+      })
+    );
 
     const response = await fetch(window.location.origin + '/upload', {
       method: 'POST',
       body: formData,
     });
 
-    const { image } = (await response.json()) as InvokeAI.ImageUploadResponse;
+    const { url, mtime, width, height } =
+      (await response.json()) as InvokeAI.ImageUploadResponse;
+
+    // const newBoundingBox = {
+    //   x: bbox[0],
+    //   y: bbox[1],
+    //   width: bbox[2],
+    //   height: bbox[3],
+    // };
 
     const newImage: InvokeAI.Image = {
       uuid: uuidv4(),
+      url,
+      mtime,
       category: saveToGallery ? 'result' : 'user',
-      ...image,
+      width: width,
+      height: height,
     };
 
     return {
       image: newImage,
       kind: saveToGallery ? 'merged_canvas' : 'temp_merged_canvas',
-      boundingBox: {
+      originalBoundingBox,
-        x: relativeX,
+      // newBoundingBox,
-        y: relativeY,
-        width: image.width,
-        height: image.height,
-      },
     };
   }
 );

frontend/src/features/gallery/gallerySlice.ts

@@ -271,7 +271,7 @@ export const gallerySlice = createSlice({
   extraReducers: (builder) => {
     builder.addCase(mergeAndUploadCanvas.fulfilled, (state, action) => {
       if (!action.payload) return;
-      const { image, kind, boundingBox } = action.payload;
+      const { image, kind, originalBoundingBox } = action.payload;
 
       if (kind === 'merged_canvas') {
         const { uuid, url, mtime } = image;

frontend/src/features/gallery/util/uploadImage.ts

@@ -23,19 +23,36 @@ export const uploadImage = createAsyncThunk(
     const formData = new FormData();
 
     formData.append('file', imageFile, imageFile.name);
-    formData.append('kind', 'init');
+    formData.append(
+      'data',
+      JSON.stringify({
+        kind: 'init',
+      })
+    );
+    // formData.append('kind', 'init');
 
     const response = await fetch(window.location.origin + '/upload', {
       method: 'POST',
       body: formData,
     });
 
-    const { image } = (await response.json()) as InvokeAI.ImageUploadResponse;
+    const { url, mtime, width, height } =
+      (await response.json()) as InvokeAI.ImageUploadResponse;
+
+    // const newBoundingBox = {
+    //   x: bbox[0],
+    //   y: bbox[1],
+    //   width: bbox[2],
+    //   height: bbox[3],
+    // };
 
     const newImage: InvokeAI.Image = {
       uuid: uuidv4(),
+      url,
+      mtime,
       category: 'user',
-      ...image,
+      width: width,
+      height: height,
     };
 
     return {