diff --git a/backend/invoke_ai_web_server.py b/backend/invoke_ai_web_server.py index bf14133681..0a3877edf0 100644 --- a/backend/invoke_ai_web_server.py +++ b/backend/invoke_ai_web_server.py @@ -8,6 +8,7 @@ import math import io import base64 import os +import json from werkzeug.utils import secure_filename from flask import Flask, redirect, send_from_directory, request, make_response @@ -107,6 +108,7 @@ class InvokeAIWebServer: @self.app.route("/upload", methods=["POST"]) def upload(): try: + data = json.loads(request.form["data"]) filename = "" # check if the post request has the file part if "file" in request.files: @@ -116,15 +118,15 @@ class InvokeAIWebServer: if file.filename == "": return make_response("No file selected", 400) filename = file.filename - elif "dataURL" in request.form: - file = dataURL_to_bytes(request.form["dataURL"]) - if "filename" not in request.form or request.form["filename"] == "": + elif "dataURL" in data: + file = dataURL_to_bytes(data["dataURL"]) + if "filename" not in data or data["filename"] == "": return make_response("No filename provided", 400) - filename = request.form["filename"] + filename = data["filename"] else: return make_response("No file or dataURL", 400) - kind = request.form["kind"] + kind = data["kind"] if kind == "init": path = self.init_image_path @@ -153,22 +155,32 @@ class InvokeAIWebServer: file_path = os.path.join(path, name) - if "dataURL" in request.form: + if "dataURL" in data: with open(file_path, "wb") as f: f.write(file) else: file.save(file_path) mtime = os.path.getmtime(file_path) - (width, height) = Image.open(file_path).size + + pil_image = Image.open(file_path) + + # visible_image_bbox = pil_image.getbbox() + # pil_image = pil_image.crop(visible_image_bbox) + # pil_image.save(file_path) + # if "cropVisible" in data and data["cropVisible"] == True: + # visible_image_bbox = pil_image.getbbox() + # pil_image = pil_image.crop(visible_image_bbox) + # pil_image.save(file_path) + + (width, height) = pil_image.size response = { - "image": { - "url": self.get_url_from_image_path(file_path), - "mtime": mtime, - "width": width, - "height": height, - }, + "url": self.get_url_from_image_path(file_path), + "mtime": mtime, + # "bbox": visible_image_bbox, + "width": width, + "height": height, } return make_response(response, 200) diff --git a/frontend/src/app/invokeai.d.ts b/frontend/src/app/invokeai.d.ts index ac57f34c04..c8028e46f4 100644 --- a/frontend/src/app/invokeai.d.ts +++ b/frontend/src/app/invokeai.d.ts @@ -179,7 +179,12 @@ export declare type ImageResultResponse = Omit & { }; export declare type ImageUploadResponse = { - image: Omit; + // image: Omit; + url: string; + mtime: number; + width: number; + height: number; + // bbox: [number, number, number, number]; }; export declare type ErrorResponse = { diff --git a/frontend/src/app/store.ts b/frontend/src/app/store.ts index 6b20d53982..4af37c195b 100644 --- a/frontend/src/app/store.ts +++ b/frontend/src/app/store.ts @@ -82,8 +82,6 @@ const rootPersistConfig = getPersistConfig({ debounce: 300, }); -// console.log(rootPersistConfig) - const persistedReducer = persistReducer(rootPersistConfig, rootReducer); // Continue with store setup @@ -95,6 +93,7 @@ export const store = configureStore({ serializableCheck: false, }).concat(socketioMiddleware()), devTools: { + // Uncommenting these very rapidly called actions makes the redux dev tools output much more readable actionsDenylist: [ // 'canvas/setCursorPosition', // 'canvas/setStageCoordinates', diff --git a/frontend/src/features/canvas/IAICanvas.tsx b/frontend/src/features/canvas/IAICanvas.tsx index 67c7677cc1..7c7820377a 100644 --- a/frontend/src/features/canvas/IAICanvas.tsx +++ b/frontend/src/features/canvas/IAICanvas.tsx @@ -1,16 +1,19 @@ // lib -import { MutableRefObject, useRef } from 'react'; +import { MutableRefObject, useCallback, useRef } from 'react'; import Konva from 'konva'; import { Layer, Stage } from 'react-konva'; import { Stage as StageType } from 'konva/lib/Stage'; // app -import { useAppSelector } from 'app/store'; +import { RootState, useAppDispatch, useAppSelector } from 'app/store'; import { baseCanvasImageSelector, currentCanvasSelector, isStagingSelector, outpaintingCanvasSelector, + setStageCoordinates, + setStageScale, + shouldLockToInitialImageSelector, } from 'features/canvas/canvasSlice'; // component @@ -35,15 +38,31 @@ import IAICanvasGrid from './IAICanvasGrid'; import IAICanvasIntermediateImage from './IAICanvasIntermediateImage'; import IAICanvasStatusText from './IAICanvasStatusText'; import IAICanvasStagingArea from './IAICanvasStagingArea'; +import IAICanvasStagingAreaToolbar from './IAICanvasStagingAreaToolbar'; +import { KonvaEventObject } from 'konva/lib/Node'; +import { + CANVAS_SCALE_BY, + MAX_CANVAS_SCALE, + MIN_CANVAS_SCALE, +} from './util/constants'; const canvasSelector = createSelector( [ + shouldLockToInitialImageSelector, currentCanvasSelector, outpaintingCanvasSelector, isStagingSelector, activeTabNameSelector, + baseCanvasImageSelector, ], - (currentCanvas, outpaintingCanvas, isStaging, activeTabName) => { + ( + shouldLockToInitialImage, + currentCanvas, + outpaintingCanvas, + isStaging, + activeTabName, + baseCanvasImage + ) => { const { isMaskEnabled, stageScale, @@ -56,6 +75,7 @@ const canvasSelector = createSelector( tool, isMovingStage, shouldShowIntermediates, + minimumStageScale, } = currentCanvas; const { shouldShowGrid } = outpaintingCanvas; @@ -89,6 +109,10 @@ const canvasSelector = createSelector( isOnOutpaintingTab: activeTabName === 'outpainting', isStaging, shouldShowIntermediates, + shouldLockToInitialImage, + activeTabName, + minimumStageScale, + baseCanvasImage, }; }, { @@ -116,8 +140,12 @@ const IAICanvas = () => { isOnOutpaintingTab, isStaging, shouldShowIntermediates, + shouldLockToInitialImage, + activeTabName, + minimumStageScale, + baseCanvasImage, } = useAppSelector(canvasSelector); - + const dispatch = useAppDispatch(); useCanvasHotkeys(); // set the closure'd refs @@ -142,6 +170,34 @@ const IAICanvas = () => { const { handleDragStart, handleDragMove, handleDragEnd } = useCanvasDragMove(); + const dragBoundFunc = useCallback( + (newCoordinates: Vector2d) => { + if (shouldLockToInitialImage && baseCanvasImage) { + newCoordinates.x = _.clamp( + newCoordinates.x, + stageDimensions.width - + Math.floor(baseCanvasImage.width * stageScale), + 0 + ); + newCoordinates.y = _.clamp( + newCoordinates.y, + stageDimensions.height - + Math.floor(baseCanvasImage.height * stageScale), + 0 + ); + } + + return newCoordinates; + }, + [ + baseCanvasImage, + shouldLockToInitialImage, + stageDimensions.height, + stageDimensions.width, + stageScale, + ] + ); + return (
@@ -157,6 +213,7 @@ const IAICanvas = () => { width={stageDimensions.width} height={stageDimensions.height} scale={{ x: stageScale, y: stageScale }} + dragBoundFunc={dragBoundFunc} onMouseDown={handleMouseDown} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseOut} @@ -205,6 +262,7 @@ const IAICanvas = () => { {isOnOutpaintingTab && } +
); diff --git a/frontend/src/features/canvas/IAICanvasBoundingBox.tsx b/frontend/src/features/canvas/IAICanvasBoundingBox.tsx index 735ab0c0b8..cf1d0dab3a 100644 --- a/frontend/src/features/canvas/IAICanvasBoundingBox.tsx +++ b/frontend/src/features/canvas/IAICanvasBoundingBox.tsx @@ -6,8 +6,11 @@ import { Vector2d } from 'konva/lib/types'; import _ from 'lodash'; import { useCallback, useEffect, useRef } from 'react'; import { Group, Rect, Transformer } from 'react-konva'; -import { useAppDispatch, useAppSelector } from 'app/store'; -import { roundToMultiple } from 'common/util/roundDownToMultiple'; +import { RootState, useAppDispatch, useAppSelector } from 'app/store'; +import { + roundDownToMultiple, + roundToMultiple, +} from 'common/util/roundDownToMultiple'; import { baseCanvasImageSelector, currentCanvasSelector, @@ -17,16 +20,24 @@ import { setIsMouseOverBoundingBox, setIsMovingBoundingBox, setIsTransformingBoundingBox, + shouldLockToInitialImageSelector, } from 'features/canvas/canvasSlice'; import { GroupConfig } from 'konva/lib/Group'; import { activeTabNameSelector } from 'features/options/optionsSelectors'; const boundingBoxPreviewSelector = createSelector( + shouldLockToInitialImageSelector, currentCanvasSelector, outpaintingCanvasSelector, baseCanvasImageSelector, activeTabNameSelector, - (currentCanvas, outpaintingCanvas, baseCanvasImage, activeTabName) => { + ( + shouldLockToInitialImage, + currentCanvas, + outpaintingCanvas, + baseCanvasImage, + activeTabName + ) => { const { boundingBoxCoordinates, boundingBoxDimensions, @@ -40,6 +51,7 @@ const boundingBoxPreviewSelector = createSelector( tool, stageCoordinates, } = currentCanvas; + const { shouldSnapToGrid } = outpaintingCanvas; return { @@ -59,6 +71,7 @@ const boundingBoxPreviewSelector = createSelector( stageCoordinates, boundingBoxStrokeWidth: (isMouseOverBoundingBox ? 8 : 1) / stageScale, hitStrokeWidth: 20 / stageScale, + shouldLockToInitialImage, }; }, { @@ -91,6 +104,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { tool, boundingBoxStrokeWidth, hitStrokeWidth, + shouldLockToInitialImage, } = useAppSelector(boundingBoxPreviewSelector); const transformerRef = useRef(null); @@ -106,7 +120,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { const handleOnDragMove = useCallback( (e: KonvaEventObject) => { - if (activeTabName === 'inpainting' || !shouldSnapToGrid) { + if (!shouldSnapToGrid) { dispatch( setBoundingBoxCoordinates({ x: Math.floor(e.target.x()), @@ -132,20 +146,27 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { }) ); }, - [activeTabName, dispatch, shouldSnapToGrid] + [dispatch, shouldSnapToGrid] ); const dragBoundFunc = useCallback( (position: Vector2d) => { - if (!baseCanvasImage && activeTabName !== 'outpainting') - return boundingBoxCoordinates; + /** + * This limits the bounding box's drag coordinates. + */ + if (!shouldLockToInitialImage) return boundingBoxCoordinates; const { x, y } = position; const maxX = - stageDimensions.width - boundingBoxDimensions.width * stageScale; + stageDimensions.width - + boundingBoxDimensions.width - + (stageDimensions.width % 64); + const maxY = - stageDimensions.height - boundingBoxDimensions.height * stageScale; + stageDimensions.height - + boundingBoxDimensions.height - + (stageDimensions.height % 64); const clampedX = Math.floor(_.clamp(x, 0, maxX)); const clampedY = Math.floor(_.clamp(y, 0, maxY)); @@ -153,14 +174,12 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { return { x: clampedX, y: clampedY }; }, [ - baseCanvasImage, - activeTabName, + shouldLockToInitialImage, boundingBoxCoordinates, stageDimensions.width, stageDimensions.height, boundingBoxDimensions.width, boundingBoxDimensions.height, - stageScale, ] ); @@ -203,7 +222,6 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { rect.scaleY(1); }, [dispatch]); - // OK const anchorDragBoundFunc = useCallback( ( oldPos: Vector2d, // old absolute position of anchor point @@ -215,49 +233,14 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { * Konva does not transform with width or height. It transforms the anchor point * and scale factor. This is then sent to the shape's onTransform listeners. * - * We need to snap the new width to steps of 64 without also snapping the - * coordinates of the bounding box to steps of 64. But because the whole + * We need to snap the new dimensions to steps of 64. But because the whole * stage is scaled, our actual desired step is actually 64 * the stage scale. */ - // Difference of the old coords from the nearest multiple the scaled step - const offsetX = oldPos.x % scaledStep; - const offsetY = oldPos.y % scaledStep; - - // Round new position to the nearest multiple of the scaled step - const closestX = roundToMultiple(newPos.x, scaledStep) + offsetX; - const closestY = roundToMultiple(newPos.y, scaledStep) + offsetY; - - // the difference between the old coord and new - const diffX = Math.abs(newPos.x - closestX); - const diffY = Math.abs(newPos.y - closestY); - - // if the difference is less than the scaled step, we want to snap - const didSnapX = diffX < scaledStep; - const didSnapY = diffY < scaledStep; - - // We may not change anything, stash the old position - let newCoordinate = { ...oldPos }; - - // Set the new coords based on what snapped - if (didSnapX && !didSnapY) { - newCoordinate = { - x: closestX, - y: oldPos.y, - }; - } else if (!didSnapX && didSnapY) { - newCoordinate = { - x: oldPos.x, - y: closestY, - }; - } else if (didSnapX && didSnapY) { - newCoordinate = { - x: closestX, - y: closestY, - }; - } - - return newCoordinate; + return { + x: roundToMultiple(newPos.x, scaledStep), + y: roundToMultiple(newPos.y, scaledStep), + }; }, [scaledStep] ); @@ -272,7 +255,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { // On the Inpainting canvas, the bounding box needs to stay in the stage if ( - activeTabName === 'inpainting' && + shouldLockToInitialImage && (newBoundBox.width + newBoundBox.x > stageDimensions.width || newBoundBox.height + newBoundBox.y > stageDimensions.height || newBoundBox.x < 0 || @@ -283,7 +266,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { return newBoundBox; }, - [activeTabName, stageDimensions.height, stageDimensions.width] + [shouldLockToInitialImage, stageDimensions.height, stageDimensions.width] ); const handleStartedTransforming = () => { @@ -337,7 +320,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => { globalCompositeOperation={'destination-out'} /> { - return { objects: currentCanvas.layerState.objects, canvasClip }; + [currentCanvasSelector], + (currentCanvas) => { + return { objects: currentCanvas.layerState.objects }; }, { memoizeOptions: { @@ -30,10 +26,10 @@ type InpaintingCanvasLinesProps = GroupConfig; */ const IAICanvasLines = (props: InpaintingCanvasLinesProps) => { const { ...rest } = props; - const { objects, canvasClip } = useAppSelector(canvasLinesSelector); + const { objects } = useAppSelector(canvasLinesSelector); return ( - + {objects.filter(isCanvasMaskLine).map((line, i) => ( { + [currentCanvasSelector], + (currentCanvas) => { const { objects } = currentCanvas.layerState; return { objects, - canvasClip, }; }, { @@ -31,12 +30,12 @@ const selector = createSelector( ); const IAICanvasObjectRenderer = () => { - const { objects, canvasClip } = useAppSelector(selector); + const { objects } = useAppSelector(selector); if (!objects) return null; return ( - + {objects.map((obj, i) => { if (isCanvasBaseImage(obj)) { return ( diff --git a/frontend/src/features/canvas/IAICanvasOutpaintingControls.tsx b/frontend/src/features/canvas/IAICanvasOutpaintingControls.tsx index 5a5ff05222..7e6f6edeb5 100644 --- a/frontend/src/features/canvas/IAICanvasOutpaintingControls.tsx +++ b/frontend/src/features/canvas/IAICanvasOutpaintingControls.tsx @@ -2,10 +2,11 @@ import { ButtonGroup } from '@chakra-ui/react'; import { createSelector } from '@reduxjs/toolkit'; import { currentCanvasSelector, + resizeAndScaleCanvas, isStagingSelector, resetCanvas, resetCanvasView, - setCanvasMode, + setShouldLockToInitialImage, setTool, } from './canvasSlice'; import { RootState, useAppDispatch, useAppSelector } from 'app/store'; @@ -30,6 +31,7 @@ import IAICanvasBrushButtonPopover from './IAICanvasBrushButtonPopover'; import IAICanvasMaskButtonPopover from './IAICanvasMaskButtonPopover'; import { mergeAndUploadCanvas } from './util/mergeAndUploadCanvas'; import IAICheckbox from 'common/components/IAICheckbox'; +import { ChangeEvent } from 'react'; export const canvasControlsSelector = createSelector( [ @@ -38,12 +40,12 @@ export const canvasControlsSelector = createSelector( isStagingSelector, ], (canvas, currentCanvas, isStaging) => { + const { shouldLockToInitialImage } = canvas; const { tool } = currentCanvas; - const { mode } = canvas; return { tool, isStaging, - mode, + shouldLockToInitialImage, }; }, { @@ -55,7 +57,16 @@ export const canvasControlsSelector = createSelector( const IAICanvasOutpaintingControls = () => { const dispatch = useAppDispatch(); - const { tool, isStaging, mode } = useAppSelector(canvasControlsSelector); + const { tool, isStaging, shouldLockToInitialImage } = useAppSelector( + canvasControlsSelector + ); + + const handleToggleShouldLockToInitialImage = ( + e: ChangeEvent + ) => { + dispatch(setShouldLockToInitialImage(e.target.checked)); + dispatch(resizeAndScaleCanvas()); + }; return (
@@ -125,10 +136,12 @@ const IAICanvasOutpaintingControls = () => { icon={} onClick={() => { if (!stageRef.current || !canvasImageLayerRef.current) return; - const clientRect = canvasImageLayerRef.current.getClientRect({skipTransform: true}); + const clientRect = canvasImageLayerRef.current.getClientRect({ + skipTransform: true, + }); dispatch( resetCanvasView({ - clientRect, + contentRect: clientRect, }) ); }} @@ -141,13 +154,9 @@ const IAICanvasOutpaintingControls = () => { /> - dispatch( - setCanvasMode(e.target.checked ? 'inpainting' : 'outpainting') - ) - } + label={'Lock Canvas to Initial Image'} + isChecked={shouldLockToInitialImage} + onChange={handleToggleShouldLockToInitialImage} />
); diff --git a/frontend/src/features/canvas/IAICanvasResizer.tsx b/frontend/src/features/canvas/IAICanvasResizer.tsx index ad2f651268..825e96e272 100644 --- a/frontend/src/features/canvas/IAICanvasResizer.tsx +++ b/frontend/src/features/canvas/IAICanvasResizer.tsx @@ -5,8 +5,9 @@ import { activeTabNameSelector } from 'features/options/optionsSelectors'; import { baseCanvasImageSelector, currentCanvasSelector, - initializeCanvas, + resizeAndScaleCanvas, resizeCanvas, + setCanvasContainerDimensions, setDoesCanvasNeedScaling, } from 'features/canvas/canvasSlice'; import { createSelector } from '@reduxjs/toolkit'; @@ -17,10 +18,14 @@ const canvasResizerSelector = createSelector( baseCanvasImageSelector, activeTabNameSelector, (canvas, currentCanvas, baseCanvasImage, activeTabName) => { - const { doesCanvasNeedScaling, mode, isCanvasInitialized } = canvas; + const { + doesCanvasNeedScaling, + shouldLockToInitialImage, + isCanvasInitialized, + } = canvas; return { doesCanvasNeedScaling, - mode, + shouldLockToInitialImage, activeTabName, baseCanvasImage, isCanvasInitialized, @@ -32,7 +37,7 @@ const IAICanvasResizer = () => { const dispatch = useAppDispatch(); const { doesCanvasNeedScaling, - mode, + shouldLockToInitialImage, activeTabName, baseCanvasImage, isCanvasInitialized, @@ -50,22 +55,17 @@ const IAICanvasResizer = () => { const { width: imageWidth, height: imageHeight } = baseCanvasImage.image; - if (!isCanvasInitialized) { - dispatch( - initializeCanvas({ - clientWidth, - clientHeight, - imageWidth, - imageHeight, - }) - ); + dispatch( + setCanvasContainerDimensions({ + width: clientWidth, + height: clientHeight, + }) + ); + + if (!isCanvasInitialized || shouldLockToInitialImage) { + dispatch(resizeAndScaleCanvas()); } else { - dispatch( - resizeCanvas({ - clientWidth, - clientHeight, - }) - ); + dispatch(resizeCanvas()); } dispatch(setDoesCanvasNeedScaling(false)); diff --git a/frontend/src/features/canvas/IAICanvasStagingArea.tsx b/frontend/src/features/canvas/IAICanvasStagingArea.tsx index 55a0505204..312a6c3f67 100644 --- a/frontend/src/features/canvas/IAICanvasStagingArea.tsx +++ b/frontend/src/features/canvas/IAICanvasStagingArea.tsx @@ -106,7 +106,7 @@ const IAICanvasStagingArea = (props: Props) => { /> )} - + {/*
{
- + */} ); }; diff --git a/frontend/src/features/canvas/IAICanvasStagingAreaToolbar.tsx b/frontend/src/features/canvas/IAICanvasStagingAreaToolbar.tsx new file mode 100644 index 0000000000..4342892ff4 --- /dev/null +++ b/frontend/src/features/canvas/IAICanvasStagingAreaToolbar.tsx @@ -0,0 +1,140 @@ +import { + background, + ButtonGroup, + ChakraProvider, + Flex, +} from '@chakra-ui/react'; +import { CacheProvider } from '@emotion/react'; +import { createSelector } from '@reduxjs/toolkit'; +import { useAppDispatch, useAppSelector } from 'app/store'; +import IAIButton from 'common/components/IAIButton'; +import IAIIconButton from 'common/components/IAIIconButton'; +import { GroupConfig } from 'konva/lib/Group'; +import _ from 'lodash'; +import { emotionCache } from 'main'; +import { useCallback, useState } from 'react'; +import { + FaArrowLeft, + FaArrowRight, + FaCheck, + FaEye, + FaEyeSlash, + FaTrash, +} from 'react-icons/fa'; +import { Group, Rect } from 'react-konva'; +import { Html } from 'react-konva-utils'; +import { + commitStagingAreaImage, + currentCanvasSelector, + discardStagedImages, + nextStagingAreaImage, + prevStagingAreaImage, +} from './canvasSlice'; +import IAICanvasImage from './IAICanvasImage'; + +const selector = createSelector( + [currentCanvasSelector], + (currentCanvas) => { + const { + layerState: { + stagingArea: { images, selectedImageIndex }, + }, + } = currentCanvas; + + return { + currentStagingAreaImage: + images.length > 0 ? images[selectedImageIndex] : undefined, + isOnFirstImage: selectedImageIndex === 0, + isOnLastImage: selectedImageIndex === images.length - 1, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: _.isEqual, + }, + } +); + +const IAICanvasStagingAreaToolbar = () => { + const dispatch = useAppDispatch(); + const { isOnFirstImage, isOnLastImage, currentStagingAreaImage } = + useAppSelector(selector); + + const [shouldShowStagedImage, setShouldShowStagedImage] = + useState(true); + + const [shouldShowStagingAreaOutline, setShouldShowStagingAreaOutline] = + useState(true); + + const handleMouseOver = useCallback(() => { + setShouldShowStagingAreaOutline(false); + }, []); + + const handleMouseOut = useCallback(() => { + setShouldShowStagingAreaOutline(true); + }, []); + + if (!currentStagingAreaImage) return null; + + return ( + + + } + onClick={() => dispatch(prevStagingAreaImage())} + onMouseOver={handleMouseOver} + onMouseOut={handleMouseOut} + data-selected={true} + isDisabled={isOnFirstImage} + /> + } + onClick={() => dispatch(nextStagingAreaImage())} + onMouseOver={handleMouseOver} + onMouseOut={handleMouseOut} + data-selected={true} + isDisabled={isOnLastImage} + /> + } + onClick={() => dispatch(commitStagingAreaImage())} + data-selected={true} + /> + : } + onClick={() => setShouldShowStagedImage(!shouldShowStagedImage)} + data-selected={true} + /> + } + onClick={() => dispatch(discardStagedImages())} + data-selected={true} + /> + + + ); +}; + +export default IAICanvasStagingAreaToolbar; diff --git a/frontend/src/features/canvas/canvasSlice.ts b/frontend/src/features/canvas/canvasSlice.ts index e5c10810cd..f2e289767d 100644 --- a/frontend/src/features/canvas/canvasSlice.ts +++ b/frontend/src/features/canvas/canvasSlice.ts @@ -21,6 +21,7 @@ import { uploadImage } from 'features/gallery/util/uploadImage'; import { setInitialCanvasImage } from './canvasReducers'; import calculateScale from './util/calculateScale'; import calculateCoordinates from './util/calculateCoordinates'; +import floorCoordinates from './util/floorCoordinates'; export interface GenericCanvasState { boundingBoxCoordinates: Vector2d; @@ -58,6 +59,7 @@ export interface GenericCanvasState { stageDimensions: Dimensions; stageScale: number; tool: CanvasTool; + minimumStageScale: number; } export type CanvasMode = 'inpainting' | 'outpainting'; @@ -147,8 +149,10 @@ export interface CanvasState { currentCanvas: ValidCanvasName; inpainting: InpaintingCanvasState; outpainting: OutpaintingCanvasState; - mode: CanvasMode; + // mode: CanvasMode; + shouldLockToInitialImage: boolean; isCanvasInitialized: boolean; + canvasContainerDimensions: Dimensions; } export const initialLayerState: CanvasLayerState = { @@ -197,14 +201,17 @@ const initialGenericCanvasState: GenericCanvasState = { stageCoordinates: { x: 0, y: 0 }, stageDimensions: { width: 0, height: 0 }, stageScale: 1, + minimumStageScale: 1, tool: 'brush', }; const initialCanvasState: CanvasState = { currentCanvas: 'inpainting', doesCanvasNeedScaling: false, - mode: 'outpainting', + shouldLockToInitialImage: false, + // mode: 'outpainting', isCanvasInitialized: false, + canvasContainerDimensions: { width: 0, height: 0 }, inpainting: { layer: 'mask', ...initialGenericCanvasState, @@ -335,80 +342,15 @@ export const canvasSlice = createSlice({ }; }, setBoundingBoxDimensions: (state, action: PayloadAction) => { - const currentCanvas = state[state.currentCanvas]; - currentCanvas.boundingBoxDimensions = action.payload; - - const { width: boundingBoxWidth, height: boundingBoxHeight } = - action.payload; - const { x: boundingBoxX, y: boundingBoxY } = - currentCanvas.boundingBoxCoordinates; - const { width: canvasWidth, height: canvasHeight } = - currentCanvas.stageDimensions; - - const scaledCanvasWidth = canvasWidth / currentCanvas.stageScale; - const scaledCanvasHeight = canvasHeight / currentCanvas.stageScale; - - const roundedCanvasWidth = roundDownToMultiple(scaledCanvasWidth, 64); - const roundedCanvasHeight = roundDownToMultiple(scaledCanvasHeight, 64); - - const roundedBoundingBoxWidth = roundDownToMultiple(boundingBoxWidth, 64); - const roundedBoundingBoxHeight = roundDownToMultiple( - boundingBoxHeight, - 64 - ); - - const overflowX = boundingBoxX + boundingBoxWidth - scaledCanvasWidth; - const overflowY = boundingBoxY + boundingBoxHeight - scaledCanvasHeight; - - const newBoundingBoxWidth = _.clamp( - roundedBoundingBoxWidth, - 64, - roundedCanvasWidth - ); - - const newBoundingBoxHeight = _.clamp( - roundedBoundingBoxHeight, - 64, - roundedCanvasHeight - ); - - const overflowCorrectedX = - overflowX > 0 ? boundingBoxX - overflowX : boundingBoxX; - - const overflowCorrectedY = - overflowY > 0 ? boundingBoxY - overflowY : boundingBoxY; - - const clampedX = _.clamp( - overflowCorrectedX, - currentCanvas.stageCoordinates.x, - roundedCanvasWidth - newBoundingBoxWidth - ); - - const clampedY = _.clamp( - overflowCorrectedY, - currentCanvas.stageCoordinates.y, - roundedCanvasHeight - newBoundingBoxHeight - ); - - currentCanvas.boundingBoxDimensions = { - width: newBoundingBoxWidth, - height: newBoundingBoxHeight, - }; - - currentCanvas.boundingBoxCoordinates = { - x: clampedX, - y: clampedY, - }; + state[state.currentCanvas].boundingBoxDimensions = action.payload; }, setBoundingBoxCoordinates: (state, action: PayloadAction) => { - const { x, y } = action.payload; - state[state.currentCanvas].boundingBoxCoordinates = { - x: Math.floor(x), - y: Math.floor(y), - }; + state[state.currentCanvas].boundingBoxCoordinates = floorCoordinates( + action.payload + ); }, setStageCoordinates: (state, action: PayloadAction) => { - state[state.currentCanvas].stageCoordinates = action.payload; + state.outpainting.stageCoordinates = floorCoordinates(action.payload); }, setBoundingBoxPreviewFill: (state, action: PayloadAction) => { state[state.currentCanvas].boundingBoxPreviewFill = action.payload; @@ -595,30 +537,52 @@ export const canvasSlice = createSlice({ state[state.currentCanvas].layerState = initialLayerState; state[state.currentCanvas].futureLayerStates = []; }, - initializeCanvas: ( + setCanvasContainerDimensions: ( state, - action: PayloadAction<{ - clientWidth: number; - clientHeight: number; - imageWidth: number; - imageHeight: number; - }> + action: PayloadAction ) => { - const { clientWidth, clientHeight, imageWidth, imageHeight } = - action.payload; + state.canvasContainerDimensions = action.payload; + }, + resizeAndScaleCanvas: (state) => { + const { width: containerWidth, height: containerHeight } = + state.canvasContainerDimensions; + + const initialCanvasImage = + state.outpainting.layerState.objects.find(isCanvasBaseImage); + + if (!initialCanvasImage) return; + + const { width: imageWidth, height: imageHeight } = initialCanvasImage; + + // const { clientWidth, clientHeight, imageWidth, imageHeight } = + // action.payload; + + const { shouldLockToInitialImage } = state; const currentCanvas = state[state.currentCanvas]; + const padding = shouldLockToInitialImage ? 1 : 0.95; + const newScale = calculateScale( - clientWidth, - clientHeight, + containerWidth, + containerHeight, imageWidth, - imageHeight + imageHeight, + padding ); + const newDimensions = { + width: shouldLockToInitialImage + ? Math.floor(imageWidth * newScale) + : Math.floor(containerWidth), + height: shouldLockToInitialImage + ? Math.floor(imageHeight * newScale) + : Math.floor(containerHeight), + }; + const newCoordinates = calculateCoordinates( - clientWidth, - clientHeight, + newDimensions.width, + newDimensions.height, 0, 0, imageWidth, @@ -627,50 +591,53 @@ export const canvasSlice = createSlice({ ); currentCanvas.stageScale = newScale; + currentCanvas.minimumStageScale = newScale; currentCanvas.stageCoordinates = newCoordinates; - currentCanvas.stageDimensions = { - width: Math.floor(clientWidth), - height: Math.floor(clientHeight), - }; + currentCanvas.stageDimensions = newDimensions; state.isCanvasInitialized = true; }, - resizeCanvas: ( - state, - action: PayloadAction<{ - clientWidth: number; - clientHeight: number; - }> - ) => { - const { clientWidth, clientHeight } = action.payload; + resizeCanvas: (state) => { + const { width: containerWidth, height: containerHeight } = + state.canvasContainerDimensions; const currentCanvas = state[state.currentCanvas]; currentCanvas.stageDimensions = { - width: Math.floor(clientWidth), - height: Math.floor(clientHeight), + width: Math.floor(containerWidth), + height: Math.floor(containerHeight), }; }, resetCanvasView: ( state, action: PayloadAction<{ - clientRect: IRect; + contentRect: IRect; }> ) => { - const { clientRect } = action.payload; + const { contentRect } = action.payload; + const currentCanvas = state[state.currentCanvas]; + const baseCanvasImage = currentCanvas.layerState.objects.find(isCanvasBaseImage); - + const { shouldLockToInitialImage } = state; if (!baseCanvasImage) return; const { stageDimensions: { width: stageWidth, height: stageHeight }, } = currentCanvas; - const { x, y, width, height } = clientRect; + const { x, y, width, height } = contentRect; + + const padding = shouldLockToInitialImage ? 1 : 0.95; + const newScale = calculateScale( + stageWidth, + stageHeight, + width, + height, + padding + ); - const newScale = calculateScale(stageWidth, stageHeight, width, height); const newCoordinates = calculateCoordinates( stageWidth, stageHeight, @@ -683,10 +650,7 @@ export const canvasSlice = createSlice({ currentCanvas.stageScale = newScale; - currentCanvas.stageCoordinates = { - x: stageWidth / 2 - (x + width / 2) * newScale, - y: stageHeight / 2 - (y + height / 2) * newScale, - }; + currentCanvas.stageCoordinates = newCoordinates; }, nextStagingAreaImage: (state) => { const currentIndex = @@ -728,14 +692,17 @@ export const canvasSlice = createSlice({ currentCanvas.futureLayerStates = []; }, - setCanvasMode: (state, action: PayloadAction) => { - state.mode = action.payload; + setShouldLockToInitialImage: (state, action: PayloadAction) => { + state.shouldLockToInitialImage = action.payload; }, + // setCanvasMode: (state, action: PayloadAction) => { + // state.mode = action.payload; + // }, }, extraReducers: (builder) => { builder.addCase(mergeAndUploadCanvas.fulfilled, (state, action) => { if (!action.payload) return; - const { image, kind, boundingBox } = action.payload; + const { image, kind, originalBoundingBox } = action.payload; if (kind === 'temp_merged_canvas') { state.outpainting.pastLayerStates.push({ @@ -748,7 +715,7 @@ export const canvasSlice = createSlice({ { kind: 'image', layer: 'base', - ...boundingBox, + ...originalBoundingBox, image, }, ]; @@ -824,10 +791,11 @@ export const { prevStagingAreaImage, commitStagingAreaImage, discardStagedImages, - setCanvasMode, - initializeCanvas, + setShouldLockToInitialImage, + resizeAndScaleCanvas, resizeCanvas, resetCanvasView, + setCanvasContainerDimensions, } = canvasSlice.actions; export default canvasSlice.reducer; @@ -847,8 +815,8 @@ export const inpaintingCanvasSelector = ( state: RootState ): InpaintingCanvasState => state.canvas.inpainting; -export const canvasModeSelector = (state: RootState): CanvasMode => - state.canvas.mode; +export const shouldLockToInitialImageSelector = (state: RootState): boolean => + state.canvas.shouldLockToInitialImage; export const baseCanvasImageSelector = createSelector( [currentCanvasSelector], @@ -857,16 +825,16 @@ export const baseCanvasImageSelector = createSelector( } ); -export const canvasClipSelector = createSelector( - [canvasModeSelector, baseCanvasImageSelector], - (canvasMode, baseCanvasImage) => { - return canvasMode === 'inpainting' - ? { - clipX: 0, - clipY: 0, - clipWidth: baseCanvasImage?.width, - clipHeight: baseCanvasImage?.height, - } - : {}; - } -); +// export const canvasClipSelector = createSelector( +// [canvasModeSelector, baseCanvasImageSelector], +// (canvasMode, baseCanvasImage) => { +// return canvasMode === 'inpainting' +// ? { +// clipX: 0, +// clipY: 0, +// clipWidth: baseCanvasImage?.width, +// clipHeight: baseCanvasImage?.height, +// } +// : {}; +// } +// ); diff --git a/frontend/src/features/canvas/hooks/useCanvasDragMove.ts b/frontend/src/features/canvas/hooks/useCanvasDragMove.ts index 268bdf6e75..1331149dec 100644 --- a/frontend/src/features/canvas/hooks/useCanvasDragMove.ts +++ b/frontend/src/features/canvas/hooks/useCanvasDragMove.ts @@ -5,20 +5,21 @@ import { KonvaEventObject } from 'konva/lib/Node'; import _ from 'lodash'; import { useCallback } from 'react'; import { + baseCanvasImageSelector, currentCanvasSelector, isStagingSelector, setIsMovingStage, setStageCoordinates, + shouldLockToInitialImageSelector, } from '../canvasSlice'; const selector = createSelector( - [currentCanvasSelector, isStagingSelector, activeTabNameSelector], - (canvas, isStaging, activeTabName) => { + [currentCanvasSelector, isStagingSelector], + (canvas, isStaging) => { const { tool } = canvas; return { tool, isStaging, - activeTabName, }; }, { memoizeOptions: { resultEqualityCheck: _.isEqual } } @@ -26,7 +27,7 @@ const selector = createSelector( const useCanvasDrag = () => { const dispatch = useAppDispatch(); - const { tool, activeTabName, isStaging } = useAppSelector(selector); + const { tool, isStaging } = useAppSelector(selector); return { handleDragStart: useCallback(() => { @@ -37,7 +38,10 @@ const useCanvasDrag = () => { handleDragMove: useCallback( (e: KonvaEventObject) => { if (!(tool === 'move' || isStaging)) return; - dispatch(setStageCoordinates(e.target.getPosition())); + + const newCoordinates = { x: e.target.x(), y: e.target.y() }; + + dispatch(setStageCoordinates(newCoordinates)); }, [dispatch, isStaging, tool] ), diff --git a/frontend/src/features/canvas/hooks/useCanvasZoom.ts b/frontend/src/features/canvas/hooks/useCanvasZoom.ts index 857b023328..63b0812466 100644 --- a/frontend/src/features/canvas/hooks/useCanvasZoom.ts +++ b/frontend/src/features/canvas/hooks/useCanvasZoom.ts @@ -1,15 +1,17 @@ import { createSelector } from '@reduxjs/toolkit'; -import { useAppDispatch, useAppSelector } from 'app/store'; +import { RootState, useAppDispatch, useAppSelector } from 'app/store'; import { activeTabNameSelector } from 'features/options/optionsSelectors'; import Konva from 'konva'; import { KonvaEventObject } from 'konva/lib/Node'; import _ from 'lodash'; import { MutableRefObject, useCallback } from 'react'; import { + baseCanvasImageSelector, currentCanvasSelector, GenericCanvasState, setStageCoordinates, setStageScale, + shouldLockToInitialImageSelector, } from '../canvasSlice'; import { CANVAS_SCALE_BY, @@ -18,13 +20,34 @@ import { } from '../util/constants'; const selector = createSelector( - [activeTabNameSelector, currentCanvasSelector], - (activeTabName, canvas: GenericCanvasState) => { - const { isMoveStageKeyHeld, stageScale } = canvas; + [ + (state: RootState) => state.canvas, + activeTabNameSelector, + currentCanvasSelector, + baseCanvasImageSelector, + shouldLockToInitialImageSelector, + ], + ( + canvas, + activeTabName, + currentCanvas, + baseCanvasImage, + shouldLockToInitialImage + ) => { + const { + isMoveStageKeyHeld, + stageScale, + stageDimensions, + minimumStageScale, + } = currentCanvas; return { isMoveStageKeyHeld, stageScale, activeTabName, + baseCanvasImage, + shouldLockToInitialImage, + stageDimensions, + minimumStageScale, }; }, { memoizeOptions: { resultEqualityCheck: _.isEqual } } @@ -32,19 +55,29 @@ const selector = createSelector( const useCanvasWheel = (stageRef: MutableRefObject) => { const dispatch = useAppDispatch(); - const { isMoveStageKeyHeld, stageScale, activeTabName } = - useAppSelector(selector); + const { + isMoveStageKeyHeld, + stageScale, + activeTabName, + baseCanvasImage, + shouldLockToInitialImage, + stageDimensions, + minimumStageScale, + } = useAppSelector(selector); return useCallback( (e: KonvaEventObject) => { // stop default scrolling - if (activeTabName !== 'outpainting') return; + if ( + activeTabName !== 'outpainting' || + !stageRef.current || + isMoveStageKeyHeld || + !baseCanvasImage + ) + return; e.evt.preventDefault(); - // const oldScale = stageRef.current.scaleX(); - if (!stageRef.current || isMoveStageKeyHeld) return; - const cursorPos = stageRef.current.getPointerPosition(); if (!cursorPos) return; @@ -64,19 +97,44 @@ const useCanvasWheel = (stageRef: MutableRefObject) => { const newScale = _.clamp( stageScale * CANVAS_SCALE_BY ** delta, - MIN_CANVAS_SCALE, + shouldLockToInitialImage ? minimumStageScale : MIN_CANVAS_SCALE, MAX_CANVAS_SCALE ); - const newPos = { + const newCoordinates = { x: cursorPos.x - mousePointTo.x * newScale, y: cursorPos.y - mousePointTo.y * newScale, }; + if (shouldLockToInitialImage) { + newCoordinates.x = _.clamp( + newCoordinates.x, + stageDimensions.width - Math.floor(baseCanvasImage.width * newScale), + 0 + ); + newCoordinates.y = _.clamp( + newCoordinates.y, + stageDimensions.height - + Math.floor(baseCanvasImage.height * newScale), + 0 + ); + } + dispatch(setStageScale(newScale)); - dispatch(setStageCoordinates(newPos)); + dispatch(setStageCoordinates(newCoordinates)); }, - [activeTabName, dispatch, isMoveStageKeyHeld, stageRef, stageScale] + [ + activeTabName, + stageRef, + isMoveStageKeyHeld, + baseCanvasImage, + stageScale, + shouldLockToInitialImage, + minimumStageScale, + dispatch, + stageDimensions.width, + stageDimensions.height, + ] ); }; diff --git a/frontend/src/features/canvas/util/floorCoordinates.ts b/frontend/src/features/canvas/util/floorCoordinates.ts new file mode 100644 index 0000000000..aa3c96ddb1 --- /dev/null +++ b/frontend/src/features/canvas/util/floorCoordinates.ts @@ -0,0 +1,10 @@ +import { Vector2d } from 'konva/lib/types'; + +const floorCoordinates = (coord: Vector2d): Vector2d => { + return { + x: Math.floor(coord.x), + y: Math.floor(coord.y), + }; +}; + +export default floorCoordinates; diff --git a/frontend/src/features/canvas/util/layerToDataURL.ts b/frontend/src/features/canvas/util/layerToDataURL.ts index 4317003025..abdbc0a21d 100644 --- a/frontend/src/features/canvas/util/layerToDataURL.ts +++ b/frontend/src/features/canvas/util/layerToDataURL.ts @@ -3,7 +3,7 @@ import Konva from 'konva'; const layerToDataURL = (layer: Konva.Layer, stageScale: number) => { const tempScale = layer.scale(); - const { x: relativeX, y: relativeY } = layer.getClientRect({ + const relativeClientRect = layer.getClientRect({ relativeTo: layer.getParent(), }); @@ -13,14 +13,27 @@ const layerToDataURL = (layer: Konva.Layer, stageScale: number) => { y: 1 / stageScale, }); - const clientRect = layer.getClientRect(); + const { x, y, width, height } = layer.getClientRect(); - const dataURL = layer.toDataURL(clientRect); + const dataURL = layer.toDataURL({ + x: Math.round(x), + y: Math.round(y), + width: Math.round(width), + height: Math.round(height), + }); // Unscale the canvas layer.scale(tempScale); - return { dataURL, relativeX, relativeY }; + return { + dataURL, + boundingBox: { + x: Math.round(relativeClientRect.x), + y: Math.round(relativeClientRect.y), + width: Math.round(width), + height: Math.round(height), + }, + }; }; export default layerToDataURL; diff --git a/frontend/src/features/canvas/util/mergeAndUploadCanvas.ts b/frontend/src/features/canvas/util/mergeAndUploadCanvas.ts index 55f69394da..9d2cacfe23 100644 --- a/frontend/src/features/canvas/util/mergeAndUploadCanvas.ts +++ b/frontend/src/features/canvas/util/mergeAndUploadCanvas.ts @@ -25,7 +25,7 @@ export const mergeAndUploadCanvas = createAsyncThunk( if (!canvasImageLayerRef.current) return; - const { dataURL, relativeX, relativeY } = layerToDataURL( + const { dataURL, boundingBox: originalBoundingBox } = layerToDataURL( canvasImageLayerRef.current, stageScale ); @@ -34,32 +34,45 @@ export const mergeAndUploadCanvas = createAsyncThunk( const formData = new FormData(); - formData.append('dataURL', dataURL); - formData.append('filename', 'merged_canvas.png'); - formData.append('kind', saveToGallery ? 'result' : 'temp'); + formData.append( + 'data', + JSON.stringify({ + dataURL, + filename: 'merged_canvas.png', + kind: saveToGallery ? 'result' : 'temp', + cropVisible: saveToGallery, + }) + ); const response = await fetch(window.location.origin + '/upload', { method: 'POST', body: formData, }); - const { image } = (await response.json()) as InvokeAI.ImageUploadResponse; + const { url, mtime, width, height } = + (await response.json()) as InvokeAI.ImageUploadResponse; + + // const newBoundingBox = { + // x: bbox[0], + // y: bbox[1], + // width: bbox[2], + // height: bbox[3], + // }; const newImage: InvokeAI.Image = { uuid: uuidv4(), + url, + mtime, category: saveToGallery ? 'result' : 'user', - ...image, + width: width, + height: height, }; return { image: newImage, kind: saveToGallery ? 'merged_canvas' : 'temp_merged_canvas', - boundingBox: { - x: relativeX, - y: relativeY, - width: image.width, - height: image.height, - }, + originalBoundingBox, + // newBoundingBox, }; } ); diff --git a/frontend/src/features/gallery/gallerySlice.ts b/frontend/src/features/gallery/gallerySlice.ts index 821b5d76ab..0b6c487c17 100644 --- a/frontend/src/features/gallery/gallerySlice.ts +++ b/frontend/src/features/gallery/gallerySlice.ts @@ -271,7 +271,7 @@ export const gallerySlice = createSlice({ extraReducers: (builder) => { builder.addCase(mergeAndUploadCanvas.fulfilled, (state, action) => { if (!action.payload) return; - const { image, kind, boundingBox } = action.payload; + const { image, kind, originalBoundingBox } = action.payload; if (kind === 'merged_canvas') { const { uuid, url, mtime } = image; diff --git a/frontend/src/features/gallery/util/uploadImage.ts b/frontend/src/features/gallery/util/uploadImage.ts index 2729138cc7..5dca9a490d 100644 --- a/frontend/src/features/gallery/util/uploadImage.ts +++ b/frontend/src/features/gallery/util/uploadImage.ts @@ -23,19 +23,36 @@ export const uploadImage = createAsyncThunk( const formData = new FormData(); formData.append('file', imageFile, imageFile.name); - formData.append('kind', 'init'); + formData.append( + 'data', + JSON.stringify({ + kind: 'init', + }) + ); + // formData.append('kind', 'init'); const response = await fetch(window.location.origin + '/upload', { method: 'POST', body: formData, }); - const { image } = (await response.json()) as InvokeAI.ImageUploadResponse; + const { url, mtime, width, height } = + (await response.json()) as InvokeAI.ImageUploadResponse; + + // const newBoundingBox = { + // x: bbox[0], + // y: bbox[1], + // width: bbox[2], + // height: bbox[3], + // }; const newImage: InvokeAI.Image = { uuid: uuidv4(), + url, + mtime, category: 'user', - ...image, + width: width, + height: height, }; return {