Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial unification of canvas

Browse Source
This commit is contained in:
psychedelicious 2022-11-17 09:56:48 +11:00 committed by blessedcoolant
parent caf8f0ae35
commit 15dd1339d2
19 changed files with 580 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
backend/invoke_ai_web_server.py
View File

@ -8,6 +8,7 @@ import math
import io
import base64
import os
import json
from werkzeug.utils import secure_filename
from flask import Flask, redirect, send_from_directory, request, make_response
@ -107,6 +108,7 @@ class InvokeAIWebServer:
@self.app.route("/upload", methods=["POST"])
def upload():
try:
data = json.loads(request.form["data"])
filename = ""
# check if the post request has the file part
if "file" in request.files:
@ -116,15 +118,15 @@ class InvokeAIWebServer:
if file.filename == "":
return make_response("No file selected", 400)
filename = file.filename
elif "dataURL" in request.form:
file = dataURL_to_bytes(request.form["dataURL"])
if "filename" not in request.form or request.form["filename"] == "":
elif "dataURL" in data:
file = dataURL_to_bytes(data["dataURL"])
if "filename" not in data or data["filename"] == "":
return make_response("No filename provided", 400)
filename = request.form["filename"]
filename = data["filename"]
else:
return make_response("No file or dataURL", 400)
kind = request.form["kind"]
kind = data["kind"]
if kind == "init":
path = self.init_image_path
@ -153,22 +155,32 @@ class InvokeAIWebServer:
file_path = os.path.join(path, name)
if "dataURL" in request.form:
if "dataURL" in data:
with open(file_path, "wb") as f:
f.write(file)
else:
file.save(file_path)
mtime = os.path.getmtime(file_path)
(width, height) = Image.open(file_path).size
pil_image = Image.open(file_path)
# visible_image_bbox = pil_image.getbbox()
# pil_image = pil_image.crop(visible_image_bbox)
# pil_image.save(file_path)
# if "cropVisible" in data and data["cropVisible"] == True:
# visible_image_bbox = pil_image.getbbox()
# pil_image = pil_image.crop(visible_image_bbox)
# pil_image.save(file_path)
(width, height) = pil_image.size
response = {
"image": {
"url": self.get_url_from_image_path(file_path),
"mtime": mtime,
"width": width,
"height": height,
},
"url": self.get_url_from_image_path(file_path),
"mtime": mtime,
# "bbox": visible_image_bbox,
"width": width,
"height": height,
}
return make_response(response, 200)

7
frontend/src/app/invokeai.d.ts vendored
View File

@ -179,7 +179,12 @@ export declare type ImageResultResponse = Omit<Image, 'uuid'> & {
};
export declare type ImageUploadResponse = {
image: Omit<Image, 'uuid' | 'metadata' | 'category'>;
// image: Omit<Image, 'uuid' | 'metadata' | 'category'>;
url: string;
mtime: number;
width: number;
height: number;
// bbox: [number, number, number, number];
};
export declare type ErrorResponse = {

3
frontend/src/app/store.ts
View File

@ -82,8 +82,6 @@ const rootPersistConfig = getPersistConfig({
debounce: 300,
});
// console.log(rootPersistConfig)
const persistedReducer = persistReducer(rootPersistConfig, rootReducer);
// Continue with store setup
@ -95,6 +93,7 @@ export const store = configureStore({
serializableCheck: false,
}).concat(socketioMiddleware()),
devTools: {
// Uncommenting these very rapidly called actions makes the redux dev tools output much more readable
actionsDenylist: [
// 'canvas/setCursorPosition',
// 'canvas/setStageCoordinates',

66
frontend/src/features/canvas/IAICanvas.tsx
View File

@ -1,16 +1,19 @@
// lib
import { MutableRefObject, useRef } from 'react';
import { MutableRefObject, useCallback, useRef } from 'react';
import Konva from 'konva';
import { Layer, Stage } from 'react-konva';
import { Stage as StageType } from 'konva/lib/Stage';
// app
import { useAppSelector } from 'app/store';
import { RootState, useAppDispatch, useAppSelector } from 'app/store';
import {
baseCanvasImageSelector,
currentCanvasSelector,
isStagingSelector,
outpaintingCanvasSelector,
setStageCoordinates,
setStageScale,
shouldLockToInitialImageSelector,
} from 'features/canvas/canvasSlice';
// component
@ -35,15 +38,31 @@ import IAICanvasGrid from './IAICanvasGrid';
import IAICanvasIntermediateImage from './IAICanvasIntermediateImage';
import IAICanvasStatusText from './IAICanvasStatusText';
import IAICanvasStagingArea from './IAICanvasStagingArea';
import IAICanvasStagingAreaToolbar from './IAICanvasStagingAreaToolbar';
import { KonvaEventObject } from 'konva/lib/Node';
import {
CANVAS_SCALE_BY,
MAX_CANVAS_SCALE,
MIN_CANVAS_SCALE,
} from './util/constants';
const canvasSelector = createSelector(
[
shouldLockToInitialImageSelector,
currentCanvasSelector,
outpaintingCanvasSelector,
isStagingSelector,
activeTabNameSelector,
baseCanvasImageSelector,
],
(currentCanvas, outpaintingCanvas, isStaging, activeTabName) => {
(
shouldLockToInitialImage,
currentCanvas,
outpaintingCanvas,
isStaging,
activeTabName,
baseCanvasImage
) => {
const {
isMaskEnabled,
stageScale,
@ -56,6 +75,7 @@ const canvasSelector = createSelector(
tool,
isMovingStage,
shouldShowIntermediates,
minimumStageScale,
} = currentCanvas;
const { shouldShowGrid } = outpaintingCanvas;
@ -89,6 +109,10 @@ const canvasSelector = createSelector(
isOnOutpaintingTab: activeTabName === 'outpainting',
isStaging,
shouldShowIntermediates,
shouldLockToInitialImage,
activeTabName,
minimumStageScale,
baseCanvasImage,
};
},
{
@ -116,8 +140,12 @@ const IAICanvas = () => {
isOnOutpaintingTab,
isStaging,
shouldShowIntermediates,
shouldLockToInitialImage,
activeTabName,
minimumStageScale,
baseCanvasImage,
} = useAppSelector(canvasSelector);
const dispatch = useAppDispatch();
useCanvasHotkeys();
// set the closure'd refs
@ -142,6 +170,34 @@ const IAICanvas = () => {
const { handleDragStart, handleDragMove, handleDragEnd } =
useCanvasDragMove();
const dragBoundFunc = useCallback(
(newCoordinates: Vector2d) => {
if (shouldLockToInitialImage && baseCanvasImage) {
newCoordinates.x = _.clamp(
newCoordinates.x,
stageDimensions.width -
Math.floor(baseCanvasImage.width * stageScale),
0
);
newCoordinates.y = _.clamp(
newCoordinates.y,
stageDimensions.height -
Math.floor(baseCanvasImage.height * stageScale),
0
);
}
return newCoordinates;
},
[
baseCanvasImage,
shouldLockToInitialImage,
stageDimensions.height,
stageDimensions.width,
stageScale,
]
);
return (
<div className="inpainting-canvas-container">
<div className="inpainting-canvas-wrapper">
@ -157,6 +213,7 @@ const IAICanvas = () => {
width={stageDimensions.width}
height={stageDimensions.height}
scale={{ x: stageScale, y: stageScale }}
dragBoundFunc={dragBoundFunc}
onMouseDown={handleMouseDown}
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseOut}
@ -205,6 +262,7 @@ const IAICanvas = () => {
</Layer>
</Stage>
{isOnOutpaintingTab && <IAICanvasStatusText />}
<IAICanvasStagingAreaToolbar />
</div>
</div>
);

95
frontend/src/features/canvas/IAICanvasBoundingBox.tsx
View File

@ -6,8 +6,11 @@ import { Vector2d } from 'konva/lib/types';
import _ from 'lodash';
import { useCallback, useEffect, useRef } from 'react';
import { Group, Rect, Transformer } from 'react-konva';
import { useAppDispatch, useAppSelector } from 'app/store';
import { roundToMultiple } from 'common/util/roundDownToMultiple';
import { RootState, useAppDispatch, useAppSelector } from 'app/store';
import {
roundDownToMultiple,
roundToMultiple,
} from 'common/util/roundDownToMultiple';
import {
baseCanvasImageSelector,
currentCanvasSelector,
@ -17,16 +20,24 @@ import {
setIsMouseOverBoundingBox,
setIsMovingBoundingBox,
setIsTransformingBoundingBox,
shouldLockToInitialImageSelector,
} from 'features/canvas/canvasSlice';
import { GroupConfig } from 'konva/lib/Group';
import { activeTabNameSelector } from 'features/options/optionsSelectors';
const boundingBoxPreviewSelector = createSelector(
shouldLockToInitialImageSelector,
currentCanvasSelector,
outpaintingCanvasSelector,
baseCanvasImageSelector,
activeTabNameSelector,
(currentCanvas, outpaintingCanvas, baseCanvasImage, activeTabName) => {
(
shouldLockToInitialImage,
currentCanvas,
outpaintingCanvas,
baseCanvasImage,
activeTabName
) => {
const {
boundingBoxCoordinates,
boundingBoxDimensions,
@ -40,6 +51,7 @@ const boundingBoxPreviewSelector = createSelector(
tool,
stageCoordinates,
} = currentCanvas;
const { shouldSnapToGrid } = outpaintingCanvas;
return {
@ -59,6 +71,7 @@ const boundingBoxPreviewSelector = createSelector(
stageCoordinates,
boundingBoxStrokeWidth: (isMouseOverBoundingBox ? 8 : 1) / stageScale,
hitStrokeWidth: 20 / stageScale,
shouldLockToInitialImage,
};
},
{
@ -91,6 +104,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
tool,
boundingBoxStrokeWidth,
hitStrokeWidth,
shouldLockToInitialImage,
} = useAppSelector(boundingBoxPreviewSelector);
const transformerRef = useRef<Konva.Transformer>(null);
@ -106,7 +120,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
const handleOnDragMove = useCallback(
(e: KonvaEventObject<DragEvent>) => {
if (activeTabName === 'inpainting' || !shouldSnapToGrid) {
if (!shouldSnapToGrid) {
dispatch(
setBoundingBoxCoordinates({
x: Math.floor(e.target.x()),
@ -132,20 +146,27 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
})
);
},
[activeTabName, dispatch, shouldSnapToGrid]
[dispatch, shouldSnapToGrid]
);
const dragBoundFunc = useCallback(
(position: Vector2d) => {
if (!baseCanvasImage && activeTabName !== 'outpainting')
return boundingBoxCoordinates;
/**
* This limits the bounding box's drag coordinates.
*/
if (!shouldLockToInitialImage) return boundingBoxCoordinates;
const { x, y } = position;
const maxX =
stageDimensions.width - boundingBoxDimensions.width * stageScale;
stageDimensions.width -
boundingBoxDimensions.width -
(stageDimensions.width % 64);
const maxY =
stageDimensions.height - boundingBoxDimensions.height * stageScale;
stageDimensions.height -
boundingBoxDimensions.height -
(stageDimensions.height % 64);
const clampedX = Math.floor(_.clamp(x, 0, maxX));
const clampedY = Math.floor(_.clamp(y, 0, maxY));
@ -153,14 +174,12 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
return { x: clampedX, y: clampedY };
},
[
baseCanvasImage,
activeTabName,
shouldLockToInitialImage,
boundingBoxCoordinates,
stageDimensions.width,
stageDimensions.height,
boundingBoxDimensions.width,
boundingBoxDimensions.height,
stageScale,
]
);
@ -203,7 +222,6 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
rect.scaleY(1);
}, [dispatch]);
// OK
const anchorDragBoundFunc = useCallback(
(
oldPos: Vector2d, // old absolute position of anchor point
@ -215,49 +233,14 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
* Konva does not transform with width or height. It transforms the anchor point
* and scale factor. This is then sent to the shape's onTransform listeners.
*
* We need to snap the new width to steps of 64 without also snapping the
* coordinates of the bounding box to steps of 64. But because the whole
* We need to snap the new dimensions to steps of 64. But because the whole
* stage is scaled, our actual desired step is actually 64 * the stage scale.
*/
// Difference of the old coords from the nearest multiple the scaled step
const offsetX = oldPos.x % scaledStep;
const offsetY = oldPos.y % scaledStep;
// Round new position to the nearest multiple of the scaled step
const closestX = roundToMultiple(newPos.x, scaledStep) + offsetX;
const closestY = roundToMultiple(newPos.y, scaledStep) + offsetY;
// the difference between the old coord and new
const diffX = Math.abs(newPos.x - closestX);
const diffY = Math.abs(newPos.y - closestY);
// if the difference is less than the scaled step, we want to snap
const didSnapX = diffX < scaledStep;
const didSnapY = diffY < scaledStep;
// We may not change anything, stash the old position
let newCoordinate = { ...oldPos };
// Set the new coords based on what snapped
if (didSnapX && !didSnapY) {
newCoordinate = {
x: closestX,
y: oldPos.y,
};
} else if (!didSnapX && didSnapY) {
newCoordinate = {
x: oldPos.x,
y: closestY,
};
} else if (didSnapX && didSnapY) {
newCoordinate = {
x: closestX,
y: closestY,
};
}
return newCoordinate;
return {
x: roundToMultiple(newPos.x, scaledStep),
y: roundToMultiple(newPos.y, scaledStep),
};
},
[scaledStep]
);
@ -272,7 +255,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
// On the Inpainting canvas, the bounding box needs to stay in the stage
if (
activeTabName === 'inpainting' &&
shouldLockToInitialImage &&
(newBoundBox.width + newBoundBox.x > stageDimensions.width ||
newBoundBox.height + newBoundBox.y > stageDimensions.height ||
newBoundBox.x < 0 ||
@ -283,7 +266,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
return newBoundBox;
},
[activeTabName, stageDimensions.height, stageDimensions.width]
[shouldLockToInitialImage, stageDimensions.height, stageDimensions.width]
);
const handleStartedTransforming = () => {
@ -337,7 +320,7 @@ const IAICanvasBoundingBox = (props: IAICanvasBoundingBoxPreviewProps) => {
globalCompositeOperation={'destination-out'}
/>
<Rect
{...(activeTabName === 'inpainting' ? { dragBoundFunc } : {})}
{...(shouldLockToInitialImage ? { dragBoundFunc } : {})}
listening={!isDrawing && tool === 'move'}
draggable={true}
fillEnabled={false}

16
frontend/src/features/canvas/IAICanvasMaskLines.tsx
View File

@ -2,17 +2,13 @@ import { GroupConfig } from 'konva/lib/Group';
import { Group, Line } from 'react-konva';
import { useAppSelector } from 'app/store';
import { createSelector } from '@reduxjs/toolkit';
import {
canvasClipSelector,
currentCanvasSelector,
isCanvasMaskLine,
} from './canvasSlice';
import { currentCanvasSelector, isCanvasMaskLine } from './canvasSlice';
import _ from 'lodash';
export const canvasLinesSelector = createSelector(
[currentCanvasSelector, canvasClipSelector],
(currentCanvas, canvasClip) => {
return { objects: currentCanvas.layerState.objects, canvasClip };
[currentCanvasSelector],
(currentCanvas) => {
return { objects: currentCanvas.layerState.objects };
},
{
memoizeOptions: {
@ -30,10 +26,10 @@ type InpaintingCanvasLinesProps = GroupConfig;
*/
const IAICanvasLines = (props: InpaintingCanvasLinesProps) => {
const { ...rest } = props;
const { objects, canvasClip } = useAppSelector(canvasLinesSelector);
const { objects } = useAppSelector(canvasLinesSelector);
return (
<Group listening={false} {...rest} {...canvasClip}>
<Group listening={false} {...rest}>
{objects.filter(isCanvasMaskLine).map((line, i) => (
<Line
key={i}

13
frontend/src/features/canvas/IAICanvasObjectRenderer.tsx
View File

@ -4,8 +4,8 @@ import _ from 'lodash';
import { Group, Line } from 'react-konva';
import {
baseCanvasImageSelector,
canvasClipSelector,
canvasModeSelector,
// canvasClipSelector,
// canvasModeSelector,
currentCanvasSelector,
isCanvasBaseImage,
isCanvasBaseLine,
@ -14,13 +14,12 @@ import IAICanvasImage from './IAICanvasImage';
import { rgbaColorToString } from './util/colorToString';
const selector = createSelector(
[currentCanvasSelector, canvasClipSelector],
(currentCanvas, canvasClip) => {
[currentCanvasSelector],
(currentCanvas) => {
const { objects } = currentCanvas.layerState;
return {
objects,
canvasClip,
};
},
{
@ -31,12 +30,12 @@ const selector = createSelector(
);
const IAICanvasObjectRenderer = () => {
const { objects, canvasClip } = useAppSelector(selector);
const { objects } = useAppSelector(selector);
if (!objects) return null;
return (
<Group name="outpainting-objects" listening={false} {...canvasClip}>
<Group name="outpainting-objects" listening={false}>
{objects.map((obj, i) => {
if (isCanvasBaseImage(obj)) {
return (

35
frontend/src/features/canvas/IAICanvasOutpaintingControls.tsx
View File

@ -2,10 +2,11 @@ import { ButtonGroup } from '@chakra-ui/react';
import { createSelector } from '@reduxjs/toolkit';
import {
currentCanvasSelector,
resizeAndScaleCanvas,
isStagingSelector,
resetCanvas,
resetCanvasView,
setCanvasMode,
setShouldLockToInitialImage,
setTool,
} from './canvasSlice';
import { RootState, useAppDispatch, useAppSelector } from 'app/store';
@ -30,6 +31,7 @@ import IAICanvasBrushButtonPopover from './IAICanvasBrushButtonPopover';
import IAICanvasMaskButtonPopover from './IAICanvasMaskButtonPopover';
import { mergeAndUploadCanvas } from './util/mergeAndUploadCanvas';
import IAICheckbox from 'common/components/IAICheckbox';
import { ChangeEvent } from 'react';
export const canvasControlsSelector = createSelector(
[
@ -38,12 +40,12 @@ export const canvasControlsSelector = createSelector(
isStagingSelector,
],
(canvas, currentCanvas, isStaging) => {
const { shouldLockToInitialImage } = canvas;
const { tool } = currentCanvas;
const { mode } = canvas;
return {
tool,
isStaging,
mode,
shouldLockToInitialImage,
};
},
{
@ -55,7 +57,16 @@ export const canvasControlsSelector = createSelector(
const IAICanvasOutpaintingControls = () => {
const dispatch = useAppDispatch();
const { tool, isStaging, mode } = useAppSelector(canvasControlsSelector);
const { tool, isStaging, shouldLockToInitialImage } = useAppSelector(
canvasControlsSelector
);
const handleToggleShouldLockToInitialImage = (
e: ChangeEvent<HTMLInputElement>
) => {
dispatch(setShouldLockToInitialImage(e.target.checked));
dispatch(resizeAndScaleCanvas());
};
return (
<div className="inpainting-settings">
@ -125,10 +136,12 @@ const IAICanvasOutpaintingControls = () => {
icon={<FaCrosshairs />}
onClick={() => {
if (!stageRef.current || !canvasImageLayerRef.current) return;
const clientRect = canvasImageLayerRef.current.getClientRect({skipTransform: true});
const clientRect = canvasImageLayerRef.current.getClientRect({
skipTransform: true,
});
dispatch(
resetCanvasView({
clientRect,
contentRect: clientRect,
})
);
}}
@ -141,13 +154,9 @@ const IAICanvasOutpaintingControls = () => {
/>
</ButtonGroup>
<IAICheckbox
label={'inpainting'}
isChecked={mode === 'inpainting'}
onChange={(e) =>
dispatch(
setCanvasMode(e.target.checked ? 'inpainting' : 'outpainting')
)
}
label={'Lock Canvas to Initial Image'}
isChecked={shouldLockToInitialImage}
onChange={handleToggleShouldLockToInitialImage}
/>
</div>
);

38
frontend/src/features/canvas/IAICanvasResizer.tsx
View File

@ -5,8 +5,9 @@ import { activeTabNameSelector } from 'features/options/optionsSelectors';
import {
baseCanvasImageSelector,
currentCanvasSelector,
initializeCanvas,
resizeAndScaleCanvas,
resizeCanvas,
setCanvasContainerDimensions,
setDoesCanvasNeedScaling,
} from 'features/canvas/canvasSlice';
import { createSelector } from '@reduxjs/toolkit';
@ -17,10 +18,14 @@ const canvasResizerSelector = createSelector(
baseCanvasImageSelector,
activeTabNameSelector,
(canvas, currentCanvas, baseCanvasImage, activeTabName) => {
const { doesCanvasNeedScaling, mode, isCanvasInitialized } = canvas;
const {
doesCanvasNeedScaling,
shouldLockToInitialImage,
isCanvasInitialized,
} = canvas;
return {
doesCanvasNeedScaling,
mode,
shouldLockToInitialImage,
activeTabName,
baseCanvasImage,
isCanvasInitialized,
@ -32,7 +37,7 @@ const IAICanvasResizer = () => {
const dispatch = useAppDispatch();
const {
doesCanvasNeedScaling,
mode,
shouldLockToInitialImage,
activeTabName,
baseCanvasImage,
isCanvasInitialized,
@ -50,22 +55,17 @@ const IAICanvasResizer = () => {
const { width: imageWidth, height: imageHeight } = baseCanvasImage.image;
if (!isCanvasInitialized) {
dispatch(
initializeCanvas({
clientWidth,
clientHeight,
imageWidth,
imageHeight,
})
);
dispatch(
setCanvasContainerDimensions({
width: clientWidth,
height: clientHeight,
})
);
if (!isCanvasInitialized || shouldLockToInitialImage) {
dispatch(resizeAndScaleCanvas());
} else {
dispatch(
resizeCanvas({
clientWidth,
clientHeight,
})
);
dispatch(resizeCanvas());
}
dispatch(setDoesCanvasNeedScaling(false));

4
frontend/src/features/canvas/IAICanvasStagingArea.tsx
View File

@ -106,7 +106,7 @@ const IAICanvasStagingArea = (props: Props) => {
/>
</Group>
)}
<Html>
{/* <Html>
<CacheProvider value={emotionCache}>
<ChakraProvider>
<div
@ -172,7 +172,7 @@ const IAICanvasStagingArea = (props: Props) => {
</div>
</ChakraProvider>
</CacheProvider>
</Html>
</Html> */}
</Group>
);
};

140
frontend/src/features/canvas/IAICanvasStagingAreaToolbar.tsx Normal file
View File

@ -0,0 +1,140 @@
import {
background,
ButtonGroup,
ChakraProvider,
Flex,
} from '@chakra-ui/react';
import { CacheProvider } from '@emotion/react';
import { createSelector } from '@reduxjs/toolkit';
import { useAppDispatch, useAppSelector } from 'app/store';
import IAIButton from 'common/components/IAIButton';
import IAIIconButton from 'common/components/IAIIconButton';
import { GroupConfig } from 'konva/lib/Group';
import _ from 'lodash';
import { emotionCache } from 'main';
import { useCallback, useState } from 'react';
import {
FaArrowLeft,
FaArrowRight,
FaCheck,
FaEye,
FaEyeSlash,
FaTrash,
} from 'react-icons/fa';
import { Group, Rect } from 'react-konva';
import { Html } from 'react-konva-utils';
import {
commitStagingAreaImage,
currentCanvasSelector,
discardStagedImages,
nextStagingAreaImage,
prevStagingAreaImage,
} from './canvasSlice';
import IAICanvasImage from './IAICanvasImage';
const selector = createSelector(
[currentCanvasSelector],
(currentCanvas) => {
const {
layerState: {
stagingArea: { images, selectedImageIndex },
},
} = currentCanvas;
return {
currentStagingAreaImage:
images.length > 0 ? images[selectedImageIndex] : undefined,
isOnFirstImage: selectedImageIndex === 0,
isOnLastImage: selectedImageIndex === images.length - 1,
};
},
{
memoizeOptions: {
resultEqualityCheck: _.isEqual,
},
}
);
const IAICanvasStagingAreaToolbar = () => {
const dispatch = useAppDispatch();
const { isOnFirstImage, isOnLastImage, currentStagingAreaImage } =
useAppSelector(selector);
const [shouldShowStagedImage, setShouldShowStagedImage] =
useState<boolean>(true);
const [shouldShowStagingAreaOutline, setShouldShowStagingAreaOutline] =
useState<boolean>(true);
const handleMouseOver = useCallback(() => {
setShouldShowStagingAreaOutline(false);
}, []);
const handleMouseOut = useCallback(() => {
setShouldShowStagingAreaOutline(true);
}, []);
if (!currentStagingAreaImage) return null;
return (
<Flex
pos={'absolute'}
bottom={'1rem'}
w={'100%'}
align={'center'}
justify={'center'}
>
<ButtonGroup isAttached>
<IAIIconButton
tooltip="Previous"
tooltipProps={{ placement: 'bottom' }}
aria-label="Previous"
icon={<FaArrowLeft />}
onClick={() => dispatch(prevStagingAreaImage())}
onMouseOver={handleMouseOver}
onMouseOut={handleMouseOut}
data-selected={true}
isDisabled={isOnFirstImage}
/>
<IAIIconButton
tooltip="Next"
tooltipProps={{ placement: 'bottom' }}
aria-label="Next"
icon={<FaArrowRight />}
onClick={() => dispatch(nextStagingAreaImage())}
onMouseOver={handleMouseOver}
onMouseOut={handleMouseOut}
data-selected={true}
isDisabled={isOnLastImage}
/>
<IAIIconButton
tooltip="Accept"
tooltipProps={{ placement: 'bottom' }}
aria-label="Accept"
icon={<FaCheck />}
onClick={() => dispatch(commitStagingAreaImage())}
data-selected={true}
/>
<IAIIconButton
tooltip="Show/Hide"
tooltipProps={{ placement: 'bottom' }}
aria-label="Show/Hide"
data-alert={!shouldShowStagedImage}
icon={shouldShowStagedImage ? <FaEye /> : <FaEyeSlash />}
onClick={() => setShouldShowStagedImage(!shouldShowStagedImage)}
data-selected={true}
/>
<IAIIconButton
tooltip="Discard All"
tooltipProps={{ placement: 'bottom' }}
aria-label="Discard All"
icon={<FaTrash />}
onClick={() => dispatch(discardStagedImages())}
data-selected={true}
/>
</ButtonGroup>
</Flex>
);
};
export default IAICanvasStagingAreaToolbar;

228
frontend/src/features/canvas/canvasSlice.ts
View File

@ -21,6 +21,7 @@ import { uploadImage } from 'features/gallery/util/uploadImage';
import { setInitialCanvasImage } from './canvasReducers';
import calculateScale from './util/calculateScale';
import calculateCoordinates from './util/calculateCoordinates';
import floorCoordinates from './util/floorCoordinates';
export interface GenericCanvasState {
boundingBoxCoordinates: Vector2d;
@ -58,6 +59,7 @@ export interface GenericCanvasState {
stageDimensions: Dimensions;
stageScale: number;
tool: CanvasTool;
minimumStageScale: number;
}
export type CanvasMode = 'inpainting' | 'outpainting';
@ -147,8 +149,10 @@ export interface CanvasState {
currentCanvas: ValidCanvasName;
inpainting: InpaintingCanvasState;
outpainting: OutpaintingCanvasState;
mode: CanvasMode;
// mode: CanvasMode;
shouldLockToInitialImage: boolean;
isCanvasInitialized: boolean;
canvasContainerDimensions: Dimensions;
}
export const initialLayerState: CanvasLayerState = {
@ -197,14 +201,17 @@ const initialGenericCanvasState: GenericCanvasState = {
stageCoordinates: { x: 0, y: 0 },
stageDimensions: { width: 0, height: 0 },
stageScale: 1,
minimumStageScale: 1,
tool: 'brush',
};
const initialCanvasState: CanvasState = {
currentCanvas: 'inpainting',
doesCanvasNeedScaling: false,
mode: 'outpainting',
shouldLockToInitialImage: false,
// mode: 'outpainting',
isCanvasInitialized: false,
canvasContainerDimensions: { width: 0, height: 0 },
inpainting: {
layer: 'mask',
...initialGenericCanvasState,
@ -335,80 +342,15 @@ export const canvasSlice = createSlice({
};
},
setBoundingBoxDimensions: (state, action: PayloadAction<Dimensions>) => {
const currentCanvas = state[state.currentCanvas];
currentCanvas.boundingBoxDimensions = action.payload;
const { width: boundingBoxWidth, height: boundingBoxHeight } =
action.payload;
const { x: boundingBoxX, y: boundingBoxY } =
currentCanvas.boundingBoxCoordinates;
const { width: canvasWidth, height: canvasHeight } =
currentCanvas.stageDimensions;
const scaledCanvasWidth = canvasWidth / currentCanvas.stageScale;
const scaledCanvasHeight = canvasHeight / currentCanvas.stageScale;
const roundedCanvasWidth = roundDownToMultiple(scaledCanvasWidth, 64);
const roundedCanvasHeight = roundDownToMultiple(scaledCanvasHeight, 64);
const roundedBoundingBoxWidth = roundDownToMultiple(boundingBoxWidth, 64);
const roundedBoundingBoxHeight = roundDownToMultiple(
boundingBoxHeight,
64
);
const overflowX = boundingBoxX + boundingBoxWidth - scaledCanvasWidth;
const overflowY = boundingBoxY + boundingBoxHeight - scaledCanvasHeight;
const newBoundingBoxWidth = _.clamp(
roundedBoundingBoxWidth,
64,
roundedCanvasWidth
);
const newBoundingBoxHeight = _.clamp(
roundedBoundingBoxHeight,
64,
roundedCanvasHeight
);
const overflowCorrectedX =
overflowX > 0 ? boundingBoxX - overflowX : boundingBoxX;
const overflowCorrectedY =
overflowY > 0 ? boundingBoxY - overflowY : boundingBoxY;
const clampedX = _.clamp(
overflowCorrectedX,
currentCanvas.stageCoordinates.x,
roundedCanvasWidth - newBoundingBoxWidth
);
const clampedY = _.clamp(
overflowCorrectedY,
currentCanvas.stageCoordinates.y,
roundedCanvasHeight - newBoundingBoxHeight
);
currentCanvas.boundingBoxDimensions = {
width: newBoundingBoxWidth,
height: newBoundingBoxHeight,
};
currentCanvas.boundingBoxCoordinates = {
x: clampedX,
y: clampedY,
};
state[state.currentCanvas].boundingBoxDimensions = action.payload;
},
setBoundingBoxCoordinates: (state, action: PayloadAction<Vector2d>) => {
const { x, y } = action.payload;
state[state.currentCanvas].boundingBoxCoordinates = {
x: Math.floor(x),
y: Math.floor(y),
};
state[state.currentCanvas].boundingBoxCoordinates = floorCoordinates(
action.payload
);
},
setStageCoordinates: (state, action: PayloadAction<Vector2d>) => {
state[state.currentCanvas].stageCoordinates = action.payload;
state.outpainting.stageCoordinates = floorCoordinates(action.payload);
},
setBoundingBoxPreviewFill: (state, action: PayloadAction<RgbaColor>) => {
state[state.currentCanvas].boundingBoxPreviewFill = action.payload;
@ -595,30 +537,52 @@ export const canvasSlice = createSlice({
state[state.currentCanvas].layerState = initialLayerState;
state[state.currentCanvas].futureLayerStates = [];
},
initializeCanvas: (
setCanvasContainerDimensions: (
state,
action: PayloadAction<{
clientWidth: number;
clientHeight: number;
imageWidth: number;
imageHeight: number;
}>
action: PayloadAction<Dimensions>
) => {
const { clientWidth, clientHeight, imageWidth, imageHeight } =
action.payload;
state.canvasContainerDimensions = action.payload;
},
resizeAndScaleCanvas: (state) => {
const { width: containerWidth, height: containerHeight } =
state.canvasContainerDimensions;
const initialCanvasImage =
state.outpainting.layerState.objects.find(isCanvasBaseImage);
if (!initialCanvasImage) return;
const { width: imageWidth, height: imageHeight } = initialCanvasImage;
// const { clientWidth, clientHeight, imageWidth, imageHeight } =
// action.payload;
const { shouldLockToInitialImage } = state;
const currentCanvas = state[state.currentCanvas];
const padding = shouldLockToInitialImage ? 1 : 0.95;
const newScale = calculateScale(
clientWidth,
clientHeight,
containerWidth,
containerHeight,
imageWidth,
imageHeight
imageHeight,
padding
);
const newDimensions = {
width: shouldLockToInitialImage
? Math.floor(imageWidth * newScale)
: Math.floor(containerWidth),
height: shouldLockToInitialImage
? Math.floor(imageHeight * newScale)
: Math.floor(containerHeight),
};
const newCoordinates = calculateCoordinates(
clientWidth,
clientHeight,
newDimensions.width,
newDimensions.height,
0,
0,
imageWidth,
@ -627,50 +591,53 @@ export const canvasSlice = createSlice({
);
currentCanvas.stageScale = newScale;
currentCanvas.minimumStageScale = newScale;
currentCanvas.stageCoordinates = newCoordinates;
currentCanvas.stageDimensions = {
width: Math.floor(clientWidth),
height: Math.floor(clientHeight),
};
currentCanvas.stageDimensions = newDimensions;
state.isCanvasInitialized = true;
},
resizeCanvas: (
state,
action: PayloadAction<{
clientWidth: number;
clientHeight: number;
}>
) => {
const { clientWidth, clientHeight } = action.payload;
resizeCanvas: (state) => {
const { width: containerWidth, height: containerHeight } =
state.canvasContainerDimensions;
const currentCanvas = state[state.currentCanvas];
currentCanvas.stageDimensions = {
width: Math.floor(clientWidth),
height: Math.floor(clientHeight),
width: Math.floor(containerWidth),
height: Math.floor(containerHeight),
};
},
resetCanvasView: (
state,
action: PayloadAction<{
clientRect: IRect;
contentRect: IRect;
}>
) => {
const { clientRect } = action.payload;
const { contentRect } = action.payload;
const currentCanvas = state[state.currentCanvas];
const baseCanvasImage =
currentCanvas.layerState.objects.find(isCanvasBaseImage);
const { shouldLockToInitialImage } = state;
if (!baseCanvasImage) return;
const {
stageDimensions: { width: stageWidth, height: stageHeight },
} = currentCanvas;
const { x, y, width, height } = clientRect;
const { x, y, width, height } = contentRect;
const padding = shouldLockToInitialImage ? 1 : 0.95;
const newScale = calculateScale(
stageWidth,
stageHeight,
width,
height,
padding
);
const newScale = calculateScale(stageWidth, stageHeight, width, height);
const newCoordinates = calculateCoordinates(
stageWidth,
stageHeight,
@ -683,10 +650,7 @@ export const canvasSlice = createSlice({
currentCanvas.stageScale = newScale;
currentCanvas.stageCoordinates = {
x: stageWidth / 2 - (x + width / 2) * newScale,
y: stageHeight / 2 - (y + height / 2) * newScale,
};
currentCanvas.stageCoordinates = newCoordinates;
},
nextStagingAreaImage: (state) => {
const currentIndex =
@ -728,14 +692,17 @@ export const canvasSlice = createSlice({
currentCanvas.futureLayerStates = [];
},
setCanvasMode: (state, action: PayloadAction<CanvasMode>) => {
state.mode = action.payload;
setShouldLockToInitialImage: (state, action: PayloadAction<boolean>) => {
state.shouldLockToInitialImage = action.payload;
},
// setCanvasMode: (state, action: PayloadAction<CanvasMode>) => {
// state.mode = action.payload;
// },
},
extraReducers: (builder) => {
builder.addCase(mergeAndUploadCanvas.fulfilled, (state, action) => {
if (!action.payload) return;
const { image, kind, boundingBox } = action.payload;
const { image, kind, originalBoundingBox } = action.payload;
if (kind === 'temp_merged_canvas') {
state.outpainting.pastLayerStates.push({
@ -748,7 +715,7 @@ export const canvasSlice = createSlice({
{
kind: 'image',
layer: 'base',
...boundingBox,
...originalBoundingBox,
image,
},
];
@ -824,10 +791,11 @@ export const {
prevStagingAreaImage,
commitStagingAreaImage,
discardStagedImages,
setCanvasMode,
initializeCanvas,
setShouldLockToInitialImage,
resizeAndScaleCanvas,
resizeCanvas,
resetCanvasView,
setCanvasContainerDimensions,
} = canvasSlice.actions;
export default canvasSlice.reducer;
@ -847,8 +815,8 @@ export const inpaintingCanvasSelector = (
state: RootState
): InpaintingCanvasState => state.canvas.inpainting;
export const canvasModeSelector = (state: RootState): CanvasMode =>
state.canvas.mode;
export const shouldLockToInitialImageSelector = (state: RootState): boolean =>
state.canvas.shouldLockToInitialImage;
export const baseCanvasImageSelector = createSelector(
[currentCanvasSelector],
@ -857,16 +825,16 @@ export const baseCanvasImageSelector = createSelector(
}
);
export const canvasClipSelector = createSelector(
[canvasModeSelector, baseCanvasImageSelector],
(canvasMode, baseCanvasImage) => {
return canvasMode === 'inpainting'
? {
clipX: 0,
clipY: 0,
clipWidth: baseCanvasImage?.width,
clipHeight: baseCanvasImage?.height,
}
: {};
}
);
// export const canvasClipSelector = createSelector(
// [canvasModeSelector, baseCanvasImageSelector],
// (canvasMode, baseCanvasImage) => {
// return canvasMode === 'inpainting'
// ? {
// clipX: 0,
// clipY: 0,
// clipWidth: baseCanvasImage?.width,
// clipHeight: baseCanvasImage?.height,
// }
// : {};
// }
// );

14
frontend/src/features/canvas/hooks/useCanvasDragMove.ts
View File

@ -5,20 +5,21 @@ import { KonvaEventObject } from 'konva/lib/Node';
import _ from 'lodash';
import { useCallback } from 'react';
import {
baseCanvasImageSelector,
currentCanvasSelector,
isStagingSelector,
setIsMovingStage,
setStageCoordinates,
shouldLockToInitialImageSelector,
} from '../canvasSlice';
const selector = createSelector(
[currentCanvasSelector, isStagingSelector, activeTabNameSelector],
(canvas, isStaging, activeTabName) => {
[currentCanvasSelector, isStagingSelector],
(canvas, isStaging) => {
const { tool } = canvas;
return {
tool,
isStaging,
activeTabName,
};
},
{ memoizeOptions: { resultEqualityCheck: _.isEqual } }
@ -26,7 +27,7 @@ const selector = createSelector(
const useCanvasDrag = () => {
const dispatch = useAppDispatch();
const { tool, activeTabName, isStaging } = useAppSelector(selector);
const { tool, isStaging } = useAppSelector(selector);
return {
handleDragStart: useCallback(() => {
@ -37,7 +38,10 @@ const useCanvasDrag = () => {
handleDragMove: useCallback(
(e: KonvaEventObject<MouseEvent>) => {
if (!(tool === 'move' || isStaging)) return;
dispatch(setStageCoordinates(e.target.getPosition()));
const newCoordinates = { x: e.target.x(), y: e.target.y() };
dispatch(setStageCoordinates(newCoordinates));
},
[dispatch, isStaging, tool]
),

86
frontend/src/features/canvas/hooks/useCanvasZoom.ts
View File

@ -1,15 +1,17 @@
import { createSelector } from '@reduxjs/toolkit';
import { useAppDispatch, useAppSelector } from 'app/store';
import { RootState, useAppDispatch, useAppSelector } from 'app/store';
import { activeTabNameSelector } from 'features/options/optionsSelectors';
import Konva from 'konva';
import { KonvaEventObject } from 'konva/lib/Node';
import _ from 'lodash';
import { MutableRefObject, useCallback } from 'react';
import {
baseCanvasImageSelector,
currentCanvasSelector,
GenericCanvasState,
setStageCoordinates,
setStageScale,
shouldLockToInitialImageSelector,
} from '../canvasSlice';
import {
CANVAS_SCALE_BY,
@ -18,13 +20,34 @@ import {
} from '../util/constants';
const selector = createSelector(
[activeTabNameSelector, currentCanvasSelector],
(activeTabName, canvas: GenericCanvasState) => {
const { isMoveStageKeyHeld, stageScale } = canvas;
[
(state: RootState) => state.canvas,
activeTabNameSelector,
currentCanvasSelector,
baseCanvasImageSelector,
shouldLockToInitialImageSelector,
],
(
canvas,
activeTabName,
currentCanvas,
baseCanvasImage,
shouldLockToInitialImage
) => {
const {
isMoveStageKeyHeld,
stageScale,
stageDimensions,
minimumStageScale,
} = currentCanvas;
return {
isMoveStageKeyHeld,
stageScale,
activeTabName,
baseCanvasImage,
shouldLockToInitialImage,
stageDimensions,
minimumStageScale,
};
},
{ memoizeOptions: { resultEqualityCheck: _.isEqual } }
@ -32,19 +55,29 @@ const selector = createSelector(
const useCanvasWheel = (stageRef: MutableRefObject<Konva.Stage | null>) => {
const dispatch = useAppDispatch();
const { isMoveStageKeyHeld, stageScale, activeTabName } =
useAppSelector(selector);
const {
isMoveStageKeyHeld,
stageScale,
activeTabName,
baseCanvasImage,
shouldLockToInitialImage,
stageDimensions,
minimumStageScale,
} = useAppSelector(selector);
return useCallback(
(e: KonvaEventObject<WheelEvent>) => {
// stop default scrolling
if (activeTabName !== 'outpainting') return;
if (
activeTabName !== 'outpainting' ||
!stageRef.current ||
isMoveStageKeyHeld ||
!baseCanvasImage
)
return;
e.evt.preventDefault();
// const oldScale = stageRef.current.scaleX();
if (!stageRef.current || isMoveStageKeyHeld) return;
const cursorPos = stageRef.current.getPointerPosition();
if (!cursorPos) return;
@ -64,19 +97,44 @@ const useCanvasWheel = (stageRef: MutableRefObject<Konva.Stage | null>) => {
const newScale = _.clamp(
stageScale * CANVAS_SCALE_BY ** delta,
MIN_CANVAS_SCALE,
shouldLockToInitialImage ? minimumStageScale : MIN_CANVAS_SCALE,
MAX_CANVAS_SCALE
);
const newPos = {
const newCoordinates = {
x: cursorPos.x - mousePointTo.x * newScale,
y: cursorPos.y - mousePointTo.y * newScale,
};
if (shouldLockToInitialImage) {
newCoordinates.x = _.clamp(
newCoordinates.x,
stageDimensions.width - Math.floor(baseCanvasImage.width * newScale),
0
);
newCoordinates.y = _.clamp(
newCoordinates.y,
stageDimensions.height -
Math.floor(baseCanvasImage.height * newScale),
0
);
}
dispatch(setStageScale(newScale));
dispatch(setStageCoordinates(newPos));
dispatch(setStageCoordinates(newCoordinates));
},
[activeTabName, dispatch, isMoveStageKeyHeld, stageRef, stageScale]
[
activeTabName,
stageRef,
isMoveStageKeyHeld,
baseCanvasImage,
stageScale,
shouldLockToInitialImage,
minimumStageScale,
dispatch,
stageDimensions.width,
stageDimensions.height,
]
);
};

10
frontend/src/features/canvas/util/floorCoordinates.ts Normal file
View File

@ -0,0 +1,10 @@
import { Vector2d } from 'konva/lib/types';
const floorCoordinates = (coord: Vector2d): Vector2d => {
return {
x: Math.floor(coord.x),
y: Math.floor(coord.y),
};
};
export default floorCoordinates;

21
frontend/src/features/canvas/util/layerToDataURL.ts
View File

@ -3,7 +3,7 @@ import Konva from 'konva';
const layerToDataURL = (layer: Konva.Layer, stageScale: number) => {
const tempScale = layer.scale();
const { x: relativeX, y: relativeY } = layer.getClientRect({
const relativeClientRect = layer.getClientRect({
relativeTo: layer.getParent(),
});
@ -13,14 +13,27 @@ const layerToDataURL = (layer: Konva.Layer, stageScale: number) => {
y: 1 / stageScale,
});
const clientRect = layer.getClientRect();
const { x, y, width, height } = layer.getClientRect();
const dataURL = layer.toDataURL(clientRect);
const dataURL = layer.toDataURL({
x: Math.round(x),
y: Math.round(y),
width: Math.round(width),
height: Math.round(height),
});
// Unscale the canvas
layer.scale(tempScale);
return { dataURL, relativeX, relativeY };
return {
dataURL,
boundingBox: {
x: Math.round(relativeClientRect.x),
y: Math.round(relativeClientRect.y),
width: Math.round(width),
height: Math.round(height),
},
};
};
export default layerToDataURL;

37
frontend/src/features/canvas/util/mergeAndUploadCanvas.ts
View File

@ -25,7 +25,7 @@ export const mergeAndUploadCanvas = createAsyncThunk(
if (!canvasImageLayerRef.current) return;
const { dataURL, relativeX, relativeY } = layerToDataURL(
const { dataURL, boundingBox: originalBoundingBox } = layerToDataURL(
canvasImageLayerRef.current,
stageScale
);
@ -34,32 +34,45 @@ export const mergeAndUploadCanvas = createAsyncThunk(
const formData = new FormData();
formData.append('dataURL', dataURL);
formData.append('filename', 'merged_canvas.png');
formData.append('kind', saveToGallery ? 'result' : 'temp');
formData.append(
'data',
JSON.stringify({
dataURL,
filename: 'merged_canvas.png',
kind: saveToGallery ? 'result' : 'temp',
cropVisible: saveToGallery,
})
);
const response = await fetch(window.location.origin + '/upload', {
method: 'POST',
body: formData,
});
const { image } = (await response.json()) as InvokeAI.ImageUploadResponse;
const { url, mtime, width, height } =
(await response.json()) as InvokeAI.ImageUploadResponse;
// const newBoundingBox = {
// x: bbox[0],
// y: bbox[1],
// width: bbox[2],
// height: bbox[3],
// };
const newImage: InvokeAI.Image = {
uuid: uuidv4(),
url,
mtime,
category: saveToGallery ? 'result' : 'user',
...image,
width: width,
height: height,
};
return {
image: newImage,
kind: saveToGallery ? 'merged_canvas' : 'temp_merged_canvas',
boundingBox: {
x: relativeX,
y: relativeY,
width: image.width,
height: image.height,
},
originalBoundingBox,
// newBoundingBox,
};
}
);

2
frontend/src/features/gallery/gallerySlice.ts
View File

@ -271,7 +271,7 @@ export const gallerySlice = createSlice({
extraReducers: (builder) => {
builder.addCase(mergeAndUploadCanvas.fulfilled, (state, action) => {
if (!action.payload) return;
const { image, kind, boundingBox } = action.payload;
const { image, kind, originalBoundingBox } = action.payload;
if (kind === 'merged_canvas') {
const { uuid, url, mtime } = image;

23
frontend/src/features/gallery/util/uploadImage.ts
View File

@ -23,19 +23,36 @@ export const uploadImage = createAsyncThunk(
const formData = new FormData();
formData.append('file', imageFile, imageFile.name);
formData.append('kind', 'init');
formData.append(
'data',
JSON.stringify({
kind: 'init',
})
);
// formData.append('kind', 'init');
const response = await fetch(window.location.origin + '/upload', {
method: 'POST',
body: formData,
});
const { image } = (await response.json()) as InvokeAI.ImageUploadResponse;
const { url, mtime, width, height } =
(await response.json()) as InvokeAI.ImageUploadResponse;
// const newBoundingBox = {
// x: bbox[0],
// y: bbox[1],
// width: bbox[2],
// height: bbox[3],
// };
const newImage: InvokeAI.Image = {
uuid: uuidv4(),
url,
mtime,
category: 'user',
...image,
width: width,
height: height,
};
return {