From d2c9140e69b3e56a4b1f9ad6c69756e9523bc478 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Mon, 15 May 2023 22:21:03 +1000 Subject: [PATCH] feat(ui): restore save/copy/download/merge functionality --- .../middleware/listenerMiddleware/index.ts | 9 + .../listeners/canvasCopiedToClipboard.ts | 33 ++++ .../listeners/canvasDownloadedAsImage.ts | 33 ++++ .../listeners/canvasMerged.ts | 88 +++++++++ .../listeners/canvasSavedToGallery.ts | 40 +++++ .../listeners/imageUploaded.ts | 28 +-- .../src/common/util/parameterTranslation.ts | 4 +- .../IAICanvasToolbar/IAICanvasToolbar.tsx | 47 ++--- .../web/src/features/canvas/store/actions.ts | 13 ++ .../src/features/canvas/util/blobToDataURL.ts | 9 + .../canvas/util/copyBlobToClipboard.ts | 10 ++ .../features/canvas/util/createMaskStage.ts | 61 +++++++ .../src/features/canvas/util/downloadBlob.ts | 11 ++ .../src/features/canvas/util/generateMask.ts | 170 ------------------ .../features/canvas/util/getBaseLayerBlob.ts | 38 ++++ .../src/features/canvas/util/getCanvasData.ts | 102 +++-------- .../canvas/util/getCanvasGenerationMode.ts | 31 ++++ .../features/canvas/util/konvaNodeToBlob.ts | 16 ++ .../canvas/util/konvaNodeToDataURL.ts | 16 ++ .../canvas/util/konvaNodeToImageData.ts | 23 +++ .../util/graphBuilders/buildCanvasGraph.ts | 54 +++--- .../UnifiedCanvasCopyToClipboard.tsx | 14 +- .../UnifiedCanvasDownloadImage.tsx | 22 +-- .../UnifiedCanvasMergeVisible.tsx | 8 +- .../UnifiedCanvasSaveToGallery.tsx | 14 +- 25 files changed, 519 insertions(+), 375 deletions(-) create mode 100644 invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasCopiedToClipboard.ts create mode 100644 invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasDownloadedAsImage.ts create mode 100644 invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasMerged.ts create mode 100644 invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasSavedToGallery.ts create mode 100644 invokeai/frontend/web/src/features/canvas/store/actions.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/blobToDataURL.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/copyBlobToClipboard.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/createMaskStage.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/downloadBlob.ts delete mode 100644 invokeai/frontend/web/src/features/canvas/util/generateMask.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/getBaseLayerBlob.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/konvaNodeToBlob.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/konvaNodeToDataURL.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/konvaNodeToImageData.ts diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts index 36bf6adfe7..f23e83a191 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts @@ -15,6 +15,10 @@ import { addUserInvokedCanvasListener } from './listeners/userInvokedCanvas'; import { addUserInvokedNodesListener } from './listeners/userInvokedNodes'; import { addUserInvokedTextToImageListener } from './listeners/userInvokedTextToImage'; import { addUserInvokedImageToImageListener } from './listeners/userInvokedImageToImage'; +import { addCanvasSavedToGalleryListener } from './listeners/canvasSavedToGallery'; +import { addCanvasDownloadedAsImageListener } from './listeners/canvasDownloadedAsImage'; +import { addCanvasCopiedToClipboardListener } from './listeners/canvasCopiedToClipboard'; +import { addCanvasMergedListener } from './listeners/canvasMerged'; export const listenerMiddleware = createListenerMiddleware(); @@ -43,3 +47,8 @@ addUserInvokedCanvasListener(); addUserInvokedNodesListener(); addUserInvokedTextToImageListener(); addUserInvokedImageToImageListener(); + +addCanvasSavedToGalleryListener(); +addCanvasDownloadedAsImageListener(); +addCanvasCopiedToClipboardListener(); +addCanvasMergedListener(); diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasCopiedToClipboard.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasCopiedToClipboard.ts new file mode 100644 index 0000000000..16642f1f32 --- /dev/null +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasCopiedToClipboard.ts @@ -0,0 +1,33 @@ +import { canvasCopiedToClipboard } from 'features/canvas/store/actions'; +import { startAppListening } from '..'; +import { log } from 'app/logging/useLogger'; +import { getBaseLayerBlob } from 'features/canvas/util/getBaseLayerBlob'; +import { addToast } from 'features/system/store/systemSlice'; +import { copyBlobToClipboard } from 'features/canvas/util/copyBlobToClipboard'; + +const moduleLog = log.child({ namespace: 'canvasCopiedToClipboardListener' }); + +export const addCanvasCopiedToClipboardListener = () => { + startAppListening({ + actionCreator: canvasCopiedToClipboard, + effect: async (action, { dispatch, getState }) => { + const state = getState(); + + const blob = await getBaseLayerBlob(state); + + if (!blob) { + moduleLog.error('Problem getting base layer blob'); + dispatch( + addToast({ + title: 'Problem Copying Canvas', + description: 'Unable to export base layer', + status: 'error', + }) + ); + return; + } + + copyBlobToClipboard(blob); + }, + }); +}; diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasDownloadedAsImage.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasDownloadedAsImage.ts new file mode 100644 index 0000000000..ef4c63b31c --- /dev/null +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasDownloadedAsImage.ts @@ -0,0 +1,33 @@ +import { canvasDownloadedAsImage } from 'features/canvas/store/actions'; +import { startAppListening } from '..'; +import { log } from 'app/logging/useLogger'; +import { downloadBlob } from 'features/canvas/util/downloadBlob'; +import { getBaseLayerBlob } from 'features/canvas/util/getBaseLayerBlob'; +import { addToast } from 'features/system/store/systemSlice'; + +const moduleLog = log.child({ namespace: 'canvasSavedToGalleryListener' }); + +export const addCanvasDownloadedAsImageListener = () => { + startAppListening({ + actionCreator: canvasDownloadedAsImage, + effect: async (action, { dispatch, getState }) => { + const state = getState(); + + const blob = await getBaseLayerBlob(state); + + if (!blob) { + moduleLog.error('Problem getting base layer blob'); + dispatch( + addToast({ + title: 'Problem Downloading Canvas', + description: 'Unable to export base layer', + status: 'error', + }) + ); + return; + } + + downloadBlob(blob, 'mergedCanvas.png'); + }, + }); +}; diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasMerged.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasMerged.ts new file mode 100644 index 0000000000..d7a58c2050 --- /dev/null +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasMerged.ts @@ -0,0 +1,88 @@ +import { canvasMerged } from 'features/canvas/store/actions'; +import { startAppListening } from '..'; +import { log } from 'app/logging/useLogger'; +import { getBaseLayerBlob } from 'features/canvas/util/getBaseLayerBlob'; +import { addToast } from 'features/system/store/systemSlice'; +import { imageUploaded } from 'services/thunks/image'; +import { v4 as uuidv4 } from 'uuid'; +import { deserializeImageResponse } from 'services/util/deserializeImageResponse'; +import { setMergedCanvas } from 'features/canvas/store/canvasSlice'; +import { getCanvasBaseLayer } from 'features/canvas/util/konvaInstanceProvider'; + +const moduleLog = log.child({ namespace: 'canvasCopiedToClipboardListener' }); + +export const addCanvasMergedListener = () => { + startAppListening({ + actionCreator: canvasMerged, + effect: async (action, { dispatch, getState, take }) => { + const state = getState(); + + const blob = await getBaseLayerBlob(state, true); + + if (!blob) { + moduleLog.error('Problem getting base layer blob'); + dispatch( + addToast({ + title: 'Problem Merging Canvas', + description: 'Unable to export base layer', + status: 'error', + }) + ); + return; + } + + const canvasBaseLayer = getCanvasBaseLayer(); + + if (!canvasBaseLayer) { + moduleLog.error('Problem getting canvas base layer'); + dispatch( + addToast({ + title: 'Problem Merging Canvas', + description: 'Unable to export base layer', + status: 'error', + }) + ); + return; + } + + const baseLayerRect = canvasBaseLayer.getClientRect({ + relativeTo: canvasBaseLayer.getParent(), + }); + + const filename = `mergedCanvas_${uuidv4()}.png`; + + dispatch( + imageUploaded({ + imageType: 'intermediates', + formData: { + file: new File([blob], filename, { type: 'image/png' }), + }, + }) + ); + + const [{ payload }] = await take( + (action): action is ReturnType => + imageUploaded.fulfilled.match(action) && + action.meta.arg.formData.file.name === filename + ); + + const mergedCanvasImage = deserializeImageResponse(payload.response); + + dispatch( + setMergedCanvas({ + kind: 'image', + layer: 'base', + image: mergedCanvasImage, + ...baseLayerRect, + }) + ); + + dispatch( + addToast({ + title: 'Canvas Merged', + status: 'success', + }) + ); + }, + }); +}; diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasSavedToGallery.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasSavedToGallery.ts new file mode 100644 index 0000000000..d8237d1d5c --- /dev/null +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasSavedToGallery.ts @@ -0,0 +1,40 @@ +import { canvasSavedToGallery } from 'features/canvas/store/actions'; +import { startAppListening } from '..'; +import { log } from 'app/logging/useLogger'; +import { imageUploaded } from 'services/thunks/image'; +import { getBaseLayerBlob } from 'features/canvas/util/getBaseLayerBlob'; +import { addToast } from 'features/system/store/systemSlice'; + +const moduleLog = log.child({ namespace: 'canvasSavedToGalleryListener' }); + +export const addCanvasSavedToGalleryListener = () => { + startAppListening({ + actionCreator: canvasSavedToGallery, + effect: async (action, { dispatch, getState }) => { + const state = getState(); + + const blob = await getBaseLayerBlob(state); + + if (!blob) { + moduleLog.error('Problem getting base layer blob'); + dispatch( + addToast({ + title: 'Problem Saving Canvas', + description: 'Unable to export base layer', + status: 'error', + }) + ); + return; + } + + dispatch( + imageUploaded({ + imageType: 'results', + formData: { + file: new File([blob], 'mergedCanvas.png', { type: 'image/png' }), + }, + }) + ); + }, + }); +}; diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts index d676ee6a1d..de06220ecd 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts @@ -6,6 +6,7 @@ import { imageUploaded } from 'services/thunks/image'; import { addToast } from 'features/system/store/systemSlice'; import { initialImageSelected } from 'features/parameters/store/actions'; import { setInitialCanvasImage } from 'features/canvas/store/canvasSlice'; +import { resultAdded } from 'features/gallery/store/resultsSlice'; export const addImageUploadedListener = () => { startAppListening({ @@ -14,24 +15,31 @@ export const addImageUploadedListener = () => { action.payload.response.image_type !== 'intermediates', effect: (action, { dispatch, getState }) => { const { response } = action.payload; + const { imageType } = action.meta.arg; const state = getState(); const image = deserializeImageResponse(response); - dispatch(uploadAdded(image)); + if (imageType === 'uploads') { + dispatch(uploadAdded(image)); - dispatch(addToast({ title: 'Image Uploaded', status: 'success' })); + dispatch(addToast({ title: 'Image Uploaded', status: 'success' })); - if (state.gallery.shouldAutoSwitchToNewImages) { - dispatch(imageSelected(image)); + if (state.gallery.shouldAutoSwitchToNewImages) { + dispatch(imageSelected(image)); + } + + if (action.meta.arg.activeTabName === 'img2img') { + dispatch(initialImageSelected(image)); + } + + if (action.meta.arg.activeTabName === 'unifiedCanvas') { + dispatch(setInitialCanvasImage(image)); + } } - if (action.meta.arg.activeTabName === 'img2img') { - dispatch(initialImageSelected(image)); - } - - if (action.meta.arg.activeTabName === 'unifiedCanvas') { - dispatch(setInitialCanvasImage(image)); + if (imageType === 'results') { + dispatch(resultAdded(image)); } }, }); diff --git a/invokeai/frontend/web/src/common/util/parameterTranslation.ts b/invokeai/frontend/web/src/common/util/parameterTranslation.ts index 83df66aab2..918d5caab3 100644 --- a/invokeai/frontend/web/src/common/util/parameterTranslation.ts +++ b/invokeai/frontend/web/src/common/util/parameterTranslation.ts @@ -8,7 +8,7 @@ import { CanvasState, isCanvasMaskLine, } from 'features/canvas/store/canvasTypes'; -import generateMask from 'features/canvas/util/generateMask'; +import createMaskStage from 'features/canvas/util/generateMask'; import { getCanvasBaseLayer } from 'features/canvas/util/konvaInstanceProvider'; import type { FacetoolType, @@ -257,7 +257,7 @@ export const frontendToBackendParameters = ( ...boundingBoxDimensions, }; - const { dataURL: maskDataURL, imageData: maskImageData } = generateMask( + const { dataURL: maskDataURL, imageData: maskImageData } = createMaskStage( isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], boundingBox ); diff --git a/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx b/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx index dd8963de7b..e111715e39 100644 --- a/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx +++ b/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx @@ -44,6 +44,12 @@ import IAICanvasRedoButton from './IAICanvasRedoButton'; import IAICanvasSettingsButtonPopover from './IAICanvasSettingsButtonPopover'; import IAICanvasToolChooserOptions from './IAICanvasToolChooserOptions'; import IAICanvasUndoButton from './IAICanvasUndoButton'; +import { + canvasCopiedToClipboard, + canvasDownloadedAsImage, + canvasMerged, + canvasSavedToGallery, +} from 'features/canvas/store/actions'; export const selector = createSelector( [systemSelector, canvasSelector, isStagingSelector], @@ -70,14 +76,8 @@ export const selector = createSelector( const IAICanvasToolbar = () => { const dispatch = useAppDispatch(); - const { - isProcessing, - isStaging, - isMaskEnabled, - layer, - tool, - shouldCropToBoundingBoxOnSave, - } = useAppSelector(selector); + const { isProcessing, isStaging, isMaskEnabled, layer, tool } = + useAppSelector(selector); const canvasBaseLayer = getCanvasBaseLayer(); const { t } = useTranslation(); @@ -183,42 +183,19 @@ const IAICanvasToolbar = () => { }; const handleMergeVisible = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: false, - shouldSetAsInitialImage: true, - }) - ); + dispatch(canvasMerged()); }; const handleSaveToGallery = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: shouldCropToBoundingBoxOnSave ? false : true, - cropToBoundingBox: shouldCropToBoundingBoxOnSave, - shouldSaveToGallery: true, - }) - ); + dispatch(canvasSavedToGallery()); }; const handleCopyImageToClipboard = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: shouldCropToBoundingBoxOnSave ? false : true, - cropToBoundingBox: shouldCropToBoundingBoxOnSave, - shouldCopy: true, - }) - ); + dispatch(canvasCopiedToClipboard()); }; const handleDownloadAsImage = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: shouldCropToBoundingBoxOnSave ? false : true, - cropToBoundingBox: shouldCropToBoundingBoxOnSave, - shouldDownload: true, - }) - ); + dispatch(canvasDownloadedAsImage()); }; const handleChangeLayer = (e: ChangeEvent) => { diff --git a/invokeai/frontend/web/src/features/canvas/store/actions.ts b/invokeai/frontend/web/src/features/canvas/store/actions.ts new file mode 100644 index 0000000000..2dce8af694 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/store/actions.ts @@ -0,0 +1,13 @@ +import { createAction } from '@reduxjs/toolkit'; + +export const canvasSavedToGallery = createAction('canvas/canvasSavedToGallery'); + +export const canvasCopiedToClipboard = createAction( + 'canvas/canvasCopiedToClipboard' +); + +export const canvasDownloadedAsImage = createAction( + 'canvas/canvasDownloadedAsImage' +); + +export const canvasMerged = createAction('canvas/canvasMerged'); diff --git a/invokeai/frontend/web/src/features/canvas/util/blobToDataURL.ts b/invokeai/frontend/web/src/features/canvas/util/blobToDataURL.ts new file mode 100644 index 0000000000..2443396105 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/blobToDataURL.ts @@ -0,0 +1,9 @@ +export const blobToDataURL = (blob: Blob): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = (_e) => resolve(reader.result as string); + reader.onerror = (_e) => reject(reader.error); + reader.onabort = (_e) => reject(new Error('Read aborted')); + reader.readAsDataURL(blob); + }); +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/copyBlobToClipboard.ts b/invokeai/frontend/web/src/features/canvas/util/copyBlobToClipboard.ts new file mode 100644 index 0000000000..e944e766b5 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/copyBlobToClipboard.ts @@ -0,0 +1,10 @@ +/** + * Copies a blob to the clipboard by calling navigator.clipboard.write(). + */ +export const copyBlobToClipboard = (blob: Blob) => { + navigator.clipboard.write([ + new ClipboardItem({ + [blob.type]: blob, + }), + ]); +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/createMaskStage.ts b/invokeai/frontend/web/src/features/canvas/util/createMaskStage.ts new file mode 100644 index 0000000000..96ac592711 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/createMaskStage.ts @@ -0,0 +1,61 @@ +import { CanvasMaskLine } from 'features/canvas/store/canvasTypes'; +import Konva from 'konva'; +import { IRect } from 'konva/lib/types'; + +/** + * Creates a stage from array of mask objects. + * We cannot just convert the mask layer to a blob because it uses a texture with transparent areas. + * So instead we create a new stage with the mask layer and composite it onto a white background. + */ +const createMaskStage = async ( + lines: CanvasMaskLine[], + boundingBox: IRect +): Promise => { + // create an offscreen canvas and add the mask to it + const { width, height } = boundingBox; + + const offscreenContainer = document.createElement('div'); + + const maskStage = new Konva.Stage({ + container: offscreenContainer, + width: width, + height: height, + }); + + const baseLayer = new Konva.Layer(); + const maskLayer = new Konva.Layer(); + + // composite the image onto the mask layer + baseLayer.add( + new Konva.Rect({ + ...boundingBox, + fill: 'white', + }) + ); + + lines.forEach((line) => + maskLayer.add( + new Konva.Line({ + points: line.points, + stroke: 'black', + strokeWidth: line.strokeWidth * 2, + tension: 0, + lineCap: 'round', + lineJoin: 'round', + shadowForStrokeEnabled: false, + globalCompositeOperation: + line.tool === 'brush' ? 'source-over' : 'destination-out', + }) + ) + ); + + maskStage.add(baseLayer); + maskStage.add(maskLayer); + + // you'd think we can't do this until we finish with the maskStage, but we can + offscreenContainer.remove(); + + return maskStage; +}; + +export default createMaskStage; diff --git a/invokeai/frontend/web/src/features/canvas/util/downloadBlob.ts b/invokeai/frontend/web/src/features/canvas/util/downloadBlob.ts new file mode 100644 index 0000000000..837e76c998 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/downloadBlob.ts @@ -0,0 +1,11 @@ +/** Download a blob as a file */ +export const downloadBlob = (blob: Blob, fileName: string) => { + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = fileName; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + a.remove(); +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/generateMask.ts b/invokeai/frontend/web/src/features/canvas/util/generateMask.ts deleted file mode 100644 index a5cd41ad10..0000000000 --- a/invokeai/frontend/web/src/features/canvas/util/generateMask.ts +++ /dev/null @@ -1,170 +0,0 @@ -// import { CanvasMaskLine } from 'features/canvas/store/canvasTypes'; -// import Konva from 'konva'; -// import { Stage } from 'konva/lib/Stage'; -// import { IRect } from 'konva/lib/types'; - -// /** -// * Generating a mask image from InpaintingCanvas.tsx is not as simple -// * as calling toDataURL() on the canvas, because the mask may be represented -// * by colored lines or transparency, or the user may have inverted the mask -// * display. -// * -// * So we need to regenerate the mask image by creating an offscreen canvas, -// * drawing the mask and compositing everything correctly to output a valid -// * mask image. -// */ -// export const getStageDataURL = (stage: Stage, boundingBox: IRect): string => { -// // create an offscreen canvas and add the mask to it -// // const { stage, offscreenContainer } = buildMaskStage(lines, boundingBox); - -// const dataURL = stage.toDataURL({ ...boundingBox }); - -// // const imageData = stage -// // .toCanvas() -// // .getContext('2d') -// // ?.getImageData( -// // boundingBox.x, -// // boundingBox.y, -// // boundingBox.width, -// // boundingBox.height -// // ); - -// // offscreenContainer.remove(); - -// // return { dataURL, imageData }; - -// return dataURL; -// }; - -// export const getStageImageData = ( -// stage: Stage, -// boundingBox: IRect -// ): ImageData | undefined => { -// const imageData = stage -// .toCanvas() -// .getContext('2d') -// ?.getImageData( -// boundingBox.x, -// boundingBox.y, -// boundingBox.width, -// boundingBox.height -// ); - -// return imageData; -// }; - -// export const buildMaskStage = ( -// lines: CanvasMaskLine[], -// boundingBox: IRect -// ): { stage: Stage; offscreenContainer: HTMLDivElement } => { -// // create an offscreen canvas and add the mask to it -// const { width, height } = boundingBox; - -// const offscreenContainer = document.createElement('div'); - -// const stage = new Konva.Stage({ -// container: offscreenContainer, -// width: width, -// height: height, -// }); - -// const baseLayer = new Konva.Layer(); -// const maskLayer = new Konva.Layer(); - -// // composite the image onto the mask layer -// baseLayer.add( -// new Konva.Rect({ -// ...boundingBox, -// fill: 'white', -// }) -// ); - -// lines.forEach((line) => -// maskLayer.add( -// new Konva.Line({ -// points: line.points, -// stroke: 'black', -// strokeWidth: line.strokeWidth * 2, -// tension: 0, -// lineCap: 'round', -// lineJoin: 'round', -// shadowForStrokeEnabled: false, -// globalCompositeOperation: -// line.tool === 'brush' ? 'source-over' : 'destination-out', -// }) -// ) -// ); - -// stage.add(baseLayer); -// stage.add(maskLayer); - -// return { stage, offscreenContainer }; -// }; - -import { CanvasMaskLine } from 'features/canvas/store/canvasTypes'; -import Konva from 'konva'; -import { IRect } from 'konva/lib/types'; -import { canvasToBlob } from './canvasToBlob'; - -/** - * Generating a mask image from InpaintingCanvas.tsx is not as simple - * as calling toDataURL() on the canvas, because the mask may be represented - * by colored lines or transparency, or the user may have inverted the mask - * display. - * - * So we need to regenerate the mask image by creating an offscreen canvas, - * drawing the mask and compositing everything correctly to output a valid - * mask image. - */ -const generateMask = async (lines: CanvasMaskLine[], boundingBox: IRect) => { - // create an offscreen canvas and add the mask to it - const { width, height } = boundingBox; - - const offscreenContainer = document.createElement('div'); - - const stage = new Konva.Stage({ - container: offscreenContainer, - width: width, - height: height, - }); - - const baseLayer = new Konva.Layer(); - const maskLayer = new Konva.Layer(); - - // composite the image onto the mask layer - baseLayer.add( - new Konva.Rect({ - ...boundingBox, - fill: 'white', - }) - ); - - lines.forEach((line) => - maskLayer.add( - new Konva.Line({ - points: line.points, - stroke: 'black', - strokeWidth: line.strokeWidth * 2, - tension: 0, - lineCap: 'round', - lineJoin: 'round', - shadowForStrokeEnabled: false, - globalCompositeOperation: - line.tool === 'brush' ? 'source-over' : 'destination-out', - }) - ) - ); - - stage.add(baseLayer); - stage.add(maskLayer); - - const maskDataURL = stage.toDataURL(boundingBox); - - const maskBlob = await canvasToBlob(stage.toCanvas(boundingBox)); - - offscreenContainer.remove(); - - return { maskDataURL, maskBlob }; -}; - -export default generateMask; diff --git a/invokeai/frontend/web/src/features/canvas/util/getBaseLayerBlob.ts b/invokeai/frontend/web/src/features/canvas/util/getBaseLayerBlob.ts new file mode 100644 index 0000000000..a576551d72 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/getBaseLayerBlob.ts @@ -0,0 +1,38 @@ +import { getCanvasBaseLayer } from './konvaInstanceProvider'; +import { RootState } from 'app/store/store'; +import { konvaNodeToBlob } from './konvaNodeToBlob'; + +export const getBaseLayerBlob = async ( + state: RootState, + withoutBoundingBox?: boolean +) => { + const canvasBaseLayer = getCanvasBaseLayer(); + + if (!canvasBaseLayer) { + return; + } + + const { + shouldCropToBoundingBoxOnSave, + boundingBoxCoordinates, + boundingBoxDimensions, + } = state.canvas; + + const clonedBaseLayer = canvasBaseLayer.clone(); + + clonedBaseLayer.scale({ x: 1, y: 1 }); + + const absPos = clonedBaseLayer.getAbsolutePosition(); + + const boundingBox = + shouldCropToBoundingBoxOnSave && !withoutBoundingBox + ? { + x: boundingBoxCoordinates.x + absPos.x, + y: boundingBoxCoordinates.y + absPos.y, + width: boundingBoxDimensions.width, + height: boundingBoxDimensions.height, + } + : clonedBaseLayer.getClientRect(); + + return konvaNodeToBlob(clonedBaseLayer, boundingBox); +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts index 28900fcc44..21a33aa349 100644 --- a/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts @@ -2,17 +2,15 @@ import { RootState } from 'app/store/store'; import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider'; import { isCanvasMaskLine } from '../store/canvasTypes'; import { log } from 'app/logging/useLogger'; -import { - areAnyPixelsBlack, - getImageDataTransparency, -} from 'common/util/arrayBuffer'; -import openBase64ImageInTab from 'common/util/openBase64ImageInTab'; -import generateMask from './generateMask'; -import { dataURLToImageData } from './dataURLToImageData'; -import { canvasToBlob } from './canvasToBlob'; +import createMaskStage from './createMaskStage'; +import { konvaNodeToImageData } from './konvaNodeToImageData'; +import { konvaNodeToBlob } from './konvaNodeToBlob'; const moduleLog = log.child({ namespace: 'getCanvasDataURLs' }); +/** + * Gets Blob and ImageData objects for the base and mask layers + */ export const getCanvasData = async (state: RootState) => { const canvasBaseLayer = getCanvasBaseLayer(); const canvasStage = getCanvasStage(); @@ -27,10 +25,6 @@ export const getCanvasData = async (state: RootState) => { boundingBoxCoordinates, boundingBoxDimensions, isMaskEnabled, - shouldPreserveMaskedArea, - boundingBoxScaleMethod: boundingBoxScale, - scaledBoundingBoxDimensions, - stageCoordinates, } = state.canvas; const boundingBox = { @@ -38,18 +32,10 @@ export const getCanvasData = async (state: RootState) => { ...boundingBoxDimensions, }; - // generationParameters.fit = false; - - // generationParameters.strength = img2imgStrength; - - // generationParameters.invert_mask = shouldPreserveMaskedArea; - - // generationParameters.bounding_box = boundingBox; - - // clone the base layer so we don't affect the actual canvas during scaling + // Clone the base layer so we don't affect the visible base layer const clonedBaseLayer = canvasBaseLayer.clone(); - // scale to 1 so we get an uninterpolated image + // Scale it to 100% so we get full resolution clonedBaseLayer.scale({ x: 1, y: 1 }); // absolute position is needed to get the bounding box coords relative to the base layer @@ -62,73 +48,25 @@ export const getCanvasData = async (state: RootState) => { height: boundingBox.height, }; - // get a dataURL of the bbox'd region (will convert this to an ImageData to check its transparency) - const baseDataURL = clonedBaseLayer.toDataURL(offsetBoundingBox); - - // get a blob (will upload this as the canvas intermediate) - const baseBlob = await canvasToBlob( - clonedBaseLayer.toCanvas(offsetBoundingBox) + // For the base layer, use the offset boundingBox + const baseBlob = await konvaNodeToBlob(clonedBaseLayer, offsetBoundingBox); + const baseImageData = await konvaNodeToImageData( + clonedBaseLayer, + offsetBoundingBox ); - // build a new mask layer and get its dataURL and blob - const { maskDataURL, maskBlob } = await generateMask( - isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], + // For the mask layer, use the normal boundingBox + const maskStage = await createMaskStage( + isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], // only include mask lines, and only if mask is enabled boundingBox ); - - // convert to ImageData (via pure jank) - const baseImageData = await dataURLToImageData( - baseDataURL, - boundingBox.width, - boundingBox.height - ); - - // convert to ImageData (via pure jank) - const maskImageData = await dataURLToImageData( - maskDataURL, - boundingBox.width, - boundingBox.height - ); - - // check transparency - const { - isPartiallyTransparent: baseIsPartiallyTransparent, - isFullyTransparent: baseIsFullyTransparent, - } = getImageDataTransparency(baseImageData.data); - - // check mask for black - const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData.data); - - if (state.system.enableImageDebugging) { - openBase64ImageInTab([ - { base64: maskDataURL, caption: 'mask b64' }, - { base64: baseDataURL, caption: 'image b64' }, - ]); - } - - // generationParameters.init_img = imageDataURL; - // generationParameters.progress_images = false; - - // if (boundingBoxScale !== 'none') { - // generationParameters.inpaint_width = scaledBoundingBoxDimensions.width; - // generationParameters.inpaint_height = scaledBoundingBoxDimensions.height; - // } - - // generationParameters.seam_size = seamSize; - // generationParameters.seam_blur = seamBlur; - // generationParameters.seam_strength = seamStrength; - // generationParameters.seam_steps = seamSteps; - // generationParameters.tile_size = tileSize; - // generationParameters.infill_method = infillMethod; - // generationParameters.force_outpaint = false; + const maskBlob = await konvaNodeToBlob(maskStage, boundingBox); + const maskImageData = await konvaNodeToImageData(maskStage, boundingBox); return { - baseDataURL, baseBlob, - maskDataURL, + baseImageData, maskBlob, - baseIsPartiallyTransparent, - baseIsFullyTransparent, - doesMaskHaveBlackPixels, + maskImageData, }; }; diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts new file mode 100644 index 0000000000..5b38ecf938 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts @@ -0,0 +1,31 @@ +import { + areAnyPixelsBlack, + getImageDataTransparency, +} from 'common/util/arrayBuffer'; + +export const getCanvasGenerationMode = ( + baseImageData: ImageData, + maskImageData: ImageData +) => { + const { + isPartiallyTransparent: baseIsPartiallyTransparent, + isFullyTransparent: baseIsFullyTransparent, + } = getImageDataTransparency(baseImageData.data); + + // check mask for black + const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData.data); + + if (baseIsPartiallyTransparent) { + if (baseIsFullyTransparent) { + return 'txt2img'; + } + + return 'outpaint'; + } else { + if (doesMaskHaveBlackPixels) { + return 'inpaint'; + } + + return 'img2img'; + } +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/konvaNodeToBlob.ts b/invokeai/frontend/web/src/features/canvas/util/konvaNodeToBlob.ts new file mode 100644 index 0000000000..8e47398764 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/konvaNodeToBlob.ts @@ -0,0 +1,16 @@ +import Konva from 'konva'; +import { IRect } from 'konva/lib/types'; +import { canvasToBlob } from './canvasToBlob'; + +/** + * Converts a Konva node to a Blob + * @param node - The Konva node to convert to a Blob + * @param boundingBox - The bounding box to crop to + * @returns A Promise that resolves with Blob of the node cropped to the bounding box + */ +export const konvaNodeToBlob = async ( + node: Konva.Node, + boundingBox: IRect +): Promise => { + return await canvasToBlob(node.toCanvas(boundingBox)); +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/konvaNodeToDataURL.ts b/invokeai/frontend/web/src/features/canvas/util/konvaNodeToDataURL.ts new file mode 100644 index 0000000000..5d0aaf5443 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/konvaNodeToDataURL.ts @@ -0,0 +1,16 @@ +import Konva from 'konva'; +import { IRect } from 'konva/lib/types'; + +/** + * Converts a Konva node to a dataURL + * @param node - The Konva node to convert to a dataURL + * @param boundingBox - The bounding box to crop to + * @returns A dataURL of the node cropped to the bounding box + */ +export const konvaNodeToDataURL = ( + node: Konva.Node, + boundingBox: IRect +): string => { + // get a dataURL of the bbox'd region + return node.toDataURL(boundingBox); +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/konvaNodeToImageData.ts b/invokeai/frontend/web/src/features/canvas/util/konvaNodeToImageData.ts new file mode 100644 index 0000000000..b8337a0cc0 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/konvaNodeToImageData.ts @@ -0,0 +1,23 @@ +import Konva from 'konva'; +import { IRect } from 'konva/lib/types'; +import { dataURLToImageData } from './dataURLToImageData'; + +/** + * Converts a Konva node to an ImageData object + * @param node - The Konva node to convert to an ImageData object + * @param boundingBox - The bounding box to crop to + * @returns A Promise that resolves with ImageData object of the node cropped to the bounding box + */ +export const konvaNodeToImageData = async ( + node: Konva.Node, + boundingBox: IRect +): Promise => { + // get a dataURL of the bbox'd region + const dataURL = node.toDataURL(boundingBox); + + return await dataURLToImageData( + dataURL, + boundingBox.width, + boundingBox.height + ); +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts index 45deed7070..227e4b3a12 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts @@ -14,9 +14,11 @@ import { buildRangeNode } from '../nodeBuilders/buildRangeNode'; import { buildIterateNode } from '../nodeBuilders/buildIterateNode'; import { buildEdges } from '../edgeBuilders/buildEdges'; import { getCanvasData } from 'features/canvas/util/getCanvasData'; -import { getGenerationMode } from '../getGenerationMode'; import { log } from 'app/logging/useLogger'; import { buildInpaintNode } from '../nodeBuilders/buildInpaintNode'; +import { getCanvasGenerationMode } from 'features/canvas/util/getCanvasGenerationMode'; +import { blobToDataURL } from 'features/canvas/util/blobToDataURL'; +import openBase64ImageInTab from 'common/util/openBase64ImageInTab'; const moduleLog = log.child({ namespace: 'buildCanvasGraph' }); @@ -61,37 +63,25 @@ export const buildCanvasGraphAndBlobs = async ( } | undefined > => { - const c = await getCanvasData(state); + const canvasData = await getCanvasData(state); - if (!c) { - moduleLog.error('Unable to create canvas graph'); + if (!canvasData) { + moduleLog.error('Unable to create canvas data'); return; } - const { - baseBlob, - maskBlob, - baseIsPartiallyTransparent, - baseIsFullyTransparent, - doesMaskHaveBlackPixels, - } = c; + const { baseBlob, baseImageData, maskBlob, maskImageData } = canvasData; - moduleLog.debug( - { - data: { - baseIsPartiallyTransparent, - baseIsFullyTransparent, - doesMaskHaveBlackPixels, - }, - }, - 'Built canvas data' - ); + const generationMode = getCanvasGenerationMode(baseImageData, maskImageData); - const generationMode = getGenerationMode( - baseIsPartiallyTransparent, - baseIsFullyTransparent, - doesMaskHaveBlackPixels - ); + if (state.system.enableImageDebugging) { + const baseDataURL = await blobToDataURL(baseBlob); + const maskDataURL = await blobToDataURL(maskBlob); + openBase64ImageInTab([ + { base64: maskDataURL, caption: 'mask b64' }, + { base64: baseDataURL, caption: 'image b64' }, + ]); + } moduleLog.debug(`Generation mode: ${generationMode}`); @@ -104,8 +94,14 @@ export const buildCanvasGraphAndBlobs = async ( } if (baseNode.type === 'inpaint') { - const { seamSize, seamBlur, seamSteps, seamStrength, tileSize } = - state.generation; + const { + seamSize, + seamBlur, + seamSteps, + seamStrength, + tileSize, + infillMethod, + } = state.generation; // generationParameters.invert_mask = shouldPreserveMaskedArea; // if (boundingBoxScale !== 'none') { @@ -117,7 +113,7 @@ export const buildCanvasGraphAndBlobs = async ( baseNode.seam_strength = seamStrength; baseNode.seam_steps = seamSteps; baseNode.tile_size = tileSize; - // baseNode.infill_method = infillMethod; + baseNode.infill_method = infillMethod as InpaintInvocation['infill_method']; // baseNode.force_outpaint = false; } diff --git a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasCopyToClipboard.tsx b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasCopyToClipboard.tsx index 4d1241c132..5e23a4c0d6 100644 --- a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasCopyToClipboard.tsx +++ b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasCopyToClipboard.tsx @@ -1,8 +1,8 @@ import { RootState } from 'app/store/store'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import IAIIconButton from 'common/components/IAIIconButton'; +import { canvasCopiedToClipboard } from 'features/canvas/store/actions'; import { isStagingSelector } from 'features/canvas/store/canvasSelectors'; -import { mergeAndUploadCanvas } from 'features/canvas/store/thunks/mergeAndUploadCanvas'; import { getCanvasBaseLayer } from 'features/canvas/util/konvaInstanceProvider'; import { useHotkeys } from 'react-hotkeys-hook'; import { useTranslation } from 'react-i18next'; @@ -16,10 +16,6 @@ export default function UnifiedCanvasCopyToClipboard() { (state: RootState) => state.system.isProcessing ); - const shouldCropToBoundingBoxOnSave = useAppSelector( - (state: RootState) => state.canvas.shouldCropToBoundingBoxOnSave - ); - const dispatch = useAppDispatch(); const { t } = useTranslation(); @@ -36,13 +32,7 @@ export default function UnifiedCanvasCopyToClipboard() { ); const handleCopyImageToClipboard = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: shouldCropToBoundingBoxOnSave ? false : true, - cropToBoundingBox: shouldCropToBoundingBoxOnSave, - shouldCopy: true, - }) - ); + dispatch(canvasCopiedToClipboard()); }; return ( diff --git a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasDownloadImage.tsx b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasDownloadImage.tsx index 2be9db2afd..1039c5f364 100644 --- a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasDownloadImage.tsx +++ b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasBeta/UnifiedCanvasToolbar/UnifiedCanvasDownloadImage.tsx @@ -1,8 +1,7 @@ -import { RootState } from 'app/store/store'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import IAIIconButton from 'common/components/IAIIconButton'; +import { canvasDownloadedAsImage } from 'features/canvas/store/actions'; import { isStagingSelector } from 'features/canvas/store/canvasSelectors'; -import { mergeAndUploadCanvas } from 'features/canvas/store/thunks/mergeAndUploadCanvas'; import { getCanvasBaseLayer } from 'features/canvas/util/konvaInstanceProvider'; import { useHotkeys } from 'react-hotkeys-hook'; import { useTranslation } from 'react-i18next'; @@ -16,14 +15,6 @@ export default function UnifiedCanvasDownloadImage() { const isStaging = useAppSelector(isStagingSelector); - const isProcessing = useAppSelector( - (state: RootState) => state.system.isProcessing - ); - - const shouldCropToBoundingBoxOnSave = useAppSelector( - (state: RootState) => state.canvas.shouldCropToBoundingBoxOnSave - ); - useHotkeys( ['shift+d'], () => { @@ -33,18 +24,13 @@ export default function UnifiedCanvasDownloadImage() { enabled: () => !isStaging, preventDefault: true, }, - [canvasBaseLayer, isProcessing] + [canvasBaseLayer] ); const handleDownloadAsImage = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: shouldCropToBoundingBoxOnSave ? false : true, - cropToBoundingBox: shouldCropToBoundingBoxOnSave, - shouldDownload: true, - }) - ); + dispatch(canvasDownloadedAsImage()); }; + return ( { - dispatch( - mergeAndUploadCanvas({ - cropVisible: false, - shouldSetAsInitialImage: true, - }) - ); + dispatch(canvasMerged()); }; return ( state.system.isProcessing ); - const shouldCropToBoundingBoxOnSave = useAppSelector( - (state: RootState) => state.canvas.shouldCropToBoundingBoxOnSave - ); const dispatch = useAppDispatch(); const { t } = useTranslation(); @@ -34,14 +31,9 @@ export default function UnifiedCanvasSaveToGallery() { ); const handleSaveToGallery = () => { - dispatch( - mergeAndUploadCanvas({ - cropVisible: shouldCropToBoundingBoxOnSave ? false : true, - cropToBoundingBox: shouldCropToBoundingBoxOnSave, - shouldSaveToGallery: true, - }) - ); + dispatch(canvasSavedToGallery()); }; + return (