From cee21ca0822309aecc549508b4c1f3cdcc33bd3c Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Wed, 3 May 2023 08:55:36 +1000 Subject: [PATCH] feat(ui): wip canvas nodes migration 2 --- .../web/src/common/util/arrayBuffer.ts | 26 ----- .../canvas/hooks/useGetCanvasNodeType.ts | 30 ++++++ .../src/features/canvas/util/generateMask.ts | 62 +++++++---- .../features/canvas/util/getCanvasDataURLs.ts | 64 +++++++---- .../features/canvas/util/getCanvasNodeType.ts | 102 ++++++++++++++++++ .../gallery/components/HoverableImage.tsx | 1 + .../UnifiedCanvas/UnifiedCanvasWorkarea.tsx | 1 + 7 files changed, 220 insertions(+), 66 deletions(-) create mode 100644 invokeai/frontend/web/src/features/canvas/hooks/useGetCanvasNodeType.ts create mode 100644 invokeai/frontend/web/src/features/canvas/util/getCanvasNodeType.ts diff --git a/invokeai/frontend/web/src/common/util/arrayBuffer.ts b/invokeai/frontend/web/src/common/util/arrayBuffer.ts index da0fe38d35..779d9b1b17 100644 --- a/invokeai/frontend/web/src/common/util/arrayBuffer.ts +++ b/invokeai/frontend/web/src/common/util/arrayBuffer.ts @@ -1,18 +1,3 @@ -export const getIsImageDataPartiallyTransparent = (imageData: ImageData) => { - let hasTransparency = false; - let isFullyTransparent = true; - const len = imageData.data.length; - let i = 3; - for (i; i < len; i += 4) { - if (imageData.data[i] !== 0) { - isFullyTransparent = false; - } else { - hasTransparency = true; - } - } - return { hasTransparency, isFullyTransparent }; -}; - export const getImageDataTransparency = (imageData: ImageData) => { let isFullyTransparent = true; let isPartiallyTransparent = false; @@ -46,14 +31,3 @@ export const areAnyPixelsBlack = (imageData: ImageData) => { } return false; }; - -export const getIsImageDataWhite = (imageData: ImageData) => { - const len = imageData.data.length; - let i = 0; - for (i; i < len; ) { - if (imageData.data[i++] !== 255) { - return false; - } - } - return true; -}; diff --git a/invokeai/frontend/web/src/features/canvas/hooks/useGetCanvasNodeType.ts b/invokeai/frontend/web/src/features/canvas/hooks/useGetCanvasNodeType.ts new file mode 100644 index 0000000000..91f8e50d29 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/hooks/useGetCanvasNodeType.ts @@ -0,0 +1,30 @@ +import { createSelector } from '@reduxjs/toolkit'; +import { useAppSelector } from 'app/store/storeHooks'; +import { canvasSelector } from '../store/canvasSelectors'; +import { useMemo } from 'react'; +import { getCanvasNodeType } from '../util/getCanvasNodeType'; + +const selector = createSelector(canvasSelector, (canvas) => { + const { + layerState: { objects }, + boundingBoxCoordinates, + boundingBoxDimensions, + stageScale, + isMaskEnabled, + } = canvas; + return { + objects, + boundingBoxCoordinates, + boundingBoxDimensions, + stageScale, + isMaskEnabled, + }; +}); + +export const useGetCanvasNodeType = () => { + const data = useAppSelector(selector); + + const nodeType = useMemo(() => getCanvasNodeType(data), [data]); + + return nodeType; +}; diff --git a/invokeai/frontend/web/src/features/canvas/util/generateMask.ts b/invokeai/frontend/web/src/features/canvas/util/generateMask.ts index 32e0fecfd0..db88dd1ad3 100644 --- a/invokeai/frontend/web/src/features/canvas/util/generateMask.ts +++ b/invokeai/frontend/web/src/features/canvas/util/generateMask.ts @@ -1,5 +1,6 @@ import { CanvasMaskLine } from 'features/canvas/store/canvasTypes'; import Konva from 'konva'; +import { Stage } from 'konva/lib/Stage'; import { IRect } from 'konva/lib/types'; /** @@ -12,10 +13,50 @@ import { IRect } from 'konva/lib/types'; * drawing the mask and compositing everything correctly to output a valid * mask image. */ -const generateMask = ( +export const getStageDataURL = (stage: Stage, boundingBox: IRect): string => { + // create an offscreen canvas and add the mask to it + // const { stage, offscreenContainer } = buildMaskStage(lines, boundingBox); + + const dataURL = stage.toDataURL({ ...boundingBox }); + + // const imageData = stage + // .toCanvas() + // .getContext('2d') + // ?.getImageData( + // boundingBox.x, + // boundingBox.y, + // boundingBox.width, + // boundingBox.height + // ); + + // offscreenContainer.remove(); + + // return { dataURL, imageData }; + + return dataURL; +}; + +export const getStageImageData = ( + stage: Stage, + boundingBox: IRect +): ImageData | undefined => { + const imageData = stage + .toCanvas() + .getContext('2d') + ?.getImageData( + boundingBox.x, + boundingBox.y, + boundingBox.width, + boundingBox.height + ); + + return imageData; +}; + +export const buildMaskStage = ( lines: CanvasMaskLine[], boundingBox: IRect -): { dataURL: string; imageData: ImageData } => { +): { stage: Stage; offscreenContainer: HTMLDivElement } => { // create an offscreen canvas and add the mask to it const { width, height } = boundingBox; @@ -57,20 +98,5 @@ const generateMask = ( stage.add(baseLayer); stage.add(maskLayer); - const dataURL = stage.toDataURL({ ...boundingBox }); - const imageData = stage - .toCanvas() - .getContext('2d') - ?.getImageData( - boundingBox.x, - boundingBox.y, - boundingBox.width, - boundingBox.height - ); - - offscreenContainer.remove(); - - return { dataURL, imageData }; + return { stage, offscreenContainer }; }; - -export default generateMask; diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasDataURLs.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasDataURLs.ts index fdc03bc48a..e309057eee 100644 --- a/invokeai/frontend/web/src/features/canvas/util/getCanvasDataURLs.ts +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasDataURLs.ts @@ -1,24 +1,27 @@ import { RootState } from 'app/store/store'; import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider'; import { isCanvasMaskLine } from '../store/canvasTypes'; -import generateMask from './generateMask'; +import { + buildMaskStage, + getStageDataURL, + getStageImageData, +} from './generateMask'; import { log } from 'app/logging/useLogger'; import { areAnyPixelsBlack, getImageDataTransparency, - getIsImageDataWhite, } from 'common/util/arrayBuffer'; import openBase64ImageInTab from 'common/util/openBase64ImageInTab'; +import { masks } from 'dateformat'; + +const moduleLog = log.child({ namespace: 'getCanvasDataURLs' }); export const getCanvasDataURLs = (state: RootState) => { const canvasBaseLayer = getCanvasBaseLayer(); const canvasStage = getCanvasStage(); if (!canvasBaseLayer || !canvasStage) { - log.error( - { namespace: 'getCanvasDataURLs' }, - 'Unable to find canvas / stage' - ); + moduleLog.error('Unable to find canvas / stage'); return; } @@ -55,30 +58,46 @@ export const getCanvasDataURLs = (state: RootState) => { const absPos = canvasBaseLayer.getAbsolutePosition(); - const { dataURL: maskDataURL, imageData: maskImageData } = generateMask( - isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], - { - x: boundingBox.x + absPos.x, - y: boundingBox.y + absPos.y, - width: boundingBox.width, - height: boundingBox.height, - } - ); - - const baseDataURL = canvasBaseLayer.toDataURL({ + const offsetBoundingBox = { x: boundingBox.x + absPos.x, y: boundingBox.y + absPos.y, width: boundingBox.width, height: boundingBox.height, - }); + }; + + const { stage: maskStage, offscreenContainer } = buildMaskStage( + isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], + offsetBoundingBox + ); + + const maskDataURL = maskStage.toDataURL(offsetBoundingBox); + + const maskImageData = maskStage + .toCanvas() + .getContext('2d') + ?.getImageData( + offsetBoundingBox.x, + offsetBoundingBox.y, + offsetBoundingBox.width, + offsetBoundingBox.height + ); + + offscreenContainer.remove(); + + if (!maskImageData) { + moduleLog.error('Unable to get mask stage context'); + return; + } + + const baseDataURL = canvasBaseLayer.toDataURL(offsetBoundingBox); const ctx = canvasBaseLayer.getContext(); const baseImageData = ctx.getImageData( - boundingBox.x + absPos.x, - boundingBox.y + absPos.y, - boundingBox.width, - boundingBox.height + offsetBoundingBox.x, + offsetBoundingBox.y, + offsetBoundingBox.width, + offsetBoundingBox.height ); const { @@ -86,6 +105,7 @@ export const getCanvasDataURLs = (state: RootState) => { isFullyTransparent: baseIsFullyTransparent, } = getImageDataTransparency(baseImageData); + // const doesMaskHaveBlackPixels = false; const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData); if (state.system.enableImageDebugging) { diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasNodeType.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasNodeType.ts new file mode 100644 index 0000000000..baa250031b --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasNodeType.ts @@ -0,0 +1,102 @@ +import { RootState } from 'app/store/store'; +import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider'; +import { + CanvasObject, + Dimensions, + isCanvasMaskLine, +} from '../store/canvasTypes'; +import { buildMaskStage, getStageImageData } from './generateMask'; +import { log } from 'app/logging/useLogger'; +import { + areAnyPixelsBlack, + getImageDataTransparency, +} from 'common/util/arrayBuffer'; +import { getNodeType } from 'features/nodes/util/getNodeType'; +import { Vector2d } from 'konva/lib/types'; + +const moduleLog = log.child({ namespace: 'getCanvasNodeTypes' }); + +export type GetCanvasNodeTypeArg = { + objects: CanvasObject[]; + boundingBoxCoordinates: Vector2d; + boundingBoxDimensions: Dimensions; + stageScale: number; + isMaskEnabled: boolean; +}; + +export const getCanvasNodeType = (arg: GetCanvasNodeTypeArg) => { + const canvasBaseLayer = getCanvasBaseLayer(); + const canvasStage = getCanvasStage(); + + if (!canvasBaseLayer || !canvasStage) { + moduleLog.error('Unable to find canvas / stage'); + return; + } + + const { + objects, + boundingBoxCoordinates, + boundingBoxDimensions, + stageScale, + isMaskEnabled, + } = arg; + + const boundingBox = { + ...boundingBoxCoordinates, + ...boundingBoxDimensions, + }; + + const tempScale = canvasBaseLayer.scale(); + + canvasBaseLayer.scale({ + x: 1 / stageScale, + y: 1 / stageScale, + }); + + const absPos = canvasBaseLayer.getAbsolutePosition(); + + const scaledBoundingBox = { + x: boundingBox.x + absPos.x, + y: boundingBox.y + absPos.y, + width: boundingBox.width, + height: boundingBox.height, + }; + + const { stage: maskStage, offscreenContainer } = buildMaskStage( + isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], + scaledBoundingBox + ); + + const maskImageData = getStageImageData(maskStage, scaledBoundingBox); + + offscreenContainer.remove(); + + if (!maskImageData) { + moduleLog.error('Unable to get mask stage context'); + return; + } + + const ctx = canvasBaseLayer.getContext(); + + const baseImageData = ctx.getImageData( + boundingBox.x + absPos.x, + boundingBox.y + absPos.y, + boundingBox.width, + boundingBox.height + ); + + canvasBaseLayer.scale(tempScale); + + const { + isPartiallyTransparent: baseIsPartiallyTransparent, + isFullyTransparent: baseIsFullyTransparent, + } = getImageDataTransparency(baseImageData); + + const doesMaskHaveBlackPixels = areAnyPixelsBlack(maskImageData); + + return getNodeType( + baseIsPartiallyTransparent, + baseIsFullyTransparent, + doesMaskHaveBlackPixels + ); +}; diff --git a/invokeai/frontend/web/src/features/gallery/components/HoverableImage.tsx b/invokeai/frontend/web/src/features/gallery/components/HoverableImage.tsx index ba100ecacc..80d2facdcf 100644 --- a/invokeai/frontend/web/src/features/gallery/components/HoverableImage.tsx +++ b/invokeai/frontend/web/src/features/gallery/components/HoverableImage.tsx @@ -134,6 +134,7 @@ const HoverableImage = memo((props: HoverableImageProps) => { const handleDragStart = useCallback( (e: DragEvent) => { + console.log('dragging'); e.dataTransfer.setData('invokeai/imageName', image.name); e.dataTransfer.setData('invokeai/imageType', image.type); e.dataTransfer.effectAllowed = 'move'; diff --git a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasWorkarea.tsx b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasWorkarea.tsx index dd32295f3c..19e8f372b2 100644 --- a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasWorkarea.tsx +++ b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasWorkarea.tsx @@ -34,6 +34,7 @@ import ParametersSlide from '../../common/ParametersSlide'; import UnifiedCanvasParameters from './UnifiedCanvasParameters'; import UnifiedCanvasContentBeta from './UnifiedCanvasBeta/UnifiedCanvasContentBeta'; import UnifiedCanvasContent from './UnifiedCanvasContent'; +import { useGetCanvasNodeType } from 'features/canvas/hooks/useGetCanvasNodeType'; const CanvasWorkspace = () => { const shouldPinParametersPanel = useAppSelector(