From 28031ead708fadf51ba38a9b162c2b0a72f851ee Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sun, 23 Jul 2023 22:34:31 +1000 Subject: [PATCH 1/2] feat(ui): display canvas generation mode in status text - use the existing logic to determine if generation is txt2img, img2img, inpaint or outpaint - technically `outpaint` and `inpaint` are the same, just display "Inpaint" if its either - debounce this by 1s to prevent jank --- .../listeners/userInvokedCanvas.ts | 2 +- .../canvas/components/IAICanvasStatusText.tsx | 3 +- .../IAICanvasToolbar/IAICanvasToolbar.tsx | 21 ++++--- .../src/features/canvas/store/canvasSlice.ts | 5 ++ .../src/features/canvas/store/canvasTypes.ts | 3 + .../src/features/canvas/util/getCanvasData.ts | 7 +-- .../canvas/util/getCanvasGenerationMode.ts | 3 +- .../Canvas/GenerationModeStatusText.tsx | 55 +++++++++++++++++++ 8 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts index 2ef62aed7b..17b2eeed46 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts @@ -40,7 +40,7 @@ export const addUserInvokedCanvasListener = () => { const state = getState(); // Build canvas blobs - const canvasBlobsAndImageData = await getCanvasData(state); + const canvasBlobsAndImageData = await getCanvasData(state.canvas); if (!canvasBlobsAndImageData) { log.error('Unable to create canvas data'); diff --git a/invokeai/frontend/web/src/features/canvas/components/IAICanvasStatusText.tsx b/invokeai/frontend/web/src/features/canvas/components/IAICanvasStatusText.tsx index 69bf628a39..8c1dfbb86f 100644 --- a/invokeai/frontend/web/src/features/canvas/components/IAICanvasStatusText.tsx +++ b/invokeai/frontend/web/src/features/canvas/components/IAICanvasStatusText.tsx @@ -2,8 +2,8 @@ import { Box, Flex } from '@chakra-ui/react'; import { createSelector } from '@reduxjs/toolkit'; import { useAppSelector } from 'app/store/storeHooks'; import { canvasSelector } from 'features/canvas/store/canvasSelectors'; +import GenerationModeStatusText from 'features/parameters/components/Parameters/Canvas/GenerationModeStatusText'; import { isEqual } from 'lodash-es'; - import { useTranslation } from 'react-i18next'; import roundToHundreth from '../util/roundToHundreth'; import IAICanvasStatusTextCursorPos from './IAICanvasStatusText/IAICanvasStatusTextCursorPos'; @@ -110,6 +110,7 @@ const IAICanvasStatusText = () => { }, }} > + { }} > - ) => { + state.generationMode = action.payload; + }, }, extraReducers: (builder) => { builder.addCase(sessionCanceled.pending, (state) => { @@ -955,6 +959,7 @@ export const { stagingAreaInitialized, canvasSessionIdChanged, setShouldAntialias, + generationModeChanged, } = canvasSlice.actions; export default canvasSlice.reducer; diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts index 48d59395ab..ba85a7e132 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts @@ -168,4 +168,7 @@ export interface CanvasState { stageDimensions: Dimensions; stageScale: number; tool: CanvasTool; + generationMode?: GenerationMode; } + +export type GenerationMode = 'txt2img' | 'img2img' | 'inpaint' | 'outpaint'; diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts index d37ee7b8d0..855420f78a 100644 --- a/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts @@ -1,6 +1,5 @@ import { logger } from 'app/logging/logger'; -import { RootState } from 'app/store/store'; -import { isCanvasMaskLine } from '../store/canvasTypes'; +import { CanvasState, isCanvasMaskLine } from '../store/canvasTypes'; import createMaskStage from './createMaskStage'; import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider'; import { konvaNodeToBlob } from './konvaNodeToBlob'; @@ -9,7 +8,7 @@ import { konvaNodeToImageData } from './konvaNodeToImageData'; /** * Gets Blob and ImageData objects for the base and mask layers */ -export const getCanvasData = async (state: RootState) => { +export const getCanvasData = async (canvasState: CanvasState) => { const log = logger('canvas'); const canvasBaseLayer = getCanvasBaseLayer(); @@ -26,7 +25,7 @@ export const getCanvasData = async (state: RootState) => { boundingBoxDimensions, isMaskEnabled, shouldPreserveMaskedArea, - } = state.canvas; + } = canvasState; const boundingBox = { ...boundingBoxCoordinates, diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts index 5b38ecf938..d3e8792690 100644 --- a/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts @@ -2,11 +2,12 @@ import { areAnyPixelsBlack, getImageDataTransparency, } from 'common/util/arrayBuffer'; +import { GenerationMode } from '../store/canvasTypes'; export const getCanvasGenerationMode = ( baseImageData: ImageData, maskImageData: ImageData -) => { +): GenerationMode => { const { isPartiallyTransparent: baseIsPartiallyTransparent, isFullyTransparent: baseIsFullyTransparent, diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx new file mode 100644 index 0000000000..5c6bbd0ba3 --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx @@ -0,0 +1,55 @@ +import { Box } from '@chakra-ui/react'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import { generationModeChanged } from 'features/canvas/store/canvasSlice'; +import { getCanvasData } from 'features/canvas/util/getCanvasData'; +import { getCanvasGenerationMode } from 'features/canvas/util/getCanvasGenerationMode'; +import { useDebounce } from 'react-use'; + +const GENERATION_MODE_NAME_MAP = { + txt2img: 'Text to Image', + img2img: 'Image to Image', + inpaint: 'Inpaint', + outpaint: 'Inpaint', +}; + +export const useGenerationMode = () => { + const dispatch = useAppDispatch(); + const canvasState = useAppSelector((state) => state.canvas); + + useDebounce( + async () => { + // Build canvas blobs + const canvasBlobsAndImageData = await getCanvasData(canvasState); + + if (!canvasBlobsAndImageData) { + return; + } + + const { baseImageData, maskImageData } = canvasBlobsAndImageData; + + // Determine the generation mode + const generationMode = getCanvasGenerationMode( + baseImageData, + maskImageData + ); + + dispatch(generationModeChanged(generationMode)); + }, + 1000, + [dispatch, canvasState, generationModeChanged] + ); +}; + +const GenerationModeStatusText = () => { + const generationMode = useAppSelector((state) => state.canvas.generationMode); + + useGenerationMode(); + + return ( + + Mode: {generationMode ? GENERATION_MODE_NAME_MAP[generationMode] : '...'} + + ); +}; + +export default GenerationModeStatusText; From 61fa960a18e2715c6b7cc4d730f500ad25545395 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:16:15 +1000 Subject: [PATCH 2/2] feat(ui): make generation mode calculation more granular --- .../listeners/userInvokedCanvas.ts | 16 ++++- .../canvas/hooks/useCanvasGenerationMode.ts | 72 +++++++++++++++++++ .../src/features/canvas/store/canvasSlice.ts | 5 -- .../src/features/canvas/util/getCanvasData.ts | 25 ++++--- .../Canvas/GenerationModeStatusText.tsx | 38 +--------- 5 files changed, 103 insertions(+), 53 deletions(-) create mode 100644 invokeai/frontend/web/src/features/canvas/hooks/useCanvasGenerationMode.ts diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts index 17b2eeed46..39bd742d7d 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts @@ -39,8 +39,22 @@ export const addUserInvokedCanvasListener = () => { const state = getState(); + const { + layerState, + boundingBoxCoordinates, + boundingBoxDimensions, + isMaskEnabled, + shouldPreserveMaskedArea, + } = state.canvas; + // Build canvas blobs - const canvasBlobsAndImageData = await getCanvasData(state.canvas); + const canvasBlobsAndImageData = await getCanvasData( + layerState, + boundingBoxCoordinates, + boundingBoxDimensions, + isMaskEnabled, + shouldPreserveMaskedArea + ); if (!canvasBlobsAndImageData) { log.error('Unable to create canvas data'); diff --git a/invokeai/frontend/web/src/features/canvas/hooks/useCanvasGenerationMode.ts b/invokeai/frontend/web/src/features/canvas/hooks/useCanvasGenerationMode.ts new file mode 100644 index 0000000000..55b04efca4 --- /dev/null +++ b/invokeai/frontend/web/src/features/canvas/hooks/useCanvasGenerationMode.ts @@ -0,0 +1,72 @@ +import { useAppSelector } from 'app/store/storeHooks'; +import { GenerationMode } from 'features/canvas/store/canvasTypes'; +import { getCanvasData } from 'features/canvas/util/getCanvasData'; +import { getCanvasGenerationMode } from 'features/canvas/util/getCanvasGenerationMode'; +import { useEffect, useState } from 'react'; +import { useDebounce } from 'react-use'; + +export const useCanvasGenerationMode = () => { + const layerState = useAppSelector((state) => state.canvas.layerState); + + const boundingBoxCoordinates = useAppSelector( + (state) => state.canvas.boundingBoxCoordinates + ); + const boundingBoxDimensions = useAppSelector( + (state) => state.canvas.boundingBoxDimensions + ); + const isMaskEnabled = useAppSelector((state) => state.canvas.isMaskEnabled); + + const shouldPreserveMaskedArea = useAppSelector( + (state) => state.canvas.shouldPreserveMaskedArea + ); + const [generationMode, setGenerationMode] = useState< + GenerationMode | undefined + >(); + + useEffect(() => { + setGenerationMode(undefined); + }, [ + layerState, + boundingBoxCoordinates, + boundingBoxDimensions, + isMaskEnabled, + shouldPreserveMaskedArea, + ]); + + useDebounce( + async () => { + // Build canvas blobs + const canvasBlobsAndImageData = await getCanvasData( + layerState, + boundingBoxCoordinates, + boundingBoxDimensions, + isMaskEnabled, + shouldPreserveMaskedArea + ); + + if (!canvasBlobsAndImageData) { + return; + } + + const { baseImageData, maskImageData } = canvasBlobsAndImageData; + + // Determine the generation mode + const generationMode = getCanvasGenerationMode( + baseImageData, + maskImageData + ); + + setGenerationMode(generationMode); + }, + 1000, + [ + layerState, + boundingBoxCoordinates, + boundingBoxDimensions, + isMaskEnabled, + shouldPreserveMaskedArea, + ] + ); + + return generationMode; +}; diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts b/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts index dc91c1c769..3163e513e9 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts @@ -30,7 +30,6 @@ import { CanvasState, CanvasTool, Dimensions, - GenerationMode, isCanvasAnyLine, isCanvasBaseImage, isCanvasMaskLine, @@ -859,9 +858,6 @@ export const canvasSlice = createSlice({ state.isMovingBoundingBox = false; state.isTransformingBoundingBox = false; }, - generationModeChanged: (state, action: PayloadAction) => { - state.generationMode = action.payload; - }, }, extraReducers: (builder) => { builder.addCase(sessionCanceled.pending, (state) => { @@ -959,7 +955,6 @@ export const { stagingAreaInitialized, canvasSessionIdChanged, setShouldAntialias, - generationModeChanged, } = canvasSlice.actions; export default canvasSlice.reducer; diff --git a/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts b/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts index 855420f78a..4e575791ed 100644 --- a/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts +++ b/invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts @@ -1,5 +1,10 @@ import { logger } from 'app/logging/logger'; -import { CanvasState, isCanvasMaskLine } from '../store/canvasTypes'; +import { Vector2d } from 'konva/lib/types'; +import { + CanvasLayerState, + Dimensions, + isCanvasMaskLine, +} from '../store/canvasTypes'; import createMaskStage from './createMaskStage'; import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider'; import { konvaNodeToBlob } from './konvaNodeToBlob'; @@ -8,7 +13,13 @@ import { konvaNodeToImageData } from './konvaNodeToImageData'; /** * Gets Blob and ImageData objects for the base and mask layers */ -export const getCanvasData = async (canvasState: CanvasState) => { +export const getCanvasData = async ( + layerState: CanvasLayerState, + boundingBoxCoordinates: Vector2d, + boundingBoxDimensions: Dimensions, + isMaskEnabled: boolean, + shouldPreserveMaskedArea: boolean +) => { const log = logger('canvas'); const canvasBaseLayer = getCanvasBaseLayer(); @@ -19,14 +30,6 @@ export const getCanvasData = async (canvasState: CanvasState) => { return; } - const { - layerState: { objects }, - boundingBoxCoordinates, - boundingBoxDimensions, - isMaskEnabled, - shouldPreserveMaskedArea, - } = canvasState; - const boundingBox = { ...boundingBoxCoordinates, ...boundingBoxDimensions, @@ -57,7 +60,7 @@ export const getCanvasData = async (canvasState: CanvasState) => { // For the mask layer, use the normal boundingBox const maskStage = await createMaskStage( - isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], // only include mask lines, and only if mask is enabled + isMaskEnabled ? layerState.objects.filter(isCanvasMaskLine) : [], // only include mask lines, and only if mask is enabled boundingBox, shouldPreserveMaskedArea ); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx index 5c6bbd0ba3..511e90f0f3 100644 --- a/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx @@ -1,9 +1,5 @@ import { Box } from '@chakra-ui/react'; -import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; -import { generationModeChanged } from 'features/canvas/store/canvasSlice'; -import { getCanvasData } from 'features/canvas/util/getCanvasData'; -import { getCanvasGenerationMode } from 'features/canvas/util/getCanvasGenerationMode'; -import { useDebounce } from 'react-use'; +import { useCanvasGenerationMode } from 'features/canvas/hooks/useCanvasGenerationMode'; const GENERATION_MODE_NAME_MAP = { txt2img: 'Text to Image', @@ -12,38 +8,8 @@ const GENERATION_MODE_NAME_MAP = { outpaint: 'Inpaint', }; -export const useGenerationMode = () => { - const dispatch = useAppDispatch(); - const canvasState = useAppSelector((state) => state.canvas); - - useDebounce( - async () => { - // Build canvas blobs - const canvasBlobsAndImageData = await getCanvasData(canvasState); - - if (!canvasBlobsAndImageData) { - return; - } - - const { baseImageData, maskImageData } = canvasBlobsAndImageData; - - // Determine the generation mode - const generationMode = getCanvasGenerationMode( - baseImageData, - maskImageData - ); - - dispatch(generationModeChanged(generationMode)); - }, - 1000, - [dispatch, canvasState, generationModeChanged] - ); -}; - const GenerationModeStatusText = () => { - const generationMode = useAppSelector((state) => state.canvas.generationMode); - - useGenerationMode(); + const generationMode = useCanvasGenerationMode(); return (