feat(ui): display canvas generation mode in status text (#3915)

## What type of PR is this? (check all applicable)

- [ ] Refactor
- [x] Feature
- [ ] Bug Fix
- [ ] Optimization
- [ ] Documentation Update
- [ ] Community Node Submission


## Have you discussed this change with the InvokeAI team?
- [ ] Yes
- [x] No, because: n/a

      
## Have you updated all relevant documentation?
- [ ] Yes
- [x] No n/a


## Description

Add a generation mode indicator to canvas.

- use the existing logic to determine if generation is txt2img, img2img,
inpaint or outpaint
- technically `outpaint` and `inpaint` are the same, just display
"Inpaint" if its either
- debounce this by 1s to prevent jank

I was going to disable controlnet conditionally when the mode is inpaint
but that involves a lot of fiddly changes to the controlnet UI
components. Instead, I'm hoping we can get inpaint moved over to latents
by next release, at which point controlnet will work.

## Related Tickets & Documents

<!--
For pull requests that relate or close an issue, please include them
below. 

For example having the text: "closes #1234" would connect the current
pull
request to issue 1234.  And when we merge the pull request, Github will
automatically close the issue.
-->

- Related Issue #
- Closes #

## QA Instructions, Screenshots, Recordings

<!-- 
Please provide steps on how to test changes, any hardware or 
software specifications as well as any other pertinent information. 
-->


https://github.com/invoke-ai/InvokeAI/assets/4822129/87464ae9-4136-4367-b992-e243ff0d05b4

## Added/updated tests?

- [ ] Yes
- [x] No : n/a

## [optional] Are there any post deployment tasks we need to perform?

n/a

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts

@@ -39,8 +39,22 @@ export const addUserInvokedCanvasListener = () => {
 
       const state = getState();
 
+      const {
+        layerState,
+        boundingBoxCoordinates,
+        boundingBoxDimensions,
+        isMaskEnabled,
+        shouldPreserveMaskedArea,
+      } = state.canvas;
+
       // Build canvas blobs
-      const canvasBlobsAndImageData = await getCanvasData(state);
+      const canvasBlobsAndImageData = await getCanvasData(
+        layerState,
+        boundingBoxCoordinates,
+        boundingBoxDimensions,
+        isMaskEnabled,
+        shouldPreserveMaskedArea
+      );
 
       if (!canvasBlobsAndImageData) {
         log.error('Unable to create canvas data');

invokeai/frontend/web/src/features/canvas/components/IAICanvasStatusText.tsx

@@ -2,8 +2,8 @@ import { Box, Flex } from '@chakra-ui/react';
 import { createSelector } from '@reduxjs/toolkit';
 import { useAppSelector } from 'app/store/storeHooks';
 import { canvasSelector } from 'features/canvas/store/canvasSelectors';
+import GenerationModeStatusText from 'features/parameters/components/Parameters/Canvas/GenerationModeStatusText';
 import { isEqual } from 'lodash-es';
-
 import { useTranslation } from 'react-i18next';
 import roundToHundreth from '../util/roundToHundreth';
 import IAICanvasStatusTextCursorPos from './IAICanvasStatusText/IAICanvasStatusTextCursorPos';
@@ -110,6 +110,7 @@ const IAICanvasStatusText = () => {
         },
       }}
     >
+      <GenerationModeStatusText />
       <Box
         style={{
           color: activeLayerColor,

invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasToolbar.tsx

@@ -2,7 +2,15 @@ import { Box, ButtonGroup, Flex } from '@chakra-ui/react';
 import { createSelector } from '@reduxjs/toolkit';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import IAIIconButton from 'common/components/IAIIconButton';
+import IAIMantineSelect from 'common/components/IAIMantineSelect';
+import { useImageUploadButton } from 'common/hooks/useImageUploadButton';
 import { useSingleAndDoubleClick } from 'common/hooks/useSingleAndDoubleClick';
+import {
+  canvasCopiedToClipboard,
+  canvasDownloadedAsImage,
+  canvasMerged,
+  canvasSavedToGallery,
+} from 'features/canvas/store/actions';
 import {
   canvasSelector,
   isStagingSelector,
@@ -21,16 +29,8 @@ import {
 } from 'features/canvas/store/canvasTypes';
 import { getCanvasBaseLayer } from 'features/canvas/util/konvaInstanceProvider';
 import { systemSelector } from 'features/system/store/systemSelectors';
+import { useCopyImageToClipboard } from 'features/ui/hooks/useCopyImageToClipboard';
 import { isEqual } from 'lodash-es';
-
-import IAIMantineSearchableSelect from 'common/components/IAIMantineSearchableSelect';
-import { useImageUploadButton } from 'common/hooks/useImageUploadButton';
-import {
-  canvasCopiedToClipboard,
-  canvasDownloadedAsImage,
-  canvasMerged,
-  canvasSavedToGallery,
-} from 'features/canvas/store/actions';
 import { useHotkeys } from 'react-hotkeys-hook';
 import { useTranslation } from 'react-i18next';
 import {
@@ -48,7 +48,6 @@ import IAICanvasRedoButton from './IAICanvasRedoButton';
 import IAICanvasSettingsButtonPopover from './IAICanvasSettingsButtonPopover';
 import IAICanvasToolChooserOptions from './IAICanvasToolChooserOptions';
 import IAICanvasUndoButton from './IAICanvasUndoButton';
-import { useCopyImageToClipboard } from 'features/ui/hooks/useCopyImageToClipboard';
 
 export const selector = createSelector(
   [systemSelector, canvasSelector, isStagingSelector],
@@ -220,7 +219,7 @@ const IAICanvasToolbar = () => {
       }}
     >
       <Box w={24}>
-        <IAIMantineSearchableSelect
+        <IAIMantineSelect
           tooltip={`${t('unifiedCanvas.layer')} (Q)`}
           value={layer}
           data={LAYER_NAMES_DICT}

invokeai/frontend/web/src/features/canvas/hooks/useCanvasGenerationMode.ts

@@ -0,0 +1,72 @@
+import { useAppSelector } from 'app/store/storeHooks';
+import { GenerationMode } from 'features/canvas/store/canvasTypes';
+import { getCanvasData } from 'features/canvas/util/getCanvasData';
+import { getCanvasGenerationMode } from 'features/canvas/util/getCanvasGenerationMode';
+import { useEffect, useState } from 'react';
+import { useDebounce } from 'react-use';
+
+export const useCanvasGenerationMode = () => {
+  const layerState = useAppSelector((state) => state.canvas.layerState);
+
+  const boundingBoxCoordinates = useAppSelector(
+    (state) => state.canvas.boundingBoxCoordinates
+  );
+  const boundingBoxDimensions = useAppSelector(
+    (state) => state.canvas.boundingBoxDimensions
+  );
+  const isMaskEnabled = useAppSelector((state) => state.canvas.isMaskEnabled);
+
+  const shouldPreserveMaskedArea = useAppSelector(
+    (state) => state.canvas.shouldPreserveMaskedArea
+  );
+  const [generationMode, setGenerationMode] = useState<
+    GenerationMode | undefined
+  >();
+
+  useEffect(() => {
+    setGenerationMode(undefined);
+  }, [
+    layerState,
+    boundingBoxCoordinates,
+    boundingBoxDimensions,
+    isMaskEnabled,
+    shouldPreserveMaskedArea,
+  ]);
+
+  useDebounce(
+    async () => {
+      // Build canvas blobs
+      const canvasBlobsAndImageData = await getCanvasData(
+        layerState,
+        boundingBoxCoordinates,
+        boundingBoxDimensions,
+        isMaskEnabled,
+        shouldPreserveMaskedArea
+      );
+
+      if (!canvasBlobsAndImageData) {
+        return;
+      }
+
+      const { baseImageData, maskImageData } = canvasBlobsAndImageData;
+
+      // Determine the generation mode
+      const generationMode = getCanvasGenerationMode(
+        baseImageData,
+        maskImageData
+      );
+
+      setGenerationMode(generationMode);
+    },
+    1000,
+    [
+      layerState,
+      boundingBoxCoordinates,
+      boundingBoxDimensions,
+      isMaskEnabled,
+      shouldPreserveMaskedArea,
+    ]
+  );
+
+  return generationMode;
+};

invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts

@@ -168,4 +168,7 @@ export interface CanvasState {
   stageDimensions: Dimensions;
   stageScale: number;
   tool: CanvasTool;
+  generationMode?: GenerationMode;
 }
+
+export type GenerationMode = 'txt2img' | 'img2img' | 'inpaint' | 'outpaint';

invokeai/frontend/web/src/features/canvas/util/getCanvasData.ts

@@ -1,6 +1,10 @@
 import { logger } from 'app/logging/logger';
-import { RootState } from 'app/store/store';
-import { isCanvasMaskLine } from '../store/canvasTypes';
+import { Vector2d } from 'konva/lib/types';
+import {
+  CanvasLayerState,
+  Dimensions,
+  isCanvasMaskLine,
+} from '../store/canvasTypes';
 import createMaskStage from './createMaskStage';
 import { getCanvasBaseLayer, getCanvasStage } from './konvaInstanceProvider';
 import { konvaNodeToBlob } from './konvaNodeToBlob';
@@ -9,7 +13,13 @@ import { konvaNodeToImageData } from './konvaNodeToImageData';
 /**
  * Gets Blob and ImageData objects for the base and mask layers
  */
-export const getCanvasData = async (state: RootState) => {
+export const getCanvasData = async (
+  layerState: CanvasLayerState,
+  boundingBoxCoordinates: Vector2d,
+  boundingBoxDimensions: Dimensions,
+  isMaskEnabled: boolean,
+  shouldPreserveMaskedArea: boolean
+) => {
   const log = logger('canvas');
 
   const canvasBaseLayer = getCanvasBaseLayer();
@@ -20,14 +30,6 @@ export const getCanvasData = async (state: RootState) => {
     return;
   }
 
-  const {
-    layerState: { objects },
-    boundingBoxCoordinates,
-    boundingBoxDimensions,
-    isMaskEnabled,
-    shouldPreserveMaskedArea,
-  } = state.canvas;
-
   const boundingBox = {
     ...boundingBoxCoordinates,
     ...boundingBoxDimensions,
@@ -58,7 +60,7 @@ export const getCanvasData = async (state: RootState) => {
 
   // For the mask layer, use the normal boundingBox
   const maskStage = await createMaskStage(
-    isMaskEnabled ? objects.filter(isCanvasMaskLine) : [], // only include mask lines, and only if mask is enabled
+    isMaskEnabled ? layerState.objects.filter(isCanvasMaskLine) : [], // only include mask lines, and only if mask is enabled
     boundingBox,
     shouldPreserveMaskedArea
   );

invokeai/frontend/web/src/features/canvas/util/getCanvasGenerationMode.ts

@@ -2,11 +2,12 @@ import {
   areAnyPixelsBlack,
   getImageDataTransparency,
 } from 'common/util/arrayBuffer';
+import { GenerationMode } from '../store/canvasTypes';
 
 export const getCanvasGenerationMode = (
   baseImageData: ImageData,
   maskImageData: ImageData
-) => {
+): GenerationMode => {
   const {
     isPartiallyTransparent: baseIsPartiallyTransparent,
     isFullyTransparent: baseIsFullyTransparent,

invokeai/frontend/web/src/features/parameters/components/Parameters/Canvas/GenerationModeStatusText.tsx

@@ -0,0 +1,21 @@
+import { Box } from '@chakra-ui/react';
+import { useCanvasGenerationMode } from 'features/canvas/hooks/useCanvasGenerationMode';
+
+const GENERATION_MODE_NAME_MAP = {
+  txt2img: 'Text to Image',
+  img2img: 'Image to Image',
+  inpaint: 'Inpaint',
+  outpaint: 'Inpaint',
+};
+
+const GenerationModeStatusText = () => {
+  const generationMode = useCanvasGenerationMode();
+
+  return (
+    <Box>
+      Mode: {generationMode ? GENERATION_MODE_NAME_MAP[generationMode] : '...'}
+    </Box>
+  );
+};
+
+export default GenerationModeStatusText;