Improves bounding box interactions

- Bounding box can now be moved by dragging any of its edges - Bounding box does not affect drawing if already drawing a stroke - Can lock bounding box to draw directly on the bounding box edges - Removes spacebar-hold behaviour due to technical issues
2024-08-30 20:32:17 +00:00 · 2022-11-01 16:28:30 +11:00 · 2022-11-01 16:28:30 +11:00 · 82dcbac28f
commit 82dcbac28f
parent d43bd4625d
8 changed files with 258 additions and 135 deletions
--- a/frontend/src/features/options/AdvancedOptions/Inpainting/BoundingBoxSettings.tsx
+++ b/frontend/src/features/options/AdvancedOptions/Inpainting/BoundingBoxSettings.tsx
@ -113,7 +113,7 @@ const BoundingBoxSettings = () => {
    <div className="inpainting-bounding-box-settings">
      <div className="inpainting-bounding-box-header">
        <p>Inpaint Box</p>
-        <IAIIconButton
+        {/* <IAIIconButton
          aria-label="Toggle Bounding Box Visibility"
          icon={
            shouldShowBoundingBox ? <BiShow size={22} /> : <BiHide size={22} />
@ -121,11 +121,12 @@ const BoundingBoxSettings = () => {
          onClick={handleShowBoundingBox}
          background={'none'}
          padding={0}
-        />
+        /> */}
      </div>
      <div className="inpainting-bounding-box-settings-items">
        <div className="inpainting-bounding-box-dimensions-slider-numberinput">
          <IAISlider
+            isDisabled={shouldLockBoundingBox}
            label="Box W"
            min={64}
            max={roundDownToMultiple(canvasDimensions.width, 64)}
@ -135,6 +136,7 @@ const BoundingBoxSettings = () => {
            width={'5rem'}
          />
          <IAINumberInput
+            isDisabled={shouldLockBoundingBox}
            value={boundingBoxDimensions.width}
            onChange={handleChangeBoundingBoxWidth}
            min={64}
@ -143,6 +145,7 @@ const BoundingBoxSettings = () => {
            width={'5rem'}
          />
          <IAIIconButton
+            isDisabled={shouldLockBoundingBox}
            size={'sm'}
            aria-label={'Reset Width'}
            tooltip={'Reset Width'}
@ -154,6 +157,7 @@ const BoundingBoxSettings = () => {
        </div>
        <div className="inpainting-bounding-box-dimensions-slider-numberinput">
          <IAISlider
+            isDisabled={shouldLockBoundingBox}
            label="Box H"
            min={64}
            max={roundDownToMultiple(canvasDimensions.height, 64)}
@ -163,6 +167,7 @@ const BoundingBoxSettings = () => {
            width={'5rem'}
          />
          <IAINumberInput
+            isDisabled={shouldLockBoundingBox}
            value={boundingBoxDimensions.height}
            onChange={handleChangeBoundingBoxHeight}
            min={64}
@ -172,6 +177,7 @@ const BoundingBoxSettings = () => {
            width={'5rem'}
          />
          <IAIIconButton
+            isDisabled={shouldLockBoundingBox}
            size={'sm'}
            aria-label={'Reset Height'}
            tooltip={'Reset Height'}
--- a/frontend/src/features/tabs/Inpainting/Inpainting.scss
+++ b/frontend/src/features/tabs/Inpainting/Inpainting.scss
@ -45,32 +45,40 @@
    height: 100%;
  }

-  .inpainting-canvas-wrapper {
+  .inpainting-canvas-spiner {
+    display: flex;
+    align-items: center;
+    width: 100%;
+    height: 100%;
+  }
+
+  .inpainting-canvas-container {
    display: flex;
    align-items: center;
    justify-content: center;
-    position: relative;
    height: 100%;
    width: 100%;
    border-radius: 0.5rem;

+    .inpainting-canvas-wrapper {
+      position: relative;
+    }
+
    .inpainting-alerts {
      position: absolute;
      top: 0;
      left: 0;
      display: flex;
-      column-gap: 0.5rem;
      z-index: 2;
-      padding: 0.5rem;
+      padding: 0.25rem;
      pointer-events: none;
-      font-size: 0.9rem;
-      font-weight: bold;

-      div {
-        background-color: var(--accent-color);
-        color: var(--text-color);
-        padding: 0.2rem 0.6rem;
-        border-radius: 0.25rem;
+      button {
+        &[data-selected='true'] {
+          svg {
+            fill: var(--accent-color);
+          }
+        }
      }
    }

--- a/frontend/src/features/tabs/Inpainting/InpaintingCanvas.tsx
+++ b/frontend/src/features/tabs/Inpainting/InpaintingCanvas.tsx
@ -34,7 +34,10 @@ import InpaintingBoundingBoxPreview, {
 } from './components/InpaintingBoundingBoxPreview';
 import { KonvaEventObject } from 'konva/lib/Node';
 import KeyboardEventManager from './components/KeyboardEventManager';
-import { useToast } from '@chakra-ui/react';
+import { Icon, IconButton, Tooltip, useToast } from '@chakra-ui/react';
+import { FaLock, FaUnlock } from 'react-icons/fa';
+import { MdInvertColors, MdInvertColorsOff } from 'react-icons/md';
+import { BiHide, BiShow } from 'react-icons/bi';

 // Use a closure allow other components to use these things... not ideal...
 export let stageRef: MutableRefObject<StageType | null>;
@ -58,6 +61,10 @@ const InpaintingCanvas = () => {
    isDrawing,
    shouldLockBoundingBox,
    boundingBoxDimensions,
+    // isTransformingBoundingBox,
+    isMouseOverBoundingBox,
+    isModifyingBoundingBox,
+    stageCursor,
  } = useAppSelector(inpaintingCanvasSelector);

  const toast = useToast();
@ -113,7 +120,7 @@ const InpaintingCanvas = () => {
    if (
      !scaledCursorPosition ||
      !maskLayerRef.current ||
-      !shouldLockBoundingBox
+      isModifyingBoundingBox
    )
      return;

@ -127,7 +134,7 @@ const InpaintingCanvas = () => {
        points: [scaledCursorPosition.x, scaledCursorPosition.y],
      })
    );
-  }, [dispatch, brushSize, tool, shouldLockBoundingBox]);
+  }, [dispatch, brushSize, tool, isModifyingBoundingBox]);

  /**
   *
@ -143,20 +150,20 @@ const InpaintingCanvas = () => {

    dispatch(setCursorPosition(scaledCursorPosition));

-    if (!maskLayerRef.current || !shouldLockBoundingBox) {
+    if (!maskLayerRef.current) {
      return;
    }

    lastCursorPosition.current = scaledCursorPosition;

-    if (!isDrawing) return;
+    if (!isDrawing || isModifyingBoundingBox) return;

    didMouseMoveRef.current = true;
    // Extend the current line
    dispatch(
      addPointToCurrentLine([scaledCursorPosition.x, scaledCursorPosition.y])
    );
-  }, [dispatch, isDrawing, shouldLockBoundingBox]);
+  }, [dispatch, isDrawing, isModifyingBoundingBox]);

  /**
   *
@ -170,7 +177,7 @@ const InpaintingCanvas = () => {
      if (
        !scaledCursorPosition ||
        !maskLayerRef.current ||
-        !shouldLockBoundingBox
+        isModifyingBoundingBox
      )
        return;

@ -187,7 +194,7 @@ const InpaintingCanvas = () => {
      didMouseMoveRef.current = false;
    }
    dispatch(setIsDrawing(false));
-  }, [dispatch, isDrawing, shouldLockBoundingBox]);
+  }, [dispatch, isDrawing, isModifyingBoundingBox]);

  /**
   *
@ -214,7 +221,7 @@ const InpaintingCanvas = () => {
        if (
          !scaledCursorPosition ||
          !maskLayerRef.current ||
-          !shouldLockBoundingBox
+          isModifyingBoundingBox
        )
          return;

@ -230,23 +237,44 @@ const InpaintingCanvas = () => {
        );
      }
    },
-    [dispatch, brushSize, tool, shouldLockBoundingBox]
+    [dispatch, brushSize, tool, isModifyingBoundingBox]
  );

  return (
-    <div className="inpainting-canvas-wrapper" tabIndex={1}>
+    <div className="inpainting-canvas-container" tabIndex={1}>
+      <div className="inpainting-canvas-wrapper">
        <div className="inpainting-alerts">
-        {!shouldShowMask && (
-          <div style={{ pointerEvents: 'none' }}>Mask Hidden (H)</div>
-        )}
-        {shouldInvertMask && (
-          <div style={{ pointerEvents: 'none' }}>Mask Inverted (Shift+M)</div>
-        )}
-        {!shouldLockBoundingBox && (
          <div style={{ pointerEvents: 'none' }}>
-            {`Transforming Bounding Box ${boundingBoxDimensions.width}x${boundingBoxDimensions.height} (M)`}
+            <IconButton
+              aria-label="Show/HideMask"
+              size="xs"
+              variant={'ghost'}
+              fontSize={'1rem'}
+              data-selected={!shouldShowMask}
+              icon={shouldShowMask ? <BiShow /> : <BiHide />}
+            />
+          </div>
+          <div style={{ pointerEvents: 'none' }}>
+            <IconButton
+              aria-label="Invert Mask"
+              size="xs"
+              variant={'ghost'}
+              fontSize={'1rem'}
+              data-selected={shouldInvertMask}
+              icon={
+                shouldInvertMask ? <MdInvertColors /> : <MdInvertColorsOff />
+              }
+            />
+          </div>
+          <div style={{ pointerEvents: 'none' }}>
+            <IconButton
+              aria-label="Bounding Box Lock"
+              size="xs"
+              variant={'ghost'}
+              data-selected={shouldLockBoundingBox}
+              icon={shouldLockBoundingBox ? <FaLock /> : <FaUnlock />}
+            />
          </div>
-        )}
        </div>

        {canvasBgImage && (
@ -260,7 +288,7 @@ const InpaintingCanvas = () => {
            onMouseUp={handleMouseUp}
            onMouseOut={handleMouseOutCanvas}
            onMouseLeave={handleMouseOutCanvas}
-          style={{ cursor: shouldShowMask ? 'none' : 'default' }}
+            style={{ ...(stageCursor ? { cursor: stageCursor } : {}) }}
            className="inpainting-canvas-stage checkerboard"
            ref={stageRef}
          >
@ -283,7 +311,9 @@ const InpaintingCanvas = () => {
                >
                  <InpaintingCanvasLines />

-                {shouldLockBoundingBox && <InpaintingCanvasBrushPreview />}
+                  {!isMouseOverBoundingBox && !isModifyingBoundingBox && (
+                    <InpaintingCanvasBrushPreview />
+                  )}

                  {shouldInvertMask && (
                    <KonvaImage
@ -305,8 +335,8 @@ const InpaintingCanvas = () => {
                    {shouldShowBoundingBoxFill && shouldShowBoundingBox && (
                      <InpaintingBoundingBoxPreviewOverlay />
                    )}
-                  {shouldShowBoundingBox && <InpaintingBoundingBoxPreview />}
-                  {shouldLockBoundingBox && (
+                    <InpaintingBoundingBoxPreview />
+                    {!isMouseOverBoundingBox && !isModifyingBoundingBox && (
                      <InpaintingCanvasBrushPreviewOutline />
                    )}
                  </Layer>
@ -318,6 +348,7 @@ const InpaintingCanvas = () => {
        <Cacher />
        <KeyboardEventManager />
      </div>
+    </div>
  );
 };

--- a/frontend/src/features/tabs/Inpainting/components/Cacher.tsx
+++ b/frontend/src/features/tabs/Inpainting/components/Cacher.tsx
@ -33,6 +33,8 @@ const Cacher = () => {
    futureLines,
    needsCache,
    isDrawing,
+    isTransformingBoundingBox,
+    isMovingBoundingBox,
  } = useAppSelector((state: RootState) => state.inpainting);

  useLayoutEffect(() => {
@ -64,6 +66,8 @@ const Cacher = () => {
    futureLines,
    needsCache,
    isDrawing,
+    isTransformingBoundingBox,
+    isMovingBoundingBox,
  ]);

  /**
--- a/frontend/src/features/tabs/Inpainting/components/InpaintingBoundingBoxPreview.tsx
+++ b/frontend/src/features/tabs/Inpainting/components/InpaintingBoundingBoxPreview.tsx
@ -4,7 +4,7 @@ import { KonvaEventObject } from 'konva/lib/Node';
 import { Box } from 'konva/lib/shapes/Transformer';
 import { Vector2d } from 'konva/lib/types';
 import _ from 'lodash';
-import { useCallback, useEffect, useRef } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
 import { Group, Rect, Transformer } from 'react-konva';
 import {
  RootState,
@ -17,6 +17,9 @@ import {
  InpaintingState,
  setBoundingBoxCoordinate,
  setBoundingBoxDimensions,
+  setIsMouseOverBoundingBox,
+  setIsMovingBoundingBox,
+  setIsTransformingBoundingBox,
 } from '../inpaintingSlice';
 import { rgbaColorToString } from '../util/colorToString';
 import {
@ -35,6 +38,10 @@ const boundingBoxPreviewSelector = createSelector(
      stageScale,
      imageToInpaint,
      shouldLockBoundingBox,
+      isDrawing,
+      isTransformingBoundingBox,
+      isMovingBoundingBox,
+      isMouseOverBoundingBox,
    } = inpainting;
    return {
      boundingBoxCoordinate,
@ -46,6 +53,10 @@ const boundingBoxPreviewSelector = createSelector(
      dash: DASH_WIDTH / stageScale, // scale dash lengths
      strokeWidth: 1 / stageScale, // scale stroke thickness
      shouldLockBoundingBox,
+      isDrawing,
+      isTransformingBoundingBox,
+      isMouseOverBoundingBox,
+      isMovingBoundingBox,
    };
  },
  {
@ -93,10 +104,13 @@ const InpaintingBoundingBoxPreview = () => {
  const {
    boundingBoxCoordinate,
    boundingBoxDimensions,
-    strokeWidth,
    stageScale,
    imageToInpaint,
    shouldLockBoundingBox,
+    isDrawing,
+    isTransformingBoundingBox,
+    isMovingBoundingBox,
+    isMouseOverBoundingBox,
  } = useAppSelector(boundingBoxPreviewSelector);

  const transformerRef = useRef<Konva.Transformer>(null);
@ -108,15 +122,6 @@ const InpaintingBoundingBoxPreview = () => {
    transformerRef.current.getLayer()?.batchDraw();
  }, [shouldLockBoundingBox]);

-  useEffect(
-    () => () => {
-      const container = stageRef.current?.container();
-      if (!container) return;
-      container.style.cursor = 'unset';
-    },
-    [shouldLockBoundingBox]
-  );
-
  const scaledStep = 64 * stageScale;

  const handleOnDragMove = useCallback(
@ -269,6 +274,29 @@ const InpaintingBoundingBoxPreview = () => {
    [imageToInpaint, stageScale]
  );

+  const handleStartedTransforming = (e: KonvaEventObject<MouseEvent>) => {
+    e.cancelBubble = true;
+    e.evt.stopImmediatePropagation();
+    dispatch(setIsTransformingBoundingBox(true));
+  };
+
+  const handleEndedTransforming = (e: KonvaEventObject<MouseEvent>) => {
+    dispatch(setIsTransformingBoundingBox(false));
+    dispatch(setIsMouseOverBoundingBox(false));
+  };
+
+  const handleStartedMoving = (e: KonvaEventObject<MouseEvent>) => {
+    e.cancelBubble = true;
+    e.evt.stopImmediatePropagation();
+    dispatch(setIsMovingBoundingBox(true));
+  };
+
+  const handleEndedModifying = (e: KonvaEventObject<MouseEvent>) => {
+    dispatch(setIsTransformingBoundingBox(false));
+    dispatch(setIsMovingBoundingBox(false));
+    dispatch(setIsMouseOverBoundingBox(false));
+  };
+
  return (
    <>
      <Rect
@ -277,23 +305,27 @@ const InpaintingBoundingBoxPreview = () => {
        width={boundingBoxDimensions.width}
        height={boundingBoxDimensions.height}
        ref={shapeRef}
-        stroke={'white'}
-        strokeWidth={strokeWidth}
-        listening={!shouldLockBoundingBox}
-        onMouseEnter={(e) => {
-          const container = e?.target?.getStage()?.container();
-          if (!container) return;
-          container.style.cursor = shouldLockBoundingBox ? 'none' : 'move';
+        stroke={isMouseOverBoundingBox ? 'rgba(255,255,255,0.3)' : 'white'}
+        strokeWidth={Math.floor((isMouseOverBoundingBox ? 8 : 1) / stageScale)}
+        fillEnabled={false}
+        hitStrokeWidth={Math.floor(13 / stageScale)}
+        listening={!isDrawing && !shouldLockBoundingBox}
+        onMouseOver={() => {
+          dispatch(setIsMouseOverBoundingBox(true));
        }}
-        onMouseLeave={(e) => {
-          const container = e?.target?.getStage()?.container();
-          if (!container) return;
-          container.style.cursor = shouldLockBoundingBox ? 'none' : 'default';
+        onMouseOut={() => {
+          !isTransformingBoundingBox &&
+            !isMovingBoundingBox &&
+            dispatch(setIsMouseOverBoundingBox(false));
        }}
-        draggable={!shouldLockBoundingBox}
+        onMouseDown={handleStartedMoving}
+        onMouseUp={handleEndedModifying}
+        draggable={true}
        onDragMove={handleOnDragMove}
        dragBoundFunc={dragBoundFunc}
        onTransform={handleOnTransform}
+        onDragEnd={handleEndedModifying}
+        onTransformEnd={handleEndedTransforming}
      />
      <Transformer
        ref={transformerRef}
@ -308,10 +340,22 @@ const InpaintingBoundingBoxPreview = () => {
        flipEnabled={false}
        ignoreStroke={true}
        keepRatio={false}
-        listening={!shouldLockBoundingBox}
+        listening={!isDrawing && !shouldLockBoundingBox}
+        onMouseDown={handleStartedTransforming}
+        onMouseUp={handleEndedTransforming}
        enabledAnchors={shouldLockBoundingBox ? [] : undefined}
        boundBoxFunc={boundBoxFunc}
        anchorDragBoundFunc={anchorDragBoundFunc}
+        onDragEnd={handleEndedModifying}
+        onTransformEnd={handleEndedTransforming}
+        onMouseOver={() => {
+          dispatch(setIsMouseOverBoundingBox(true));
+        }}
+        onMouseOut={() => {
+          !isTransformingBoundingBox &&
+            !isMovingBoundingBox &&
+            dispatch(setIsMouseOverBoundingBox(false));
+        }}
      />
    </>
  );
--- a/frontend/src/features/tabs/Inpainting/components/InpaintingCanvasBrushPreview.tsx
+++ b/frontend/src/features/tabs/Inpainting/components/InpaintingCanvasBrushPreview.tsx
@ -40,18 +40,10 @@ const inpaintingCanvasBrushPreviewSelector = createSelector(
 * Draws a black circle around the canvas brush preview.
 */
 const InpaintingCanvasBrushPreview = () => {
-  const {
-    cursorPosition,
-    width,
-    height,
-    shouldShowBrushPreview,
-    brushSize,
-    maskColorString,
-    tool,
-    shouldShowBrush,
-  } = useAppSelector(inpaintingCanvasBrushPreviewSelector);
+  const { cursorPosition, width, height, brushSize, maskColorString, tool } =
+    useAppSelector(inpaintingCanvasBrushPreviewSelector);

-  if (!shouldShowBrush || !(cursorPosition || shouldShowBrushPreview)) {
+  if (!cursorPosition) {
    return null;
  }

--- a/frontend/src/features/tabs/Inpainting/inpaintingSlice.ts
+++ b/frontend/src/features/tabs/Inpainting/inpaintingSlice.ts
@ -51,6 +51,9 @@ export interface InpaintingState {
  needsCache: boolean;
  stageScale: number;
  isDrawing: boolean;
+  isTransformingBoundingBox: boolean;
+  isMouseOverBoundingBox: boolean;
+  isMovingBoundingBox: boolean;
  shouldUseInpaintReplace: boolean;
  inpaintReplace: number;
  shouldLockBoundingBox: boolean;
@ -63,7 +66,7 @@ const initialInpaintingState: InpaintingState = {
  canvasDimensions: { width: 0, height: 0 },
  boundingBoxDimensions: { width: 512, height: 512 },
  boundingBoxCoordinate: { x: 0, y: 0 },
-  boundingBoxPreviewFill: { r: 0, g: 0, b: 0, a: 0.7 },
+  boundingBoxPreviewFill: { r: 0, g: 0, b: 0, a: 0.5 },
  shouldShowBoundingBox: true,
  shouldShowBoundingBoxFill: true,
  cursorPosition: null,
@ -77,6 +80,9 @@ const initialInpaintingState: InpaintingState = {
  shouldShowBrushPreview: false,
  needsCache: false,
  isDrawing: false,
+  isTransformingBoundingBox: false,
+  isMouseOverBoundingBox: false,
+  isMovingBoundingBox: false,
  stageScale: 1,
  shouldUseInpaintReplace: false,
  inpaintReplace: 1,
@ -319,6 +325,15 @@ export const inpaintingSlice = createSlice({
    setShouldShowBoundingBox: (state, action: PayloadAction<boolean>) => {
      state.shouldShowBoundingBox = action.payload;
    },
+    setIsTransformingBoundingBox: (state, action: PayloadAction<boolean>) => {
+      state.isTransformingBoundingBox = action.payload;
+    },
+    setIsMovingBoundingBox: (state, action: PayloadAction<boolean>) => {
+      state.isMovingBoundingBox = action.payload;
+    },
+    setIsMouseOverBoundingBox: (state, action: PayloadAction<boolean>) => {
+      state.isMouseOverBoundingBox = action.payload;
+    },
  },
 });

@ -354,6 +369,9 @@ export const {
  setInpaintReplace,
  setShouldLockBoundingBox,
  toggleShouldLockBoundingBox,
+  setIsMovingBoundingBox,
+  setIsTransformingBoundingBox,
+  setIsMouseOverBoundingBox,
 } = inpaintingSlice.actions;

 export default inpaintingSlice.reducer;
--- a/frontend/src/features/tabs/Inpainting/inpaintingSliceSelectors.ts
+++ b/frontend/src/features/tabs/Inpainting/inpaintingSliceSelectors.ts
@ -78,7 +78,23 @@ export const inpaintingCanvasSelector = createSelector(
      isDrawing,
      shouldLockBoundingBox,
      boundingBoxDimensions,
+      isTransformingBoundingBox,
+      isMouseOverBoundingBox,
+      isMovingBoundingBox,
    } = inpainting;
+
+    let stageCursor: string | undefined = '';
+
+    if (isTransformingBoundingBox) {
+      stageCursor = undefined;
+    } else if (isMovingBoundingBox || isMouseOverBoundingBox) {
+      stageCursor = 'move';
+    } else if (shouldShowMask) {
+      stageCursor = 'none';
+    } else {
+      stageCursor = 'default';
+    }
+
    return {
      tool,
      brushSize,
@ -93,6 +109,10 @@ export const inpaintingCanvasSelector = createSelector(
      isDrawing,
      shouldLockBoundingBox,
      boundingBoxDimensions,
+      isTransformingBoundingBox,
+      isModifyingBoundingBox: isTransformingBoundingBox || isMovingBoundingBox,
+      stageCursor,
+      isMouseOverBoundingBox,
    };
  },
  {