feat(ui): CL zoom and pan, some rendering optimizations

invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx

@@ -1,23 +1,34 @@
-import { Flex } from '@invoke-ai/ui-library';
+import { Box, Flex, Heading } from '@invoke-ai/ui-library';
 import { useStore } from '@nanostores/react';
 import { createSelector } from '@reduxjs/toolkit';
 import { logger } from 'app/logging/logger';
 import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import { BRUSH_SPACING_PCT, MAX_BRUSH_SPACING_PX, MIN_BRUSH_SPACING_PX } from 'features/controlLayers/konva/constants';
+import {
+  BRUSH_SPACING_PCT,
+  MAX_BRUSH_SPACING_PX,
+  MIN_BRUSH_SPACING_PX,
+  TRANSPARENCY_CHECKER_PATTERN,
+} from 'features/controlLayers/konva/constants';
 import { setStageEventHandlers } from 'features/controlLayers/konva/events';
 import { debouncedRenderers, renderers as normalRenderers } from 'features/controlLayers/konva/renderers/layers';
+import { renderImageDimsPreview } from 'features/controlLayers/konva/renderers/toolPreview';
 import {
   $brushColor,
   $brushSize,
   $brushSpacingPx,
   $isDrawing,
+  $isMouseDown,
+  $isSpaceDown,
   $lastAddedPoint,
   $lastCursorPos,
   $lastMouseDownPos,
   $selectedLayer,
   $shouldInvertBrushSizeScrollDirection,
+  $stagePos,
+  $stageScale,
   $tool,
+  $toolBuffer,
   brushLineAdded,
   brushSizeChanged,
   eraserLineAdded,
@@ -38,6 +49,7 @@ import Konva from 'konva';
 import type { IRect } from 'konva/lib/types';
 import { clamp } from 'lodash-es';
 import { memo, useCallback, useLayoutEffect, useMemo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
 import { getImageDTO } from 'services/api/endpoints/images';
 import { useDevicePixelRatio } from 'use-device-pixel-ratio';
 import { v4 as uuidv4 } from 'uuid';
@@ -63,6 +75,11 @@ const selectSelectedLayer = createSelector(selectControlLayersSlice, (controlLay
   return controlLayers.present.layers.find((l) => l.id === controlLayers.present.selectedLayerId) ?? null;
 });
 
+const selectLayerCount = createSelector(
+  selectControlLayersSlice,
+  (controlLayers) => controlLayers.present.layers.length
+);
+
 const useStageRenderer = (
   stage: Konva.Stage,
   container: HTMLDivElement | null,
@@ -74,11 +91,11 @@ const useStageRenderer = (
   const tool = useStore($tool);
   const lastCursorPos = useStore($lastCursorPos);
   const lastMouseDownPos = useStore($lastMouseDownPos);
+  const isMouseDown = useStore($isMouseDown);
+  const stageScale = useStore($stageScale);
   const isDrawing = useStore($isDrawing);
   const brushColor = useAppSelector(selectBrushColor);
   const selectedLayer = useAppSelector(selectSelectedLayer);
-  const layerIds = useMemo(() => state.layers.map((l) => l.id), [state.layers]);
-  const layerCount = useMemo(() => state.layers.length, [state.layers]);
   const renderers = useMemo(() => (asPreview ? debouncedRenderers : normalRenderers), [asPreview]);
   const dpr = useDevicePixelRatio({ round: false });
   const shouldInvertBrushSizeScrollDirection = useAppSelector((s) => s.canvas.shouldInvertBrushSizeScrollDirection);
@@ -166,28 +183,23 @@ const useStageRenderer = (
       return;
     }
 
-    const cancelShape = (e: KeyboardEvent) => {
-      // Cancel shape drawing on escape
-      if (e.key === 'Escape') {
-        $isDrawing.set(false);
-        $lastMouseDownPos.set(null);
-      }
-    };
-
-    container.addEventListener('keydown', cancelShape);
-
     const cleanup = setStageEventHandlers({
       stage,
       $tool,
+      $toolBuffer,
       $isDrawing,
+      $isMouseDown,
       $lastMouseDownPos,
       $lastCursorPos,
       $lastAddedPoint,
+      $stageScale,
+      $stagePos,
       $brushSize,
       $brushColor,
       $brushSpacingPx,
       $selectedLayer,
       $shouldInvertBrushSizeScrollDirection,
+      $isSpaceDown,
       onBrushSizeChanged,
       onBrushLineAdded,
       onEraserLineAdded,
@@ -198,7 +210,6 @@ const useStageRenderer = (
     return () => {
       log.trace('Removing stage listeners');
       cleanup();
-      container.removeEventListener('keydown', cancelShape);
     };
   }, [
     asPreview,
@@ -251,7 +262,8 @@ const useStageRenderer = (
       lastCursorPos,
       lastMouseDownPos,
       state.brushSize,
-      isDrawing
+      isDrawing,
+      isMouseDown
     );
   }, [
     asPreview,
@@ -265,6 +277,32 @@ const useStageRenderer = (
     state.brushSize,
     renderers,
     isDrawing,
+    isMouseDown,
+  ]);
+
+  useLayoutEffect(() => {
+    if (asPreview) {
+      // Preview should not display tool
+      return;
+    }
+    log.trace('Rendering tool preview');
+    renderImageDimsPreview(stage, state.size.width, state.size.height, stageScale);
+  }, [
+    asPreview,
+    stage,
+    tool,
+    brushColor,
+    selectedLayer,
+    state.globalMaskLayerOpacity,
+    lastCursorPos,
+    lastMouseDownPos,
+    state.brushSize,
+    renderers,
+    isDrawing,
+    isMouseDown,
+    state.size.width,
+    state.size.height,
+    stageScale,
   ]);
 
   useLayoutEffect(() => {
@@ -290,29 +328,6 @@ const useStageRenderer = (
     debouncedRenderers.updateBboxes(stage, state.layers, onBboxChanged);
   }, [stage, asPreview, state.layers, onBboxChanged]);
 
-  useLayoutEffect(() => {
-    if (asPreview) {
-      // The preview should not have a background
-      return;
-    }
-    log.trace('Rendering background');
-    renderers.renderBackground(stage, state.size.width, state.size.height);
-  }, [stage, asPreview, state.size.width, state.size.height, renderers]);
-
-  useLayoutEffect(() => {
-    log.trace('Arranging layers');
-    renderers.arrangeLayers(stage, layerIds);
-  }, [stage, layerIds, renderers]);
-
-  useLayoutEffect(() => {
-    if (asPreview) {
-      // The preview should not display the no layers message
-      return;
-    }
-    log.trace('Rendering no layers message');
-    renderers.renderNoLayersMessage(stage, layerCount, state.size.width, state.size.height);
-  }, [stage, layerCount, renderers, asPreview, state.size.width, state.size.height]);
-
   useLayoutEffect(() => {
     Konva.pixelRatio = dpr;
   }, [dpr]);
@@ -323,8 +338,15 @@ type Props = {
 };
 
 export const StageComponent = memo(({ asPreview = false }: Props) => {
+  const { t } = useTranslation();
+  const layerCount = useAppSelector(selectLayerCount);
   const [stage] = useState(
-    () => new Konva.Stage({ id: uuidv4(), container: document.createElement('div'), listening: !asPreview })
+    () =>
+      new Konva.Stage({
+        id: uuidv4(),
+        container: document.createElement('div'),
+        listening: !asPreview,
+      })
   );
   const [container, setContainer] = useState<HTMLDivElement | null>(null);
   const [wrapper, setWrapper] = useState<HTMLDivElement | null>(null);
@@ -341,14 +363,29 @@ export const StageComponent = memo(({ asPreview = false }: Props) => {
 
   return (
     <Flex overflow="hidden" w="full" h="full">
-      <Flex ref={wrapperRef} w="full" h="full" alignItems="center" justifyContent="center">
+      <Flex position="relative" ref={wrapperRef} w="full" h="full" alignItems="center" justifyContent="center">
+        <Box
+          position="absolute"
+          w="full"
+          h="full"
+          borderRadius="base"
+          backgroundImage={TRANSPARENCY_CHECKER_PATTERN}
+          backgroundRepeat="repeat"
+          opacity={0.2}
+        />
+        {layerCount === 0 && !asPreview && (
+          <Flex position="absolute" w="full" h="full" alignItems="center" justifyContent="center">
+            <Heading color="base.200">{t('controlLayers.noLayersAdded')}</Heading>
+          </Flex>
+        )}
+
         <Flex
           ref={containerRef}
           tabIndex={-1}
-          bg="base.850"
           borderRadius="base"
           overflow="hidden"
           data-testid="control-layers-canvas"
+          border="1px solid red"
         />
       </Flex>
     </Flex>

invokeai/frontend/web/src/features/controlLayers/components/ToolChooser.tsx

@@ -11,7 +11,7 @@ import {
 import { useCallback } from 'react';
 import { useHotkeys } from 'react-hotkeys-hook';
 import { useTranslation } from 'react-i18next';
-import { PiArrowsOutCardinalBold, PiEraserBold, PiPaintBrushBold, PiRectangleBold } from 'react-icons/pi';
+import { PiArrowsOutCardinalBold, PiEraserBold, PiHandBold, PiPaintBrushBold, PiRectangleBold } from 'react-icons/pi';
 
 const selectIsDisabled = createSelector(selectControlLayersSlice, (controlLayers) => {
   const selectedLayer = controlLayers.present.layers.find((l) => l.id === controlLayers.present.selectedLayerId);
@@ -41,6 +41,10 @@ export const ToolChooser: React.FC = () => {
     $tool.set('move');
   }, []);
   useHotkeys('v', setToolToMove, { enabled: !isDisabled }, [isDisabled]);
+  const setToolToView = useCallback(() => {
+    $tool.set('view');
+  }, []);
+  useHotkeys('h', setToolToView, { enabled: !isDisabled }, [isDisabled]);
 
   const resetSelectedLayer = useCallback(() => {
     if (selectedLayerId === null) {
@@ -89,6 +93,14 @@ export const ToolChooser: React.FC = () => {
         onClick={setToolToMove}
         isDisabled={isDisabled}
       />
+      <IconButton
+        aria-label={`${t('unifiedCanvas.view')} (H)`}
+        tooltip={`${t('unifiedCanvas.view')} (H)`}
+        icon={<PiHandBold />}
+        variant={tool === 'view' ? 'solid' : 'outline'}
+        onClick={setToolToView}
+        isDisabled={isDisabled}
+      />
     </ButtonGroup>
   );
 };

invokeai/frontend/web/src/features/controlLayers/konva/constants.ts

@@ -39,3 +39,18 @@ export const MAX_BRUSH_SPACING_PX = 15;
  * The debounce time in milliseconds for debounced renderers.
  */
 export const DEBOUNCE_MS = 300;
+
+/**
+ * Konva wheel zoom exponential scale factor
+ */
+export const CANVAS_SCALE_BY = 0.999;
+
+/**
+ * Minimum (furthest-zoomed-out) scale
+ */
+export const MIN_CANVAS_SCALE = 0.1;
+
+/**
+ * Maximum (furthest-zoomed-in) scale
+ */
+export const MAX_CANVAS_SCALE = 20;

invokeai/frontend/web/src/features/controlLayers/konva/events.ts

@@ -1,4 +1,5 @@
 import { calculateNewBrushSize } from 'features/canvas/hooks/useCanvasZoom';
+import { CANVAS_SCALE_BY, MAX_CANVAS_SCALE, MIN_CANVAS_SCALE } from 'features/canvas/util/constants';
 import { getIsMouseDown, getScaledFlooredCursorPosition, snapPosToStage } from 'features/controlLayers/konva/util';
 import {
   type AddBrushLineArg,
@@ -11,23 +12,29 @@ import {
 } from 'features/controlLayers/store/types';
 import type Konva from 'konva';
 import type { Vector2d } from 'konva/lib/types';
+import { clamp } from 'lodash-es';
 import type { WritableAtom } from 'nanostores';
 import type { RgbaColor } from 'react-colorful';
 
-import { TOOL_PREVIEW_LAYER_ID } from './naming';
+import { TOOL_PREVIEW_TOOL_GROUP_ID } from './naming';
 
 type SetStageEventHandlersArg = {
   stage: Konva.Stage;
   $tool: WritableAtom<Tool>;
+  $toolBuffer: WritableAtom<Tool | null>;
   $isDrawing: WritableAtom<boolean>;
+  $isMouseDown: WritableAtom<boolean>;
   $lastMouseDownPos: WritableAtom<Vector2d | null>;
   $lastCursorPos: WritableAtom<Vector2d | null>;
   $lastAddedPoint: WritableAtom<Vector2d | null>;
+  $stageScale: WritableAtom<number>;
+  $stagePos: WritableAtom<Vector2d>;
   $brushColor: WritableAtom<RgbaColor>;
   $brushSize: WritableAtom<number>;
   $brushSpacingPx: WritableAtom<number>;
   $selectedLayer: WritableAtom<Layer | null>;
   $shouldInvertBrushSizeScrollDirection: WritableAtom<boolean>;
+  $isSpaceDown: WritableAtom<boolean>;
   onBrushLineAdded: (arg: AddBrushLineArg) => void;
   onEraserLineAdded: (arg: AddEraserLineArg) => void;
   onPointAddedToLine: (arg: AddPointToLineArg) => void;
@@ -80,15 +87,20 @@ const maybeAddNextPoint = (
 export const setStageEventHandlers = ({
   stage,
   $tool,
+  $toolBuffer,
   $isDrawing,
+  $isMouseDown,
   $lastMouseDownPos,
   $lastCursorPos,
   $lastAddedPoint,
+  $stagePos,
+  $stageScale,
   $brushColor,
   $brushSize,
   $brushSpacingPx,
   $selectedLayer,
   $shouldInvertBrushSizeScrollDirection,
+  $isSpaceDown,
   onBrushLineAdded,
   onEraserLineAdded,
   onPointAddedToLine,
@@ -102,7 +114,7 @@ export const setStageEventHandlers = ({
       return;
     }
     const tool = $tool.get();
-    stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`)?.visible(tool === 'brush' || tool === 'eraser');
+    stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_TOOL_GROUP_ID}`)?.visible(tool === 'brush' || tool === 'eraser');
   });
 
   //#region mousedown
@@ -111,6 +123,7 @@ export const setStageEventHandlers = ({
     if (!stage) {
       return;
     }
+    $isMouseDown.set(true);
     const tool = $tool.get();
     const pos = updateLastCursorPos(stage, $lastCursorPos);
     const selectedLayer = $selectedLayer.get();
@@ -120,6 +133,12 @@ export const setStageEventHandlers = ({
     if (selectedLayer.type !== 'regional_guidance_layer' && selectedLayer.type !== 'raster_layer') {
       return;
     }
+
+    if ($isSpaceDown.get()) {
+      // No drawing when space is down - we are panning the stage
+      return;
+    }
+
     if (tool === 'brush') {
       onBrushLineAdded({
         layerId: selectedLayer.id,
@@ -151,6 +170,7 @@ export const setStageEventHandlers = ({
     if (!stage) {
       return;
     }
+    $isMouseDown.set(false);
     const pos = $lastCursorPos.get();
     const selectedLayer = $selectedLayer.get();
 
@@ -160,6 +180,12 @@ export const setStageEventHandlers = ({
     if (selectedLayer.type !== 'regional_guidance_layer' && selectedLayer.type !== 'raster_layer') {
       return;
     }
+
+    if ($isSpaceDown.get()) {
+      // No drawing when space is down - we are panning the stage
+      return;
+    }
+
     const tool = $tool.get();
 
     if (tool === 'rect') {
@@ -193,7 +219,7 @@ export const setStageEventHandlers = ({
     const pos = updateLastCursorPos(stage, $lastCursorPos);
     const selectedLayer = $selectedLayer.get();
 
-    stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`)?.visible(tool === 'brush' || tool === 'eraser');
+    stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_TOOL_GROUP_ID}`)?.visible(tool === 'brush' || tool === 'eraser');
 
     if (!pos || !selectedLayer) {
       return;
@@ -202,6 +228,11 @@ export const setStageEventHandlers = ({
       return;
     }
 
+    if ($isSpaceDown.get()) {
+      // No drawing when space is down - we are panning the stage
+      return;
+    }
+
     if (!getIsMouseDown(e)) {
       return;
     }
@@ -246,7 +277,7 @@ export const setStageEventHandlers = ({
     const selectedLayer = $selectedLayer.get();
     const tool = $tool.get();
 
-    stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`)?.visible(false);
+    stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_TOOL_GROUP_ID}`)?.visible(false);
 
     if (!pos || !selectedLayer) {
       return;
@@ -254,6 +285,10 @@ export const setStageEventHandlers = ({
     if (selectedLayer.type !== 'regional_guidance_layer' && selectedLayer.type !== 'raster_layer') {
       return;
     }
+    if ($isSpaceDown.get()) {
+      // No drawing when space is down - we are panning the stage
+      return;
+    }
     if (getIsMouseDown(e)) {
       if (tool === 'brush') {
         onPointAddedToLine({ layerId: selectedLayer.id, point: [pos.x, pos.y] });
@@ -267,28 +302,73 @@ export const setStageEventHandlers = ({
 
   stage.on('wheel', (e) => {
     e.evt.preventDefault();
-    const tool = $tool.get();
-    const selectedLayer = $selectedLayer.get();
-
-    if (tool !== 'brush' && tool !== 'eraser') {
-      return;
-    }
-    if (!selectedLayer) {
-      return;
-    }
-    if (selectedLayer.type !== 'regional_guidance_layer' && selectedLayer.type !== 'raster_layer') {
-      return;
-    }
-    // Invert the delta if the property is set to true
-    let delta = e.evt.deltaY;
-    if ($shouldInvertBrushSizeScrollDirection.get()) {
-      delta = -delta;
-    }
 
     if (e.evt.ctrlKey || e.evt.metaKey) {
+      let delta = e.evt.deltaY;
+      if ($shouldInvertBrushSizeScrollDirection.get()) {
+        delta = -delta;
+      }
+      // Holding ctrl or meta while scrolling changes the brush size
       onBrushSizeChanged(calculateNewBrushSize($brushSize.get(), delta));
+    } else {
+      // We need the absolute cursor position - not the scaled position
+      const cursorPos = stage.getPointerPosition();
+      if (!cursorPos) {
+        return;
+      }
+      // Stage's x and y scale are always the same
+      const stageScale = stage.scaleX();
+      // When wheeling on trackpad, e.evt.ctrlKey is true - in that case, let's reverse the direction
+      const delta = e.evt.ctrlKey ? -e.evt.deltaY : e.evt.deltaY;
+      const mousePointTo = {
+        x: (cursorPos.x - stage.x()) / stageScale,
+        y: (cursorPos.y - stage.y()) / stageScale,
+      };
+      const newScale = clamp(stageScale * CANVAS_SCALE_BY ** delta, MIN_CANVAS_SCALE, MAX_CANVAS_SCALE);
+      const newPos = {
+        x: cursorPos.x - mousePointTo.x * newScale,
+        y: cursorPos.y - mousePointTo.y * newScale,
+      };
+
+      stage.scaleX(newScale);
+      stage.scaleY(newScale);
+      stage.position(newPos);
+      $stageScale.set(newScale);
+      $stagePos.set(newPos);
     }
   });
 
-  return () => stage.off('mousedown mouseup mousemove mouseenter mouseleave wheel');
+  const onKeyDown = (e: KeyboardEvent) => {
+    if (e.repeat) {
+      return;
+    }
+    // Cancel shape drawing on escape
+    if (e.key === 'Escape') {
+      $isDrawing.set(false);
+      $lastMouseDownPos.set(null);
+    } else if (e.key === ' ') {
+      $toolBuffer.set($tool.get());
+      $tool.set('view');
+    }
+  };
+  window.addEventListener('keydown', onKeyDown);
+
+  const onKeyUp = (e: KeyboardEvent) => {
+    // Cancel shape drawing on escape
+    if (e.repeat) {
+      return;
+    }
+    if (e.key === ' ') {
+      const toolBuffer = $toolBuffer.get();
+      $tool.set(toolBuffer ?? 'move');
+      $toolBuffer.set(null);
+    }
+  };
+  window.addEventListener('keyup', onKeyUp);
+
+  return () => {
+    stage.off('mousedown mouseup mousemove mouseenter mouseleave wheel');
+    window.removeEventListener('keydown', onKeyDown);
+    window.removeEventListener('keyup', onKeyUp);
+  };
 };

invokeai/frontend/web/src/features/controlLayers/konva/naming.ts

@@ -4,14 +4,14 @@
 
 // IDs for singleton Konva layers and objects
 export const TOOL_PREVIEW_LAYER_ID = 'tool_preview_layer';
+export const TOOL_PREVIEW_TOOL_GROUP_ID = 'tool_preview_layer.tool_group';
 export const TOOL_PREVIEW_BRUSH_GROUP_ID = 'tool_preview_layer.brush_group';
 export const TOOL_PREVIEW_BRUSH_FILL_ID = 'tool_preview_layer.brush_fill';
 export const TOOL_PREVIEW_BRUSH_BORDER_INNER_ID = 'tool_preview_layer.brush_border_inner';
 export const TOOL_PREVIEW_BRUSH_BORDER_OUTER_ID = 'tool_preview_layer.brush_border_outer';
 export const TOOL_PREVIEW_RECT_ID = 'tool_preview_layer.rect';
-export const BACKGROUND_LAYER_ID = 'background_layer';
+export const TOOL_PREVIEW_IMAGE_DIMS_RECT = 'tool_preview_layer.image_dims_rect';
-export const BACKGROUND_RECT_ID = 'background_layer.rect';
+
-export const NO_LAYERS_MESSAGE_LAYER_ID = 'no_layers_message';
 
 // Names for Konva layers and objects (comparable to CSS classes)
 export const LAYER_BBOX_NAME = 'layer.bbox';

invokeai/frontend/web/src/features/controlLayers/konva/renderers/background.ts

@@ -1,67 +0,0 @@
-import { TRANSPARENCY_CHECKER_PATTERN } from 'features/controlLayers/konva/constants';
-import { BACKGROUND_LAYER_ID, BACKGROUND_RECT_ID } from 'features/controlLayers/konva/naming';
-import Konva from 'konva';
-import { assert } from 'tsafe';
-
-/**
- * The stage background is a semi-transparent checkerboard pattern. We use konva's `fillPatternImage` to apply the
- * a data URL of the pattern image to the background rect. Some scaling and positioning is required to ensure the
- * everything lines up correctly.
- */
-
-/**
- * Creates the background layer for the stage.
- * @param stage The konva stage
- */
-const createBackgroundLayer = (stage: Konva.Stage): Konva.Layer => {
-  const layer = new Konva.Layer({
-    id: BACKGROUND_LAYER_ID,
-  });
-  const background = new Konva.Rect({
-    id: BACKGROUND_RECT_ID,
-    x: stage.x(),
-    y: 0,
-    width: stage.width() / stage.scaleX(),
-    height: stage.height() / stage.scaleY(),
-    listening: false,
-    opacity: 0.2,
-  });
-  layer.add(background);
-  stage.add(layer);
-  const image = new Image();
-  image.onload = () => {
-    background.fillPatternImage(image);
-  };
-  image.src = TRANSPARENCY_CHECKER_PATTERN;
-  return layer;
-};
-
-/**
- * Renders the background layer for the stage.
- * @param stage The konva stage
- * @param width The unscaled width of the canvas
- * @param height The unscaled height of the canvas
- */
-export const renderBackground = (stage: Konva.Stage, width: number, height: number): void => {
-  const layer = stage.findOne<Konva.Layer>(`#${BACKGROUND_LAYER_ID}`) ?? createBackgroundLayer(stage);
-
-  const background = layer.findOne<Konva.Rect>(`#${BACKGROUND_RECT_ID}`);
-  assert(background, 'Background rect not found');
-  // ensure background rect is in the top-left of the canvas
-  background.absolutePosition({ x: 0, y: 0 });
-
-  // set the dimensions of the background rect to match the canvas - not the stage!!!
-  background.size({
-    width: width / stage.scaleX(),
-    height: height / stage.scaleY(),
-  });
-
-  // Calculate the amount the stage is moved - including the effect of scaling
-  const stagePos = {
-    x: -stage.x() / stage.scaleX(),
-    y: -stage.y() / stage.scaleY(),
-  };
-
-  // Apply that movement to the fill pattern
-  background.fillPatternOffset(stagePos);
-};

invokeai/frontend/web/src/features/controlLayers/konva/renderers/layers.ts

@@ -1,10 +1,8 @@
 import { DEBOUNCE_MS } from 'features/controlLayers/konva/constants';
-import { BACKGROUND_LAYER_ID, TOOL_PREVIEW_LAYER_ID } from 'features/controlLayers/konva/naming';
+import { TOOL_PREVIEW_LAYER_ID } from 'features/controlLayers/konva/naming';
-import { renderBackground } from 'features/controlLayers/konva/renderers/background';
 import { updateBboxes } from 'features/controlLayers/konva/renderers/bbox';
 import { renderCALayer } from 'features/controlLayers/konva/renderers/caLayer';
 import { renderIILayer } from 'features/controlLayers/konva/renderers/iiLayer';
-import { renderNoLayersMessage } from 'features/controlLayers/konva/renderers/noLayersMessage';
 import { renderRasterLayer } from 'features/controlLayers/konva/renderers/rasterLayer';
 import { renderRGLayer } from 'features/controlLayers/konva/renderers/rgLayer';
 import { renderToolPreview } from 'features/controlLayers/konva/renderers/toolPreview';
@@ -25,23 +23,6 @@ import type { ImageDTO } from 'services/api/types';
  * Logic for rendering arranging and rendering all layers.
  */
 
-/**
- * Arranges all layers in the z-axis by updating their z-indices.
- * @param stage The konva stage
- * @param layerIds An array of redux layer ids, in their z-index order
- */
-const arrangeLayers = (stage: Konva.Stage, layerIds: string[]): void => {
-  let nextZIndex = 0;
-  // Background is the first layer
-  stage.findOne<Konva.Layer>(`#${BACKGROUND_LAYER_ID}`)?.zIndex(nextZIndex++);
-  // Then arrange the redux layers in order
-  for (const layerId of layerIds) {
-    stage.findOne<Konva.Layer>(`#${layerId}`)?.zIndex(nextZIndex++);
-  }
-  // Finally, the tool preview layer is always on top
-  stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`)?.zIndex(nextZIndex++);
-};
-
 /**
  * Renders the layers on the stage.
  * @param stage The konva stage
@@ -66,7 +47,8 @@ const renderLayers = (
       konvaLayer.destroy();
     }
   }
-
+  // We'll need to ensure the tool preview layer is on top of the rest of the layers
+  let toolLayerZIndex = 0;
   for (const layer of layerStates) {
     if (isRegionalGuidanceLayer(layer)) {
       renderRGLayer(stage, layer, globalMaskLayerOpacity, tool, onLayerPosChanged);
@@ -81,7 +63,11 @@ const renderLayers = (
       renderRasterLayer(stage, layer, tool, onLayerPosChanged);
     }
     // IP Adapter layers are not rendered
+    // Increment the z-index for the tool layer
+    toolLayerZIndex++;
   }
+  // Arrange the tool preview layer
+  stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`)?.zIndex(toolLayerZIndex);
 };
 
 /**
@@ -90,9 +76,6 @@ const renderLayers = (
 export const renderers = {
   renderToolPreview,
   renderLayers,
-  renderBackground,
-  renderNoLayersMessage,
-  arrangeLayers,
   updateBboxes,
 };
 
@@ -104,9 +87,6 @@ export const renderers = {
 const getDebouncedRenderers = (ms = DEBOUNCE_MS): typeof renderers => ({
   renderToolPreview: debounce(renderToolPreview, ms),
   renderLayers: debounce(renderLayers, ms),
-  renderBackground: debounce(renderBackground, ms),
-  renderNoLayersMessage: debounce(renderNoLayersMessage, ms),
-  arrangeLayers: debounce(arrangeLayers, ms),
   updateBboxes: debounce(updateBboxes, ms),
 });
 

invokeai/frontend/web/src/features/controlLayers/konva/renderers/noLayersMessage.ts

@@ -1,53 +0,0 @@
-import { NO_LAYERS_MESSAGE_LAYER_ID } from 'features/controlLayers/konva/naming';
-import { t } from 'i18next';
-import Konva from 'konva';
-
-/**
- * Logic for creating and rendering a fallback message when there are no layers to render.
- */
-
-/**
- * Creates the "no layers" fallback layer
- * @param stage The konva stage
- */
-const createNoLayersMessageLayer = (stage: Konva.Stage): Konva.Layer => {
-  const noLayersMessageLayer = new Konva.Layer({
-    id: NO_LAYERS_MESSAGE_LAYER_ID,
-    opacity: 0.7,
-    listening: false,
-  });
-  const text = new Konva.Text({
-    x: 0,
-    y: 0,
-    align: 'center',
-    verticalAlign: 'middle',
-    text: t('controlLayers.noLayersAdded', 'No Layers Added'),
-    fontFamily: '"Inter Variable", sans-serif',
-    fontStyle: '600',
-    fill: 'white',
-  });
-  noLayersMessageLayer.add(text);
-  stage.add(noLayersMessageLayer);
-  return noLayersMessageLayer;
-};
-
-/**
- * Renders the "no layers" message when there are no layers to render
- * @param stage The konva stage
- * @param layerCount The current number of layers
- * @param width The target width of the text
- * @param height The target height of the text
- */
-export const renderNoLayersMessage = (stage: Konva.Stage, layerCount: number, width: number, height: number): void => {
-  const noLayersMessageLayer =
-    stage.findOne<Konva.Layer>(`#${NO_LAYERS_MESSAGE_LAYER_ID}`) ?? createNoLayersMessageLayer(stage);
-  if (layerCount === 0) {
-    noLayersMessageLayer.findOne<Konva.Text>('Text')?.setAttrs({
-      width,
-      height,
-      fontSize: 32 / stage.scaleX(),
-    });
-  } else {
-    noLayersMessageLayer?.destroy();
-  }
-};

invokeai/frontend/web/src/features/controlLayers/konva/renderers/objects.ts

@@ -1,5 +1,10 @@
 import { rgbaColorToString } from 'features/canvas/util/colorToString';
-import { getLayerBboxId, getObjectGroupId, LAYER_BBOX_NAME } from 'features/controlLayers/konva/naming';
+import {
+  getLayerBboxId,
+  getObjectGroupId,
+  LAYER_BBOX_NAME,
+  TOOL_PREVIEW_IMAGE_DIMS_RECT,
+} from 'features/controlLayers/konva/naming';
 import type { BrushLine, EraserLine, ImageObject, Layer, RectShape } from 'features/controlLayers/store/types';
 import { DEFAULT_RGBA_COLOR } from 'features/controlLayers/store/types';
 import { t } from 'i18next';
@@ -198,3 +203,18 @@ export const createObjectGroup = (konvaLayer: Konva.Layer, name: string): Konva.
   konvaLayer.add(konvaObjectGroup);
   return konvaObjectGroup;
 };
+
+export const createImageDimsPreview = (konvaLayer: Konva.Layer, width: number, height: number): Konva.Rect => {
+  const imageDimsPreview = new Konva.Rect({
+    id: TOOL_PREVIEW_IMAGE_DIMS_RECT,
+    x: 0,
+    y: 0,
+    width,
+    height,
+    stroke: 'rgb(255,0,255)',
+    strokeWidth: 1 / konvaLayer.getStage().scaleX(),
+    listening: false,
+  });
+  konvaLayer.add(imageDimsPreview);
+  return imageDimsPreview;
+};

invokeai/frontend/web/src/features/controlLayers/konva/renderers/rasterLayer.ts

@@ -16,7 +16,7 @@ import {
   createObjectGroup,
   createRectShape,
 } from 'features/controlLayers/konva/renderers/objects';
-import { getScaledFlooredCursorPosition, mapId, selectRasterObjects } from 'features/controlLayers/konva/util';
+import { mapId, selectRasterObjects } from 'features/controlLayers/konva/util';
 import type { RasterLayer, Tool } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 
@@ -51,24 +51,6 @@ const createRasterLayer = (
     });
   }
 
-  // The dragBoundFunc limits how far the layer can be dragged
-  konvaLayer.dragBoundFunc(function (pos) {
-    const cursorPos = getScaledFlooredCursorPosition(stage);
-    if (!cursorPos) {
-      return this.getAbsolutePosition();
-    }
-    // Prevent the user from dragging the layer out of the stage bounds by constaining the cursor position to the stage bounds
-    if (
-      cursorPos.x < 0 ||
-      cursorPos.x > stage.width() / stage.scaleX() ||
-      cursorPos.y < 0 ||
-      cursorPos.y > stage.height() / stage.scaleY()
-    ) {
-      return this.getAbsolutePosition();
-    }
-    return pos;
-  });
-
   stage.add(konvaLayer);
 
   return konvaLayer;
@@ -156,6 +138,7 @@ export const renderRasterLayer = async (
       width: layerState.bbox.width,
       height: layerState.bbox.height,
       stroke: layerState.isSelected ? BBOX_SELECTED_STROKE : '',
+      strokeWidth: 1 / stage.scaleX(),
     });
   } else {
     bboxRect.visible(false);

invokeai/frontend/web/src/features/controlLayers/konva/renderers/rgLayer.ts

@@ -17,7 +17,7 @@ import {
   createObjectGroup,
   createRectShape,
 } from 'features/controlLayers/konva/renderers/objects';
-import { getScaledFlooredCursorPosition, mapId, selectVectorMaskObjects } from 'features/controlLayers/konva/util';
+import { mapId, selectVectorMaskObjects } from 'features/controlLayers/konva/util';
 import type { RegionalGuidanceLayer, Tool } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 
@@ -65,24 +65,6 @@ const createRGLayer = (
     });
   }
 
-  // The dragBoundFunc limits how far the layer can be dragged
-  konvaLayer.dragBoundFunc(function (pos) {
-    const cursorPos = getScaledFlooredCursorPosition(stage);
-    if (!cursorPos) {
-      return this.getAbsolutePosition();
-    }
-    // Prevent the user from dragging the layer out of the stage bounds by constaining the cursor position to the stage bounds
-    if (
-      cursorPos.x < 0 ||
-      cursorPos.x > stage.width() / stage.scaleX() ||
-      cursorPos.y < 0 ||
-      cursorPos.y > stage.height() / stage.scaleY()
-    ) {
-      return this.getAbsolutePosition();
-    }
-    return pos;
-  });
-
   stage.add(konvaLayer);
 
   return konvaLayer;

invokeai/frontend/web/src/features/controlLayers/konva/renderers/toolPreview.ts

@@ -9,8 +9,10 @@ import {
   TOOL_PREVIEW_BRUSH_BORDER_OUTER_ID,
   TOOL_PREVIEW_BRUSH_FILL_ID,
   TOOL_PREVIEW_BRUSH_GROUP_ID,
+  TOOL_PREVIEW_IMAGE_DIMS_RECT,
   TOOL_PREVIEW_LAYER_ID,
   TOOL_PREVIEW_RECT_ID,
+  TOOL_PREVIEW_TOOL_GROUP_ID,
 } from 'features/controlLayers/konva/naming';
 import { selectRenderableLayers, snapPosToStage } from 'features/controlLayers/konva/util';
 import type { Layer, RgbaColor, Tool } from 'features/controlLayers/store/types';
@@ -26,9 +28,13 @@ import { assert } from 'tsafe';
  * Creates the singleton tool preview layer and all its objects.
  * @param stage The konva stage
  */
-const createToolPreviewLayer = (stage: Konva.Stage): Konva.Layer => {
+const getToolPreviewLayer = (stage: Konva.Stage): Konva.Layer => {
+  let toolPreviewLayer = stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`);
+  if (toolPreviewLayer) {
+    return toolPreviewLayer;
+  }
   // Initialize the brush preview layer & add to the stage
-  const toolPreviewLayer = new Konva.Layer({ id: TOOL_PREVIEW_LAYER_ID, visible: false, listening: false });
+  toolPreviewLayer = new Konva.Layer({ id: TOOL_PREVIEW_LAYER_ID, listening: false });
   stage.add(toolPreviewLayer);
 
   // Create the brush preview group & circles
@@ -55,7 +61,6 @@ const createToolPreviewLayer = (stage: Konva.Stage): Konva.Layer => {
     strokeEnabled: true,
   });
   brushPreviewGroup.add(brushPreviewBorderOuter);
-  toolPreviewLayer.add(brushPreviewGroup);
 
   // Create the rect preview - this is a rectangle drawn from the last mouse down position to the current cursor position
   const rectPreview = new Konva.Rect({
@@ -64,11 +69,38 @@ const createToolPreviewLayer = (stage: Konva.Stage): Konva.Layer => {
     stroke: BBOX_SELECTED_STROKE,
     strokeWidth: 1,
   });
-  toolPreviewLayer.add(rectPreview);
+
+  const toolGroup = new Konva.Group({ id: TOOL_PREVIEW_TOOL_GROUP_ID });
+
+  toolGroup.add(rectPreview);
+  toolGroup.add(brushPreviewGroup);
+
+  const imageDimsPreview = new Konva.Rect({
+    id: TOOL_PREVIEW_IMAGE_DIMS_RECT,
+    x: 0,
+    y: 0,
+    width: 0,
+    height: 0,
+    stroke: 'rgb(255,0,255)',
+    strokeWidth: 1 / toolPreviewLayer.getStage().scaleX(),
+    listening: false,
+  });
+
+  toolPreviewLayer.add(toolGroup);
+  toolPreviewLayer.add(imageDimsPreview);
 
   return toolPreviewLayer;
 };
 
+export const renderImageDimsPreview = (stage: Konva.Stage, width: number, height: number, stageScale: number): void => {
+  const imageDimsPreview = stage.findOne<Konva.Rect>(`#${TOOL_PREVIEW_IMAGE_DIMS_RECT}`);
+  imageDimsPreview?.setAttrs({
+    width,
+    height,
+    strokeWidth: 1 / stageScale,
+  });
+};
+
 /**
  * Renders the brush preview for the selected tool.
  * @param stage The konva stage
@@ -89,11 +121,15 @@ export const renderToolPreview = (
   cursorPos: Vector2d | null,
   lastMouseDownPos: Vector2d | null,
   brushSize: number,
-  isDrawing: boolean
+  isDrawing: boolean,
+  isMouseDown: boolean
 ): void => {
   const layerCount = stage.find(selectRenderableLayers).length;
   // Update the stage's pointer style
-  if (layerCount === 0) {
+  if (tool === 'view') {
+    // View gets a hand
+    stage.container().style.cursor = isMouseDown ? 'grabbing' : 'grab';
+  } else if (layerCount === 0) {
     // We have no layers, so we should not render any tool
     stage.container().style.cursor = 'default';
   } else if (selectedLayerType !== 'regional_guidance_layer' && selectedLayerType !== 'raster_layer') {
@@ -103,70 +139,74 @@ export const renderToolPreview = (
     // Move tool gets a pointer
     stage.container().style.cursor = 'default';
   } else if (tool === 'rect') {
-    // Move rect gets a crosshair
+    // Rect gets a crosshair
     stage.container().style.cursor = 'crosshair';
   } else {
     // Else we hide the native cursor and use the konva-rendered brush preview
     stage.container().style.cursor = 'none';
   }
 
-  const toolPreviewLayer = stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`) ?? createToolPreviewLayer(stage);
+  stage.draggable(tool === 'view');
+
+  const toolPreviewLayer = getToolPreviewLayer(stage);
+  const toolGroup = toolPreviewLayer.findOne<Konva.Group>(`#${TOOL_PREVIEW_TOOL_GROUP_ID}`);
+
+  assert(toolGroup, 'Tool group not found');
 
   if (!cursorPos || layerCount === 0) {
     // We can bail early if the mouse isn't over the stage or there are no layers
-    toolPreviewLayer.visible(false);
+    toolGroup.visible(false);
-    return;
-  }
-
-  toolPreviewLayer.visible(true);
-
-  const brushPreviewGroup = stage.findOne<Konva.Group>(`#${TOOL_PREVIEW_BRUSH_GROUP_ID}`);
-  assert(brushPreviewGroup, 'Brush preview group not found');
-
-  const rectPreview = stage.findOne<Konva.Rect>(`#${TOOL_PREVIEW_RECT_ID}`);
-  assert(rectPreview, 'Rect preview not found');
-
-  // No need to render the brush preview if the cursor position or color is missing
-  if (cursorPos && (tool === 'brush' || tool === 'eraser')) {
-    // Update the fill circle
-    const brushPreviewFill = brushPreviewGroup.findOne<Konva.Circle>(`#${TOOL_PREVIEW_BRUSH_FILL_ID}`);
-    brushPreviewFill?.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
-      radius: brushSize / 2,
-      fill: isDrawing ? '' : rgbaColorToString(brushColor),
-      globalCompositeOperation: tool === 'brush' ? 'source-over' : 'destination-out',
-    });
-
-    // Update the inner border of the brush preview
-    const brushPreviewInner = toolPreviewLayer.findOne<Konva.Circle>(`#${TOOL_PREVIEW_BRUSH_BORDER_INNER_ID}`);
-    brushPreviewInner?.setAttrs({ x: cursorPos.x, y: cursorPos.y, radius: brushSize / 2 });
-
-    // Update the outer border of the brush preview
-    const brushPreviewOuter = toolPreviewLayer.findOne<Konva.Circle>(`#${TOOL_PREVIEW_BRUSH_BORDER_OUTER_ID}`);
-    brushPreviewOuter?.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
-      radius: brushSize / 2 + 1,
-    });
-
-    brushPreviewGroup.visible(true);
   } else {
-    brushPreviewGroup.visible(false);
+    toolGroup.visible(true);
-  }
 
-  if (cursorPos && lastMouseDownPos && tool === 'rect') {
+    const brushPreviewGroup = stage.findOne<Konva.Group>(`#${TOOL_PREVIEW_BRUSH_GROUP_ID}`);
-    const snappedPos = snapPosToStage(cursorPos, stage);
+    assert(brushPreviewGroup, 'Brush preview group not found');
-    const rectPreview = toolPreviewLayer.findOne<Konva.Rect>(`#${TOOL_PREVIEW_RECT_ID}`);
+
-    rectPreview?.setAttrs({
+    const rectPreview = stage.findOne<Konva.Rect>(`#${TOOL_PREVIEW_RECT_ID}`);
-      x: Math.min(snappedPos.x, lastMouseDownPos.x),
+    assert(rectPreview, 'Rect preview not found');
-      y: Math.min(snappedPos.y, lastMouseDownPos.y),
+
-      width: Math.abs(snappedPos.x - lastMouseDownPos.x),
+    // No need to render the brush preview if the cursor position or color is missing
-      height: Math.abs(snappedPos.y - lastMouseDownPos.y),
+    if (cursorPos && (tool === 'brush' || tool === 'eraser')) {
-      fill: rgbaColorToString(brushColor),
+      // Update the fill circle
-    });
+      const brushPreviewFill = brushPreviewGroup.findOne<Konva.Circle>(`#${TOOL_PREVIEW_BRUSH_FILL_ID}`);
-    rectPreview?.visible(true);
+      brushPreviewFill?.setAttrs({
-  } else {
+        x: cursorPos.x,
-    rectPreview?.visible(false);
+        y: cursorPos.y,
+        radius: brushSize / 2,
+        fill: isDrawing ? '' : rgbaColorToString(brushColor),
+        globalCompositeOperation: tool === 'brush' ? 'source-over' : 'destination-out',
+      });
+
+      // Update the inner border of the brush preview
+      const brushPreviewInner = toolPreviewLayer.findOne<Konva.Circle>(`#${TOOL_PREVIEW_BRUSH_BORDER_INNER_ID}`);
+      brushPreviewInner?.setAttrs({ x: cursorPos.x, y: cursorPos.y, radius: brushSize / 2 });
+
+      // Update the outer border of the brush preview
+      const brushPreviewOuter = toolPreviewLayer.findOne<Konva.Circle>(`#${TOOL_PREVIEW_BRUSH_BORDER_OUTER_ID}`);
+      brushPreviewOuter?.setAttrs({
+        x: cursorPos.x,
+        y: cursorPos.y,
+        radius: brushSize / 2 + 1,
+      });
+
+      brushPreviewGroup.visible(true);
+    } else {
+      brushPreviewGroup.visible(false);
+    }
+
+    if (cursorPos && lastMouseDownPos && tool === 'rect') {
+      const snappedPos = snapPosToStage(cursorPos, stage);
+      const rectPreview = toolPreviewLayer.findOne<Konva.Rect>(`#${TOOL_PREVIEW_RECT_ID}`);
+      rectPreview?.setAttrs({
+        x: Math.min(snappedPos.x, lastMouseDownPos.x),
+        y: Math.min(snappedPos.y, lastMouseDownPos.y),
+        width: Math.abs(snappedPos.x - lastMouseDownPos.x),
+        height: Math.abs(snappedPos.y - lastMouseDownPos.y),
+        fill: rgbaColorToString(brushColor),
+      });
+      rectPreview?.visible(true);
+    } else {
+      rectPreview?.visible(false);
+    }
   }
 };

invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts

@@ -979,11 +979,16 @@ const migrateControlLayersState = (state: any): any => {
 
 // Ephemeral interaction state
 export const $isDrawing = atom(false);
+export const $isMouseDown = atom(false);
 export const $lastMouseDownPos = atom<Vector2d | null>(null);
 export const $tool = atom<Tool>('brush');
+export const $toolBuffer = atom<Tool | null>(null);
 export const $lastCursorPos = atom<Vector2d | null>(null);
 export const $isPreviewVisible = atom(true);
 export const $lastAddedPoint = atom<Vector2d | null>(null);
+export const $isSpaceDown = atom(false);
+export const $stageScale = atom<number>(1);
+export const $stagePos = atom<Vector2d>({ x: 0, y: 0 });
 
 // Some nanostores that are manually synced to redux state to provide imperative access
 // TODO(psyche): This is a hack, figure out another way to handle this...

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -23,7 +23,7 @@ import type { IRect } from 'konva/lib/types';
 import type { ImageDTO } from 'services/api/types';
 import { z } from 'zod';
 
-const zTool = z.enum(['brush', 'eraser', 'move', 'rect']);
+const zTool = z.enum(['brush', 'eraser', 'move', 'rect', 'view']);
 export type Tool = z.infer<typeof zTool>;
 const zDrawingTool = zTool.extract(['brush', 'eraser']);