fix(ui): incorrect rect/brush/eraser positions

invokeai/frontend/web/src/features/controlLayers/konva/events.ts

@@ -50,6 +50,7 @@ type Arg = {
   setStageAttrs: (attrs: StageAttrs) => void;
   getSelectedEntity: () => CanvasEntity | null;
   getSpaceKey: () => boolean;
+  setSpaceKey: (val: boolean) => void;
   getDocument: () => CanvasV2State['document'];
   getBbox: () => CanvasV2State['bbox'];
   onBrushLineAdded: (arg: BrushLineAddedArg, entityType: CanvasEntity['type']) => void;
@@ -151,6 +152,7 @@ export const setStageEventHandlers = ({
   setStageAttrs,
   getSelectedEntity,
   getSpaceKey,
+  setSpaceKey,
   getDocument,
   getBbox,
   onBrushLineAdded,
@@ -174,7 +176,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -190,7 +192,6 @@ export const setStageEventHandlers = ({
     const toolState = getToolState();
     const pos = updateLastCursorPos(stage, setLastCursorPos);
     const selectedEntity = getSelectedEntity();
-
     if (
       pos &&
       selectedEntity &&
@@ -308,7 +309,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -339,8 +340,8 @@ export const setStageEventHandlers = ({
             {
               id: selectedEntity.id,
               rect: {
-                x: Math.min(pos.x, lastMouseDownPos.x),
+                x: Math.min(pos.x - selectedEntity.x, lastMouseDownPos.x - selectedEntity.x),
-                y: Math.min(pos.y, lastMouseDownPos.y),
+                y: Math.min(pos.y - selectedEntity.y, lastMouseDownPos.y - selectedEntity.y),
                 width: Math.abs(pos.x - lastMouseDownPos.x),
                 height: Math.abs(pos.y - lastMouseDownPos.y),
               },
@@ -361,7 +362,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -472,7 +473,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -516,7 +517,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -571,7 +572,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -594,7 +595,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -616,7 +617,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -638,6 +639,7 @@ export const setStageEventHandlers = ({
       // Select the view tool on space key down
       setToolBuffer(getToolState().selected);
       setTool('view');
+      setSpaceKey(true);
     } else if (e.key === 'r') {
       const stageAttrs = fitDocumentToStage(stage, getDocument());
       setStageAttrs(stageAttrs);
@@ -651,7 +653,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );
@@ -670,6 +672,7 @@ export const setStageEventHandlers = ({
       const toolBuffer = getToolState().selectedBuffer;
       setTool(toolBuffer ?? 'move');
       setToolBuffer(null);
+      setSpaceKey(false);
     }
     renderToolPreview(
       stage,
@@ -677,7 +680,7 @@ export const setStageEventHandlers = ({
       getCurrentFill(),
       getSelectedEntity(),
       getLastCursorPos(),
-      getLastAddedPoint(),
+      getLastMouseDownPos(),
       getIsDrawing(),
       getIsMouseDown()
     );

invokeai/frontend/web/src/features/controlLayers/konva/renderers/layers.ts

@@ -1,98 +1,6 @@
-import { DEBOUNCE_MS } from 'features/controlLayers/konva/constants';
 import { BACKGROUND_LAYER_ID, PREVIEW_LAYER_ID } from 'features/controlLayers/konva/naming';
-import { updateBboxes } from 'features/controlLayers/konva/renderers/bbox';
+import type { ControlAdapterEntity, LayerEntity, RegionEntity } from 'features/controlLayers/store/types';
-import { renderCALayer } from 'features/controlLayers/konva/renderers/caLayer';
-import { renderBboxPreview, renderToolPreview } from 'features/controlLayers/konva/renderers/previewLayer';
-import { renderRasterLayer } from 'features/controlLayers/konva/renderers/rasterLayer';
-import { renderRGLayer } from 'features/controlLayers/konva/renderers/rgLayer';
-import { mapId, selectRenderableLayers } from 'features/controlLayers/konva/util';
-import type {
-  CanvasEntity,
-  ControlAdapterEntity,
-  LayerEntity,
-  PosChangedArg,
-  RegionEntity,
-  Tool,
-} from 'features/controlLayers/store/types';
 import type Konva from 'konva';
-import { debounce } from 'lodash-es';
-import type { ImageDTO } from 'services/api/types';
-
-/**
- * Logic for rendering arranging and rendering all layers.
- */
-
-/**
- * Renders the layers on the stage.
- * @param stage The konva stage
- * @param layers Array of all layer states
- * @param rgGlobalOpacity The global mask layer opacity
- * @param tool The current tool
- * @param getImageDTO A function to retrieve an image DTO from the server, used to update the image source
- * @param onPosChanged Callback for when the layer's position changes
- */
-const renderLayers = (
-  stage: Konva.Stage,
-  layers: LayerEntity[],
-  controlAdapters: ControlAdapterEntity[],
-  regions: RegionEntity[],
-  rgGlobalOpacity: number,
-  tool: Tool,
-  selectedEntity: CanvasEntity | null,
-  getImageDTO: (imageName: string) => Promise<ImageDTO | null>,
-  onPosChanged?: (arg: PosChangedArg, entityType: CanvasEntity['type']) => void
-): void => {
-  const renderableIds = [...layers.map(mapId), ...controlAdapters.map(mapId), ...regions.map(mapId)];
-  // Remove un-rendered layers
-  for (const konvaLayer of stage.find<Konva.Layer>(selectRenderableLayers)) {
-    if (!renderableIds.includes(konvaLayer.id())) {
-      konvaLayer.destroy();
-    }
-  }
-  // We'll need to ensure the tool preview layer is on top of the rest of the layers
-  let zIndex = 1;
-  for (const layer of layers) {
-    renderRasterLayer(stage, layer, tool, zIndex, onPosChanged);
-    zIndex++;
-  }
-  for (const ca of controlAdapters) {
-    renderCALayer(stage, ca, zIndex, getImageDTO);
-    zIndex++;
-  }
-  for (const rg of regions) {
-    renderRGLayer(stage, rg, rgGlobalOpacity, tool, zIndex, selectedEntity, onPosChanged);
-    zIndex++;
-  }
-  // Arrange the tool preview layer
-  stage.findOne<Konva.Layer>(`#${PREVIEW_LAYER_ID}`)?.zIndex(zIndex);
-};
-
-/**
- * All the renderers for the Konva stage.
- */
-export const renderers = {
-  renderToolPreview,
-  renderBboxPreview,
-  renderLayers,
-  updateBboxes,
-};
-
-/**
- * Gets the renderers with debouncing applied.
- * @param ms The debounce time in milliseconds
- * @returns The renderers with debouncing applied
- */
-const getDebouncedRenderers = (ms = DEBOUNCE_MS): typeof renderers => ({
-  renderToolPreview: debounce(renderToolPreview, ms),
-  renderBboxPreview: debounce(renderBboxPreview, ms),
-  renderLayers: debounce(renderLayers, ms),
-  updateBboxes: debounce(updateBboxes, ms),
-});
-
-/**
- * All the renderers for the Konva stage, debounced.
- */
-export const debouncedRenderers: typeof renderers = getDebouncedRenderers();
 
 export const arrangeEntities = (
   stage: Konva.Stage,

invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts

@@ -17,12 +17,6 @@ import { renderLayers } from 'features/controlLayers/konva/renderers/rasterLayer
 import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer';
 import { fitDocumentToStage } from 'features/controlLayers/konva/renderers/stage';
 import {
-  $isDrawing,
-  $isMouseDown,
-  $lastAddedPoint,
-  $lastCursorPos,
-  $lastMouseDownPos,
-  $spaceKey,
   $stageAttrs,
   bboxChanged,
   brushWidthChanged,
@@ -48,7 +42,6 @@ import type {
   BboxChangedArg,
   BrushLineAddedArg,
   CanvasEntity,
-  CanvasEntityIdentifier,
   CanvasV2State,
   EraserLineAddedArg,
   PointAddedToLineArg,
@@ -57,7 +50,7 @@ import type {
   Tool,
 } from 'features/controlLayers/store/types';
 import type Konva from 'konva';
-import type { IRect } from 'konva/lib/types';
+import type { IRect, Vector2d } from 'konva/lib/types';
 import { debounce } from 'lodash-es';
 import type { RgbaColor } from 'react-colorful';
 import { getImageDTO } from 'services/api/endpoints/images';
@@ -200,46 +193,77 @@ export const initializeRenderer = (
 
   const { getState, subscribe, dispatch } = store;
 
-  // Create closures for the rendering functions, used to check if specific parts of state have changed so we only
-  // render what needs to be rendered.
-  let prevCanvasV2 = getState().canvasV2;
-  let selectedEntityIdentifier: CanvasEntityIdentifier | null = prevCanvasV2.selectedEntityIdentifier;
-  let selectedEntity: CanvasEntity | null = _getSelectedEntity(prevCanvasV2);
-  let currentFill: RgbaColor = _getCurrentFill(prevCanvasV2, selectedEntity);
-  let didSelectedEntityChange: boolean = false;
-
   // On the first render, we need to render everything.
   let isFirstRender = true;
 
-  // Stage event listeners use a fully imperative approach to event handling, using these helpers to get state.
+  // Stage interaction listeners need helpers to get and update current state. Some of the state is read-only, like
+  // bbox, document and tool state, while interaction state is read-write.
+
+  // Read-only state, derived from redux
+  let prevCanvasV2 = getState().canvasV2;
+  let prevSelectedEntity: CanvasEntity | null = _getSelectedEntity(prevCanvasV2);
+  let prevCurrentFill: RgbaColor = _getCurrentFill(prevCanvasV2, prevSelectedEntity);
+  const getSelectedEntity = () => prevSelectedEntity;
+  const getCurrentFill = () => prevCurrentFill;
   const getBbox = () => getState().canvasV2.bbox;
   const getDocument = () => getState().canvasV2.document;
   const getToolState = () => getState().canvasV2.tool;
-  const getSelectedEntity = () => selectedEntity;
-  const getCurrentFill = () => currentFill;
 
-  // Calculating bounding boxes is expensive, must be debounced to not block the UI thread.
+  // Read-write state, ephemeral interaction state
-  // TODO(psyche): Figure out how to do this in a worker. Probably means running the renderer in a worker and sending
+  let isDrawing = false;
-  // the entire state over when needed.
+  const getIsDrawing = () => isDrawing;
-  const debouncedUpdateBboxes = debounce(updateBboxes, 300);
+  const setIsDrawing = (val: boolean) => {
+    isDrawing = val;
+  };
+
+  let isMouseDown = false;
+  const getIsMouseDown = () => isMouseDown;
+  const setIsMouseDown = (val: boolean) => {
+    isMouseDown = val;
+  };
+
+  let lastAddedPoint: Vector2d | null = null;
+  const getLastAddedPoint = () => lastAddedPoint;
+  const setLastAddedPoint = (val: Vector2d | null) => {
+    lastAddedPoint = val;
+  };
+
+  let lastMouseDownPos: Vector2d | null = null;
+  const getLastMouseDownPos = () => lastMouseDownPos;
+  const setLastMouseDownPos = (val: Vector2d | null) => {
+    lastMouseDownPos = val;
+  };
+
+  let lastCursorPos: Vector2d | null = null;
+  const getLastCursorPos = () => lastCursorPos;
+  const setLastCursorPos = (val: Vector2d | null) => {
+    lastCursorPos = val;
+  };
+
+  let spaceKey = false;
+  const getSpaceKey = () => spaceKey;
+  const setSpaceKey = (val: boolean) => {
+    spaceKey = val;
+  };
 
   const cleanupListeners = setStageEventHandlers({
     stage,
     getToolState,
     setTool,
     setToolBuffer,
-    getIsDrawing: $isDrawing.get,
+    getIsDrawing,
-    setIsDrawing: $isDrawing.set,
+    setIsDrawing,
-    getIsMouseDown: $isMouseDown.get,
+    getIsMouseDown,
-    setIsMouseDown: $isMouseDown.set,
+    setIsMouseDown,
     getSelectedEntity,
-    getLastAddedPoint: $lastAddedPoint.get,
+    getLastAddedPoint,
-    setLastAddedPoint: $lastAddedPoint.set,
+    setLastAddedPoint,
-    getLastCursorPos: $lastCursorPos.get,
+    getLastCursorPos,
-    setLastCursorPos: $lastCursorPos.set,
+    setLastCursorPos,
-    getLastMouseDownPos: $lastMouseDownPos.get,
+    getLastMouseDownPos,
-    setLastMouseDownPos: $lastMouseDownPos.set,
+    setLastMouseDownPos,
-    getSpaceKey: $spaceKey.get,
+    getSpaceKey,
+    setSpaceKey,
     setStageAttrs: $stageAttrs.set,
     getDocument,
     getBbox,
@@ -252,6 +276,11 @@ export const initializeRenderer = (
     getCurrentFill,
   });
 
+  // Calculating bounding boxes is expensive, must be debounced to not block the UI thread during a user interaction.
+  // TODO(psyche): Figure out how to do this in a worker. Probably means running the renderer in a worker and sending
+  // the entire state over when needed.
+  const debouncedUpdateBboxes = debounce(updateBboxes, 300);
+
   const renderCanvas = () => {
     const { canvasV2 } = store.getState();
 
@@ -260,20 +289,8 @@ export const initializeRenderer = (
       return;
     }
 
-    // We can save some cycles for specific renderers if we track whether the selected entity has changed.
+    const selectedEntity = _getSelectedEntity(canvasV2);
-    if (canvasV2.selectedEntityIdentifier !== selectedEntityIdentifier) {
+    const currentFill = _getCurrentFill(canvasV2, selectedEntity);
-      selectedEntityIdentifier = canvasV2.selectedEntityIdentifier;
-      selectedEntity = _getSelectedEntity(canvasV2);
-      didSelectedEntityChange = true;
-    } else {
-      didSelectedEntityChange = false;
-    }
-
-    // The current fill is either the tool fill or, if a regional guidance region is selected, the mask fill for that
-    // region. We need to manually sync this state.
-    if (isFirstRender || canvasV2.tool.fill !== prevCanvasV2.tool.fill || didSelectedEntityChange) {
-      currentFill = _getCurrentFill(canvasV2, selectedEntity);
-    }
 
     if (
       isFirstRender ||
@@ -288,8 +305,7 @@ export const initializeRenderer = (
       isFirstRender ||
       canvasV2.regions !== prevCanvasV2.regions ||
       canvasV2.settings.maskOpacity !== prevCanvasV2.settings.maskOpacity ||
-      canvasV2.tool.selected !== prevCanvasV2.tool.selected ||
+      canvasV2.tool.selected !== prevCanvasV2.tool.selected
-      didSelectedEntityChange
     ) {
       logIfDebugging('Rendering regions');
       renderRegions(
@@ -297,7 +313,7 @@ export const initializeRenderer = (
         canvasV2.regions,
         canvasV2.settings.maskOpacity,
         canvasV2.tool.selected,
-        selectedEntity,
+        canvasV2.selectedEntityIdentifier,
         onPosChanged
       );
     }
@@ -348,6 +364,8 @@ export const initializeRenderer = (
     }
 
     prevCanvasV2 = canvasV2;
+    prevSelectedEntity = selectedEntity;
+    prevCurrentFill = currentFill;
 
     if (isFirstRender) {
       isFirstRender = false;

invokeai/frontend/web/src/features/controlLayers/konva/renderers/rgLayer.ts

@@ -18,7 +18,13 @@ import {
   getEraserLine,
 } from 'features/controlLayers/konva/renderers/objects';
 import { mapId, selectVectorMaskObjects } from 'features/controlLayers/konva/util';
-import type { CanvasEntity, PosChangedArg, RegionEntity, Tool } from 'features/controlLayers/store/types';
+import type {
+  CanvasEntity,
+  CanvasEntityIdentifier,
+  PosChangedArg,
+  RegionEntity,
+  Tool,
+} from 'features/controlLayers/store/types';
 import Konva from 'konva';
 
 /**
@@ -83,7 +89,7 @@ export const renderRGLayer = (
   rg: RegionEntity,
   globalMaskLayerOpacity: number,
   tool: Tool,
-  selectedEntity: CanvasEntity | null,
+  selectedEntityIdentifier: CanvasEntityIdentifier | null,
   onPosChanged?: (arg: PosChangedArg, entityType: CanvasEntity['type']) => void
 ): void => {
   const konvaLayer = stage.findOne<Konva.Layer>(`#${rg.id}`) ?? createRGLayer(stage, rg, onPosChanged);
@@ -171,7 +177,7 @@ export const renderRGLayer = (
 
   const compositingRect =
     konvaLayer.findOne<Konva.Rect>(`.${COMPOSITING_RECT_NAME}`) ?? createCompositingRect(konvaLayer);
-  const isSelected = selectedEntity?.id === rg.id;
+  const isSelected = selectedEntityIdentifier?.id === rg.id;
 
   /**
    * When the group is selected, we use a rect of the selected preview color, composited over the shapes. This allows
@@ -237,7 +243,7 @@ export const renderRegions = (
   regions: RegionEntity[],
   maskOpacity: number,
   tool: Tool,
-  selectedEntity: CanvasEntity | null,
+  selectedEntityIdentifier: CanvasEntityIdentifier | null,
   onPosChanged?: (arg: PosChangedArg, entityType: CanvasEntity['type']) => void
 ): void => {
   // Destroy nonexistent layers
@@ -247,6 +253,6 @@ export const renderRegions = (
     }
   }
   for (const rg of regions) {
-    renderRGLayer(stage, rg, maskOpacity, tool, selectedEntity, onPosChanged);
+    renderRGLayer(stage, rg, maskOpacity, tool, selectedEntityIdentifier, onPosChanged);
   }
 };

invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts

@@ -15,10 +15,9 @@ import { settingsReducers } from 'features/controlLayers/store/settingsReducers'
 import { toolReducers } from 'features/controlLayers/store/toolReducers';
 import { initialAspectRatioState } from 'features/parameters/components/ImageSize/constants';
 import type { AspectRatioState } from 'features/parameters/components/ImageSize/types';
-import type { Vector2d } from 'konva/lib/types';
 import { atom } from 'nanostores';
 
-import type { CanvasEntity, CanvasEntityIdentifier, CanvasV2State, RgbaColor, StageAttrs } from './types';
+import type { CanvasEntityIdentifier, CanvasV2State, StageAttrs } from './types';
 import { DEFAULT_RGBA_COLOR } from './types';
 
 const initialState: CanvasV2State = {
@@ -306,14 +305,8 @@ const migrate = (state: any): any => {
   return state;
 };
 
-// Ephemeral interaction state
+// Ephemeral state that does not need to be in redux
-export const $isDrawing = atom(false);
-export const $isMouseDown = atom(false);
-export const $lastMouseDownPos = atom<Vector2d | null>(null);
-export const $lastCursorPos = atom<Vector2d | null>(null);
 export const $isPreviewVisible = atom(true);
-export const $lastAddedPoint = atom<Vector2d | null>(null);
-export const $spaceKey = atom(false);
 export const $stageAttrs = atom<StageAttrs>({
   x: 0,
   y: 0,
@@ -322,14 +315,6 @@ export const $stageAttrs = atom<StageAttrs>({
   scale: 0,
 });
 
-// Some nanostores that are manually synced to redux state to provide imperative access
-// TODO(psyche):
-export const $toolState = atom<CanvasV2State['tool']>(deepClone(initialState.tool));
-export const $currentFill = atom<RgbaColor>(DEFAULT_RGBA_COLOR);
-export const $selectedEntity = atom<CanvasEntity | null>(null);
-export const $bbox = atom<CanvasV2State['bbox']>(deepClone(initialState.bbox));
-export const $document = atom<CanvasV2State['document']>(deepClone(initialState.document));
-
 export const canvasV2PersistConfig: PersistConfig<CanvasV2State> = {
   name: canvasV2Slice.name,
   initialState,

invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts

@@ -2,8 +2,8 @@ import { getStore } from 'app/store/nanostores/store';
 import { deepClone } from 'common/util/deepClone';
 import openBase64ImageInTab from 'common/util/openBase64ImageInTab';
 import { RG_LAYER_NAME } from 'features/controlLayers/konva/naming';
-import { renderers } from 'features/controlLayers/konva/renderers/layers';
+import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer';
-import { blobToDataURL } from "features/controlLayers/konva/util";
+import { blobToDataURL } from 'features/controlLayers/konva/util';
 import { rgMaskImageUploaded } from 'features/controlLayers/store/canvasV2Slice';
 import type { Dimensions, IPAdapterEntity, RegionEntity } from 'features/controlLayers/store/types';
 import {
@@ -260,7 +260,7 @@ export const getRGMaskBlobs = async (
 ): Promise<Record<string, Blob>> => {
   const container = document.createElement('div');
   const stage = new Konva.Stage({ container, ...documentSize });
-  renderers.renderLayers(stage, [], [], regions, 1, 'brush', null, getImageDTO);
+  renderRegions(stage, regions, 1, 'brush', null);
   const konvaLayers = stage.find<Konva.Layer>(`.${RG_LAYER_NAME}`);
   const blobs: Record<string, Blob> = {};