From f65ce6a01902c6adf9eabfc32996d7b0030daa9d Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:21:48 +1000 Subject: [PATCH] fix(ui): incorrect rect/brush/eraser positions --- .../features/controlLayers/konva/events.ts | 29 +++-- .../controlLayers/konva/renderers/layers.ts | 94 +------------- .../controlLayers/konva/renderers/renderer.ts | 120 ++++++++++-------- .../controlLayers/konva/renderers/rgLayer.ts | 16 ++- .../controlLayers/store/canvasV2Slice.ts | 19 +-- .../nodes/util/graph/generation/addRegions.ts | 6 +- 6 files changed, 102 insertions(+), 182 deletions(-) diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/events.ts b/invokeai/frontend/web/src/features/controlLayers/konva/events.ts index 6614d1c9dc..19178a6651 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/events.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/events.ts @@ -50,6 +50,7 @@ type Arg = { setStageAttrs: (attrs: StageAttrs) => void; getSelectedEntity: () => CanvasEntity | null; getSpaceKey: () => boolean; + setSpaceKey: (val: boolean) => void; getDocument: () => CanvasV2State['document']; getBbox: () => CanvasV2State['bbox']; onBrushLineAdded: (arg: BrushLineAddedArg, entityType: CanvasEntity['type']) => void; @@ -151,6 +152,7 @@ export const setStageEventHandlers = ({ setStageAttrs, getSelectedEntity, getSpaceKey, + setSpaceKey, getDocument, getBbox, onBrushLineAdded, @@ -174,7 +176,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -190,7 +192,6 @@ export const setStageEventHandlers = ({ const toolState = getToolState(); const pos = updateLastCursorPos(stage, setLastCursorPos); const selectedEntity = getSelectedEntity(); - if ( pos && selectedEntity && @@ -308,7 +309,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -339,8 +340,8 @@ export const setStageEventHandlers = ({ { id: selectedEntity.id, rect: { - x: Math.min(pos.x, lastMouseDownPos.x), - y: Math.min(pos.y, lastMouseDownPos.y), + x: Math.min(pos.x - selectedEntity.x, lastMouseDownPos.x - selectedEntity.x), + y: Math.min(pos.y - selectedEntity.y, lastMouseDownPos.y - selectedEntity.y), width: Math.abs(pos.x - lastMouseDownPos.x), height: Math.abs(pos.y - lastMouseDownPos.y), }, @@ -361,7 +362,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -472,7 +473,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -516,7 +517,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -571,7 +572,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -594,7 +595,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -616,7 +617,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -638,6 +639,7 @@ export const setStageEventHandlers = ({ // Select the view tool on space key down setToolBuffer(getToolState().selected); setTool('view'); + setSpaceKey(true); } else if (e.key === 'r') { const stageAttrs = fitDocumentToStage(stage, getDocument()); setStageAttrs(stageAttrs); @@ -651,7 +653,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); @@ -670,6 +672,7 @@ export const setStageEventHandlers = ({ const toolBuffer = getToolState().selectedBuffer; setTool(toolBuffer ?? 'move'); setToolBuffer(null); + setSpaceKey(false); } renderToolPreview( stage, @@ -677,7 +680,7 @@ export const setStageEventHandlers = ({ getCurrentFill(), getSelectedEntity(), getLastCursorPos(), - getLastAddedPoint(), + getLastMouseDownPos(), getIsDrawing(), getIsMouseDown() ); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/layers.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/layers.ts index 7ed3fa3170..d0b148da4d 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/layers.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/layers.ts @@ -1,98 +1,6 @@ -import { DEBOUNCE_MS } from 'features/controlLayers/konva/constants'; import { BACKGROUND_LAYER_ID, PREVIEW_LAYER_ID } from 'features/controlLayers/konva/naming'; -import { updateBboxes } from 'features/controlLayers/konva/renderers/bbox'; -import { renderCALayer } from 'features/controlLayers/konva/renderers/caLayer'; -import { renderBboxPreview, renderToolPreview } from 'features/controlLayers/konva/renderers/previewLayer'; -import { renderRasterLayer } from 'features/controlLayers/konva/renderers/rasterLayer'; -import { renderRGLayer } from 'features/controlLayers/konva/renderers/rgLayer'; -import { mapId, selectRenderableLayers } from 'features/controlLayers/konva/util'; -import type { - CanvasEntity, - ControlAdapterEntity, - LayerEntity, - PosChangedArg, - RegionEntity, - Tool, -} from 'features/controlLayers/store/types'; +import type { ControlAdapterEntity, LayerEntity, RegionEntity } from 'features/controlLayers/store/types'; import type Konva from 'konva'; -import { debounce } from 'lodash-es'; -import type { ImageDTO } from 'services/api/types'; - -/** - * Logic for rendering arranging and rendering all layers. - */ - -/** - * Renders the layers on the stage. - * @param stage The konva stage - * @param layers Array of all layer states - * @param rgGlobalOpacity The global mask layer opacity - * @param tool The current tool - * @param getImageDTO A function to retrieve an image DTO from the server, used to update the image source - * @param onPosChanged Callback for when the layer's position changes - */ -const renderLayers = ( - stage: Konva.Stage, - layers: LayerEntity[], - controlAdapters: ControlAdapterEntity[], - regions: RegionEntity[], - rgGlobalOpacity: number, - tool: Tool, - selectedEntity: CanvasEntity | null, - getImageDTO: (imageName: string) => Promise, - onPosChanged?: (arg: PosChangedArg, entityType: CanvasEntity['type']) => void -): void => { - const renderableIds = [...layers.map(mapId), ...controlAdapters.map(mapId), ...regions.map(mapId)]; - // Remove un-rendered layers - for (const konvaLayer of stage.find(selectRenderableLayers)) { - if (!renderableIds.includes(konvaLayer.id())) { - konvaLayer.destroy(); - } - } - // We'll need to ensure the tool preview layer is on top of the rest of the layers - let zIndex = 1; - for (const layer of layers) { - renderRasterLayer(stage, layer, tool, zIndex, onPosChanged); - zIndex++; - } - for (const ca of controlAdapters) { - renderCALayer(stage, ca, zIndex, getImageDTO); - zIndex++; - } - for (const rg of regions) { - renderRGLayer(stage, rg, rgGlobalOpacity, tool, zIndex, selectedEntity, onPosChanged); - zIndex++; - } - // Arrange the tool preview layer - stage.findOne(`#${PREVIEW_LAYER_ID}`)?.zIndex(zIndex); -}; - -/** - * All the renderers for the Konva stage. - */ -export const renderers = { - renderToolPreview, - renderBboxPreview, - renderLayers, - updateBboxes, -}; - -/** - * Gets the renderers with debouncing applied. - * @param ms The debounce time in milliseconds - * @returns The renderers with debouncing applied - */ -const getDebouncedRenderers = (ms = DEBOUNCE_MS): typeof renderers => ({ - renderToolPreview: debounce(renderToolPreview, ms), - renderBboxPreview: debounce(renderBboxPreview, ms), - renderLayers: debounce(renderLayers, ms), - updateBboxes: debounce(updateBboxes, ms), -}); - -/** - * All the renderers for the Konva stage, debounced. - */ -export const debouncedRenderers: typeof renderers = getDebouncedRenderers(); export const arrangeEntities = ( stage: Konva.Stage, diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts index d339de9d5b..0e9c8fc53e 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts @@ -17,12 +17,6 @@ import { renderLayers } from 'features/controlLayers/konva/renderers/rasterLayer import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer'; import { fitDocumentToStage } from 'features/controlLayers/konva/renderers/stage'; import { - $isDrawing, - $isMouseDown, - $lastAddedPoint, - $lastCursorPos, - $lastMouseDownPos, - $spaceKey, $stageAttrs, bboxChanged, brushWidthChanged, @@ -48,7 +42,6 @@ import type { BboxChangedArg, BrushLineAddedArg, CanvasEntity, - CanvasEntityIdentifier, CanvasV2State, EraserLineAddedArg, PointAddedToLineArg, @@ -57,7 +50,7 @@ import type { Tool, } from 'features/controlLayers/store/types'; import type Konva from 'konva'; -import type { IRect } from 'konva/lib/types'; +import type { IRect, Vector2d } from 'konva/lib/types'; import { debounce } from 'lodash-es'; import type { RgbaColor } from 'react-colorful'; import { getImageDTO } from 'services/api/endpoints/images'; @@ -200,46 +193,77 @@ export const initializeRenderer = ( const { getState, subscribe, dispatch } = store; - // Create closures for the rendering functions, used to check if specific parts of state have changed so we only - // render what needs to be rendered. - let prevCanvasV2 = getState().canvasV2; - let selectedEntityIdentifier: CanvasEntityIdentifier | null = prevCanvasV2.selectedEntityIdentifier; - let selectedEntity: CanvasEntity | null = _getSelectedEntity(prevCanvasV2); - let currentFill: RgbaColor = _getCurrentFill(prevCanvasV2, selectedEntity); - let didSelectedEntityChange: boolean = false; - // On the first render, we need to render everything. let isFirstRender = true; - // Stage event listeners use a fully imperative approach to event handling, using these helpers to get state. + // Stage interaction listeners need helpers to get and update current state. Some of the state is read-only, like + // bbox, document and tool state, while interaction state is read-write. + + // Read-only state, derived from redux + let prevCanvasV2 = getState().canvasV2; + let prevSelectedEntity: CanvasEntity | null = _getSelectedEntity(prevCanvasV2); + let prevCurrentFill: RgbaColor = _getCurrentFill(prevCanvasV2, prevSelectedEntity); + const getSelectedEntity = () => prevSelectedEntity; + const getCurrentFill = () => prevCurrentFill; const getBbox = () => getState().canvasV2.bbox; const getDocument = () => getState().canvasV2.document; const getToolState = () => getState().canvasV2.tool; - const getSelectedEntity = () => selectedEntity; - const getCurrentFill = () => currentFill; - // Calculating bounding boxes is expensive, must be debounced to not block the UI thread. - // TODO(psyche): Figure out how to do this in a worker. Probably means running the renderer in a worker and sending - // the entire state over when needed. - const debouncedUpdateBboxes = debounce(updateBboxes, 300); + // Read-write state, ephemeral interaction state + let isDrawing = false; + const getIsDrawing = () => isDrawing; + const setIsDrawing = (val: boolean) => { + isDrawing = val; + }; + + let isMouseDown = false; + const getIsMouseDown = () => isMouseDown; + const setIsMouseDown = (val: boolean) => { + isMouseDown = val; + }; + + let lastAddedPoint: Vector2d | null = null; + const getLastAddedPoint = () => lastAddedPoint; + const setLastAddedPoint = (val: Vector2d | null) => { + lastAddedPoint = val; + }; + + let lastMouseDownPos: Vector2d | null = null; + const getLastMouseDownPos = () => lastMouseDownPos; + const setLastMouseDownPos = (val: Vector2d | null) => { + lastMouseDownPos = val; + }; + + let lastCursorPos: Vector2d | null = null; + const getLastCursorPos = () => lastCursorPos; + const setLastCursorPos = (val: Vector2d | null) => { + lastCursorPos = val; + }; + + let spaceKey = false; + const getSpaceKey = () => spaceKey; + const setSpaceKey = (val: boolean) => { + spaceKey = val; + }; const cleanupListeners = setStageEventHandlers({ stage, getToolState, setTool, setToolBuffer, - getIsDrawing: $isDrawing.get, - setIsDrawing: $isDrawing.set, - getIsMouseDown: $isMouseDown.get, - setIsMouseDown: $isMouseDown.set, + getIsDrawing, + setIsDrawing, + getIsMouseDown, + setIsMouseDown, getSelectedEntity, - getLastAddedPoint: $lastAddedPoint.get, - setLastAddedPoint: $lastAddedPoint.set, - getLastCursorPos: $lastCursorPos.get, - setLastCursorPos: $lastCursorPos.set, - getLastMouseDownPos: $lastMouseDownPos.get, - setLastMouseDownPos: $lastMouseDownPos.set, - getSpaceKey: $spaceKey.get, + getLastAddedPoint, + setLastAddedPoint, + getLastCursorPos, + setLastCursorPos, + getLastMouseDownPos, + setLastMouseDownPos, + getSpaceKey, + setSpaceKey, setStageAttrs: $stageAttrs.set, getDocument, getBbox, @@ -252,6 +276,11 @@ export const initializeRenderer = ( getCurrentFill, }); + // Calculating bounding boxes is expensive, must be debounced to not block the UI thread during a user interaction. + // TODO(psyche): Figure out how to do this in a worker. Probably means running the renderer in a worker and sending + // the entire state over when needed. + const debouncedUpdateBboxes = debounce(updateBboxes, 300); + const renderCanvas = () => { const { canvasV2 } = store.getState(); @@ -260,20 +289,8 @@ export const initializeRenderer = ( return; } - // We can save some cycles for specific renderers if we track whether the selected entity has changed. - if (canvasV2.selectedEntityIdentifier !== selectedEntityIdentifier) { - selectedEntityIdentifier = canvasV2.selectedEntityIdentifier; - selectedEntity = _getSelectedEntity(canvasV2); - didSelectedEntityChange = true; - } else { - didSelectedEntityChange = false; - } - - // The current fill is either the tool fill or, if a regional guidance region is selected, the mask fill for that - // region. We need to manually sync this state. - if (isFirstRender || canvasV2.tool.fill !== prevCanvasV2.tool.fill || didSelectedEntityChange) { - currentFill = _getCurrentFill(canvasV2, selectedEntity); - } + const selectedEntity = _getSelectedEntity(canvasV2); + const currentFill = _getCurrentFill(canvasV2, selectedEntity); if ( isFirstRender || @@ -288,8 +305,7 @@ export const initializeRenderer = ( isFirstRender || canvasV2.regions !== prevCanvasV2.regions || canvasV2.settings.maskOpacity !== prevCanvasV2.settings.maskOpacity || - canvasV2.tool.selected !== prevCanvasV2.tool.selected || - didSelectedEntityChange + canvasV2.tool.selected !== prevCanvasV2.tool.selected ) { logIfDebugging('Rendering regions'); renderRegions( @@ -297,7 +313,7 @@ export const initializeRenderer = ( canvasV2.regions, canvasV2.settings.maskOpacity, canvasV2.tool.selected, - selectedEntity, + canvasV2.selectedEntityIdentifier, onPosChanged ); } @@ -348,6 +364,8 @@ export const initializeRenderer = ( } prevCanvasV2 = canvasV2; + prevSelectedEntity = selectedEntity; + prevCurrentFill = currentFill; if (isFirstRender) { isFirstRender = false; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/rgLayer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/rgLayer.ts index 916c6c9c47..148f6925e6 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/rgLayer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/rgLayer.ts @@ -18,7 +18,13 @@ import { getEraserLine, } from 'features/controlLayers/konva/renderers/objects'; import { mapId, selectVectorMaskObjects } from 'features/controlLayers/konva/util'; -import type { CanvasEntity, PosChangedArg, RegionEntity, Tool } from 'features/controlLayers/store/types'; +import type { + CanvasEntity, + CanvasEntityIdentifier, + PosChangedArg, + RegionEntity, + Tool, +} from 'features/controlLayers/store/types'; import Konva from 'konva'; /** @@ -83,7 +89,7 @@ export const renderRGLayer = ( rg: RegionEntity, globalMaskLayerOpacity: number, tool: Tool, - selectedEntity: CanvasEntity | null, + selectedEntityIdentifier: CanvasEntityIdentifier | null, onPosChanged?: (arg: PosChangedArg, entityType: CanvasEntity['type']) => void ): void => { const konvaLayer = stage.findOne(`#${rg.id}`) ?? createRGLayer(stage, rg, onPosChanged); @@ -171,7 +177,7 @@ export const renderRGLayer = ( const compositingRect = konvaLayer.findOne(`.${COMPOSITING_RECT_NAME}`) ?? createCompositingRect(konvaLayer); - const isSelected = selectedEntity?.id === rg.id; + const isSelected = selectedEntityIdentifier?.id === rg.id; /** * When the group is selected, we use a rect of the selected preview color, composited over the shapes. This allows @@ -237,7 +243,7 @@ export const renderRegions = ( regions: RegionEntity[], maskOpacity: number, tool: Tool, - selectedEntity: CanvasEntity | null, + selectedEntityIdentifier: CanvasEntityIdentifier | null, onPosChanged?: (arg: PosChangedArg, entityType: CanvasEntity['type']) => void ): void => { // Destroy nonexistent layers @@ -247,6 +253,6 @@ export const renderRegions = ( } } for (const rg of regions) { - renderRGLayer(stage, rg, maskOpacity, tool, selectedEntity, onPosChanged); + renderRGLayer(stage, rg, maskOpacity, tool, selectedEntityIdentifier, onPosChanged); } }; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts b/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts index 652366ca1e..70efc6186c 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts @@ -15,10 +15,9 @@ import { settingsReducers } from 'features/controlLayers/store/settingsReducers' import { toolReducers } from 'features/controlLayers/store/toolReducers'; import { initialAspectRatioState } from 'features/parameters/components/ImageSize/constants'; import type { AspectRatioState } from 'features/parameters/components/ImageSize/types'; -import type { Vector2d } from 'konva/lib/types'; import { atom } from 'nanostores'; -import type { CanvasEntity, CanvasEntityIdentifier, CanvasV2State, RgbaColor, StageAttrs } from './types'; +import type { CanvasEntityIdentifier, CanvasV2State, StageAttrs } from './types'; import { DEFAULT_RGBA_COLOR } from './types'; const initialState: CanvasV2State = { @@ -306,14 +305,8 @@ const migrate = (state: any): any => { return state; }; -// Ephemeral interaction state -export const $isDrawing = atom(false); -export const $isMouseDown = atom(false); -export const $lastMouseDownPos = atom(null); -export const $lastCursorPos = atom(null); +// Ephemeral state that does not need to be in redux export const $isPreviewVisible = atom(true); -export const $lastAddedPoint = atom(null); -export const $spaceKey = atom(false); export const $stageAttrs = atom({ x: 0, y: 0, @@ -322,14 +315,6 @@ export const $stageAttrs = atom({ scale: 0, }); -// Some nanostores that are manually synced to redux state to provide imperative access -// TODO(psyche): -export const $toolState = atom(deepClone(initialState.tool)); -export const $currentFill = atom(DEFAULT_RGBA_COLOR); -export const $selectedEntity = atom(null); -export const $bbox = atom(deepClone(initialState.bbox)); -export const $document = atom(deepClone(initialState.document)); - export const canvasV2PersistConfig: PersistConfig = { name: canvasV2Slice.name, initialState, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts index a245d2c854..168c941033 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts @@ -2,8 +2,8 @@ import { getStore } from 'app/store/nanostores/store'; import { deepClone } from 'common/util/deepClone'; import openBase64ImageInTab from 'common/util/openBase64ImageInTab'; import { RG_LAYER_NAME } from 'features/controlLayers/konva/naming'; -import { renderers } from 'features/controlLayers/konva/renderers/layers'; -import { blobToDataURL } from "features/controlLayers/konva/util"; +import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer'; +import { blobToDataURL } from 'features/controlLayers/konva/util'; import { rgMaskImageUploaded } from 'features/controlLayers/store/canvasV2Slice'; import type { Dimensions, IPAdapterEntity, RegionEntity } from 'features/controlLayers/store/types'; import { @@ -260,7 +260,7 @@ export const getRGMaskBlobs = async ( ): Promise> => { const container = document.createElement('div'); const stage = new Konva.Stage({ container, ...documentSize }); - renderers.renderLayers(stage, [], [], regions, 1, 'brush', null, getImageDTO); + renderRegions(stage, regions, 1, 'brush', null); const konvaLayers = stage.find(`.${RG_LAYER_NAME}`); const blobs: Record = {};