From 8a69fbd336d76677be33075c0ce04475d8d36a18 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat, 20 Apr 2024 11:06:25 +1000 Subject: [PATCH] perf(ui): more bbox optimizations - Keep track of whether the bbox needs to be recalculated (e.g. had lines/points added) - Keep track of whether the bbox has eraser strokes - if yes, we need to do the full pixel-perfect bbox calculation, otherwise we can use the faster getClientRect - Use comparison rather than Math.min/max in bbox calculation (slightly faster) - Return `null` if no pixel data at all in bbox --- .../components/StageComponent.tsx | 6 +-- .../store/regionalPromptsSlice.ts | 14 ++++++- .../src/features/regionalPrompts/util/bbox.ts | 42 ++++++++++++++----- .../regionalPrompts/util/renderers.ts | 27 ++++++++++-- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/invokeai/frontend/web/src/features/regionalPrompts/components/StageComponent.tsx b/invokeai/frontend/web/src/features/regionalPrompts/components/StageComponent.tsx index e336d49bea..f06d62305e 100644 --- a/invokeai/frontend/web/src/features/regionalPrompts/components/StageComponent.tsx +++ b/invokeai/frontend/web/src/features/regionalPrompts/components/StageComponent.tsx @@ -49,7 +49,7 @@ const useStageRenderer = (container: HTMLDivElement | null, wrapper: HTMLDivElem ); const onBboxChanged = useCallback( - (layerId: string, bbox: IRect) => { + (layerId: string, bbox: IRect | null) => { dispatch(rpLayerBboxChanged({ layerId, bbox })); }, [dispatch] @@ -138,8 +138,8 @@ const useStageRenderer = (container: HTMLDivElement | null, wrapper: HTMLDivElem if (!stage) { return; } - renderBbox(stage, tool, state.selectedLayerId, onBboxChanged); - }, [dispatch, stage, tool, state.selectedLayerId, onBboxChanged]); + renderBbox(stage, state.layers, state.selectedLayerId, tool, onBboxChanged); + }, [dispatch, stage, state.layers, state.selectedLayerId, tool, onBboxChanged]); }; const $container = atom(null); diff --git a/invokeai/frontend/web/src/features/regionalPrompts/store/regionalPromptsSlice.ts b/invokeai/frontend/web/src/features/regionalPrompts/store/regionalPromptsSlice.ts index 12e693c182..34616df99a 100644 --- a/invokeai/frontend/web/src/features/regionalPrompts/store/regionalPromptsSlice.ts +++ b/invokeai/frontend/web/src/features/regionalPrompts/store/regionalPromptsSlice.ts @@ -53,6 +53,8 @@ export type RegionalPromptLayer = LayerBase & { x: number; y: number; bbox: IRect | null; + bboxNeedsUpdate: boolean; + hasEraserStrokes: boolean; kind: 'regionalPromptLayer'; objects: LayerObject[]; positivePrompt: string; @@ -104,6 +106,8 @@ export const regionalPromptsSlice = createSlice({ x: 0, y: 0, autoNegative: 'off', + bboxNeedsUpdate: false, + hasEraserStrokes: false, }; state.layers.push(layer); state.selectedLayerId = layer.id; @@ -154,6 +158,8 @@ export const regionalPromptsSlice = createSlice({ layer.objects = []; layer.bbox = null; layer.isVisible = true; + layer.hasEraserStrokes = false; + layer.bboxNeedsUpdate = false; } }, rpLayerTranslated: (state, action: PayloadAction<{ layerId: string; x: number; y: number }>) => { @@ -169,6 +175,7 @@ export const regionalPromptsSlice = createSlice({ const layer = state.layers.find((l) => l.id === layerId); if (isRPLayer(layer)) { layer.bbox = bbox; + layer.bboxNeedsUpdate = false; } }, allLayersDeleted: (state) => { @@ -218,6 +225,10 @@ export const regionalPromptsSlice = createSlice({ points: [points[0] - layer.x, points[1] - layer.y, points[2] - layer.x, points[3] - layer.y], strokeWidth: state.brushSize, }); + layer.bboxNeedsUpdate = true; + if (!layer.hasEraserStrokes) { + layer.hasEraserStrokes = tool === 'eraser'; + } } }, prepare: (payload: { layerId: string; points: [number, number, number, number]; tool: DrawingTool }) => ({ @@ -236,6 +247,7 @@ export const regionalPromptsSlice = createSlice({ // Points must be offset by the layer's x and y coordinates // TODO: Handle this in the event listener lastLine.points.push(point[0] - layer.x, point[1] - layer.y); + layer.bboxNeedsUpdate = true; } }, rpLayerAutoNegativeChanged: ( @@ -364,7 +376,7 @@ const undoableGroupByMatcher = isAnyOf( rpLayerPositivePromptChanged, rpLayerNegativePromptChanged, rpLayerTranslated, - rpLayerColorChanged, + rpLayerColorChanged ); const LINE_1 = 'LINE_1'; diff --git a/invokeai/frontend/web/src/features/regionalPrompts/util/bbox.ts b/invokeai/frontend/web/src/features/regionalPrompts/util/bbox.ts index 0243f89f3e..0f6e1a2f36 100644 --- a/invokeai/frontend/web/src/features/regionalPrompts/util/bbox.ts +++ b/invokeai/frontend/web/src/features/regionalPrompts/util/bbox.ts @@ -6,31 +6,49 @@ import type { Node as KonvaNodeType, NodeConfig as KonvaNodeConfigType } from 'k import type { IRect } from 'konva/lib/types'; import { assert } from 'tsafe'; +type Extents = { + minX: number; + minY: number; + maxX: number; + maxY: number; +}; + /** * Get the bounding box of an image. * @param imageData The ImageData object to get the bounding box of. * @returns The minimum and maximum x and y values of the image's bounding box. */ -const getImageDataBbox = (imageData: ImageData) => { +const getImageDataBbox = (imageData: ImageData): Extents | null => { const { data, width, height } = imageData; let minX = width; let minY = height; - let maxX = 0; - let maxY = 0; + let maxX = -1; + let maxY = -1; + let alpha = 0; + let isEmpty = true; for (let y = 0; y < height; y++) { for (let x = 0; x < width; x++) { - const alpha = data[(y * width + x) * 4 + 3] ?? 0; + alpha = data[(y * width + x) * 4 + 3] ?? 0; if (alpha > 0) { - minX = Math.min(minX, x); - maxX = Math.max(maxX, x); - minY = Math.min(minY, y); - maxY = Math.max(maxY, y); + isEmpty = false; + if (x < minX) { + minX = x; + } + if (x > maxX) { + maxX = x; + } + if (y < minY) { + minY = y; + } + if (y > maxY) { + maxY = y; + } } } } - return { minX, minY, maxX, maxY }; + return isEmpty ? null : { minX, minY, maxX, maxY }; }; /** @@ -43,7 +61,7 @@ export const getKonvaLayerBbox = ( layer: KonvaLayerType, filterChildren?: (item: KonvaNodeType) => boolean, preview: boolean = false -): IRect => { +): IRect | null => { // To calculate the layer's bounding box, we must first export it to a pixel array, then do some math. // // Though it is relatively fast, we can't use Konva's `getClientRect`. It programmatically determines the rect @@ -89,6 +107,10 @@ export const getKonvaLayerBbox = ( // Calculate the layer's bounding box. const layerBbox = getImageDataBbox(layerImageData); + if (!layerBbox) { + return null; + } + // Correct the bounding box to be relative to the layer's position. const correctedLayerBbox = { x: layerBbox.minX - stage.x() + layerRect.x - layer.x(), diff --git a/invokeai/frontend/web/src/features/regionalPrompts/util/renderers.ts b/invokeai/frontend/web/src/features/regionalPrompts/util/renderers.ts index ea81fd4e90..1715b659ae 100644 --- a/invokeai/frontend/web/src/features/regionalPrompts/util/renderers.ts +++ b/invokeai/frontend/web/src/features/regionalPrompts/util/renderers.ts @@ -302,6 +302,8 @@ export const renderLayers = ( const selectPromptLayerObjectGroup = (item: Node) => item.name() !== REGIONAL_PROMPT_LAYER_OBJECT_GROUP_NAME; +const GET_CLIENT_RECT_CONFIG = { skipTransform: true }; + /** * * @param stage The konva stage to render on. @@ -312,9 +314,10 @@ const selectPromptLayerObjectGroup = (item: Node) => */ export const renderBbox = ( stage: Konva.Stage, - tool: RPTool, + reduxLayers: Layer[], selectedLayerIdId: string | null, - onBboxChanged: (layerId: string, bbox: IRect) => void + tool: RPTool, + onBboxChanged: (layerId: string, bbox: IRect | null) => void ) => { // Hide all bounding boxes for (const bboxRect of stage.find(`.${REGIONAL_PROMPT_LAYER_BBOX_NAME}`)) { @@ -327,11 +330,27 @@ export const renderBbox = ( return; } + const reduxLayer = reduxLayers.find((layer) => layer.id === selectedLayerIdId); + assert(reduxLayer, `Selected layer ${selectedLayerIdId} not found in redux layers`); + const konvaLayer = stage.findOne(`#${selectedLayerIdId}`); assert(konvaLayer, `Selected layer ${selectedLayerIdId} not found in stage`); - const bbox = getKonvaLayerBbox(konvaLayer, selectPromptLayerObjectGroup); - onBboxChanged(selectedLayerIdId, bbox); + let bbox = reduxLayer.bbox; + + // We only need to recalculate the bbox if the layer has changed and it has objects + if (reduxLayer.bboxNeedsUpdate && reduxLayer.objects.length) { + // We only need to use the pixel-perfect bounding box if the layer has eraser strokes + bbox = reduxLayer.hasEraserStrokes + ? getKonvaLayerBbox(konvaLayer, selectPromptLayerObjectGroup) + : konvaLayer.getClientRect(GET_CLIENT_RECT_CONFIG); + + onBboxChanged(selectedLayerIdId, bbox); + } + + if (!bbox) { + return; + } let rect = konvaLayer.findOne(`.${REGIONAL_PROMPT_LAYER_BBOX_NAME}`); if (!rect) {