From ba6362dc9d6528b7480fbb199265ddc67a522468 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Mon, 5 Aug 2024 18:43:42 +1000 Subject: [PATCH] feat(ui): move bbox calculation to transformer --- .../controlLayers/konva/CanvasLayer.ts | 115 ++------------- .../controlLayers/konva/CanvasTransformer.ts | 134 +++++++++++++++++- 2 files changed, 137 insertions(+), 112 deletions(-) diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts index c865aca7c3..70fe8d4107 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts @@ -1,20 +1,19 @@ import { getStore } from 'app/store/nanostores/store'; import { deepClone } from 'common/util/deepClone'; -import { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; +import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer'; import { CanvasTransformer } from 'features/controlLayers/konva/CanvasTransformer'; -import { getEmptyRect, konvaNodeToBlob, previewBlob } from 'features/controlLayers/konva/util'; +import { konvaNodeToBlob, previewBlob } from 'features/controlLayers/konva/util'; import { layerRasterized } from 'features/controlLayers/store/canvasV2Slice'; import type { CanvasLayerState, CanvasV2State, Coordinate, GetLoggingContext, - Rect, } from 'features/controlLayers/store/types'; import { imageDTOToImageObject } from 'features/controlLayers/store/types'; import Konva from 'konva'; -import { debounce, get } from 'lodash-es'; +import { get } from 'lodash-es'; import type { Logger } from 'roarr'; import { uploadImage } from 'services/api/endpoints/images'; @@ -40,10 +39,6 @@ export class CanvasLayer { isFirstRender: boolean = true; bboxNeedsUpdate: boolean = true; - isPendingBboxCalculation: boolean = false; - - rect: Rect = getEmptyRect(); - bbox: Rect = getEmptyRect(); constructor(state: CanvasLayerState, manager: CanvasManager) { this.id = state.id; @@ -105,7 +100,7 @@ export class CanvasLayer { // this.transformer.syncInteractionState(); if (this.isFirstRender) { - await this.updateBbox(); + await this.transformer.updateBbox(); } this.state = state; @@ -123,13 +118,13 @@ export class CanvasLayer { const position = get(arg, 'position', this.state.position); this.konva.objectGroup.setAttrs({ - x: position.x + this.bbox.x, - y: position.y + this.bbox.y, - offsetX: this.bbox.x, - offsetY: this.bbox.y, + x: position.x + this.transformer.pixelRect.x, + y: position.y + this.transformer.pixelRect.y, + offsetX: this.transformer.pixelRect.x, + offsetY: this.transformer.pixelRect.y, }); - this.transformer.update(position, this.bbox); + this.transformer.update(position, this.transformer.pixelRect); }; updateObjects = async (arg?: { objects: CanvasLayerState['objects'] }) => { @@ -140,7 +135,7 @@ export class CanvasLayer { const didUpdate = await this.renderer.render(objects); if (didUpdate) { - this.calculateBbox(); + this.transformer.requestRectCalculation(); } this.isFirstRender = false; @@ -152,35 +147,6 @@ export class CanvasLayer { this.konva.objectGroup.opacity(opacity); }; - updateBbox = () => { - this.log.trace('Updating bbox'); - - if (this.isPendingBboxCalculation) { - return; - } - - // If the bbox has no width or height, that means the layer is fully transparent. This can happen if it is only - // eraser lines, fully clipped brush lines or if it has been fully erased. - if (this.bbox.width === 0 || this.bbox.height === 0) { - // We shouldn't reset on the first render - the bbox will be calculated on the next render - if (!this.isFirstRender && !this.renderer.hasObjects()) { - // The layer is fully transparent but has objects - reset it - this.manager.stateApi.onEntityReset({ id: this.id }, 'layer'); - } - this.transformer.syncInteractionState(); - return; - } - - this.transformer.syncInteractionState(); - this.transformer.update(this.state.position, this.bbox); - this.konva.objectGroup.setAttrs({ - x: this.state.position.x + this.bbox.x, - y: this.state.position.y + this.bbox.y, - offsetX: this.bbox.x, - offsetY: this.bbox.y, - }); - }; - resetScale = () => { const attrs = { scaleX: 1, @@ -210,73 +176,12 @@ export class CanvasLayer { dispatch(layerRasterized({ id: this.id, imageObject, position: { x: Math.round(rect.x), y: Math.round(rect.y) } })); }; - calculateBbox = debounce(() => { - this.log.debug('Calculating bbox'); - - this.isPendingBboxCalculation = true; - - if (!this.renderer.hasObjects()) { - this.log.trace('No objects, resetting bbox'); - this.rect = getEmptyRect(); - this.bbox = getEmptyRect(); - this.isPendingBboxCalculation = false; - this.updateBbox(); - return; - } - - const rect = this.konva.objectGroup.getClientRect({ skipTransform: true }); - - if (!this.renderer.needsPixelBbox()) { - this.rect = deepClone(rect); - this.bbox = deepClone(rect); - this.isPendingBboxCalculation = false; - this.log.trace({ bbox: this.bbox, rect: this.rect }, 'Got bbox from client rect'); - this.updateBbox(); - return; - } - - // We have eraser strokes - we must calculate the bbox using pixel data - - const clone = this.konva.objectGroup.clone(); - const canvas = clone.toCanvas(); - const ctx = canvas.getContext('2d'); - if (!ctx) { - return; - } - const imageData = ctx.getImageData(0, 0, rect.width, rect.height); - this.manager.requestBbox( - { buffer: imageData.data.buffer, width: imageData.width, height: imageData.height }, - (extents) => { - if (extents) { - const { minX, minY, maxX, maxY } = extents; - this.rect = deepClone(rect); - this.bbox = { - x: rect.x + minX, - y: rect.y + minY, - width: maxX - minX, - height: maxY - minY, - }; - } else { - this.bbox = getEmptyRect(); - this.rect = getEmptyRect(); - } - this.isPendingBboxCalculation = false; - this.log.trace({ bbox: this.bbox, rect: this.rect, extents }, `Got bbox from worker`); - this.updateBbox(); - clone.destroy(); - } - ); - }, CanvasManager.BBOX_DEBOUNCE_MS); - repr = () => { return { id: this.id, type: CanvasLayer.TYPE, state: deepClone(this.state), - rect: deepClone(this.rect), - bbox: deepClone(this.bbox), bboxNeedsUpdate: this.bboxNeedsUpdate, - isPendingBboxCalculation: this.isPendingBboxCalculation, transformer: this.transformer.repr(), renderer: this.renderer.repr(), }; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts index 432f9c4ca2..5911a36b87 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts @@ -1,8 +1,9 @@ import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer'; -import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; -import { getPrefixedId } from 'features/controlLayers/konva/util'; +import { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; +import { getEmptyRect, getPrefixedId } from 'features/controlLayers/konva/util'; import type { Coordinate, GetLoggingContext, Rect } from 'features/controlLayers/store/types'; import Konva from 'konva'; +import { debounce } from 'lodash-es'; import type { Logger } from 'roarr'; /** @@ -36,7 +37,26 @@ export class CanvasTransformer { getLoggingContext: GetLoggingContext; /** - * A list of subscriptions that should be cleaned up when the transformer is destroyed. + * The rect of the parent, _including_ transparent regions. + * It is calculated via Konva's getClientRect method, which is fast but includes transparent regions. + */ + nodeRect = getEmptyRect(); + + /** + * The rect of the parent, _excluding_ transparent regions. + * If the parent's nodes have no possibility of transparent regions, this will be calculated the same way as nodeRect. + * If the parent's nodes may have transparent regions, this will be calculated manually by rasterizing the parent and + * checking the pixel data. + */ + pixelRect = getEmptyRect(); + + /** + * Whether the transformer is currently calculating the rect of the parent. + */ + isPendingRectCalculation: boolean = false; + + /** + * A set of subscriptions that should be cleaned up when the transformer is destroyed. */ subscriptions: Set<() => void> = new Set(); @@ -315,7 +335,7 @@ export class CanvasTransformer { }); // The object group is translated by the difference between the interaction rect's new and old positions (which is - // stored as this.bbox) + // stored as this.pixelRect) this.parent.konva.objectGroup.setAttrs({ x: this.konva.proxyRect.x(), y: this.konva.proxyRect.y(), @@ -329,8 +349,8 @@ export class CanvasTransformer { } const position = { - x: this.konva.proxyRect.x() - this.parent.bbox.x, - y: this.konva.proxyRect.y() - this.parent.bbox.y, + x: this.konva.proxyRect.x() - this.pixelRect.x, + y: this.konva.proxyRect.y() - this.pixelRect.y, }; this.log.trace({ position }, 'Position changed'); @@ -403,6 +423,13 @@ export class CanvasTransformer { syncInteractionState = () => { this.log.trace('Syncing interaction state'); + if (this.isPendingRectCalculation || this.pixelRect.width === 0 || this.pixelRect.height === 0) { + // If the rect is being calculated, or if the rect has no width or height, we can't interact with the transformer + this.parent.konva.layer.listening(false); + this.setInteractionMode('off'); + return; + } + const toolState = this.manager.stateApi.getToolState(); const isSelected = this.manager.stateApi.getIsSelected(this.parent.id); @@ -486,7 +513,7 @@ export class CanvasTransformer { this.setInteractionMode('off'); this.parent.resetScale(); this.parent.updatePosition(); - this.parent.updateBbox(); + this.updateBbox(); this.syncInteractionState(); }; @@ -514,6 +541,99 @@ export class CanvasTransformer { } }; + updateBbox = () => { + this.log.trace('Updating bbox'); + + if (this.isPendingRectCalculation) { + this.syncInteractionState(); + return; + } + + // If the bbox has no width or height, that means the layer is fully transparent. This can happen if it is only + // eraser lines, fully clipped brush lines or if it has been fully erased. + if (this.pixelRect.width === 0 || this.pixelRect.height === 0) { + // We shouldn't reset on the first render - the bbox will be calculated on the next render + if (!this.parent.renderer.hasObjects()) { + // The layer is fully transparent but has objects - reset it + this.manager.stateApi.onEntityReset({ id: this.parent.id }, this.parent.type); + } + this.syncInteractionState(); + return; + } + + this.syncInteractionState(); + this.update(this.parent.state.position, this.pixelRect); + this.parent.konva.objectGroup.setAttrs({ + x: this.parent.state.position.x + this.pixelRect.x, + y: this.parent.state.position.y + this.pixelRect.y, + offsetX: this.pixelRect.x, + offsetY: this.pixelRect.y, + }); + }; + + calculateRect = debounce(() => { + this.log.debug('Calculating bbox'); + + this.isPendingRectCalculation = true; + + if (!this.parent.renderer.hasObjects()) { + this.log.trace('No objects, resetting bbox'); + this.nodeRect = getEmptyRect(); + this.pixelRect = getEmptyRect(); + this.isPendingRectCalculation = false; + this.updateBbox(); + return; + } + + const rect = this.parent.konva.objectGroup.getClientRect({ skipTransform: true }); + + if (!this.parent.renderer.needsPixelBbox()) { + this.nodeRect = { ...rect }; + this.pixelRect = { ...rect }; + this.isPendingRectCalculation = false; + this.log.trace({ nodeRect: this.nodeRect, pixelRect: this.pixelRect }, 'Got bbox from client rect'); + this.updateBbox(); + return; + } + + // We have eraser strokes - we must calculate the bbox using pixel data + + const clone = this.parent.konva.objectGroup.clone(); + const canvas = clone.toCanvas(); + const ctx = canvas.getContext('2d'); + if (!ctx) { + return; + } + const imageData = ctx.getImageData(0, 0, rect.width, rect.height); + this.manager.requestBbox( + { buffer: imageData.data.buffer, width: imageData.width, height: imageData.height }, + (extents) => { + if (extents) { + const { minX, minY, maxX, maxY } = extents; + this.nodeRect = { ...rect }; + this.pixelRect = { + x: rect.x + minX, + y: rect.y + minY, + width: maxX - minX, + height: maxY - minY, + }; + } else { + this.nodeRect = getEmptyRect(); + this.pixelRect = getEmptyRect(); + } + this.isPendingRectCalculation = false; + this.log.trace({ nodeRect: this.nodeRect, pixelRect: this.pixelRect, extents }, `Got bbox from worker`); + this.updateBbox(); + clone.destroy(); + } + ); + }, CanvasManager.BBOX_DEBOUNCE_MS); + + requestRectCalculation = () => { + this.isPendingRectCalculation = true; + this.calculateRect(); + }; + _enableTransform = () => { this.isTransformEnabled = true; this.konva.transformer.visible(true);