diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts b/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts index c61857fe98..7612fbef38 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts @@ -1,12 +1,6 @@ import { getImageDataTransparency } from 'common/util/arrayBuffer'; import { CanvasBackground } from 'features/controlLayers/konva/renderers/background'; -import { - CanvasBbox, - CanvasDocumentSizeOverlay, - CanvasPreview, - CanvasStagingArea, - CanvasTool, -} from 'features/controlLayers/konva/renderers/preview'; +import { CanvasPreview } from 'features/controlLayers/konva/renderers/preview'; import { konvaNodeToBlob, konvaNodeToImageData, previewBlob } from 'features/controlLayers/konva/util'; import type { BrushLineAddedArg, @@ -30,10 +24,14 @@ import { getImageDTO as defaultGetImageDTO, uploadImage as defaultUploadImage } import type { ImageCategory, ImageDTO } from 'services/api/types'; import { assert } from 'tsafe'; +import { CanvasBbox } from './renderers/bbox'; import { CanvasControlAdapter } from './renderers/controlAdapters'; +import { CanvasDocumentSizeOverlay } from './renderers/documentSizeOverlay'; import { CanvasInpaintMask } from './renderers/inpaintMask'; import { CanvasLayer } from './renderers/layers'; import { CanvasRegion } from './renderers/regions'; +import { CanvasStagingArea } from './renderers/stagingArea'; +import { CanvasTool } from './renderers/tool'; export type StateApi = { getToolState: () => CanvasV2State['tool']; @@ -157,10 +155,10 @@ export class KonvaNodeManager { const { entities } = this.stateApi.getLayersState(); const toolState = this.stateApi.getToolState(); - for (const adapter of this.layers.values()) { - if (!entities.find((l) => l.id === adapter.id)) { - adapter.destroy(); - this.layers.delete(adapter.id); + for (const canvasLayer of this.layers.values()) { + if (!entities.find((l) => l.id === canvasLayer.id)) { + canvasLayer.destroy(); + this.layers.delete(canvasLayer.id); } } @@ -182,10 +180,10 @@ export class KonvaNodeManager { const selectedEntity = this.stateApi.getSelectedEntity(); // Destroy the konva nodes for nonexistent entities - for (const adapter of this.regions.values()) { - if (!entities.find((rg) => rg.id === adapter.id)) { - adapter.destroy(); - this.regions.delete(adapter.id); + for (const canvasRegion of this.regions.values()) { + if (!entities.find((rg) => rg.id === canvasRegion.id)) { + canvasRegion.destroy(); + this.regions.delete(canvasRegion.id); } } @@ -212,10 +210,10 @@ export class KonvaNodeManager { renderControlAdapters() { const { entities } = this.stateApi.getControlAdaptersState(); - for (const adapter of this.controlAdapters.values()) { - if (!entities.find((ca) => ca.id === adapter.id)) { - adapter.destroy(); - this.controlAdapters.delete(adapter.id); + for (const canvasControlAdapter of this.controlAdapters.values()) { + if (!entities.find((ca) => ca.id === canvasControlAdapter.id)) { + canvasControlAdapter.destroy(); + this.controlAdapters.delete(canvasControlAdapter.id); } } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts index 5c226d017a..f6e20708e3 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts @@ -1,244 +1,236 @@ -import openBase64ImageInTab from 'common/util/openBase64ImageInTab'; +import { roundToMultiple, roundToMultipleMin } from 'common/util/roundDownToMultiple'; import { - CA_LAYER_IMAGE_NAME, - LAYER_BBOX_NAME, - RASTER_LAYER_OBJECT_GROUP_NAME, - RG_LAYER_OBJECT_GROUP_NAME, + PREVIEW_GENERATION_BBOX_DUMMY_RECT, + PREVIEW_GENERATION_BBOX_GROUP, + PREVIEW_GENERATION_BBOX_TRANSFORMER } from 'features/controlLayers/konva/naming'; -import { createBboxRect } from 'features/controlLayers/konva/renderers/objects'; -import { imageDataToDataURL } from 'features/controlLayers/konva/util'; -import type { - BboxChangedArg, - CanvasEntity, - ControlAdapterEntity, - LayerEntity, - RegionEntity, -} from 'features/controlLayers/store/types'; +import type { CanvasV2State } from 'features/controlLayers/store/types'; import Konva from 'konva'; import type { IRect } from 'konva/lib/types'; +import { atom } from 'nanostores'; import { assert } from 'tsafe'; -/** - * Logic to create and render bounding boxes for layers. - * Some utils are included for calculating bounding boxes. - */ -type Extents = { - minX: number; - minY: number; - maxX: number; - maxY: number; -}; +export class CanvasBbox { + group: Konva.Group; + rect: Konva.Rect; + transformer: Konva.Transformer; -const GET_CLIENT_RECT_CONFIG = { skipTransform: true }; + ALL_ANCHORS: string[] = [ + 'top-left', + 'top-center', + 'top-right', + 'middle-right', + 'middle-left', + 'bottom-left', + 'bottom-center', + 'bottom-right', + ]; + CORNER_ANCHORS: string[] = ['top-left', 'top-right', 'bottom-left', 'bottom-right']; + NO_ANCHORS: string[] = []; -/** - * Get the bounding box of an image. - * @param imageData The ImageData object to get the bounding box of. - * @returns The minimum and maximum x and y values of the image's bounding box, or null if the image has no pixels. - */ -const getImageDataBbox = (imageData: ImageData): Extents | null => { - const { data, width, height } = imageData; - let minX = width; - let minY = height; - let maxX = -1; - let maxY = -1; - let alpha = 0; - let isEmpty = true; + constructor( + getBbox: () => IRect, + onBboxTransformed: (bbox: IRect) => void, + getShiftKey: () => boolean, + getCtrlKey: () => boolean, + getMetaKey: () => boolean, + getAltKey: () => boolean + ) { + // Create a stash to hold onto the last aspect ratio of the bbox - this allows for locking the aspect ratio when + // transforming the bbox. + const bbox = getBbox(); + const $aspectRatioBuffer = atom(bbox.width / bbox.height); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - alpha = data[(y * width + x) * 4 + 3] ?? 0; - if (alpha > 0) { - isEmpty = false; - if (x < minX) { - minX = x; - } - if (x > maxX) { - maxX = x; - } - if (y < minY) { - minY = y; - } - if (y > maxY) { - maxY = y; - } + // Use a transformer for the generation bbox. Transformers need some shape to transform, we will use a fully + // transparent rect for this purpose. + this.group = new Konva.Group({ id: PREVIEW_GENERATION_BBOX_GROUP, listening: false }); + this.rect = new Konva.Rect({ + id: PREVIEW_GENERATION_BBOX_DUMMY_RECT, + listening: false, + strokeEnabled: false, + draggable: true, + ...getBbox(), + }); + this.rect.on('dragmove', () => { + const gridSize = getCtrlKey() || getMetaKey() ? 8 : 64; + const oldBbox = getBbox(); + const newBbox: IRect = { + ...oldBbox, + x: roundToMultiple(this.rect.x(), gridSize), + y: roundToMultiple(this.rect.y(), gridSize), + }; + this.rect.setAttrs(newBbox); + if (oldBbox.x !== newBbox.x || oldBbox.y !== newBbox.y) { + onBboxTransformed(newBbox); } - } - } + }); - return isEmpty ? null : { minX, minY, maxX, maxY }; -}; - -/** - * Clones a regional guidance konva layer onto an offscreen stage/canvas. This allows the pixel data for a given layer - * to be captured, manipulated or analyzed without interference from other layers. - * @param layer The konva layer to clone. - * @param filterChildren A callback to filter out unwanted children - * @returns The cloned stage and layer. - */ -const getIsolatedLayerClone = ( - layer: Konva.Layer, - filterChildren: (node: Konva.Node) => boolean -): { stageClone: Konva.Stage; layerClone: Konva.Layer } => { - const stage = layer.getStage(); - - // Construct an offscreen canvas with the same dimensions as the layer's stage. - const offscreenStageContainer = document.createElement('div'); - const stageClone = new Konva.Stage({ - container: offscreenStageContainer, - x: stage.x(), - y: stage.y(), - width: stage.width(), - height: stage.height(), - }); - - // Clone the layer and filter out unwanted children. - const layerClone = layer.clone(); - stageClone.add(layerClone); - - for (const child of layerClone.getChildren()) { - if (filterChildren(child) && child.hasChildren()) { - // We need to cache the group to ensure it composites out eraser strokes correctly - child.opacity(1); - child.cache(); - } else { - // Filter out unwanted children. - child.destroy(); - } - } - - return { stageClone, layerClone }; -}; - -/** - * Get the bounding box of a regional prompt konva layer. This function has special handling for regional prompt layers. - * @param layer The konva layer to get the bounding box of. - * @param preview Whether to open a new tab displaying the rendered layer, which is used to calculate the bbox. - */ -const getLayerBboxPixels = ( - layer: Konva.Layer, - filterChildren: (node: Konva.Node) => boolean, - preview: boolean = false -): IRect | null => { - // To calculate the layer's bounding box, we must first export it to a pixel array, then do some math. - // - // Though it is relatively fast, we can't use Konva's `getClientRect`. It programmatically determines the rect - // by calculating the extents of individual shapes from their "vector" shape data. - // - // This doesn't work when some shapes are drawn with composite operations that "erase" pixels, like eraser lines. - // These shapes' extents are still calculated as if they were solid, leading to a bounding box that is too large. - const { stageClone, layerClone } = getIsolatedLayerClone(layer, filterChildren); - - // Get a worst-case rect using the relatively fast `getClientRect`. - const layerRect = layerClone.getClientRect(); - if (layerRect.width === 0 || layerRect.height === 0) { - return null; - } - // Capture the image data with the above rect. - const layerImageData = stageClone - .toCanvas(layerRect) - .getContext('2d') - ?.getImageData(0, 0, layerRect.width, layerRect.height); - assert(layerImageData, "Unable to get layer's image data"); - - if (preview) { - openBase64ImageInTab([{ base64: imageDataToDataURL(layerImageData), caption: layer.id() }]); - } - - // Calculate the layer's bounding box. - const layerBbox = getImageDataBbox(layerImageData); - - if (!layerBbox) { - return null; - } - - // Correct the bounding box to be relative to the layer's position. - const correctedLayerBbox = { - x: layerBbox.minX - Math.floor(stageClone.x()) + layerRect.x - Math.floor(layer.x()), - y: layerBbox.minY - Math.floor(stageClone.y()) + layerRect.y - Math.floor(layer.y()), - width: layerBbox.maxX - layerBbox.minX, - height: layerBbox.maxY - layerBbox.minY, - }; - - return correctedLayerBbox; -}; - -/** - * Get the bounding box of a konva layer. This function is faster than `getLayerBboxPixels` but less accurate. It - * should only be used when there are no eraser strokes or shapes in the layer. - * @param layer The konva layer to get the bounding box of. - * @returns The bounding box of the layer. - */ -export const getLayerBboxFast = (layer: Konva.Layer): IRect => { - const bbox = layer.getClientRect(GET_CLIENT_RECT_CONFIG); - return { - x: Math.floor(bbox.x), - y: Math.floor(bbox.y), - width: Math.floor(bbox.width), - height: Math.floor(bbox.height), - }; -}; - -const filterRGChildren = (node: Konva.Node): boolean => node.name() === RG_LAYER_OBJECT_GROUP_NAME; -const filterLayerChildren = (node: Konva.Node): boolean => node.name() === RASTER_LAYER_OBJECT_GROUP_NAME; -const filterCAChildren = (node: Konva.Node): boolean => node.name() === CA_LAYER_IMAGE_NAME; - -/** - * Calculates the bbox of each regional guidance layer. Only calculates if the mask has changed. - * @param stage The konva stage - * @param entityStates An array of layers to calculate bboxes for - * @param onBboxChanged Callback for when the bounding box changes - */ -export const updateBboxes = ( - stage: Konva.Stage, - layers: LayerEntity[], - controlAdapters: ControlAdapterEntity[], - regions: RegionEntity[], - onBboxChanged: (arg: BboxChangedArg, entityType: CanvasEntity['type']) => void -): void => { - for (const entityState of [...layers, ...controlAdapters, ...regions]) { - const konvaLayer = stage.findOne(`#${entityState.id}`); - assert(konvaLayer, `Layer ${entityState.id} not found in stage`); - // We only need to recalculate the bbox if the layer has changed - if (entityState.bboxNeedsUpdate) { - const bboxRect = konvaLayer.findOne(`.${LAYER_BBOX_NAME}`) ?? createBboxRect(entityState, konvaLayer); - - // Hide the bbox while we calculate the new bbox, else the bbox will be included in the calculation - const visible = bboxRect.visible(); - bboxRect.visible(false); - - if (entityState.type === 'layer') { - if (entityState.objects.length === 0) { - // No objects - no bbox to calculate - onBboxChanged({ id: entityState.id, bbox: null }, 'layer'); - } else { - onBboxChanged({ id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterLayerChildren) }, 'layer'); + this.transformer = new Konva.Transformer({ + id: PREVIEW_GENERATION_BBOX_TRANSFORMER, + borderDash: [5, 5], + borderStroke: 'rgba(212,216,234,1)', + borderEnabled: true, + rotateEnabled: false, + keepRatio: false, + ignoreStroke: true, + listening: false, + flipEnabled: false, + anchorFill: 'rgba(212,216,234,1)', + anchorStroke: 'rgb(42,42,42)', + anchorSize: 12, + anchorCornerRadius: 3, + shiftBehavior: 'none', // we will implement our own shift behavior + centeredScaling: false, + anchorStyleFunc: (anchor) => { + // Make the x/y resize anchors little bars + if (anchor.hasName('top-center') || anchor.hasName('bottom-center')) { + anchor.height(8); + anchor.offsetY(4); + anchor.width(30); + anchor.offsetX(15); } - } else if (entityState.type === 'control_adapter') { - if (!entityState.imageObject && !entityState.processedImageObject) { - // No objects - no bbox to calculate - onBboxChanged({ id: entityState.id, bbox: null }, 'control_adapter'); - } else { - onBboxChanged( - { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterCAChildren) }, - 'control_adapter' - ); - } - } else if (entityState.type === 'regional_guidance') { - if (entityState.objects.length === 0) { - // No objects - no bbox to calculate - onBboxChanged({ id: entityState.id, bbox: null }, 'regional_guidance'); - } else { - onBboxChanged( - { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterRGChildren) }, - 'regional_guidance' - ); + if (anchor.hasName('middle-left') || anchor.hasName('middle-right')) { + anchor.height(30); + anchor.offsetY(15); + anchor.width(8); + anchor.offsetX(4); } + }, + anchorDragBoundFunc: (_oldAbsPos, newAbsPos) => { + // This function works with absolute position - that is, a position in "physical" pixels on the screen, as opposed + // to konva's internal coordinate system. + const stage = this.transformer.getStage(); + assert(stage, 'Stage must exist'); + + // We need to snap the anchors to the grid. If the user is holding ctrl/meta, we use the finer 8px grid. + const gridSize = getCtrlKey() || getMetaKey() ? 8 : 64; + // Because we are working in absolute coordinates, we need to scale the grid size by the stage scale. + const scaledGridSize = gridSize * stage.scaleX(); + // To snap the anchor to the grid, we need to calculate an offset from the stage's absolute position. + const stageAbsPos = stage.getAbsolutePosition(); + // The offset is the remainder of the stage's absolute position divided by the scaled grid size. + const offsetX = stageAbsPos.x % scaledGridSize; + const offsetY = stageAbsPos.y % scaledGridSize; + // Finally, calculate the position by rounding to the grid and adding the offset. + return { + x: roundToMultiple(newAbsPos.x, scaledGridSize) + offsetX, + y: roundToMultiple(newAbsPos.y, scaledGridSize) + offsetY, + }; + }, + }); + + this.transformer.on('transform', () => { + // In the transform callback, we calculate the bbox's new dims and pos and update the konva object. + // Some special handling is needed depending on the anchor being dragged. + const anchor = this.transformer.getActiveAnchor(); + if (!anchor) { + // Pretty sure we should always have an anchor here? + return; } - // Restore the visibility of the bbox - bboxRect.visible(visible); - } + const alt = getAltKey(); + const ctrl = getCtrlKey(); + const meta = getMetaKey(); + const shift = getShiftKey(); + + // Grid size depends on the modifier keys + let gridSize = ctrl || meta ? 8 : 64; + + // Alt key indicates we are using centered scaling. We need to double the gride size used when calculating the + // new dimensions so that each size scales in the correct increments and doesn't mis-place the bbox. For example, if + // we snapped the width and height to 8px increments, the bbox would be mis-placed by 4px in the x and y axes. + // Doubling the grid size ensures the bbox's coords remain aligned to the 8px/64px grid. + if (getAltKey()) { + gridSize = gridSize * 2; + } + + // The coords should be correct per the anchorDragBoundFunc. + let x = this.rect.x(); + let y = this.rect.y(); + + // Konva transforms by scaling the dims, not directly changing width and height. At this point, the width and height + // *have not changed*, only the scale has changed. To get the final height, we need to scale the dims and then snap + // them to the grid. + let width = roundToMultipleMin(this.rect.width() * this.rect.scaleX(), gridSize); + let height = roundToMultipleMin(this.rect.height() * this.rect.scaleY(), gridSize); + + // If shift is held and we are resizing from a corner, retain aspect ratio - needs special handling. We skip this + // if alt/opt is held - this requires math too big for my brain. + if (shift && this.CORNER_ANCHORS.includes(anchor) && !alt) { + // Fit the bbox to the last aspect ratio + let fittedWidth = Math.sqrt(width * height * $aspectRatioBuffer.get()); + let fittedHeight = fittedWidth / $aspectRatioBuffer.get(); + fittedWidth = roundToMultipleMin(fittedWidth, gridSize); + fittedHeight = roundToMultipleMin(fittedHeight, gridSize); + + // We need to adjust the x and y coords to have the resize occur from the right origin. + if (anchor === 'top-left') { + // The transform origin is the bottom-right anchor. Both x and y need to be updated. + x = x - (fittedWidth - width); + y = y - (fittedHeight - height); + } + if (anchor === 'top-right') { + // The transform origin is the bottom-left anchor. Only y needs to be updated. + y = y - (fittedHeight - height); + } + if (anchor === 'bottom-left') { + // The transform origin is the top-right anchor. Only x needs to be updated. + x = x - (fittedWidth - width); + } + // Update the width and height to the fitted dims. + width = fittedWidth; + height = fittedHeight; + } + + const bbox = { + x: Math.round(x), + y: Math.round(y), + width, + height, + }; + + // Update the bboxRect's attrs directly with the new transform, and reset its scale to 1. + // TODO(psyche): In `renderBboxPreview()` we also call setAttrs, need to do it twice to ensure it renders correctly. + // Gotta be a way to avoid setting it twice... + this.rect.setAttrs({ ...bbox, scaleX: 1, scaleY: 1 }); + + // Update the bbox in internal state. + onBboxTransformed(bbox); + + // Update the aspect ratio buffer whenever the shift key is not held - this allows for a nice UX where you can start + // a transform, get the right aspect ratio, then hold shift to lock it in. + if (!shift) { + $aspectRatioBuffer.set(bbox.width / bbox.height); + } + }); + + this.transformer.on('transformend', () => { + // Always update the aspect ratio buffer when the transform ends, so if the next transform starts with shift held, + // we have the correct aspect ratio to start from. + $aspectRatioBuffer.set(this.rect.width() / this.rect.height()); + }); + + // The transformer will always be transforming the dummy rect + this.transformer.nodes([this.rect]); + this.group.add(this.rect); + this.group.add(this.transformer); } -}; + + render(bbox: CanvasV2State['bbox'], toolState: CanvasV2State['tool']) { + this.group.listening(toolState.selected === 'bbox'); + this.rect.setAttrs({ + x: bbox.x, + y: bbox.y, + width: bbox.width, + height: bbox.height, + scaleX: 1, + scaleY: 1, + listening: toolState.selected === 'bbox', + }); + this.transformer.setAttrs({ + listening: toolState.selected === 'bbox', + enabledAnchors: toolState.selected === 'bbox' ? this.ALL_ANCHORS : this.NO_ANCHORS, + }); + } +} diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/documentSizeOverlay.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/documentSizeOverlay.ts new file mode 100644 index 0000000000..abf87c485b --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/documentSizeOverlay.ts @@ -0,0 +1,67 @@ +import { getArbitraryBaseColor } from '@invoke-ai/ui-library'; +import { DOCUMENT_FIT_PADDING_PX } from 'features/controlLayers/konva/constants'; +import type { CanvasV2State, StageAttrs } from 'features/controlLayers/store/types'; +import Konva from 'konva'; + +export class CanvasDocumentSizeOverlay { + group: Konva.Group; + outerRect: Konva.Rect; + innerRect: Konva.Rect; + padding: number; + + constructor(padding?: number) { + this.padding = padding ?? DOCUMENT_FIT_PADDING_PX; + this.group = new Konva.Group({ id: 'document_overlay_group', listening: false }); + this.outerRect = new Konva.Rect({ + id: 'document_overlay_outer_rect', + listening: false, + fill: getArbitraryBaseColor(10), + opacity: 0.7, + }); + this.innerRect = new Konva.Rect({ + id: 'document_overlay_inner_rect', + listening: false, + fill: 'white', + globalCompositeOperation: 'destination-out', + }); + this.group.add(this.outerRect); + this.group.add(this.innerRect); + } + + render(stage: Konva.Stage, document: CanvasV2State['document']) { + this.group.zIndex(0); + + const x = stage.x(); + const y = stage.y(); + const width = stage.width(); + const height = stage.height(); + const scale = stage.scaleX(); + + this.outerRect.setAttrs({ + offsetX: x / scale, + offsetY: y / scale, + width: width / scale, + height: height / scale, + }); + + this.innerRect.setAttrs({ + x: 0, + y: 0, + width: document.width, + height: document.height, + }); + } + + fitToStage(stage: Konva.Stage, document: CanvasV2State['document'], setStageAttrs: (attrs: StageAttrs) => void) { + // Fit & center the document on the stage + const width = stage.width(); + const height = stage.height(); + const docWidthWithBuffer = document.width + this.padding * 2; + const docHeightWithBuffer = document.height + this.padding * 2; + const scale = Math.min(Math.min(width / docWidthWithBuffer, height / docHeightWithBuffer), 1); + const x = (width - docWidthWithBuffer * scale) / 2 + this.padding * scale; + const y = (height - docHeightWithBuffer * scale) / 2 + this.padding * scale; + stage.setAttrs({ x, y, width, height, scaleX: scale, scaleY: scale }); + setStageAttrs({ x, y, width, height, scale }); + } +} diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts new file mode 100644 index 0000000000..5c226d017a --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts @@ -0,0 +1,244 @@ +import openBase64ImageInTab from 'common/util/openBase64ImageInTab'; +import { + CA_LAYER_IMAGE_NAME, + LAYER_BBOX_NAME, + RASTER_LAYER_OBJECT_GROUP_NAME, + RG_LAYER_OBJECT_GROUP_NAME, +} from 'features/controlLayers/konva/naming'; +import { createBboxRect } from 'features/controlLayers/konva/renderers/objects'; +import { imageDataToDataURL } from 'features/controlLayers/konva/util'; +import type { + BboxChangedArg, + CanvasEntity, + ControlAdapterEntity, + LayerEntity, + RegionEntity, +} from 'features/controlLayers/store/types'; +import Konva from 'konva'; +import type { IRect } from 'konva/lib/types'; +import { assert } from 'tsafe'; + +/** + * Logic to create and render bounding boxes for layers. + * Some utils are included for calculating bounding boxes. + */ + +type Extents = { + minX: number; + minY: number; + maxX: number; + maxY: number; +}; + +const GET_CLIENT_RECT_CONFIG = { skipTransform: true }; + +/** + * Get the bounding box of an image. + * @param imageData The ImageData object to get the bounding box of. + * @returns The minimum and maximum x and y values of the image's bounding box, or null if the image has no pixels. + */ +const getImageDataBbox = (imageData: ImageData): Extents | null => { + const { data, width, height } = imageData; + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + let alpha = 0; + let isEmpty = true; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + alpha = data[(y * width + x) * 4 + 3] ?? 0; + if (alpha > 0) { + isEmpty = false; + if (x < minX) { + minX = x; + } + if (x > maxX) { + maxX = x; + } + if (y < minY) { + minY = y; + } + if (y > maxY) { + maxY = y; + } + } + } + } + + return isEmpty ? null : { minX, minY, maxX, maxY }; +}; + +/** + * Clones a regional guidance konva layer onto an offscreen stage/canvas. This allows the pixel data for a given layer + * to be captured, manipulated or analyzed without interference from other layers. + * @param layer The konva layer to clone. + * @param filterChildren A callback to filter out unwanted children + * @returns The cloned stage and layer. + */ +const getIsolatedLayerClone = ( + layer: Konva.Layer, + filterChildren: (node: Konva.Node) => boolean +): { stageClone: Konva.Stage; layerClone: Konva.Layer } => { + const stage = layer.getStage(); + + // Construct an offscreen canvas with the same dimensions as the layer's stage. + const offscreenStageContainer = document.createElement('div'); + const stageClone = new Konva.Stage({ + container: offscreenStageContainer, + x: stage.x(), + y: stage.y(), + width: stage.width(), + height: stage.height(), + }); + + // Clone the layer and filter out unwanted children. + const layerClone = layer.clone(); + stageClone.add(layerClone); + + for (const child of layerClone.getChildren()) { + if (filterChildren(child) && child.hasChildren()) { + // We need to cache the group to ensure it composites out eraser strokes correctly + child.opacity(1); + child.cache(); + } else { + // Filter out unwanted children. + child.destroy(); + } + } + + return { stageClone, layerClone }; +}; + +/** + * Get the bounding box of a regional prompt konva layer. This function has special handling for regional prompt layers. + * @param layer The konva layer to get the bounding box of. + * @param preview Whether to open a new tab displaying the rendered layer, which is used to calculate the bbox. + */ +const getLayerBboxPixels = ( + layer: Konva.Layer, + filterChildren: (node: Konva.Node) => boolean, + preview: boolean = false +): IRect | null => { + // To calculate the layer's bounding box, we must first export it to a pixel array, then do some math. + // + // Though it is relatively fast, we can't use Konva's `getClientRect`. It programmatically determines the rect + // by calculating the extents of individual shapes from their "vector" shape data. + // + // This doesn't work when some shapes are drawn with composite operations that "erase" pixels, like eraser lines. + // These shapes' extents are still calculated as if they were solid, leading to a bounding box that is too large. + const { stageClone, layerClone } = getIsolatedLayerClone(layer, filterChildren); + + // Get a worst-case rect using the relatively fast `getClientRect`. + const layerRect = layerClone.getClientRect(); + if (layerRect.width === 0 || layerRect.height === 0) { + return null; + } + // Capture the image data with the above rect. + const layerImageData = stageClone + .toCanvas(layerRect) + .getContext('2d') + ?.getImageData(0, 0, layerRect.width, layerRect.height); + assert(layerImageData, "Unable to get layer's image data"); + + if (preview) { + openBase64ImageInTab([{ base64: imageDataToDataURL(layerImageData), caption: layer.id() }]); + } + + // Calculate the layer's bounding box. + const layerBbox = getImageDataBbox(layerImageData); + + if (!layerBbox) { + return null; + } + + // Correct the bounding box to be relative to the layer's position. + const correctedLayerBbox = { + x: layerBbox.minX - Math.floor(stageClone.x()) + layerRect.x - Math.floor(layer.x()), + y: layerBbox.minY - Math.floor(stageClone.y()) + layerRect.y - Math.floor(layer.y()), + width: layerBbox.maxX - layerBbox.minX, + height: layerBbox.maxY - layerBbox.minY, + }; + + return correctedLayerBbox; +}; + +/** + * Get the bounding box of a konva layer. This function is faster than `getLayerBboxPixels` but less accurate. It + * should only be used when there are no eraser strokes or shapes in the layer. + * @param layer The konva layer to get the bounding box of. + * @returns The bounding box of the layer. + */ +export const getLayerBboxFast = (layer: Konva.Layer): IRect => { + const bbox = layer.getClientRect(GET_CLIENT_RECT_CONFIG); + return { + x: Math.floor(bbox.x), + y: Math.floor(bbox.y), + width: Math.floor(bbox.width), + height: Math.floor(bbox.height), + }; +}; + +const filterRGChildren = (node: Konva.Node): boolean => node.name() === RG_LAYER_OBJECT_GROUP_NAME; +const filterLayerChildren = (node: Konva.Node): boolean => node.name() === RASTER_LAYER_OBJECT_GROUP_NAME; +const filterCAChildren = (node: Konva.Node): boolean => node.name() === CA_LAYER_IMAGE_NAME; + +/** + * Calculates the bbox of each regional guidance layer. Only calculates if the mask has changed. + * @param stage The konva stage + * @param entityStates An array of layers to calculate bboxes for + * @param onBboxChanged Callback for when the bounding box changes + */ +export const updateBboxes = ( + stage: Konva.Stage, + layers: LayerEntity[], + controlAdapters: ControlAdapterEntity[], + regions: RegionEntity[], + onBboxChanged: (arg: BboxChangedArg, entityType: CanvasEntity['type']) => void +): void => { + for (const entityState of [...layers, ...controlAdapters, ...regions]) { + const konvaLayer = stage.findOne(`#${entityState.id}`); + assert(konvaLayer, `Layer ${entityState.id} not found in stage`); + // We only need to recalculate the bbox if the layer has changed + if (entityState.bboxNeedsUpdate) { + const bboxRect = konvaLayer.findOne(`.${LAYER_BBOX_NAME}`) ?? createBboxRect(entityState, konvaLayer); + + // Hide the bbox while we calculate the new bbox, else the bbox will be included in the calculation + const visible = bboxRect.visible(); + bboxRect.visible(false); + + if (entityState.type === 'layer') { + if (entityState.objects.length === 0) { + // No objects - no bbox to calculate + onBboxChanged({ id: entityState.id, bbox: null }, 'layer'); + } else { + onBboxChanged({ id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterLayerChildren) }, 'layer'); + } + } else if (entityState.type === 'control_adapter') { + if (!entityState.imageObject && !entityState.processedImageObject) { + // No objects - no bbox to calculate + onBboxChanged({ id: entityState.id, bbox: null }, 'control_adapter'); + } else { + onBboxChanged( + { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterCAChildren) }, + 'control_adapter' + ); + } + } else if (entityState.type === 'regional_guidance') { + if (entityState.objects.length === 0) { + // No objects - no bbox to calculate + onBboxChanged({ id: entityState.id, bbox: null }, 'regional_guidance'); + } else { + onBboxChanged( + { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterRGChildren) }, + 'regional_guidance' + ); + } + } + + // Restore the visibility of the bbox + bboxRect.visible(visible); + } + } +}; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts index 93d56be444..db4f7b6cbf 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts @@ -1,7 +1,7 @@ import { rgbColorToString } from 'common/util/colorCodeTransformers'; import { getObjectGroupId } from 'features/controlLayers/konva/naming'; import type { StateApi } from 'features/controlLayers/konva/nodeManager'; -import { getLayerBboxFast } from 'features/controlLayers/konva/renderers/bbox'; +import { getLayerBboxFast } from 'features/controlLayers/konva/renderers/entityBbox'; import { KonvaBrushLine, KonvaEraserLine, KonvaRect } from 'features/controlLayers/konva/renderers/objects'; import { mapId } from 'features/controlLayers/konva/util'; import type { CanvasEntityIdentifier, InpaintMaskEntity, Tool } from 'features/controlLayers/store/types'; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/preview.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/preview.ts index 5df04d4d6c..871e21353e 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/preview.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/preview.ts @@ -1,588 +1,9 @@ -import { getArbitraryBaseColor } from '@invoke-ai/ui-library'; -import { rgbaColorToString } from 'common/util/colorCodeTransformers'; -import { roundToMultiple, roundToMultipleMin } from 'common/util/roundDownToMultiple'; -import { - BRUSH_BORDER_INNER_COLOR, - BRUSH_BORDER_OUTER_COLOR, - BRUSH_ERASER_BORDER_WIDTH, - DOCUMENT_FIT_PADDING_PX, -} from 'features/controlLayers/konva/constants'; -import { - PREVIEW_GENERATION_BBOX_DUMMY_RECT, - PREVIEW_GENERATION_BBOX_GROUP, - PREVIEW_GENERATION_BBOX_TRANSFORMER, - PREVIEW_RECT_ID, -} from 'features/controlLayers/konva/naming'; -import { KonvaImage } from 'features/controlLayers/konva/renderers/objects'; -import type { CanvasEntity, CanvasV2State, Position, RgbaColor, StageAttrs } from 'features/controlLayers/store/types'; import Konva from 'konva'; -import type { IRect } from 'konva/lib/types'; -import { atom } from 'nanostores'; -import { assert } from 'tsafe'; -export class CanvasDocumentSizeOverlay { - group: Konva.Group; - outerRect: Konva.Rect; - innerRect: Konva.Rect; - padding: number; - - constructor(padding?: number) { - this.padding = padding ?? DOCUMENT_FIT_PADDING_PX; - this.group = new Konva.Group({ id: 'document_overlay_group', listening: false }); - this.outerRect = new Konva.Rect({ - id: 'document_overlay_outer_rect', - listening: false, - fill: getArbitraryBaseColor(10), - opacity: 0.7, - }); - this.innerRect = new Konva.Rect({ - id: 'document_overlay_inner_rect', - listening: false, - fill: 'white', - globalCompositeOperation: 'destination-out', - }); - this.group.add(this.outerRect); - this.group.add(this.innerRect); - } - - render(stage: Konva.Stage, document: CanvasV2State['document']) { - this.group.zIndex(0); - - const x = stage.x(); - const y = stage.y(); - const width = stage.width(); - const height = stage.height(); - const scale = stage.scaleX(); - - this.outerRect.setAttrs({ - offsetX: x / scale, - offsetY: y / scale, - width: width / scale, - height: height / scale, - }); - - this.innerRect.setAttrs({ - x: 0, - y: 0, - width: document.width, - height: document.height, - }); - } - - fitToStage(stage: Konva.Stage, document: CanvasV2State['document'], setStageAttrs: (attrs: StageAttrs) => void) { - // Fit & center the document on the stage - const width = stage.width(); - const height = stage.height(); - const docWidthWithBuffer = document.width + this.padding * 2; - const docHeightWithBuffer = document.height + this.padding * 2; - const scale = Math.min(Math.min(width / docWidthWithBuffer, height / docHeightWithBuffer), 1); - const x = (width - docWidthWithBuffer * scale) / 2 + this.padding * scale; - const y = (height - docHeightWithBuffer * scale) / 2 + this.padding * scale; - stage.setAttrs({ x, y, width, height, scaleX: scale, scaleY: scale }); - setStageAttrs({ x, y, width, height, scale }); - } -} - -export class CanvasStagingArea { - group: Konva.Group; - image: KonvaImage | null; - - constructor() { - this.group = new Konva.Group({ listening: false }); - this.image = null; - } - - async render(stagingArea: CanvasV2State['stagingArea']) { - if (!stagingArea || stagingArea.selectedImageIndex === null) { - if (this.image) { - this.image.destroy(); - this.image = null; - } - return; - } - - if (stagingArea.selectedImageIndex !== null) { - const imageDTO = stagingArea.images[stagingArea.selectedImageIndex]; - assert(imageDTO, 'Image must exist'); - if (this.image) { - if (!this.image.isLoading && !this.image.isError && this.image.imageName !== imageDTO.image_name) { - await this.image.updateImageSource(imageDTO.image_name); - } - } else { - const { image_name, width, height } = imageDTO; - this.image = new KonvaImage({ - imageObject: { - id: 'staging-area-image', - type: 'image', - x: stagingArea.bbox.x, - y: stagingArea.bbox.y, - width, - height, - filters: [], - image: { - name: image_name, - width, - height, - }, - }, - }); - this.group.add(this.image.konvaImageGroup); - await this.image.updateImageSource(imageDTO.image_name); - } - } - } -} - -export class CanvasTool { - group: Konva.Group; - brush: { - group: Konva.Group; - fillCircle: Konva.Circle; - innerBorderCircle: Konva.Circle; - outerBorderCircle: Konva.Circle; - }; - eraser: { - group: Konva.Group; - fillCircle: Konva.Circle; - innerBorderCircle: Konva.Circle; - outerBorderCircle: Konva.Circle; - }; - rect: { - group: Konva.Group; - fillRect: Konva.Rect; - }; - - constructor() { - this.group = new Konva.Group(); - - // Create the brush preview group & circles - this.brush = { - group: new Konva.Group(), - fillCircle: new Konva.Circle({ - listening: false, - strokeEnabled: false, - }), - innerBorderCircle: new Konva.Circle({ - listening: false, - stroke: BRUSH_BORDER_INNER_COLOR, - strokeWidth: BRUSH_ERASER_BORDER_WIDTH, - strokeEnabled: true, - }), - outerBorderCircle: new Konva.Circle({ - listening: false, - stroke: BRUSH_BORDER_OUTER_COLOR, - strokeWidth: BRUSH_ERASER_BORDER_WIDTH, - strokeEnabled: true, - }), - }; - this.brush.group.add(this.brush.fillCircle); - this.brush.group.add(this.brush.innerBorderCircle); - this.brush.group.add(this.brush.outerBorderCircle); - this.group.add(this.brush.group); - - this.eraser = { - group: new Konva.Group(), - fillCircle: new Konva.Circle({ - listening: false, - strokeEnabled: false, - fill: 'white', - globalCompositeOperation: 'destination-out', - }), - innerBorderCircle: new Konva.Circle({ - listening: false, - stroke: BRUSH_BORDER_INNER_COLOR, - strokeWidth: BRUSH_ERASER_BORDER_WIDTH, - strokeEnabled: true, - }), - outerBorderCircle: new Konva.Circle({ - listening: false, - stroke: BRUSH_BORDER_OUTER_COLOR, - strokeWidth: BRUSH_ERASER_BORDER_WIDTH, - strokeEnabled: true, - }), - }; - this.eraser.group.add(this.eraser.fillCircle); - this.eraser.group.add(this.eraser.innerBorderCircle); - this.eraser.group.add(this.eraser.outerBorderCircle); - this.group.add(this.eraser.group); - - // Create the rect preview - this is a rectangle drawn from the last mouse down position to the current cursor position - this.rect = { - group: new Konva.Group(), - fillRect: new Konva.Rect({ - id: PREVIEW_RECT_ID, - listening: false, - strokeEnabled: false, - }), - }; - this.rect.group.add(this.rect.fillRect); - this.group.add(this.rect.group); - } - - scaleTool(stage: Konva.Stage, toolState: CanvasV2State['tool']) { - const scale = stage.scaleX(); - - const brushRadius = toolState.brush.width / 2; - this.brush.innerBorderCircle.strokeWidth(BRUSH_ERASER_BORDER_WIDTH / scale); - this.brush.outerBorderCircle.setAttrs({ - strokeWidth: BRUSH_ERASER_BORDER_WIDTH / scale, - radius: brushRadius + BRUSH_ERASER_BORDER_WIDTH / scale, - }); - - const eraserRadius = toolState.eraser.width / 2; - this.eraser.innerBorderCircle.strokeWidth(BRUSH_ERASER_BORDER_WIDTH / scale); - this.eraser.outerBorderCircle.setAttrs({ - strokeWidth: BRUSH_ERASER_BORDER_WIDTH / scale, - radius: eraserRadius + BRUSH_ERASER_BORDER_WIDTH / scale, - }); - } - - render( - stage: Konva.Stage, - renderedEntityCount: number, - toolState: CanvasV2State['tool'], - currentFill: RgbaColor, - selectedEntity: CanvasEntity | null, - cursorPos: Position | null, - lastMouseDownPos: Position | null, - isDrawing: boolean, - isMouseDown: boolean - ) { - const tool = toolState.selected; - const isDrawableEntity = - selectedEntity?.type === 'regional_guidance' || - selectedEntity?.type === 'layer' || - selectedEntity?.type === 'inpaint_mask'; - - // Update the stage's pointer style - if (tool === 'view') { - // View gets a hand - stage.container().style.cursor = isMouseDown ? 'grabbing' : 'grab'; - } else if (renderedEntityCount === 0) { - // We have no layers, so we should not render any tool - stage.container().style.cursor = 'default'; - } else if (!isDrawableEntity) { - // Non-drawable layers don't have tools - stage.container().style.cursor = 'not-allowed'; - } else if (tool === 'move') { - // Move tool gets a pointer - stage.container().style.cursor = 'default'; - } else if (tool === 'rect') { - // Rect gets a crosshair - stage.container().style.cursor = 'crosshair'; - } else if (tool === 'brush' || tool === 'eraser') { - // Hide the native cursor and use the konva-rendered brush preview - stage.container().style.cursor = 'none'; - } else if (tool === 'bbox') { - stage.container().style.cursor = 'default'; - } - - stage.draggable(tool === 'view'); - - if (!cursorPos || renderedEntityCount === 0 || !isDrawableEntity) { - // We can bail early if the mouse isn't over the stage or there are no layers - this.group.visible(false); - } else { - this.group.visible(true); - - // No need to render the brush preview if the cursor position or color is missing - if (cursorPos && tool === 'brush') { - const scale = stage.scaleX(); - // Update the fill circle - const radius = toolState.brush.width / 2; - this.brush.fillCircle.setAttrs({ - x: cursorPos.x, - y: cursorPos.y, - radius, - fill: isDrawing ? '' : rgbaColorToString(currentFill), - }); - - // Update the inner border of the brush preview - this.brush.innerBorderCircle.setAttrs({ x: cursorPos.x, y: cursorPos.y, radius }); - - // Update the outer border of the brush preview - this.brush.outerBorderCircle.setAttrs({ - x: cursorPos.x, - y: cursorPos.y, - radius: radius + BRUSH_ERASER_BORDER_WIDTH / scale, - }); - - this.scaleTool(stage, toolState); - - this.brush.group.visible(true); - this.eraser.group.visible(false); - this.rect.group.visible(false); - } else if (cursorPos && tool === 'eraser') { - const scale = stage.scaleX(); - // Update the fill circle - const radius = toolState.eraser.width / 2; - this.eraser.fillCircle.setAttrs({ - x: cursorPos.x, - y: cursorPos.y, - radius, - fill: 'white', - }); - - // Update the inner border of the eraser preview - this.eraser.innerBorderCircle.setAttrs({ x: cursorPos.x, y: cursorPos.y, radius }); - - // Update the outer border of the eraser preview - this.eraser.outerBorderCircle.setAttrs({ - x: cursorPos.x, - y: cursorPos.y, - radius: radius + BRUSH_ERASER_BORDER_WIDTH / scale, - }); - - this.scaleTool(stage, toolState); - - this.brush.group.visible(false); - this.eraser.group.visible(true); - this.rect.group.visible(false); - } else if (cursorPos && lastMouseDownPos && tool === 'rect') { - this.rect.fillRect.setAttrs({ - x: Math.min(cursorPos.x, lastMouseDownPos.x), - y: Math.min(cursorPos.y, lastMouseDownPos.y), - width: Math.abs(cursorPos.x - lastMouseDownPos.x), - height: Math.abs(cursorPos.y - lastMouseDownPos.y), - fill: rgbaColorToString(currentFill), - visible: true, - }); - this.brush.group.visible(false); - this.eraser.group.visible(false); - this.rect.group.visible(true); - } else { - this.brush.group.visible(false); - this.eraser.group.visible(false); - this.rect.group.visible(false); - } - } - } -} - -export class CanvasBbox { - group: Konva.Group; - rect: Konva.Rect; - transformer: Konva.Transformer; - - ALL_ANCHORS: string[] = [ - 'top-left', - 'top-center', - 'top-right', - 'middle-right', - 'middle-left', - 'bottom-left', - 'bottom-center', - 'bottom-right', - ]; - CORNER_ANCHORS: string[] = ['top-left', 'top-right', 'bottom-left', 'bottom-right']; - NO_ANCHORS: string[] = []; - - constructor( - getBbox: () => IRect, - onBboxTransformed: (bbox: IRect) => void, - getShiftKey: () => boolean, - getCtrlKey: () => boolean, - getMetaKey: () => boolean, - getAltKey: () => boolean - ) { - // Create a stash to hold onto the last aspect ratio of the bbox - this allows for locking the aspect ratio when - // transforming the bbox. - const bbox = getBbox(); - const $aspectRatioBuffer = atom(bbox.width / bbox.height); - - // Use a transformer for the generation bbox. Transformers need some shape to transform, we will use a fully - // transparent rect for this purpose. - this.group = new Konva.Group({ id: PREVIEW_GENERATION_BBOX_GROUP, listening: false }); - this.rect = new Konva.Rect({ - id: PREVIEW_GENERATION_BBOX_DUMMY_RECT, - listening: false, - strokeEnabled: false, - draggable: true, - ...getBbox(), - }); - this.rect.on('dragmove', () => { - const gridSize = getCtrlKey() || getMetaKey() ? 8 : 64; - const oldBbox = getBbox(); - const newBbox: IRect = { - ...oldBbox, - x: roundToMultiple(this.rect.x(), gridSize), - y: roundToMultiple(this.rect.y(), gridSize), - }; - this.rect.setAttrs(newBbox); - if (oldBbox.x !== newBbox.x || oldBbox.y !== newBbox.y) { - onBboxTransformed(newBbox); - } - }); - - this.transformer = new Konva.Transformer({ - id: PREVIEW_GENERATION_BBOX_TRANSFORMER, - borderDash: [5, 5], - borderStroke: 'rgba(212,216,234,1)', - borderEnabled: true, - rotateEnabled: false, - keepRatio: false, - ignoreStroke: true, - listening: false, - flipEnabled: false, - anchorFill: 'rgba(212,216,234,1)', - anchorStroke: 'rgb(42,42,42)', - anchorSize: 12, - anchorCornerRadius: 3, - shiftBehavior: 'none', // we will implement our own shift behavior - centeredScaling: false, - anchorStyleFunc: (anchor) => { - // Make the x/y resize anchors little bars - if (anchor.hasName('top-center') || anchor.hasName('bottom-center')) { - anchor.height(8); - anchor.offsetY(4); - anchor.width(30); - anchor.offsetX(15); - } - if (anchor.hasName('middle-left') || anchor.hasName('middle-right')) { - anchor.height(30); - anchor.offsetY(15); - anchor.width(8); - anchor.offsetX(4); - } - }, - anchorDragBoundFunc: (_oldAbsPos, newAbsPos) => { - // This function works with absolute position - that is, a position in "physical" pixels on the screen, as opposed - // to konva's internal coordinate system. - const stage = this.transformer.getStage(); - assert(stage, 'Stage must exist'); - - // We need to snap the anchors to the grid. If the user is holding ctrl/meta, we use the finer 8px grid. - const gridSize = getCtrlKey() || getMetaKey() ? 8 : 64; - // Because we are working in absolute coordinates, we need to scale the grid size by the stage scale. - const scaledGridSize = gridSize * stage.scaleX(); - // To snap the anchor to the grid, we need to calculate an offset from the stage's absolute position. - const stageAbsPos = stage.getAbsolutePosition(); - // The offset is the remainder of the stage's absolute position divided by the scaled grid size. - const offsetX = stageAbsPos.x % scaledGridSize; - const offsetY = stageAbsPos.y % scaledGridSize; - // Finally, calculate the position by rounding to the grid and adding the offset. - return { - x: roundToMultiple(newAbsPos.x, scaledGridSize) + offsetX, - y: roundToMultiple(newAbsPos.y, scaledGridSize) + offsetY, - }; - }, - }); - - this.transformer.on('transform', () => { - // In the transform callback, we calculate the bbox's new dims and pos and update the konva object. - - // Some special handling is needed depending on the anchor being dragged. - const anchor = this.transformer.getActiveAnchor(); - if (!anchor) { - // Pretty sure we should always have an anchor here? - return; - } - - const alt = getAltKey(); - const ctrl = getCtrlKey(); - const meta = getMetaKey(); - const shift = getShiftKey(); - - // Grid size depends on the modifier keys - let gridSize = ctrl || meta ? 8 : 64; - - // Alt key indicates we are using centered scaling. We need to double the gride size used when calculating the - // new dimensions so that each size scales in the correct increments and doesn't mis-place the bbox. For example, if - // we snapped the width and height to 8px increments, the bbox would be mis-placed by 4px in the x and y axes. - // Doubling the grid size ensures the bbox's coords remain aligned to the 8px/64px grid. - if (getAltKey()) { - gridSize = gridSize * 2; - } - - // The coords should be correct per the anchorDragBoundFunc. - let x = this.rect.x(); - let y = this.rect.y(); - - // Konva transforms by scaling the dims, not directly changing width and height. At this point, the width and height - // *have not changed*, only the scale has changed. To get the final height, we need to scale the dims and then snap - // them to the grid. - let width = roundToMultipleMin(this.rect.width() * this.rect.scaleX(), gridSize); - let height = roundToMultipleMin(this.rect.height() * this.rect.scaleY(), gridSize); - - // If shift is held and we are resizing from a corner, retain aspect ratio - needs special handling. We skip this - // if alt/opt is held - this requires math too big for my brain. - if (shift && this.CORNER_ANCHORS.includes(anchor) && !alt) { - // Fit the bbox to the last aspect ratio - let fittedWidth = Math.sqrt(width * height * $aspectRatioBuffer.get()); - let fittedHeight = fittedWidth / $aspectRatioBuffer.get(); - fittedWidth = roundToMultipleMin(fittedWidth, gridSize); - fittedHeight = roundToMultipleMin(fittedHeight, gridSize); - - // We need to adjust the x and y coords to have the resize occur from the right origin. - if (anchor === 'top-left') { - // The transform origin is the bottom-right anchor. Both x and y need to be updated. - x = x - (fittedWidth - width); - y = y - (fittedHeight - height); - } - if (anchor === 'top-right') { - // The transform origin is the bottom-left anchor. Only y needs to be updated. - y = y - (fittedHeight - height); - } - if (anchor === 'bottom-left') { - // The transform origin is the top-right anchor. Only x needs to be updated. - x = x - (fittedWidth - width); - } - // Update the width and height to the fitted dims. - width = fittedWidth; - height = fittedHeight; - } - - const bbox = { - x: Math.round(x), - y: Math.round(y), - width, - height, - }; - - // Update the bboxRect's attrs directly with the new transform, and reset its scale to 1. - // TODO(psyche): In `renderBboxPreview()` we also call setAttrs, need to do it twice to ensure it renders correctly. - // Gotta be a way to avoid setting it twice... - this.rect.setAttrs({ ...bbox, scaleX: 1, scaleY: 1 }); - - // Update the bbox in internal state. - onBboxTransformed(bbox); - - // Update the aspect ratio buffer whenever the shift key is not held - this allows for a nice UX where you can start - // a transform, get the right aspect ratio, then hold shift to lock it in. - if (!shift) { - $aspectRatioBuffer.set(bbox.width / bbox.height); - } - }); - - this.transformer.on('transformend', () => { - // Always update the aspect ratio buffer when the transform ends, so if the next transform starts with shift held, - // we have the correct aspect ratio to start from. - $aspectRatioBuffer.set(this.rect.width() / this.rect.height()); - }); - - // The transformer will always be transforming the dummy rect - this.transformer.nodes([this.rect]); - this.group.add(this.rect); - this.group.add(this.transformer); - } - - render(bbox: CanvasV2State['bbox'], toolState: CanvasV2State['tool']) { - this.group.listening(toolState.selected === 'bbox'); - this.rect.setAttrs({ - x: bbox.x, - y: bbox.y, - width: bbox.width, - height: bbox.height, - scaleX: 1, - scaleY: 1, - listening: toolState.selected === 'bbox', - }); - this.transformer.setAttrs({ - listening: toolState.selected === 'bbox', - enabledAnchors: toolState.selected === 'bbox' ? this.ALL_ANCHORS : this.NO_ANCHORS, - }); - } -} +import type { CanvasBbox } from './bbox'; +import type { CanvasDocumentSizeOverlay } from './documentSizeOverlay'; +import type { CanvasStagingArea } from './stagingArea'; +import type { CanvasTool } from './tool'; export class CanvasPreview { konvaLayer: Konva.Layer; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts index cc0862ecd2..7f6ebd5ddb 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts @@ -1,7 +1,7 @@ import { rgbColorToString } from 'common/util/colorCodeTransformers'; import { getObjectGroupId } from 'features/controlLayers/konva/naming'; import type { StateApi } from 'features/controlLayers/konva/nodeManager'; -import { getLayerBboxFast } from 'features/controlLayers/konva/renderers/bbox'; +import { getLayerBboxFast } from 'features/controlLayers/konva/renderers/entityBbox'; import { KonvaBrushLine, KonvaEraserLine, KonvaRect } from 'features/controlLayers/konva/renderers/objects'; import { mapId } from 'features/controlLayers/konva/util'; import type { CanvasEntityIdentifier, RegionEntity, Tool } from 'features/controlLayers/store/types'; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts index eb0f83fc73..7b23fff52d 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts @@ -5,7 +5,7 @@ import { $isDebugging } from 'app/store/nanostores/isDebugging'; import type { RootState } from 'app/store/store'; import { setStageEventHandlers } from 'features/controlLayers/konva/events'; import { KonvaNodeManager, setNodeManager } from 'features/controlLayers/konva/nodeManager'; -import { updateBboxes } from 'features/controlLayers/konva/renderers/bbox'; +import { updateBboxes } from 'features/controlLayers/konva/renderers/entityBbox'; import { $stageAttrs, bboxChanged, diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stagingArea.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stagingArea.ts index d4178e6739..2b2deed489 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stagingArea.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stagingArea.ts @@ -1,41 +1,55 @@ -import type { KonvaNodeManager } from 'features/controlLayers/konva/nodeManager'; -import { createImageObjectGroup, updateImageSource } from 'features/controlLayers/konva/renderers/objects'; -import { imageDTOToImageObject, imageDTOToImageWithDims } from 'features/controlLayers/store/types'; +import { KonvaImage } from 'features/controlLayers/konva/renderers/objects'; +import type { CanvasV2State } from 'features/controlLayers/store/types'; import Konva from 'konva'; import { assert } from 'tsafe'; -export const createStagingArea = (): KonvaNodeManager['preview']['stagingArea'] => { - const group = new Konva.Group({ id: 'staging_area_group', listening: false }); - return { group, image: null }; -}; -export const getRenderStagingArea = async (manager: KonvaNodeManager) => { - const { getStagingAreaState } = manager.stateApi; - const stagingArea = getStagingAreaState(); +export class CanvasStagingArea { + group: Konva.Group; + image: KonvaImage | null; - if (!stagingArea || stagingArea.selectedImageIndex === null) { - if (manager.preview.stagingArea.image) { - manager.preview.stagingArea.image.konvaImageGroup.visible(false); - manager.preview.stagingArea.image = null; - } - return; + constructor() { + this.group = new Konva.Group({ listening: false }); + this.image = null; } - if (stagingArea.selectedImageIndex) { - const imageDTO = stagingArea.images[stagingArea.selectedImageIndex]; - assert(imageDTO, 'Image must exist'); - if (manager.preview.stagingArea.image) { - if (manager.preview.stagingArea.image.imageName !== imageDTO.image_name) { - await updateImageSource({ - objectRecord: manager.preview.stagingArea.image, - image: imageDTOToImageWithDims(imageDTO), - }); + async render(stagingArea: CanvasV2State['stagingArea']) { + if (!stagingArea || stagingArea.selectedImageIndex === null) { + if (this.image) { + this.image.destroy(); + this.image = null; + } + return; + } + + if (stagingArea.selectedImageIndex !== null) { + const imageDTO = stagingArea.images[stagingArea.selectedImageIndex]; + assert(imageDTO, 'Image must exist'); + if (this.image) { + if (!this.image.isLoading && !this.image.isError && this.image.imageName !== imageDTO.image_name) { + await this.image.updateImageSource(imageDTO.image_name); + } + } else { + const { image_name, width, height } = imageDTO; + this.image = new KonvaImage({ + imageObject: { + id: 'staging-area-image', + type: 'image', + x: stagingArea.bbox.x, + y: stagingArea.bbox.y, + width, + height, + filters: [], + image: { + name: image_name, + width, + height, + }, + }, + }); + this.group.add(this.image.konvaImageGroup); + await this.image.updateImageSource(imageDTO.image_name); } - } else { - manager.preview.stagingArea.image = await createImageObjectGroup({ - obj: imageDTOToImageObject(imageDTO), - name: imageDTO.image_name, - }); } } -}; +} diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/tool.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/tool.ts new file mode 100644 index 0000000000..d7b05d0cb3 --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/tool.ts @@ -0,0 +1,235 @@ +import { rgbaColorToString } from 'common/util/colorCodeTransformers'; +import { + BRUSH_BORDER_INNER_COLOR, + BRUSH_BORDER_OUTER_COLOR, + BRUSH_ERASER_BORDER_WIDTH +} from 'features/controlLayers/konva/constants'; +import { PREVIEW_RECT_ID } from 'features/controlLayers/konva/naming'; +import type { CanvasEntity, CanvasV2State, Position, RgbaColor } from 'features/controlLayers/store/types'; +import Konva from 'konva'; + + +export class CanvasTool { + group: Konva.Group; + brush: { + group: Konva.Group; + fillCircle: Konva.Circle; + innerBorderCircle: Konva.Circle; + outerBorderCircle: Konva.Circle; + }; + eraser: { + group: Konva.Group; + fillCircle: Konva.Circle; + innerBorderCircle: Konva.Circle; + outerBorderCircle: Konva.Circle; + }; + rect: { + group: Konva.Group; + fillRect: Konva.Rect; + }; + + constructor() { + this.group = new Konva.Group(); + + // Create the brush preview group & circles + this.brush = { + group: new Konva.Group(), + fillCircle: new Konva.Circle({ + listening: false, + strokeEnabled: false, + }), + innerBorderCircle: new Konva.Circle({ + listening: false, + stroke: BRUSH_BORDER_INNER_COLOR, + strokeWidth: BRUSH_ERASER_BORDER_WIDTH, + strokeEnabled: true, + }), + outerBorderCircle: new Konva.Circle({ + listening: false, + stroke: BRUSH_BORDER_OUTER_COLOR, + strokeWidth: BRUSH_ERASER_BORDER_WIDTH, + strokeEnabled: true, + }), + }; + this.brush.group.add(this.brush.fillCircle); + this.brush.group.add(this.brush.innerBorderCircle); + this.brush.group.add(this.brush.outerBorderCircle); + this.group.add(this.brush.group); + + this.eraser = { + group: new Konva.Group(), + fillCircle: new Konva.Circle({ + listening: false, + strokeEnabled: false, + fill: 'white', + globalCompositeOperation: 'destination-out', + }), + innerBorderCircle: new Konva.Circle({ + listening: false, + stroke: BRUSH_BORDER_INNER_COLOR, + strokeWidth: BRUSH_ERASER_BORDER_WIDTH, + strokeEnabled: true, + }), + outerBorderCircle: new Konva.Circle({ + listening: false, + stroke: BRUSH_BORDER_OUTER_COLOR, + strokeWidth: BRUSH_ERASER_BORDER_WIDTH, + strokeEnabled: true, + }), + }; + this.eraser.group.add(this.eraser.fillCircle); + this.eraser.group.add(this.eraser.innerBorderCircle); + this.eraser.group.add(this.eraser.outerBorderCircle); + this.group.add(this.eraser.group); + + // Create the rect preview - this is a rectangle drawn from the last mouse down position to the current cursor position + this.rect = { + group: new Konva.Group(), + fillRect: new Konva.Rect({ + id: PREVIEW_RECT_ID, + listening: false, + strokeEnabled: false, + }), + }; + this.rect.group.add(this.rect.fillRect); + this.group.add(this.rect.group); + } + + scaleTool(stage: Konva.Stage, toolState: CanvasV2State['tool']) { + const scale = stage.scaleX(); + + const brushRadius = toolState.brush.width / 2; + this.brush.innerBorderCircle.strokeWidth(BRUSH_ERASER_BORDER_WIDTH / scale); + this.brush.outerBorderCircle.setAttrs({ + strokeWidth: BRUSH_ERASER_BORDER_WIDTH / scale, + radius: brushRadius + BRUSH_ERASER_BORDER_WIDTH / scale, + }); + + const eraserRadius = toolState.eraser.width / 2; + this.eraser.innerBorderCircle.strokeWidth(BRUSH_ERASER_BORDER_WIDTH / scale); + this.eraser.outerBorderCircle.setAttrs({ + strokeWidth: BRUSH_ERASER_BORDER_WIDTH / scale, + radius: eraserRadius + BRUSH_ERASER_BORDER_WIDTH / scale, + }); + } + + render( + stage: Konva.Stage, + renderedEntityCount: number, + toolState: CanvasV2State['tool'], + currentFill: RgbaColor, + selectedEntity: CanvasEntity | null, + cursorPos: Position | null, + lastMouseDownPos: Position | null, + isDrawing: boolean, + isMouseDown: boolean + ) { + const tool = toolState.selected; + const isDrawableEntity = selectedEntity?.type === 'regional_guidance' || + selectedEntity?.type === 'layer' || + selectedEntity?.type === 'inpaint_mask'; + + // Update the stage's pointer style + if (tool === 'view') { + // View gets a hand + stage.container().style.cursor = isMouseDown ? 'grabbing' : 'grab'; + } else if (renderedEntityCount === 0) { + // We have no layers, so we should not render any tool + stage.container().style.cursor = 'default'; + } else if (!isDrawableEntity) { + // Non-drawable layers don't have tools + stage.container().style.cursor = 'not-allowed'; + } else if (tool === 'move') { + // Move tool gets a pointer + stage.container().style.cursor = 'default'; + } else if (tool === 'rect') { + // Rect gets a crosshair + stage.container().style.cursor = 'crosshair'; + } else if (tool === 'brush' || tool === 'eraser') { + // Hide the native cursor and use the konva-rendered brush preview + stage.container().style.cursor = 'none'; + } else if (tool === 'bbox') { + stage.container().style.cursor = 'default'; + } + + stage.draggable(tool === 'view'); + + if (!cursorPos || renderedEntityCount === 0 || !isDrawableEntity) { + // We can bail early if the mouse isn't over the stage or there are no layers + this.group.visible(false); + } else { + this.group.visible(true); + + // No need to render the brush preview if the cursor position or color is missing + if (cursorPos && tool === 'brush') { + const scale = stage.scaleX(); + // Update the fill circle + const radius = toolState.brush.width / 2; + this.brush.fillCircle.setAttrs({ + x: cursorPos.x, + y: cursorPos.y, + radius, + fill: isDrawing ? '' : rgbaColorToString(currentFill), + }); + + // Update the inner border of the brush preview + this.brush.innerBorderCircle.setAttrs({ x: cursorPos.x, y: cursorPos.y, radius }); + + // Update the outer border of the brush preview + this.brush.outerBorderCircle.setAttrs({ + x: cursorPos.x, + y: cursorPos.y, + radius: radius + BRUSH_ERASER_BORDER_WIDTH / scale, + }); + + this.scaleTool(stage, toolState); + + this.brush.group.visible(true); + this.eraser.group.visible(false); + this.rect.group.visible(false); + } else if (cursorPos && tool === 'eraser') { + const scale = stage.scaleX(); + // Update the fill circle + const radius = toolState.eraser.width / 2; + this.eraser.fillCircle.setAttrs({ + x: cursorPos.x, + y: cursorPos.y, + radius, + fill: 'white', + }); + + // Update the inner border of the eraser preview + this.eraser.innerBorderCircle.setAttrs({ x: cursorPos.x, y: cursorPos.y, radius }); + + // Update the outer border of the eraser preview + this.eraser.outerBorderCircle.setAttrs({ + x: cursorPos.x, + y: cursorPos.y, + radius: radius + BRUSH_ERASER_BORDER_WIDTH / scale, + }); + + this.scaleTool(stage, toolState); + + this.brush.group.visible(false); + this.eraser.group.visible(true); + this.rect.group.visible(false); + } else if (cursorPos && lastMouseDownPos && tool === 'rect') { + this.rect.fillRect.setAttrs({ + x: Math.min(cursorPos.x, lastMouseDownPos.x), + y: Math.min(cursorPos.y, lastMouseDownPos.y), + width: Math.abs(cursorPos.x - lastMouseDownPos.x), + height: Math.abs(cursorPos.y - lastMouseDownPos.y), + fill: rgbaColorToString(currentFill), + visible: true, + }); + this.brush.group.visible(false); + this.eraser.group.visible(false); + this.rect.group.visible(true); + } else { + this.brush.group.visible(false); + this.eraser.group.visible(false); + this.rect.group.visible(false); + } + } + } +}