From d26095dfa198856c9ad29b882a4d269374f59b1f Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:56:30 +1000 Subject: [PATCH] feat(ui): region mask rendering --- .../components/HeadsUpDisplay.tsx | 4 +- .../{CanvasLayer.ts => CanvasLayerAdapter.ts} | 21 +-- .../controlLayers/konva/CanvasManager.ts | 123 ++++++++++-------- ...vasInpaintMask.ts => CanvasMaskAdapter.ts} | 36 +++-- .../konva/CanvasObjectRenderer.ts | 39 +++--- .../controlLayers/konva/CanvasStateApi.ts | 3 + .../controlLayers/konva/CanvasTransformer.ts | 41 +++--- .../features/controlLayers/konva/events.ts | 24 +++- .../controlLayers/store/canvasV2Slice.ts | 7 +- .../controlLayers/store/regionsReducers.ts | 11 ++ .../src/features/controlLayers/store/types.ts | 22 +++- 11 files changed, 200 insertions(+), 131 deletions(-) rename invokeai/frontend/web/src/features/controlLayers/konva/{CanvasLayer.ts => CanvasLayerAdapter.ts} (90%) rename invokeai/frontend/web/src/features/controlLayers/konva/{CanvasInpaintMask.ts => CanvasMaskAdapter.ts} (80%) diff --git a/invokeai/frontend/web/src/features/controlLayers/components/HeadsUpDisplay.tsx b/invokeai/frontend/web/src/features/controlLayers/components/HeadsUpDisplay.tsx index abf799c893..5f3bcab13c 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/HeadsUpDisplay.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/HeadsUpDisplay.tsx @@ -24,10 +24,10 @@ export const HeadsUpDisplay = memo(() => { return ( - + diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayerAdapter.ts similarity index 90% rename from invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts rename to invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayerAdapter.ts index 6e4da97d0d..7eae3240bc 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayerAdapter.ts @@ -7,13 +7,9 @@ import Konva from 'konva'; import { get } from 'lodash-es'; import type { Logger } from 'roarr'; -export class CanvasLayer { - static TYPE = 'layer' as const; - static KONVA_LAYER_NAME = `${CanvasLayer.TYPE}_layer`; - static KONVA_OBJECT_GROUP_NAME = `${CanvasLayer.TYPE}_object-group`; - +export class CanvasLayerAdapter { id: string; - type = CanvasLayer.TYPE; + type: CanvasLayerState['type']; manager: CanvasManager; log: Logger; getLoggingContext: GetLoggingContext; @@ -29,8 +25,9 @@ export class CanvasLayer { isFirstRender: boolean = true; bboxNeedsUpdate: boolean = true; - constructor(state: CanvasLayerState, manager: CanvasManager) { + constructor(state: CanvasLayerAdapter['state'], manager: CanvasLayerAdapter['manager']) { this.id = state.id; + this.type = state.type; this.manager = manager; this.getLoggingContext = this.manager.buildGetLoggingContext(this); this.log = this.manager.buildLogger(this.getLoggingContext); @@ -39,7 +36,7 @@ export class CanvasLayer { this.konva = { layer: new Konva.Layer({ id: this.id, - name: CanvasLayer.KONVA_LAYER_NAME, + name: `${this.type}:layer`, listening: false, imageSmoothingEnabled: false, }), @@ -59,7 +56,11 @@ export class CanvasLayer { this.konva.layer.destroy(); }; - update = async (arg?: { state: CanvasLayerState; toolState: CanvasV2State['tool']; isSelected: boolean }) => { + update = async (arg?: { + state: CanvasLayerAdapter['state']; + toolState: CanvasV2State['tool']; + isSelected: boolean; + }) => { const state = get(arg, 'state', this.state); if (!this.isFirstRender && state === this.state) { @@ -119,7 +120,7 @@ export class CanvasLayer { repr = () => { return { id: this.id, - type: CanvasLayer.TYPE, + type: this.type, state: deepClone(this.state), bboxNeedsUpdate: this.bboxNeedsUpdate, transformer: this.transformer.repr(), diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts index b28197c70b..6cac07952c 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts @@ -26,7 +26,6 @@ import { $lastProgressEvent, $shouldShowStagedImage } from 'features/controlLaye import { type CanvasControlAdapterState, type CanvasEntityIdentifier, - type CanvasEntityState, type CanvasInpaintMaskState, type CanvasLayerState, type CanvasRegionalGuidanceState, @@ -42,15 +41,14 @@ import { atom } from 'nanostores'; import type { Logger } from 'roarr'; import { getImageDTO as defaultGetImageDTO, uploadImage as defaultUploadImage } from 'services/api/endpoints/images'; import type { ImageCategory, ImageDTO } from 'services/api/types'; -import { assert } from 'tsafe'; import { CanvasBackground } from './CanvasBackground'; import { CanvasBbox } from './CanvasBbox'; import { CanvasControlAdapter } from './CanvasControlAdapter'; -import { CanvasInpaintMask } from './CanvasInpaintMask'; -import { CanvasLayer } from './CanvasLayer'; +import { CanvasLayerAdapter } from './CanvasLayerAdapter'; +import { CanvasMaskAdapter } from './CanvasMaskAdapter'; import { CanvasPreview } from './CanvasPreview'; -import { CanvasRegion } from './CanvasRegion'; +import type { CanvasRegion } from './CanvasRegion'; import { CanvasStagingArea } from './CanvasStagingArea'; import { CanvasStateApi } from './CanvasStateApi'; import { CanvasTool } from './CanvasTool'; @@ -86,20 +84,28 @@ type Util = { type EntityStateAndAdapter = | { + id: string; + type: CanvasLayerState['type']; state: CanvasLayerState; - adapter: CanvasLayer; + adapter: CanvasLayerAdapter; } | { + id: string; + type: CanvasInpaintMaskState['type']; state: CanvasInpaintMaskState; - adapter: CanvasInpaintMask; + adapter: CanvasMaskAdapter; } | { + id: string; + type: CanvasControlAdapterState['type']; state: CanvasControlAdapterState; adapter: CanvasControlAdapter; } | { + id: string; + type: CanvasRegionalGuidanceState['type']; state: CanvasRegionalGuidanceState; - adapter: CanvasRegion; + adapter: CanvasMaskAdapter; }; export const $canvasManager = atom(null); @@ -111,9 +117,9 @@ export class CanvasManager { stage: Konva.Stage; container: HTMLDivElement; controlAdapters: Map; - layers: Map; - regions: Map; - inpaintMask: CanvasInpaintMask; + layers: Map; + regions: Map; + inpaintMask: CanvasMaskAdapter; initialImage: CanvasInitialImage; util: Util; stateApi: CanvasStateApi; @@ -221,7 +227,7 @@ export class CanvasManager { (a, b) => a?.state === b?.state && a?.adapter === b?.adapter ); - this.inpaintMask = new CanvasInpaintMask(this.stateApi.getInpaintMaskState(), this); + this.inpaintMask = new CanvasMaskAdapter(this.stateApi.getInpaintMaskState(), this); this.stage.add(this.inpaintMask.konva.layer); } @@ -248,28 +254,6 @@ export class CanvasManager { await this.initialImage.render(this.stateApi.getInitialImageState()); } - async renderRegions() { - const { entities } = this.stateApi.getRegionsState(); - - // Destroy the konva nodes for nonexistent entities - for (const canvasRegion of this.regions.values()) { - if (!entities.find((rg) => rg.id === canvasRegion.id)) { - canvasRegion.destroy(); - this.regions.delete(canvasRegion.id); - } - } - - for (const entity of entities) { - let adapter = this.regions.get(entity.id); - if (!adapter) { - adapter = new CanvasRegion(entity, this); - this.regions.set(adapter.id, adapter); - this.stage.add(adapter.konva.layer); - } - await adapter.render(entity); - } - } - async renderProgressPreview() { await this.preview.progressPreview.render(this.stateApi.$lastProgressEvent.get()); } @@ -320,8 +304,10 @@ export class CanvasManager { this.stage.width(this.container.offsetWidth); this.stage.height(this.container.offsetHeight); this.stateApi.$stageAttrs.set({ - position: { x: this.stage.x(), y: this.stage.y() }, - dimensions: { width: this.stage.width(), height: this.stage.height() }, + x: this.stage.x(), + y: this.stage.y(), + width: this.stage.width(), + height: this.stage.height(), scale: this.stage.scaleX(), }); this.background.render(); @@ -330,8 +316,13 @@ export class CanvasManager { getEntity(identifier: CanvasEntityIdentifier): EntityStateAndAdapter | null { const state = this.stateApi.getState(); - let entityState: CanvasEntityState | null = null; - let entityAdapter: CanvasLayer | CanvasRegion | CanvasControlAdapter | CanvasInpaintMask | null = null; + let entityState: + | CanvasLayerState + | CanvasControlAdapterState + | CanvasRegionalGuidanceState + | CanvasInpaintMaskState + | null = null; + let entityAdapter: CanvasLayerAdapter | CanvasControlAdapter | CanvasRegion | CanvasMaskAdapter | null = null; if (identifier.type === 'layer') { entityState = state.layers.entities.find((i) => i.id === identifier.id) ?? null; @@ -348,7 +339,12 @@ export class CanvasManager { } if (entityState && entityAdapter && entityState.type === entityAdapter.type) { - return { state: entityState, adapter: entityAdapter } as EntityStateAndAdapter; + return { + id: entityState.id, + type: entityState.type, + state: entityState, + adapter: entityAdapter, + } as EntityStateAndAdapter; // TODO(psyche): make TS happy w/o this cast } return null; @@ -400,6 +396,8 @@ export class CanvasManager { return this.layers.get(id) ?? null; } else if (type === 'inpaint_mask') { return this.inpaintMask; + } else if (type === 'regional_guidance') { + return this.regions.get(id) ?? null; } return null; @@ -413,14 +411,14 @@ export class CanvasManager { if (this.getIsTransforming()) { return; } - const layer = this.getSelectedEntity(); + const entity = this.getSelectedEntity(); + if (!entity) { + this.log.warn('No entity selected to transform'); + return; + } // TODO(psyche): Support other entity types - assert( - layer && (layer.adapter instanceof CanvasLayer || layer.adapter instanceof CanvasInpaintMask), - 'No selected layer' - ); - layer.adapter.transformer.startTransform(); - this.transformingEntity.publish({ id: layer.state.id, type: layer.state.type }); + entity.adapter.transformer.startTransform(); + this.transformingEntity.publish({ id: entity.id, type: entity.type }); } async applyTransform() { @@ -460,7 +458,7 @@ export class CanvasManager { for (const entityState of state.layers.entities) { let adapter = this.layers.get(entityState.id); if (!adapter) { - adapter = new CanvasLayer(entityState, this); + adapter = new CanvasLayerAdapter(entityState, this); this.layers.set(adapter.id, adapter); this.stage.add(adapter.konva.layer); } @@ -491,7 +489,28 @@ export class CanvasManager { state.selectedEntityIdentifier?.id !== this._prevState.selectedEntityIdentifier?.id ) { this.log.debug('Rendering regions'); - await this.renderRegions(); + + // Destroy the konva nodes for nonexistent entities + for (const canvasRegion of this.regions.values()) { + if (!state.regions.entities.find((rg) => rg.id === canvasRegion.id)) { + canvasRegion.destroy(); + this.regions.delete(canvasRegion.id); + } + } + + for (const entityState of state.regions.entities) { + let adapter = this.regions.get(entityState.id); + if (!adapter) { + adapter = new CanvasMaskAdapter(entityState, this); + this.regions.set(adapter.id, adapter); + this.stage.add(adapter.konva.layer); + } + await adapter.update({ + state: entityState, + toolState: state.tool, + isSelected: state.selectedEntityIdentifier?.id === entityState.id, + }); + } } if ( @@ -697,14 +716,14 @@ export class CanvasManager { | CanvasImageRenderer | CanvasTransformer | CanvasObjectRenderer - | CanvasLayer - | CanvasInpaintMask + | CanvasLayerAdapter + | CanvasMaskAdapter | CanvasStagingArea ): GetLoggingContext => { if ( - instance instanceof CanvasLayer || + instance instanceof CanvasLayerAdapter || instance instanceof CanvasStagingArea || - instance instanceof CanvasInpaintMask + instance instanceof CanvasMaskAdapter ) { return (extra?: JSONObject): JSONObject => { return { diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasInpaintMask.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasMaskAdapter.ts similarity index 80% rename from invokeai/frontend/web/src/features/controlLayers/konva/CanvasInpaintMask.ts rename to invokeai/frontend/web/src/features/controlLayers/konva/CanvasMaskAdapter.ts index fe6e43f193..9481fa2857 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasInpaintMask.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasMaskAdapter.ts @@ -1,23 +1,24 @@ import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer'; import { CanvasTransformer } from 'features/controlLayers/konva/CanvasTransformer'; -import type { CanvasInpaintMaskState, CanvasV2State, GetLoggingContext } from 'features/controlLayers/store/types'; +import type { + CanvasInpaintMaskState, + CanvasRegionalGuidanceState, + CanvasV2State, + GetLoggingContext, +} from 'features/controlLayers/store/types'; import Konva from 'konva'; import { get } from 'lodash-es'; import type { Logger } from 'roarr'; -export class CanvasInpaintMask { - static TYPE = 'inpaint_mask' as const; - static NAME_PREFIX = 'inpaint-mask'; - static KONVA_LAYER_NAME = `${CanvasInpaintMask.NAME_PREFIX}_layer`; - - id = CanvasInpaintMask.TYPE; - type = CanvasInpaintMask.TYPE; +export class CanvasMaskAdapter { + id: string; + type: CanvasInpaintMaskState['type'] | CanvasRegionalGuidanceState['type']; manager: CanvasManager; log: Logger; getLoggingContext: GetLoggingContext; - state: CanvasInpaintMaskState; + state: CanvasInpaintMaskState | CanvasRegionalGuidanceState; maskOpacity: number; transformer: CanvasTransformer; @@ -29,15 +30,18 @@ export class CanvasInpaintMask { layer: Konva.Layer; }; - constructor(state: CanvasInpaintMaskState, manager: CanvasManager) { + constructor(state: CanvasMaskAdapter['state'], manager: CanvasMaskAdapter['manager']) { + this.id = state.id; + this.type = state.type; this.manager = manager; this.getLoggingContext = this.manager.buildGetLoggingContext(this); this.log = this.manager.buildLogger(this.getLoggingContext); - this.log.debug({ state }, 'Creating inpaint mask'); + this.log.debug({ state }, 'Creating mask'); this.konva = { layer: new Konva.Layer({ - name: CanvasInpaintMask.KONVA_LAYER_NAME, + id: this.id, + name: `${this.type}:layer`, listening: false, imageSmoothingEnabled: false, }), @@ -51,14 +55,18 @@ export class CanvasInpaintMask { } destroy = (): void => { - this.log.debug('Destroying inpaint mask'); + this.log.debug('Destroying mask'); // We need to call the destroy method on all children so they can do their own cleanup. this.transformer.destroy(); this.renderer.destroy(); this.konva.layer.destroy(); }; - update = async (arg?: { state: CanvasInpaintMaskState; toolState: CanvasV2State['tool']; isSelected: boolean }) => { + update = async (arg?: { + state: CanvasMaskAdapter['state']; + toolState: CanvasV2State['tool']; + isSelected: boolean; + }) => { const state = get(arg, 'state', this.state); const maskOpacity = this.manager.stateApi.getMaskOpacity(); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts index 17e411fa30..d56f2e7636 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts @@ -4,9 +4,9 @@ import { deepClone } from 'common/util/deepClone'; import { CanvasBrushLineRenderer } from 'features/controlLayers/konva/CanvasBrushLine'; import { CanvasEraserLineRenderer } from 'features/controlLayers/konva/CanvasEraserLine'; import { CanvasImageRenderer } from 'features/controlLayers/konva/CanvasImage'; -import type { CanvasInpaintMask } from 'features/controlLayers/konva/CanvasInpaintMask'; -import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer'; +import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter'; import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; +import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter'; import { CanvasRectRenderer } from 'features/controlLayers/konva/CanvasRect'; import { getPrefixedId, konvaNodeToBlob, previewBlob } from 'features/controlLayers/konva/util'; import { @@ -36,11 +36,11 @@ type AnyObjectState = CanvasBrushLineState | CanvasEraserLineState | CanvasImage */ export class CanvasObjectRenderer { static TYPE = 'object_renderer'; - static KONVA_OBJECT_GROUP_NAME = 'object-group'; - static KONVA_COMPOSITING_RECT_NAME = 'compositing-rect'; + static KONVA_OBJECT_GROUP_NAME = `${CanvasObjectRenderer.TYPE}:object_group`; + static KONVA_COMPOSITING_RECT_NAME = `${CanvasObjectRenderer.TYPE}:compositing_rect`; id: string; - parent: CanvasLayer | CanvasInpaintMask; + parent: CanvasLayerAdapter | CanvasMaskAdapter; manager: CanvasManager; log: Logger; getLoggingContext: (extra?: JSONObject) => JSONObject; @@ -87,7 +87,7 @@ export class CanvasObjectRenderer { compositingRect: Konva.Rect | null; }; - constructor(parent: CanvasLayer | CanvasInpaintMask) { + constructor(parent: CanvasLayerAdapter | CanvasMaskAdapter) { this.id = getPrefixedId(CanvasObjectRenderer.TYPE); this.parent = parent; this.manager = parent.manager; @@ -102,7 +102,7 @@ export class CanvasObjectRenderer { this.parent.konva.layer.add(this.konva.objectGroup); - if (this.parent.type === 'inpaint_mask') { + if (this.parent.type === 'inpaint_mask' || this.parent.type === 'regional_guidance') { this.konva.compositingRect = new Konva.Rect({ name: CanvasObjectRenderer.KONVA_COMPOSITING_RECT_NAME, listening: false, @@ -122,13 +122,13 @@ export class CanvasObjectRenderer { // The compositing rect must cover the whole stage at all times. When the stage is scaled, moved or resized, we // need to update the compositing rect to match the stage. this.subscriptions.add( - this.manager.stateApi.$stageAttrs.listen((attrs) => { + this.manager.stateApi.$stageAttrs.listen(({ x, y, width, height, scale }) => { if (this.konva.compositingRect) { this.konva.compositingRect.setAttrs({ - x: -attrs.position.x / attrs.scale, - y: -attrs.position.y / attrs.scale, - width: attrs.dimensions.width / attrs.scale, - height: attrs.dimensions.height / attrs.scale, + x: -x / scale, + y: -y / scale, + width: width / scale, + height: height / scale, }); } }) @@ -168,9 +168,14 @@ export class CanvasObjectRenderer { assert(this.konva.compositingRect, 'Missing compositing rect'); const rgbColor = rgbColorToString(fill); + const { x, y, width, height, scale } = this.manager.stateApi.$stageAttrs.get(); this.konva.compositingRect.setAttrs({ fill: rgbColor, opacity, + x: -x / scale, + y: -y / scale, + width: width / scale, + height: height / scale, }); }; @@ -288,11 +293,11 @@ export class CanvasObjectRenderer { this.buffer.id = getPrefixedId(this.buffer.type); if (this.buffer.type === 'brush_line') { - this.manager.stateApi.addBrushLine({ id: this.parent.id, brushLine: this.buffer }, this.parent.state.type); + this.manager.stateApi.addBrushLine({ id: this.parent.id, brushLine: this.buffer }, this.parent.type); } else if (this.buffer.type === 'eraser_line') { - this.manager.stateApi.addEraserLine({ id: this.parent.id, eraserLine: this.buffer }, this.parent.state.type); + this.manager.stateApi.addEraserLine({ id: this.parent.id, eraserLine: this.buffer }, this.parent.type); } else if (this.buffer.type === 'rect') { - this.manager.stateApi.addRect({ id: this.parent.id, rect: this.buffer }, this.parent.state.type); + this.manager.stateApi.addRect({ id: this.parent.id, rect: this.buffer }, this.parent.type); } else { this.log.warn({ buffer: this.buffer }, 'Invalid buffer object type'); } @@ -340,13 +345,13 @@ export class CanvasObjectRenderer { const rect = interactionRectClone.getClientRect(); const blob = await konvaNodeToBlob(objectGroupClone, rect); if (this.manager._isDebugging) { - previewBlob(blob, 'Rasterized layer'); + previewBlob(blob, 'Rasterized entity'); } const imageDTO = await uploadImage(blob, `${this.id}_rasterized.png`, 'other', true); const imageObject = imageDTOToImageObject(imageDTO); await this.renderObject(imageObject, true); this.manager.stateApi.rasterizeEntity( - { id: this.id, imageObject, position: { x: Math.round(rect.x), y: Math.round(rect.y) } }, + { id: this.parent.id, imageObject, position: { x: Math.round(rect.x), y: Math.round(rect.y) } }, this.parent.type ); }; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts index f8e6436051..7f738d3fcf 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts @@ -31,6 +31,7 @@ import { layerRectAdded, layerReset, layerTranslated, + regionMaskRasterized, rgBrushLineAdded, rgEraserLineAdded, rgImageCacheChanged, @@ -122,6 +123,8 @@ export class CanvasStateApi { this._store.dispatch(layerRasterized(arg)); } else if (entityType === 'inpaint_mask') { this._store.dispatch(inpaintMaskRasterized(arg)); + } else if (entityType === 'regional_guidance') { + this._store.dispatch(regionMaskRasterized(arg)); } else { assert(false, 'Rasterizing not supported for this entity type'); } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts index a96b3ce12d..cf6960052b 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts @@ -1,6 +1,6 @@ -import type { CanvasInpaintMask } from 'features/controlLayers/konva/CanvasInpaintMask'; -import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer'; +import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter'; import { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; +import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter'; import { getEmptyRect, getPrefixedId } from 'features/controlLayers/konva/util'; import type { Coordinate, GetLoggingContext, Rect } from 'features/controlLayers/store/types'; import Konva from 'konva'; @@ -17,9 +17,10 @@ import type { Logger } from 'roarr'; */ export class CanvasTransformer { static TYPE = 'entity_transformer'; - static TRANSFORMER_NAME = `${CanvasTransformer.TYPE}:transformer`; - static PROXY_RECT_NAME = `${CanvasTransformer.TYPE}:proxy_rect`; - static BBOX_OUTLINE_NAME = `${CanvasTransformer.TYPE}:bbox_outline`; + static KONVA_TRANSFORMER_NAME = `${CanvasTransformer.TYPE}:transformer`; + static KONVA_PROXY_RECT_NAME = `${CanvasTransformer.TYPE}:proxy_rect`; + static KONVA_OUTLINE_RECT_NAME = `${CanvasTransformer.TYPE}:outline_rect`; + static STROKE_COLOR = 'hsl(200 76% 50% / 1)'; // invokeBlue.500 static ANCHOR_FILL_COLOR = CanvasTransformer.STROKE_COLOR; static ANCHOR_STROKE_COLOR = 'hsl(200 76% 77% / 1)'; // invokeBlue.200 @@ -32,7 +33,7 @@ export class CanvasTransformer { static ANCHOR_HIT_PADDING = 10; id: string; - parent: CanvasLayer | CanvasInpaintMask; + parent: CanvasLayerAdapter | CanvasMaskAdapter; manager: CanvasManager; log: Logger; getLoggingContext: GetLoggingContext; @@ -87,10 +88,10 @@ export class CanvasTransformer { konva: { transformer: Konva.Transformer; proxyRect: Konva.Rect; - bboxOutline: Konva.Rect; + outlineRect: Konva.Rect; }; - constructor(parent: CanvasLayer | CanvasInpaintMask) { + constructor(parent: CanvasLayerAdapter | CanvasMaskAdapter) { this.id = getPrefixedId(CanvasTransformer.TYPE); this.parent = parent; this.manager = parent.manager; @@ -99,16 +100,16 @@ export class CanvasTransformer { this.log = this.manager.buildLogger(this.getLoggingContext); this.konva = { - bboxOutline: new Konva.Rect({ + outlineRect: new Konva.Rect({ listening: false, draggable: false, - name: CanvasTransformer.BBOX_OUTLINE_NAME, + name: CanvasTransformer.KONVA_OUTLINE_RECT_NAME, stroke: CanvasTransformer.STROKE_COLOR, perfectDrawEnabled: false, strokeHitEnabled: false, }), transformer: new Konva.Transformer({ - name: CanvasTransformer.TRANSFORMER_NAME, + name: CanvasTransformer.KONVA_TRANSFORMER_NAME, // Visibility and listening are managed via activate() and deactivate() visible: false, listening: false, @@ -227,7 +228,7 @@ export class CanvasTransformer { }, }), proxyRect: new Konva.Rect({ - name: CanvasTransformer.PROXY_RECT_NAME, + name: CanvasTransformer.KONVA_PROXY_RECT_NAME, listening: false, draggable: true, }), @@ -330,7 +331,7 @@ export class CanvasTransformer { // The bbox should be updated to reflect the new position of the interaction rect, taking into account its padding // and border - this.konva.bboxOutline.setAttrs({ + this.konva.outlineRect.setAttrs({ x: this.konva.proxyRect.x() - this.manager.getScaledBboxPadding(), y: this.konva.proxyRect.y() - this.manager.getScaledBboxPadding(), }); @@ -392,7 +393,7 @@ export class CanvasTransformer { }) ); - this.parent.konva.layer.add(this.konva.bboxOutline); + this.parent.konva.layer.add(this.konva.outlineRect); this.parent.konva.layer.add(this.konva.proxyRect); this.parent.konva.layer.add(this.konva.transformer); } @@ -406,7 +407,7 @@ export class CanvasTransformer { const onePixel = this.manager.getScaledPixel(); const bboxPadding = this.manager.getScaledBboxPadding(); - this.konva.bboxOutline.setAttrs({ + this.konva.outlineRect.setAttrs({ x: position.x + bbox.x - bboxPadding, y: position.y + bbox.y - bboxPadding, width: bbox.width + bboxPadding * 2, @@ -473,7 +474,7 @@ export class CanvasTransformer { const onePixel = this.manager.getScaledPixel(); const bboxPadding = this.manager.getScaledBboxPadding(); - this.konva.bboxOutline.setAttrs({ + this.konva.outlineRect.setAttrs({ x: this.konva.proxyRect.x() - bboxPadding, y: this.konva.proxyRect.y() - bboxPadding, width: this.konva.proxyRect.width() * this.konva.proxyRect.scaleX() + bboxPadding * 2, @@ -539,7 +540,7 @@ export class CanvasTransformer { rotation: 0, }; this.parent.renderer.konva.objectGroup.setAttrs(attrs); - this.konva.bboxOutline.setAttrs(attrs); + this.konva.outlineRect.setAttrs(attrs); this.konva.proxyRect.setAttrs(attrs); }; @@ -706,11 +707,11 @@ export class CanvasTransformer { }; _showBboxOutline = () => { - this.konva.bboxOutline.visible(true); + this.konva.outlineRect.visible(true); }; _hideBboxOutline = () => { - this.konva.bboxOutline.visible(false); + this.konva.outlineRect.visible(false); }; /** @@ -735,7 +736,7 @@ export class CanvasTransformer { this.log.trace('Cleaning up listener'); cleanup(); } - this.konva.bboxOutline.destroy(); + this.konva.outlineRect.destroy(); this.konva.transformer.destroy(); this.konva.proxyRect.destroy(); }; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/events.ts b/invokeai/frontend/web/src/features/controlLayers/konva/events.ts index 75b8ac9b17..486d3d422c 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/events.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/events.ts @@ -487,8 +487,10 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => { stage.scaleY(newScale); stage.position(newPos); $stageAttrs.set({ - position: newPos, - dimensions: { width: stage.width(), height: stage.height() }, + x: newPos.x, + y: newPos.y, + width: stage.width(), + height: stage.height(), scale: newScale, }); manager.background.render(); @@ -500,8 +502,10 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => { //#region dragmove stage.on('dragmove', () => { $stageAttrs.set({ - position: { x: Math.floor(stage.x()), y: Math.floor(stage.y()) }, - dimensions: { width: stage.width(), height: stage.height() }, + x: Math.floor(stage.x()), + y: Math.floor(stage.y()), + width: stage.width(), + height: stage.height(), scale: stage.scaleX(), }); manager.background.render(); @@ -512,8 +516,10 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => { stage.on('dragend', () => { // Stage position should always be an integer, else we get fractional pixels which are blurry $stageAttrs.set({ - position: { x: Math.floor(stage.x()), y: Math.floor(stage.y()) }, - dimensions: { width: stage.width(), height: stage.height() }, + x: Math.floor(stage.x()), + y: Math.floor(stage.y()), + width: stage.width(), + height: stage.height(), scale: stage.scaleX(), }); manager.preview.tool.render(); @@ -529,7 +535,11 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => { } if (e.key === 'Escape') { // Cancel shape drawing on escape - $lastMouseDownPos.set(null); + const selectedEntity = getSelectedEntity(); + if (selectedEntity) { + selectedEntity.adapter.renderer.clearBuffer(); + $lastMouseDownPos.set(null); + } } else if (e.key === ' ') { // Select the view tool on space key down setToolBuffer(getToolState().selected); diff --git a/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts b/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts index e5c5395e4a..2614339936 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts @@ -289,6 +289,7 @@ export const { rgBrushLineAdded, rgEraserLineAdded, rgRectAdded, + regionMaskRasterized, // Compositing setInfillMethod, setInfillTileSize, @@ -371,8 +372,10 @@ const migrate = (state: any): any => { // Ephemeral state that does not need to be in redux export const $isPreviewVisible = atom(true); export const $stageAttrs = atom({ - position: { x: 0, y: 0 }, - dimensions: { width: 0, height: 0 }, + x: 0, + y: 0, + width: 0, + height: 0, scale: 0, }); export const $shouldShowStagedImage = atom(true); diff --git a/invokeai/frontend/web/src/features/controlLayers/store/regionsReducers.ts b/invokeai/frontend/web/src/features/controlLayers/store/regionsReducers.ts index b847a24e86..d4847310db 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/regionsReducers.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/regionsReducers.ts @@ -6,6 +6,7 @@ import type { CanvasRectState, CanvasV2State, CLIPVisionModelV2, + EntityRasterizedArg, IPMethodV2, PositionChangedArg, ScaleChangedArg, @@ -361,4 +362,14 @@ export const regionsReducers = { rg.bboxNeedsUpdate = true; state.layers.imageCache = null; }, + regionMaskRasterized: (state, action: PayloadAction) => { + const { id, imageObject, position } = action.payload; + const rg = selectRG(state, id); + if (!rg) { + return; + } + rg.objects = [imageObject]; + rg.position = position; + rg.imageCache = null; + }, } satisfies SliceCaseReducers; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/types.ts b/invokeai/frontend/web/src/features/controlLayers/store/types.ts index 520f1086ae..5fa10f9d85 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/types.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/types.ts @@ -1,7 +1,7 @@ import type { JSONObject } from 'common/types'; import type { CanvasControlAdapter } from 'features/controlLayers/konva/CanvasControlAdapter'; -import { CanvasInpaintMask } from 'features/controlLayers/konva/CanvasInpaintMask'; -import { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer'; +import { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter'; +import { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter'; import { CanvasRegion } from 'features/controlLayers/konva/CanvasRegion'; import { getObjectId } from 'features/controlLayers/konva/util'; import { zModelIdentifierField } from 'features/nodes/types/common'; @@ -658,7 +658,7 @@ export const zCanvasRegionalGuidanceState = z.object({ position: zCoordinate, bbox: zRect.nullable(), bboxNeedsUpdate: z.boolean(), - objects: z.array(zMaskObject), + objects: z.array(zCanvasObjectState), positivePrompt: zParameterPositivePrompt.nullable(), negativePrompt: zParameterNegativePrompt.nullable(), ipAdapters: z.array(zCanvasIPAdapterState), @@ -933,7 +933,13 @@ export type CanvasV2State = { }; }; -export type StageAttrs = { position: Coordinate; dimensions: Dimensions; scale: number }; +export type StageAttrs = { + x: Coordinate['x']; + y: Coordinate['y']; + width: Dimensions['width']; + height: Dimensions['height']; + scale: number; +}; export type PositionChangedArg = { id: string; position: Coordinate }; export type ScaleChangedArg = { id: string; scale: Coordinate; position: Coordinate }; export type BboxChangedArg = { id: string; bbox: Rect | null }; @@ -969,9 +975,11 @@ export function isDrawableEntity( } export function isDrawableEntityAdapter( - adapter: CanvasLayer | CanvasRegion | CanvasControlAdapter | CanvasInpaintMask -): adapter is CanvasLayer | CanvasRegion | CanvasInpaintMask { - return adapter instanceof CanvasLayer || adapter instanceof CanvasRegion || adapter instanceof CanvasInpaintMask; + adapter: CanvasLayerAdapter | CanvasRegion | CanvasControlAdapter | CanvasMaskAdapter +): adapter is CanvasLayerAdapter | CanvasRegion | CanvasMaskAdapter { + return ( + adapter instanceof CanvasLayerAdapter || adapter instanceof CanvasRegion || adapter instanceof CanvasMaskAdapter + ); } export function isDrawableEntityType(