From 54e1eae5098bbf81a5fc1fb0446834f018ef62ae Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Wed, 3 Jul 2024 19:42:48 +1000 Subject: [PATCH] fix(ui): inpaint mask rendering --- .../controlLayers/konva/nodeManager.ts | 10 +- .../konva/renderers/entityBbox.ts | 14 +- .../konva/renderers/inpaintMask.ts | 259 +++++++++--------- .../controlLayers/konva/renderers/regions.ts | 4 +- .../controlLayers/konva/renderers/renderer.ts | 4 + .../controlLayers/store/canvasV2Slice.ts | 1 + .../store/inpaintMaskReducers.ts | 25 +- .../controlLayers/store/layersReducers.ts | 3 +- 8 files changed, 176 insertions(+), 144 deletions(-) diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts b/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts index bea047c4c8..7dc7415b3e 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/nodeManager.ts @@ -150,7 +150,7 @@ export class KonvaNodeManager { this.background = new CanvasBackground(); this.stage.add(this.background.layer); - this.inpaintMask = new CanvasInpaintMask(this.stateApi.getInpaintMaskState(), this.stateApi.onPosChanged); + this.inpaintMask = new CanvasInpaintMask(this.stateApi.getInpaintMaskState(), this); this.stage.add(this.inpaintMask.layer); this.layers = new Map(); @@ -206,11 +206,7 @@ export class KonvaNodeManager { renderInpaintMask() { const inpaintMaskState = this.stateApi.getInpaintMaskState(); - const toolState = this.stateApi.getToolState(); - const selectedEntity = this.stateApi.getSelectedEntity(); - const maskOpacity = this.stateApi.getMaskOpacity(); - - this.inpaintMask.render(inpaintMaskState, toolState.selected, selectedEntity, maskOpacity); + this.inpaintMask.render(inpaintMaskState); } renderControlAdapters() { @@ -250,7 +246,7 @@ export class KonvaNodeManager { for (const rg of regions) { this.regions.get(rg.id)?.layer.zIndex(++zIndex); } - this.inpaintMask?.layer.zIndex(++zIndex); + this.inpaintMask.layer.zIndex(++zIndex); this.preview.layer.zIndex(++zIndex); } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts index 5c226d017a..1669df159a 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/entityBbox.ts @@ -165,13 +165,13 @@ const getLayerBboxPixels = ( }; /** - * Get the bounding box of a konva layer. This function is faster than `getLayerBboxPixels` but less accurate. It - * should only be used when there are no eraser strokes or shapes in the layer. - * @param layer The konva layer to get the bounding box of. - * @returns The bounding box of the layer. + * Get the bounding box of a konva node. This function is faster than `getLayerBboxPixels` but less accurate. It + * should only be used when there are no eraser strokes or shapes in the node. + * @param node The konva node to get the bounding box of. + * @returns The bounding box of the node. */ -export const getLayerBboxFast = (layer: Konva.Layer): IRect => { - const bbox = layer.getClientRect(GET_CLIENT_RECT_CONFIG); +export const getNodeBboxFast = (node: Konva.Node): IRect => { + const bbox = node.getClientRect(GET_CLIENT_RECT_CONFIG); return { x: Math.floor(bbox.x), y: Math.floor(bbox.y), @@ -210,7 +210,7 @@ export const updateBboxes = ( if (entityState.type === 'layer') { if (entityState.objects.length === 0) { - // No objects - no bbox to calculate + // No objects - no bbox to calculate onBboxChanged({ id: entityState.id, bbox: null }, 'layer'); } else { onBboxChanged({ id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterLayerChildren) }, 'layer'); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts index 85b6efff90..4f9c022240 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/inpaintMask.ts @@ -1,42 +1,62 @@ import { rgbColorToString } from 'common/util/colorCodeTransformers'; import { getObjectGroupId } from 'features/controlLayers/konva/naming'; -import type { StateApi } from 'features/controlLayers/konva/nodeManager'; -import { getLayerBboxFast } from 'features/controlLayers/konva/renderers/entityBbox'; +import type { KonvaNodeManager } from 'features/controlLayers/konva/nodeManager'; +import { getNodeBboxFast } from 'features/controlLayers/konva/renderers/entityBbox'; import { KonvaBrushLine, KonvaEraserLine, KonvaRect } from 'features/controlLayers/konva/renderers/objects'; import { mapId } from 'features/controlLayers/konva/util'; -import type { CanvasEntityIdentifier, InpaintMaskEntity, Tool } from 'features/controlLayers/store/types'; +import { type InpaintMaskEntity, isDrawingTool } from 'features/controlLayers/store/types'; import Konva from 'konva'; import { assert } from 'tsafe'; import { v4 as uuidv4 } from 'uuid'; export class CanvasInpaintMask { id: string; + manager: KonvaNodeManager; layer: Konva.Layer; group: Konva.Group; + objectsGroup: Konva.Group; compositingRect: Konva.Rect; + transformer: Konva.Transformer; objects: Map; - constructor(entity: InpaintMaskEntity, onPosChanged: StateApi['onPosChanged']) { + constructor(entity: InpaintMaskEntity, manager: KonvaNodeManager) { this.id = entity.id; - + this.manager = manager; this.layer = new Konva.Layer({ id: entity.id, draggable: true, dragDistance: 0, }); - // When a drag on the layer finishes, update the layer's position in state. During the drag, konva handles changing - // the position - we do not need to call this on the `dragmove` event. - this.layer.on('dragend', function (e) { - onPosChanged({ id: entity.id, x: Math.floor(e.target.x()), y: Math.floor(e.target.y()) }, 'inpaint_mask'); - }); this.group = new Konva.Group({ id: getObjectGroupId(this.layer.id(), uuidv4()), listening: false, }); + this.objectsGroup = new Konva.Group({}); + this.group.add(this.objectsGroup); this.layer.add(this.group); + + this.transformer = new Konva.Transformer({ + shouldOverdrawWholeArea: true, + draggable: true, + dragDistance: 0, + enabledAnchors: ['top-left', 'top-right', 'bottom-left', 'bottom-right'], + rotateEnabled: false, + flipEnabled: false, + }); + this.transformer.on('transformend', () => { + this.manager.stateApi.onScaleChanged( + { id: this.id, scale: this.group.scaleX(), x: this.group.x(), y: this.group.y() }, + 'inpaint_mask' + ); + }); + this.transformer.on('dragend', () => { + this.manager.stateApi.onPosChanged({ id: this.id, x: this.group.x(), y: this.group.y() }, 'inpaint_mask'); + }); + this.layer.add(this.transformer); + this.compositingRect = new Konva.Rect({ listening: false }); - this.layer.add(this.compositingRect); + this.group.add(this.compositingRect); this.objects = new Map(); } @@ -44,24 +64,16 @@ export class CanvasInpaintMask { this.layer.destroy(); } - async render( - inpaintMaskState: InpaintMaskEntity, - selectedTool: Tool, - selectedEntityIdentifier: CanvasEntityIdentifier | null, - maskOpacity: number - ) { + async render(inpaintMaskState: InpaintMaskEntity) { // Update the layer's position and listening state - this.layer.setAttrs({ - listening: selectedTool === 'move', // The layer only listens when using the move tool - otherwise the stage is handling mouse events - x: Math.floor(inpaintMaskState.x), - y: Math.floor(inpaintMaskState.y), + this.group.setAttrs({ + x: inpaintMaskState.x, + y: inpaintMaskState.y, + scaleX: 1, + scaleY: 1, }); - // Convert the color to a string, stripping the alpha - the object group will handle opacity. - const rgbColor = rgbColorToString(inpaintMaskState.fill); - - // We use caching to handle "global" layer opacity, but caching is expensive and we should only do it when required. - let groupNeedsCache = false; + let didDraw = false; const objectIds = inpaintMaskState.objects.map(mapId); // Destroy any objects that are no longer in state @@ -69,7 +81,7 @@ export class CanvasInpaintMask { if (!objectIds.includes(object.id)) { this.objects.delete(object.id); object.destroy(); - groupNeedsCache = true; + didDraw = true; } } @@ -81,13 +93,12 @@ export class CanvasInpaintMask { if (!brushLine) { brushLine = new KonvaBrushLine(obj); this.objects.set(brushLine.id, brushLine); - this.group.add(brushLine.konvaLineGroup); - groupNeedsCache = true; - } - - if (obj.points.length !== brushLine.konvaLine.points().length) { - brushLine.konvaLine.points(obj.points); - groupNeedsCache = true; + this.objectsGroup.add(brushLine.konvaLineGroup); + didDraw = true; + } else { + if (brushLine.update(obj)) { + didDraw = true; + } } } else if (obj.type === 'eraser_line') { let eraserLine = this.objects.get(obj.id); @@ -96,13 +107,12 @@ export class CanvasInpaintMask { if (!eraserLine) { eraserLine = new KonvaEraserLine(obj); this.objects.set(eraserLine.id, eraserLine); - this.group.add(eraserLine.konvaLineGroup); - groupNeedsCache = true; - } - - if (obj.points.length !== eraserLine.konvaLine.points().length) { - eraserLine.konvaLine.points(obj.points); - groupNeedsCache = true; + this.objectsGroup.add(eraserLine.konvaLineGroup); + didDraw = true; + } else { + if (eraserLine.update(obj)) { + didDraw = true; + } } } else if (obj.type === 'rect_shape') { let rect = this.objects.get(obj.id); @@ -111,8 +121,12 @@ export class CanvasInpaintMask { if (!rect) { rect = new KonvaRect(obj); this.objects.set(rect.id, rect); - this.group.add(rect.konvaRect); - groupNeedsCache = true; + this.objectsGroup.add(rect.konvaRect); + didDraw = true; + } else { + if (rect.update(obj)) { + didDraw = true; + } } } } @@ -120,96 +134,91 @@ export class CanvasInpaintMask { // Only update layer visibility if it has changed. if (this.layer.visible() !== inpaintMaskState.isEnabled) { this.layer.visible(inpaintMaskState.isEnabled); - groupNeedsCache = true; } - if (this.objects.size === 0) { - // No objects - clear the cache to reset the previous pixel data - this.group.clearCache(); - return; - } - - // We must clear the cache first so Konva will re-draw the group with the new compositing rect - if (this.group.isCached()) { - this.group.clearCache(); - } // The user is allowed to reduce mask opacity to 0, but we need the opacity for the compositing rect to work this.group.opacity(1); - this.compositingRect.setAttrs({ - // The rect should be the size of the layer - use the fast method if we don't have a pixel-perfect bbox already - ...(!inpaintMaskState.bboxNeedsUpdate && inpaintMaskState.bbox - ? inpaintMaskState.bbox - : getLayerBboxFast(this.layer)), - fill: rgbColor, - opacity: maskOpacity, - // Draw this rect only where there are non-transparent pixels under it (e.g. the mask shapes) - globalCompositeOperation: 'source-in', - visible: true, - // This rect must always be on top of all other shapes - zIndex: this.objects.size + 1, - }); + if (didDraw) { + // Convert the color to a string, stripping the alpha - the object group will handle opacity. + const rgbColor = rgbColorToString(inpaintMaskState.fill); + const maskOpacity = this.manager.stateApi.getMaskOpacity(); - // const isSelected = selectedEntityIdentifier?.id === inpaintMaskState.id; + this.compositingRect.setAttrs({ + // The rect should be the size of the layer - use the fast method if we don't have a pixel-perfect bbox already + ...getNodeBboxFast(this.objectsGroup), + fill: rgbColor, + opacity: maskOpacity, + // Draw this rect only where there are non-transparent pixels under it (e.g. the mask shapes) + globalCompositeOperation: 'source-in', + visible: true, + // This rect must always be on top of all other shapes + zIndex: this.objects.size + 1, + }); + } - // /** - // * When the group is selected, we use a rect of the selected preview color, composited over the shapes. This allows - // * shapes to render as a "raster" layer with all pixels drawn at the same color and opacity. - // * - // * Without this special handling, each shape is drawn individually with the given opacity, atop the other shapes. The - // * effect is like if you have a Photoshop Group consisting of many shapes, each of which has the given opacity. - // * Overlapping shapes will have their colors blended together, and the final color is the result of all the shapes. - // * - // * Instead, with the special handling, the effect is as if you drew all the shapes at 100% opacity, flattened them to - // * a single raster image, and _then_ applied the 50% opacity. - // */ - // if (isSelected && selectedTool !== 'move') { - // // We must clear the cache first so Konva will re-draw the group with the new compositing rect - // if (this.konvaObjectGroup.isCached()) { - // this.konvaObjectGroup.clearCache(); - // } - // // The user is allowed to reduce mask opacity to 0, but we need the opacity for the compositing rect to work - // this.konvaObjectGroup.opacity(1); + this.updateGroup(didDraw); + } - // this.compositingRect.setAttrs({ - // // The rect should be the size of the layer - use the fast method if we don't have a pixel-perfect bbox already - // ...(!inpaintMaskState.bboxNeedsUpdate && inpaintMaskState.bbox - // ? inpaintMaskState.bbox - // : getLayerBboxFast(this.konvaLayer)), - // fill: rgbColor, - // opacity: maskOpacity, - // // Draw this rect only where there are non-transparent pixels under it (e.g. the mask shapes) - // globalCompositeOperation: 'source-in', - // visible: true, - // // This rect must always be on top of all other shapes - // zIndex: this.objects.size + 1, - // }); - // } else { - // // The compositing rect should only be shown when the layer is selected. - // this.compositingRect.visible(false); - // // Cache only if needed - or if we are on this code path and _don't_ have a cache - // if (groupNeedsCache || !this.konvaObjectGroup.isCached()) { - // this.konvaObjectGroup.cache(); - // } - // // Updating group opacity does not require re-caching - // this.konvaObjectGroup.opacity(maskOpacity); - // } + updateGroup(didDraw: boolean) { + const isSelected = this.manager.stateApi.getIsSelected(this.id); + const selectedTool = this.manager.stateApi.getToolState().selected; - // const bboxRect = - // regionMap.konvaLayer.findOne(`.${LAYER_BBOX_NAME}`) ?? createBboxRect(rg, regionMap.konvaLayer); - // if (rg.bbox) { - // const active = !rg.bboxNeedsUpdate && isSelected && tool === 'move'; - // bboxRect.setAttrs({ - // visible: active, - // listening: active, - // x: rg.bbox.x, - // y: rg.bbox.y, - // width: rg.bbox.width, - // height: rg.bbox.height, - // stroke: isSelected ? BBOX_SELECTED_STROKE : '', - // }); - // } else { - // bboxRect.visible(false); - // } + if (this.objects.size === 0) { + // If the layer is totally empty, reset the cache and bail out. + this.layer.listening(false); + this.transformer.nodes([]); + if (this.group.isCached()) { + this.group.clearCache(); + } + return; + } + + if (isSelected && selectedTool === 'move') { + // When the layer is selected and being moved, we should always cache it. + // We should update the cache if we drew to the layer. + if (!this.group.isCached() || didDraw) { + this.group.cache(); + } + // Activate the transformer + this.layer.listening(true); + this.transformer.nodes([this.group]); + this.transformer.forceUpdate(); + return; + } + + if (isSelected && selectedTool !== 'move') { + // If the layer is selected but not using the move tool, we don't want the layer to be listening. + this.layer.listening(false); + // The transformer also does not need to be active. + this.transformer.nodes([]); + if (isDrawingTool(selectedTool)) { + // We are using a drawing tool (brush, eraser, rect). These tools change the layer's rendered appearance, so we + // should never be cached. + if (this.group.isCached()) { + this.group.clearCache(); + } + } else { + // We are using a non-drawing tool (move, view, bbox), so we should cache the layer. + // We should update the cache if we drew to the layer. + if (!this.group.isCached() || didDraw) { + this.group.cache(); + } + } + return; + } + + if (!isSelected) { + // Unselected layers should not be listening + this.layer.listening(false); + // The transformer also does not need to be active. + this.transformer.nodes([]); + // Update the layer's cache if it's not already cached or we drew to it. + if (!this.group.isCached() || didDraw) { + this.group.cache(); + } + + return; + } } } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts index 0a2c5353d4..2e8f154a98 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/regions.ts @@ -1,7 +1,7 @@ import { rgbColorToString } from 'common/util/colorCodeTransformers'; import { getObjectGroupId } from 'features/controlLayers/konva/naming'; import type { StateApi } from 'features/controlLayers/konva/nodeManager'; -import { getLayerBboxFast } from 'features/controlLayers/konva/renderers/entityBbox'; +import { getNodeBboxFast } from 'features/controlLayers/konva/renderers/entityBbox'; import { KonvaBrushLine, KonvaEraserLine, KonvaRect } from 'features/controlLayers/konva/renderers/objects'; import { mapId } from 'features/controlLayers/konva/util'; import type { CanvasEntityIdentifier, RegionEntity, Tool } from 'features/controlLayers/store/types'; @@ -138,7 +138,7 @@ export class CanvasRegion { this.compositingRect.setAttrs({ // The rect should be the size of the layer - use the fast method if we don't have a pixel-perfect bbox already - ...(!regionState.bboxNeedsUpdate && regionState.bbox ? regionState.bbox : getLayerBboxFast(this.layer)), + ...(!regionState.bboxNeedsUpdate && regionState.bbox ? regionState.bbox : getNodeBboxFast(this.layer)), fill: rgbColor, opacity: maskOpacity, // Draw this rect only where there are non-transparent pixels under it (e.g. the mask shapes) diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts index 9539a3403b..1ecd2621b8 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts @@ -21,6 +21,7 @@ import { imImageCacheChanged, imLinePointAdded, imRectAdded, + imScaled, imTranslated, layerBboxChanged, layerBrushLineAdded, @@ -107,6 +108,8 @@ export const initializeRenderer = ( logIfDebugging('onScaleChanged'); if (entityType === 'layer') { dispatch(layerScaled(arg)); + } else if (entityType === 'inpaint_mask') { + dispatch(imScaled(arg)); } }; const onBboxChanged = (arg: BboxChangedArg, entityType: CanvasEntity['type']) => { @@ -441,6 +444,7 @@ export const initializeRenderer = ( canvasV2.layers.entities !== prevCanvasV2.layers.entities || canvasV2.controlAdapters.entities !== prevCanvasV2.controlAdapters.entities || canvasV2.regions.entities !== prevCanvasV2.regions.entities || + canvasV2.inpaintMask !== prevCanvasV2.inpaintMask || canvasV2.selectedEntityIdentifier?.id !== prevCanvasV2.selectedEntityIdentifier?.id ) { logIfDebugging('Arranging entities'); diff --git a/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts b/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts index 186d47946c..e9dabfafee 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts @@ -336,6 +336,7 @@ export const { imEraserLineAdded, imLinePointAdded, imRectAdded, + imScaled, // Staging stagingAreaStartedStaging, stagingAreaImageAdded, diff --git a/invokeai/frontend/web/src/features/controlLayers/store/inpaintMaskReducers.ts b/invokeai/frontend/web/src/features/controlLayers/store/inpaintMaskReducers.ts index 1b7a9346ee..58343c3ebf 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/inpaintMaskReducers.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/inpaintMaskReducers.ts @@ -1,7 +1,7 @@ import type { PayloadAction, SliceCaseReducers } from '@reduxjs/toolkit'; import { getBrushLineId, getEraserLineId, getRectShapeId } from 'features/controlLayers/konva/naming'; -import type { CanvasV2State, InpaintMaskEntity } from 'features/controlLayers/store/types'; -import { imageDTOToImageWithDims,RGBA_RED } from 'features/controlLayers/store/types'; +import type { CanvasV2State, InpaintMaskEntity, ScaleChangedArg } from 'features/controlLayers/store/types'; +import { imageDTOToImageWithDims, RGBA_RED } from 'features/controlLayers/store/types'; import type { IRect } from 'konva/lib/types'; import type { ImageDTO } from 'services/api/types'; import { v4 as uuidv4 } from 'uuid'; @@ -29,6 +29,27 @@ export const inpaintMaskReducers = { state.inpaintMask.x = x; state.inpaintMask.y = y; }, + imScaled: (state, action: PayloadAction) => { + const { scale, x, y } = action.payload; + for (const obj of state.inpaintMask.objects) { + if (obj.type === 'brush_line') { + obj.points = obj.points.map((point) => point * scale); + obj.strokeWidth *= scale; + } else if (obj.type === 'eraser_line') { + obj.points = obj.points.map((point) => point * scale); + obj.strokeWidth *= scale; + } else if (obj.type === 'rect_shape') { + obj.x *= scale; + obj.y *= scale; + obj.height *= scale; + obj.width *= scale; + } + } + state.inpaintMask.x = x; + state.inpaintMask.y = y; + state.inpaintMask.bboxNeedsUpdate = true; + state.inpaintMask.imageCache = null; + }, imBboxChanged: (state, action: PayloadAction<{ bbox: IRect | null }>) => { const { bbox } = action.payload; state.inpaintMask.bbox = bbox; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/layersReducers.ts b/invokeai/frontend/web/src/features/controlLayers/store/layersReducers.ts index c9ed79f069..c39a70fe01 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/layersReducers.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/layersReducers.ts @@ -14,6 +14,7 @@ import type { LayerEntity, PointAddedToLineArg, RectShapeAddedArg, + ScaleChangedArg, } from './types'; import { imageDTOToImageObject, imageDTOToImageWithDims, isLine } from './types'; @@ -174,7 +175,7 @@ export const layersReducers = { payload: { ...payload, lineId: uuidv4() }, }), }, - layerScaled: (state, action: PayloadAction<{ id: string; scale: number; x: number; y: number }>) => { + layerScaled: (state, action: PayloadAction) => { const { id, scale, x, y } = action.payload; const layer = selectLayer(state, id); if (!layer) {