feat(ui): wip transform mode 2

invokeai/frontend/web/src/features/controlLayers/components/ControlLayersToolbar.tsx

@@ -24,7 +24,7 @@ export const ControlLayersToolbar = memo(() => {
       return;
     }
     for (const l of canvasManager.layers.values()) {
-      l.getBbox();
+      l.calculateBbox();
     }
   }, [canvasManager]);
   return (

invokeai/frontend/web/src/features/controlLayers/konva/CanvasBrushLine.ts

@@ -1,4 +1,5 @@
 import { rgbaColorToString } from 'common/util/colorCodeTransformers';
+import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer';
 import type { BrushLine } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 
@@ -7,18 +8,25 @@ export class CanvasBrushLine {
   static GROUP_NAME = `${CanvasBrushLine.NAME_PREFIX}_group`;
   static LINE_NAME = `${CanvasBrushLine.NAME_PREFIX}_line`;
 
-  private state: BrushLine;
+  state: BrushLine;
 
+  type = 'brush_line';
   id: string;
   konva: {
     group: Konva.Group;
     line: Konva.Line;
   };
 
-  constructor(state: BrushLine) {
-    this.state = state;
-    const { id, strokeWidth, clip, color, points } = this.state;
+  parent: CanvasLayer;
+
+  constructor(state: BrushLine, parent: CanvasLayer) {
+    const { id, strokeWidth, clip, color, points } = state;
+
     this.id = id;
+
+    this.parent = parent;
+    this.parent.log.trace(`Creating brush line ${this.id}`);
+
     this.konva = {
       group: new Konva.Group({
         name: CanvasBrushLine.GROUP_NAME,
@@ -46,6 +54,7 @@ export class CanvasBrushLine {
 
   async update(state: BrushLine, force?: boolean): Promise<boolean> {
     if (force || this.state !== state) {
+      this.parent.log.trace(`Updating brush line ${this.id}`);
       const { points, color, clip, strokeWidth } = state;
       this.konva.line.setAttrs({
         // A line with only one point will not be rendered, so we duplicate the points to make it visible
@@ -62,6 +71,7 @@ export class CanvasBrushLine {
   }
 
   destroy() {
+    this.parent.log.trace(`Destroying brush line ${this.id}`);
     this.konva.group.destroy();
   }
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEraserLine.ts

@@ -1,4 +1,5 @@
 import { rgbaColorToString } from 'common/util/colorCodeTransformers';
+import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer';
 import type { EraserLine } from 'features/controlLayers/store/types';
 import { RGBA_RED } from 'features/controlLayers/store/types';
 import Konva from 'konva';
@@ -8,17 +9,25 @@ export class CanvasEraserLine {
   static GROUP_NAME = `${CanvasEraserLine.NAME_PREFIX}_group`;
   static LINE_NAME = `${CanvasEraserLine.NAME_PREFIX}_line`;
 
-  private state: EraserLine;
+  state: EraserLine;
 
+  type = 'eraser_line';
   id: string;
   konva: {
     group: Konva.Group;
     line: Konva.Line;
   };
 
-  constructor(state: EraserLine) {
+  parent: CanvasLayer;
+
+  constructor(state: EraserLine, parent: CanvasLayer) {
     const { id, strokeWidth, clip, points } = state;
+
     this.id = id;
+
+    this.parent = parent;
+    this.parent.log.trace(`Creating eraser line ${this.id}`);
+
     this.konva = {
       group: new Konva.Group({
         name: CanvasEraserLine.GROUP_NAME,
@@ -46,6 +55,7 @@ export class CanvasEraserLine {
 
   async update(state: EraserLine, force?: boolean): Promise<boolean> {
     if (force || this.state !== state) {
+      this.parent.log.trace(`Updating eraser line ${this.id}`);
       const { points, clip, strokeWidth } = state;
       this.konva.line.setAttrs({
         // A line with only one point will not be rendered, so we duplicate the points to make it visible
@@ -61,6 +71,7 @@ export class CanvasEraserLine {
   }
 
   destroy() {
+    this.parent.log.trace(`Destroying eraser line ${this.id}`);
     this.konva.group.destroy();
   }
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasImage.ts

@@ -1,3 +1,4 @@
+import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer';
 import { FILTER_MAP } from 'features/controlLayers/konva/filters';
 import { loadImage } from 'features/controlLayers/konva/util';
 import type { ImageObject } from 'features/controlLayers/store/types';
@@ -14,7 +15,9 @@ export class CanvasImage {
   static PLACEHOLDER_RECT_NAME = `${CanvasImage.NAME_PREFIX}_placeholder-rect`;
   static PLACEHOLDER_TEXT_NAME = `${CanvasImage.NAME_PREFIX}_placeholder-text`;
 
-  private state: ImageObject;
+  state: ImageObject;
+
+  type = 'image';
 
   id: string;
   konva: {
@@ -26,8 +29,15 @@ export class CanvasImage {
   isLoading: boolean;
   isError: boolean;
 
-  constructor(state: ImageObject) {
+  parent: CanvasLayer;
+
+  constructor(state: ImageObject, parent: CanvasLayer) {
     const { id, width, height, x, y } = state;
+    this.id = id;
+
+    this.parent = parent;
+    this.parent.log.trace(`Creating image ${this.id}`);
+
     this.konva = {
       group: new Konva.Group({ name: CanvasImage.GROUP_NAME, listening: false, x, y }),
       placeholder: {
@@ -58,7 +68,6 @@ export class CanvasImage {
     this.konva.placeholder.group.add(this.konva.placeholder.text);
     this.konva.group.add(this.konva.placeholder.group);
 
-    this.id = id;
     this.imageName = null;
     this.image = null;
     this.isLoading = false;
@@ -68,6 +77,8 @@ export class CanvasImage {
 
   async updateImageSource(imageName: string) {
     try {
+      this.parent.log.trace(`Updating image source ${this.id}`);
+
       this.isLoading = true;
       this.konva.group.visible(true);
 
@@ -119,6 +130,8 @@ export class CanvasImage {
 
   async update(state: ImageObject, force?: boolean): Promise<boolean> {
     if (this.state !== state || force) {
+      this.parent.log.trace(`Updating image ${this.id}`);
+
       const { width, height, x, y, image, filters } = state;
       if (this.state.image.name !== image.name || force) {
         await this.updateImageSource(image.name);
@@ -141,6 +154,7 @@ export class CanvasImage {
   }
 
   destroy() {
+    this.parent.log.trace(`Destroying image ${this.id}`);
     this.konva.group.destroy();
   }
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts

@@ -4,6 +4,7 @@ import { CanvasEraserLine } from 'features/controlLayers/konva/CanvasEraserLine'
 import { CanvasImage } from 'features/controlLayers/konva/CanvasImage';
 import { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasRect } from 'features/controlLayers/konva/CanvasRect';
+import { getBrushLineId, getEraserLineId, getRectShapeId } from 'features/controlLayers/konva/naming';
 import { konvaNodeToBlob, mapId, previewBlob } from 'features/controlLayers/konva/util';
 import { layerRasterized } from 'features/controlLayers/store/canvasV2Slice';
 import type {
@@ -19,6 +20,7 @@ import { debounce, get } from 'lodash-es';
 import type { Logger } from 'roarr';
 import { uploadImage } from 'services/api/endpoints/images';
 import { assert } from 'tsafe';
+import { v4 as uuidv4 } from 'uuid';
 
 export class CanvasLayer {
   static NAME_PREFIX = 'layer';
@@ -82,7 +84,7 @@ export class CanvasLayer {
         name: CanvasLayer.INTERACTION_RECT_NAME,
         listening: false,
         draggable: true,
-        fill: 'rgba(255,0,0,0.5)',
+        // fill: 'rgba(255,0,0,0.5)',
       }),
     };
 
@@ -150,6 +152,7 @@ export class CanvasLayer {
         scaleY: this.konva.interactionRect.scaleY(),
         rotation: this.konva.interactionRect.rotation(),
       });
+
       console.log('objectGroup', {
         x: this.konva.objectGroup.x(),
         y: this.konva.objectGroup.y(),
@@ -241,7 +244,6 @@ export class CanvasLayer {
   getDrawingBuffer() {
     return this.drawingBuffer;
   }
-
   async setDrawingBuffer(obj: BrushLine | EraserLine | RectShape | null) {
     if (obj) {
       this.drawingBuffer = obj;
@@ -258,11 +260,17 @@ export class CanvasLayer {
     const drawingBuffer = this.drawingBuffer;
     this.setDrawingBuffer(null);
 
+    // We need to give the objects a fresh ID else they will be considered the same object when they are re-rendered as
+    // a non-buffer object, and we won't trigger things like bbox calculation
+
     if (drawingBuffer.type === 'brush_line') {
+      drawingBuffer.id = getBrushLineId(this.id, uuidv4());
       this.manager.stateApi.onBrushLineAdded({ id: this.id, brushLine: drawingBuffer }, 'layer');
     } else if (drawingBuffer.type === 'eraser_line') {
+      drawingBuffer.id = getEraserLineId(this.id, uuidv4());
       this.manager.stateApi.onEraserLineAdded({ id: this.id, eraserLine: drawingBuffer }, 'layer');
     } else if (drawingBuffer.type === 'rect_shape') {
+      drawingBuffer.id = getRectShapeId(this.id, uuidv4());
       this.manager.stateApi.onRectShapeAdded({ id: this.id, rectShape: drawingBuffer }, 'layer');
     }
   }
@@ -328,26 +336,33 @@ export class CanvasLayer {
     const objects = get(arg, 'objects', this.state.objects);
 
     const objectIds = objects.map(mapId);
+
+    let didUpdate = false;
+
     // Destroy any objects that are no longer in state
     for (const object of this.objects.values()) {
       if (!objectIds.includes(object.id) && object.id !== this.drawingBuffer?.id) {
         this.objects.delete(object.id);
         object.destroy();
-        this.bboxNeedsUpdate = true;
+        didUpdate = true;
       }
     }
 
     for (const obj of objects) {
       if (await this._renderObject(obj)) {
-        this.bboxNeedsUpdate = true;
+        didUpdate = true;
       }
     }
 
     if (this.drawingBuffer) {
       if (await this._renderObject(this.drawingBuffer)) {
-        this.bboxNeedsUpdate = true;
+        didUpdate = true;
       }
     }
+
+    if (didUpdate) {
+      this.calculateBbox();
+    }
   }
 
   async updateOpacity(arg?: { opacity: number }) {
@@ -410,6 +425,14 @@ export class CanvasLayer {
   async updateBbox() {
     this.log.trace('Updating bbox');
 
+    // If the bbox has no width or height, that means the layer is fully transparent. This can happen if it is only
+    // eraser lines, fully clipped brush lines or if it has been fully erased. In this case, we should reset the layer
+    // so we aren't drawing shapes that do not render anything.
+    if (this.width === 0 || this.height === 0) {
+      this.manager.stateApi.onEntityReset({ id: this.id }, 'layer');
+      return;
+    }
+
     const onePixel = this.manager.getScaledPixel();
     const bboxPadding = this.manager.getScaledBboxPadding();
 
@@ -450,23 +473,19 @@ export class CanvasLayer {
       assert(brushLine instanceof CanvasBrushLine || brushLine === undefined);
 
       if (!brushLine) {
-        console.log('creating new brush line');
-        brushLine = new CanvasBrushLine(obj);
+        brushLine = new CanvasBrushLine(obj, this);
         this.objects.set(brushLine.id, brushLine);
         this.konva.objectGroup.add(brushLine.konva.group);
         return true;
       } else {
-        console.log('updating brush line');
-        if (await brushLine.update(obj, force)) {
-          return true;
-        }
+        return await brushLine.update(obj, force);
       }
     } else if (obj.type === 'eraser_line') {
       let eraserLine = this.objects.get(obj.id);
       assert(eraserLine instanceof CanvasEraserLine || eraserLine === undefined);
 
       if (!eraserLine) {
-        eraserLine = new CanvasEraserLine(obj);
+        eraserLine = new CanvasEraserLine(obj, this);
         this.objects.set(eraserLine.id, eraserLine);
         this.konva.objectGroup.add(eraserLine.konva.group);
         return true;
@@ -480,12 +499,12 @@ export class CanvasLayer {
       assert(rect instanceof CanvasRect || rect === undefined);
 
       if (!rect) {
-        rect = new CanvasRect(obj);
+        rect = new CanvasRect(obj, this);
         this.objects.set(rect.id, rect);
         this.konva.objectGroup.add(rect.konva.group);
         return true;
       } else {
-        if (rect.update(obj, force)) {
+        if (await rect.update(obj, force)) {
           return true;
         }
       }
@@ -494,7 +513,7 @@ export class CanvasLayer {
       assert(image instanceof CanvasImage || image === undefined);
 
       if (!image) {
-        image = new CanvasImage(obj);
+        image = new CanvasImage(obj, this);
         this.objects.set(image.id, image);
         this.konva.objectGroup.add(image.konva.group);
         await image.updateImageSource(obj.image.name);
@@ -510,6 +529,7 @@ export class CanvasLayer {
   }
 
   async startTransform() {
+    this.log.debug('Starting transform');
     this.isTransforming = true;
 
     // When transforming, we want the stage to still be movable if the view tool is selected. If the transformer or
@@ -538,6 +558,8 @@ export class CanvasLayer {
   }
 
   async applyTransform() {
+    this.log.debug('Applying transform');
+
     this.isTransforming = false;
     const objectGroupClone = this.konva.objectGroup.clone();
     const rect = {
@@ -556,6 +578,8 @@ export class CanvasLayer {
   }
 
   async cancelTransform() {
+    this.log.debug('Canceling transform');
+
     this.isTransforming = false;
     this.resetScale();
     await this.updatePosition({ position: this.state.position });
@@ -566,7 +590,9 @@ export class CanvasLayer {
     });
   }
 
-  getBbox = debounce(() => {
+  calculateBbox = debounce(() => {
+    this.log.debug('Calculating bbox');
+
     if (this.objects.size === 0) {
       this.offsetX = 0;
       this.offsetY = 0;
@@ -581,9 +607,21 @@ export class CanvasLayer {
 
     console.log('getBbox rect', rect);
 
-    // If there are no eraser strokes, we can use the client rect directly
+    /**
+     * In some cases, we can use konva's getClientRect as the bbox, but there are some cases where we need to calculate
+     * the bbox using pixel data:
+     *
+     * - Eraser lines are normal lines, except they composite as transparency. Konva's getClientRect includes them when
+     *  calculating the bbox.
+     * - Clipped portions of lines will be included in the client rect.
+     *
+     * TODO(psyche): Using pixel data is slow. Is it possible to be clever and somehow subtract the eraser lines and
+     * clipped areas from the client rect?
+     */
     for (const obj of this.objects.values()) {
-      if (obj instanceof CanvasEraserLine) {
+      const isEraserLine = obj instanceof CanvasEraserLine;
+      const hasClip = obj instanceof CanvasBrushLine && obj.state.clip;
+      if (isEraserLine || hasClip) {
         needsPixelBbox = true;
         break;
       }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasRect.ts

@@ -1,4 +1,5 @@
 import { rgbaColorToString } from 'common/util/colorCodeTransformers';
+import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer';
 import type { RectShape } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 
@@ -7,7 +8,9 @@ export class CanvasRect {
   static GROUP_NAME = `${CanvasRect.NAME_PREFIX}_group`;
   static RECT_NAME = `${CanvasRect.NAME_PREFIX}_rect`;
 
-  private state: RectShape;
+  state: RectShape;
+
+  type = 'rect';
 
   id: string;
   konva: {
@@ -15,9 +18,16 @@ export class CanvasRect {
     rect: Konva.Rect;
   };
 
-  constructor(state: RectShape) {
+  parent: CanvasLayer;
+
+  constructor(state: RectShape, parent: CanvasLayer) {
     const { id, x, y, width, height, color } = state;
+
     this.id = id;
+
+    this.parent = parent;
+    this.parent.log.trace(`Creating rect ${this.id}`);
+
     this.konva = {
       group: new Konva.Group({ name: CanvasRect.GROUP_NAME, listening: false }),
       rect: new Konva.Rect({
@@ -35,8 +45,9 @@ export class CanvasRect {
     this.state = state;
   }
 
-  update(state: RectShape, force?: boolean): boolean {
+  async update(state: RectShape, force?: boolean): Promise<boolean> {
     if (this.state !== state || force) {
+      this.parent.log.trace(`Updating rect ${this.id}`);
       const { x, y, width, height, color } = state;
       this.konva.rect.setAttrs({
         x,
@@ -53,6 +64,7 @@ export class CanvasRect {
   }
 
   destroy() {
+    this.parent.log.trace(`Destroying rect ${this.id}`);
     this.konva.group.destroy();
   }
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts

@@ -31,6 +31,7 @@ import {
   layerEraserLineAdded,
   layerImageCacheChanged,
   layerRectShapeAdded,
+  layerReset,
   layerScaled,
   layerTranslated,
   rgBboxChanged,
@@ -70,7 +71,12 @@ export class CanvasStateApi {
   getState = () => {
     return this.store.getState().canvasV2;
   };
-
+  onEntityReset = (arg: { id: string }, entityType: CanvasEntity['type']) => {
+    log.debug('onEntityReset');
+    if (entityType === 'layer') {
+      this.store.dispatch(layerReset(arg));
+    }
+  };
   onPosChanged = (arg: PositionChangedArg, entityType: CanvasEntity['type']) => {
     log.debug('onPosChanged');
     if (entityType === 'layer') {

invokeai/frontend/web/src/features/controlLayers/konva/events.ts

@@ -188,7 +188,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
             await selectedEntityAdapter.finalizeDrawingBuffer();
           }
           await selectedEntityAdapter.setDrawingBuffer({
-            id: getBrushLineId(selectedEntityAdapter.id, uuidv4()),
+            id: getBrushLineId(selectedEntityAdapter.id, uuidv4(), true),
             type: 'brush_line',
             points: [
               // The last point of the last line is already normalized to the entity's coordinates
@@ -206,7 +206,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
             await selectedEntityAdapter.finalizeDrawingBuffer();
           }
           await selectedEntityAdapter.setDrawingBuffer({
-            id: getBrushLineId(selectedEntityAdapter.id, uuidv4()),
+            id: getBrushLineId(selectedEntityAdapter.id, uuidv4(), true),
             type: 'brush_line',
             points: [pos.x - selectedEntity.position.x, pos.y - selectedEntity.position.y],
             strokeWidth: toolState.brush.width,
@@ -225,7 +225,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
             await selectedEntityAdapter.finalizeDrawingBuffer();
           }
           await selectedEntityAdapter.setDrawingBuffer({
-            id: getBrushLineId(selectedEntityAdapter.id, uuidv4()),
+            id: getEraserLineId(selectedEntityAdapter.id, uuidv4(), true),
             type: 'eraser_line',
             points: [
               // The last point of the last line is already normalized to the entity's coordinates
@@ -242,7 +242,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
             await selectedEntityAdapter.finalizeDrawingBuffer();
           }
           await selectedEntityAdapter.setDrawingBuffer({
-            id: getEraserLineId(selectedEntityAdapter.id, uuidv4()),
+            id: getEraserLineId(selectedEntityAdapter.id, uuidv4(), true),
             type: 'eraser_line',
             points: [pos.x - selectedEntity.position.x, pos.y - selectedEntity.position.y],
             strokeWidth: toolState.eraser.width,
@@ -257,7 +257,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
           await selectedEntityAdapter.finalizeDrawingBuffer();
         }
         await selectedEntityAdapter.setDrawingBuffer({
-          id: getRectShapeId(selectedEntityAdapter.id, uuidv4()),
+          id: getRectShapeId(selectedEntityAdapter.id, uuidv4(), true),
           type: 'rect_shape',
           x: pos.x - selectedEntity.position.x,
           y: pos.y - selectedEntity.position.y,
@@ -357,7 +357,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
             await selectedEntityAdapter.finalizeDrawingBuffer();
           }
           await selectedEntityAdapter.setDrawingBuffer({
-            id: getBrushLineId(selectedEntityAdapter.id, uuidv4()),
+            id: getBrushLineId(selectedEntityAdapter.id, uuidv4(), true),
             type: 'brush_line',
             points: [pos.x - selectedEntity.position.x, pos.y - selectedEntity.position.y],
             strokeWidth: toolState.brush.width,
@@ -389,7 +389,7 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
             await selectedEntityAdapter.finalizeDrawingBuffer();
           }
           await selectedEntityAdapter.setDrawingBuffer({
-            id: getEraserLineId(selectedEntityAdapter.id, uuidv4()),
+            id: getEraserLineId(selectedEntityAdapter.id, uuidv4(), true),
             type: 'eraser_line',
             points: [pos.x - selectedEntity.position.x, pos.y - selectedEntity.position.y],
             strokeWidth: toolState.eraser.width,

invokeai/frontend/web/src/features/controlLayers/konva/naming.ts

@@ -5,9 +5,12 @@
 // Getters for non-singleton layer and object IDs
 export const getRGId = (entityId: string) => `region_${entityId}`;
 export const getLayerId = (entityId: string) => `layer_${entityId}`;
-export const getBrushLineId = (entityId: string, lineId: string) => `${entityId}.brush_line_${lineId}`;
-export const getEraserLineId = (entityId: string, lineId: string) => `${entityId}.eraser_line_${lineId}`;
-export const getRectShapeId = (entityId: string, rectId: string) => `${entityId}.rect_${rectId}`;
+export const getBrushLineId = (entityId: string, lineId: string, isBuffer?: boolean) =>
+  `${entityId}.${isBuffer ? 'buffer_' : ''}brush_line_${lineId}`;
+export const getEraserLineId = (entityId: string, lineId: string, isBuffer?: boolean) =>
+  `${entityId}.${isBuffer ? 'buffer_' : ''}eraser_line_${lineId}`;
+export const getRectShapeId = (entityId: string, rectId: string, isBuffer?: boolean) =>
+  `${entityId}.${isBuffer ? 'buffer_' : ''}rect_${rectId}`;
 export const getImageObjectId = (entityId: string, imageId: string) => `${entityId}.image_${imageId}`;
 export const getObjectGroupId = (entityId: string, groupId: string) => `${entityId}.objectGroup_${groupId}`;
 export const getLayerBboxId = (entityId: string) => `${entityId}.bbox`;