feat(ui): move all caching out of redux

While we lose the benefit of the caches persisting across reloads, this is a much simpler way to handle things. If we need a persistent cache, we can explore it in the future.

invokeai/frontend/web/package.json

@@ -75,6 +75,7 @@
     "jsondiffpatch": "^0.6.0",
     "konva": "^9.3.14",
     "lodash-es": "^4.17.21",
+    "lru-cache": "^11.0.0",
     "nanoid": "^5.0.7",
     "nanostores": "^0.11.2",
     "new-github-issue-url": "^1.0.0",

invokeai/frontend/web/pnpm-lock.yaml

@@ -74,6 +74,9 @@ dependencies:
   lodash-es:
     specifier: ^4.17.21
     version: 4.17.21
+  lru-cache:
+    specifier: ^11.0.0
+    version: 11.0.0
   nanoid:
     specifier: ^5.0.7
     version: 5.0.7
@@ -8704,6 +8707,11 @@ packages:
     engines: {node: 14 || >=16.14}
     dev: true
 
+  /lru-cache@11.0.0:
+    resolution: {integrity: sha512-Qv32eSV1RSCfhY3fpPE2GNZ8jgM9X7rdAfemLWqTUxwiyIC4jJ6Sy0fZ8H+oLWevO6i4/bizg7c8d8i6bxrzbA==}
+    engines: {node: 20 || >=22}
+    dev: false
+
   /lru-cache@5.1.1:
     resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}
     dependencies:

invokeai/frontend/web/src/features/controlLayers/components/ControlLayersSettingsPopover.tsx

@@ -14,11 +14,7 @@ import { useStore } from '@nanostores/react';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { CanvasSettingsBackgroundStyle } from 'features/controlLayers/components/CanvasSettingsBackgroundStyle';
 import { $canvasManager } from 'features/controlLayers/konva/CanvasManager';
-import {
-  clipToBboxChanged,
-  invertScrollChanged,
-  rasterizationCachesInvalidated,
-} from 'features/controlLayers/store/canvasV2Slice';
+import { clipToBboxChanged, invertScrollChanged } from 'features/controlLayers/store/canvasV2Slice';
 import type { ChangeEvent } from 'react';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
@@ -38,23 +34,22 @@ const ControlLayersSettingsPopover = () => {
     (e: ChangeEvent<HTMLInputElement>) => dispatch(clipToBboxChanged(e.target.checked)),
     [dispatch]
   );
-  const invalidateRasterizationCaches = useCallback(() => {
-    dispatch(rasterizationCachesInvalidated());
-  }, [dispatch]);
+  const clearCaches = useCallback(() => {
+    canvasManager?.clearCaches();
+  }, [canvasManager]);
   const calculateBboxes = useCallback(() => {
     if (!canvasManager) {
       return;
     }
-    for (const adapter of canvasManager.rasterLayerAdapters.values()) {
+    const adapters = [
+      ...canvasManager.rasterLayerAdapters.values(),
+      ...canvasManager.controlLayerAdapters.values(),
+      ...canvasManager.regionalGuidanceAdapters.values(),
+      ...canvasManager.inpaintMaskAdapters.values(),
+    ];
+    for (const adapter of adapters) {
       adapter.transformer.requestRectCalculation();
     }
-    for (const adapter of canvasManager.controlLayerAdapters.values()) {
-      adapter.transformer.requestRectCalculation();
-    }
-    for (const adapter of canvasManager.regionalGuidanceAdapters.values()) {
-      adapter.transformer.requestRectCalculation();
-    }
-    canvasManager.inpaintMaskAdapters.transformer.requestRectCalculation();
   }, [canvasManager]);
   return (
     <Popover isLazy>
@@ -73,8 +68,8 @@ const ControlLayersSettingsPopover = () => {
               <Checkbox isChecked={clipToBbox} onChange={onChangeClipToBbox} />
             </FormControl>
             <CanvasSettingsBackgroundStyle />
-            <Button onClick={invalidateRasterizationCaches} size="sm">
-              Invalidate Rasterization Caches
+            <Button onClick={clearCaches} size="sm">
+              Clear Caches
             </Button>
             <Button onClick={calculateBboxes} size="sm">
               Calculate Bboxes

invokeai/frontend/web/src/features/controlLayers/konva/CanvasFilter.ts

@@ -60,7 +60,7 @@ export class CanvasFilter {
     this.log.trace({ config }, 'Previewing filter');
     const dispatch = this.manager.stateApi._store.dispatch;
     const rect = adapter.transformer.getRelativeRect();
-    const imageDTO = await adapter.renderer.rasterize(rect, false);
+    const imageDTO = await adapter.renderer.rasterize({ rect });
     // TODO(psyche): I can't get TS to be happy, it thinkgs `config` is `never` but it should be inferred from the generic... I'll just cast it for now
     const filterNode = IMAGE_FILTERS[config.type].buildNode(imageDTO, config as never);
     const enqueueBatchArg: BatchConfig = {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayerAdapter.ts

@@ -1,4 +1,4 @@
-import type { JSONObject } from 'common/types';
+import type { JSONObject, SerializableObject } from 'common/types';
 import { deepClone } from 'common/util/deepClone';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer';
@@ -12,8 +12,11 @@ import type {
 } from 'features/controlLayers/store/types';
 import { getEntityIdentifier } from 'features/controlLayers/store/types';
 import Konva from 'konva';
-import { get } from 'lodash-es';
+import type { GroupConfig } from 'konva/lib/Group';
+import { get, omit } from 'lodash-es';
 import type { Logger } from 'roarr';
+import stableHash from 'stable-hash';
+import { assert } from 'tsafe';
 
 export class CanvasLayerAdapter {
   readonly type = 'layer_adapter';
@@ -147,13 +150,36 @@ export class CanvasLayerAdapter {
     return { ...this.manager.getLoggingContext(), path: this.path.join('.') };
   };
 
-  getCanvas = (rect: Rect): HTMLCanvasElement => {
+  getCanvas = (rect?: Rect): HTMLCanvasElement => {
     // TODO(psyche) - cache this - maybe with package `memoizee`? Would require careful review of cache invalidation
     this.log.trace({ rect }, 'Getting canvas');
-    const canvas = this.renderer.getCanvas(rect);
+    // The opacity may have been changed in response to user selecting a different entity category, so we must restore
+    // the original opacity before rendering the canvas
+    const attrs: GroupConfig = { opacity: this.state.opacity };
+    const canvas = this.renderer.getCanvas(rect, attrs);
     return canvas;
   };
 
+  getHashableState = (): SerializableObject => {
+    if (this.state.type === 'control_layer') {
+      const keysToOmit: (keyof CanvasControlLayerState)[] = ['name', 'controlAdapter', 'withTransparencyEffect'];
+      return omit(this.state, keysToOmit);
+    } else if (this.state.type === 'raster_layer') {
+      const keysToOmit: (keyof CanvasRasterLayerState)[] = ['name'];
+      return omit(this.state, keysToOmit);
+    } else {
+      assert(false, 'Unexpected layer type');
+    }
+  };
+
+  hash = (extra?: SerializableObject): string => {
+    const arg = {
+      state: this.getHashableState(),
+      extra,
+    };
+    return stableHash(arg);
+  };
+
   logDebugInfo(msg = 'Debug info') {
     const info = {
       repr: this.repr(),

invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts

@@ -1,13 +1,12 @@
 import type { AppSocket } from 'app/hooks/useSocketIO';
 import { logger } from 'app/logging/logger';
 import type { AppStore } from 'app/store/store';
-import type { JSONObject } from 'common/types';
+import type { JSONObject, SerializableObject } from 'common/types';
 import { CanvasFilter } from 'features/controlLayers/konva/CanvasFilter';
 import { MAX_CANVAS_SCALE, MIN_CANVAS_SCALE } from 'features/controlLayers/konva/constants';
 import {
   canvasToBlob,
   canvasToImageData,
-  getHash,
   getImageDataTransparency,
   getPrefixedId,
   getRectUnion,
@@ -18,10 +17,12 @@ import type { Extents, ExtentsResult, GetBboxTask, WorkerLogMessage } from 'feat
 import type { CanvasV2State, Coordinate, Dimensions, GenerationMode, Rect } from 'features/controlLayers/store/types';
 import type Konva from 'konva';
 import { clamp } from 'lodash-es';
+import { LRUCache } from 'lru-cache';
 import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
 import { getImageDTO, uploadImage } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
+import stableHash from 'stable-hash';
 import { assert } from 'tsafe';
 
 import { CanvasBackground } from './CanvasBackground';
@@ -58,6 +59,10 @@ export class CanvasManager {
   isFirstRender: boolean = true;
   _isDebugging: boolean = false;
 
+  imageNameCache = new LRUCache<string, string>({ max: 100 });
+  canvasCache = new LRUCache<string, HTMLCanvasElement>({ max: 32 });
+  generationModeCache = new LRUCache<string, GenerationMode>({ max: 100 });
+
   _worker: Worker = new Worker(new URL('./worker.ts', import.meta.url), { type: 'module', name: 'worker' });
   _tasks: Map<string, { task: GetBboxTask; onComplete: (extents: Extents | null) => void }> = new Map();
 
@@ -579,7 +584,41 @@ export class CanvasManager {
     return pixels / this.getStageScale();
   }
 
+  clearCaches = () => {
+    this.canvasCache.clear();
+    this.imageNameCache.clear();
+    this.generationModeCache.clear();
+  };
+
+  getCompositeRasterLayerEntityIds = (): string[] => {
+    const ids = [];
+    for (const adapter of this.rasterLayerAdapters.values()) {
+      if (adapter.state.isEnabled && adapter.renderer.hasObjects()) {
+        ids.push(adapter.id);
+      }
+    }
+    return ids;
+  };
+
+  getCompositeInpaintMaskEntityIds = (): string[] => {
+    const ids = [];
+    for (const adapter of this.inpaintMaskAdapters.values()) {
+      if (adapter.state.isEnabled && adapter.renderer.hasObjects()) {
+        ids.push(adapter.id);
+      }
+    }
+    return ids;
+  };
+
   getCompositeRasterLayerCanvas = (rect: Rect): HTMLCanvasElement => {
+    const hash = this.getCompositeRasterLayerHash({ rect });
+    const cachedCanvas = this.canvasCache.get(hash);
+
+    if (cachedCanvas) {
+      this.log.trace({ rect }, 'Using cached composite inpaint mask canvas');
+      return cachedCanvas;
+    }
+
     this.log.trace({ rect }, 'Building composite raster layer canvas');
 
     const canvas = document.createElement('canvas');
@@ -589,22 +628,29 @@ export class CanvasManager {
     const ctx = canvas.getContext('2d');
     assert(ctx !== null);
 
-    for (const { id } of this.stateApi.getRasterLayersState().entities) {
+    for (const id of this.getCompositeRasterLayerEntityIds()) {
       const adapter = this.rasterLayerAdapters.get(id);
       if (!adapter) {
         this.log.warn({ id }, 'Raster layer adapter not found');
         continue;
       }
-      if (adapter.state.isEnabled && adapter.renderer.hasObjects()) {
         this.log.trace({ id }, 'Drawing raster layer to composite canvas');
         const adapterCanvas = adapter.getCanvas(rect);
         ctx.drawImage(adapterCanvas, 0, 0);
       }
-    }
+    this.canvasCache.set(hash, canvas);
     return canvas;
   };
 
   getCompositeInpaintMaskCanvas = (rect: Rect): HTMLCanvasElement => {
+    const hash = this.getCompositeInpaintMaskHash({ rect });
+    const cachedCanvas = this.canvasCache.get(hash);
+
+    if (cachedCanvas) {
+      this.log.trace({ rect }, 'Using cached composite inpaint mask canvas');
+      return cachedCanvas;
+    }
+
     this.log.trace({ rect }, 'Building composite inpaint mask canvas');
 
     const canvas = document.createElement('canvas');
@@ -614,35 +660,55 @@ export class CanvasManager {
     const ctx = canvas.getContext('2d');
     assert(ctx !== null);
 
-    for (const { id } of this.stateApi.getInpaintMasksState().entities) {
+    for (const id of this.getCompositeInpaintMaskEntityIds()) {
       const adapter = this.inpaintMaskAdapters.get(id);
       if (!adapter) {
         this.log.warn({ id }, 'Inpaint mask adapter not found');
         continue;
       }
-      if (adapter.state.isEnabled && adapter.renderer.hasObjects()) {
         this.log.trace({ id }, 'Drawing inpaint mask to composite canvas');
         const adapterCanvas = adapter.getCanvas(rect);
         ctx.drawImage(adapterCanvas, 0, 0);
       }
-    }
+    this.canvasCache.set(hash, canvas);
     return canvas;
   };
 
-  getCompositeInpaintMaskImageCache = (hash: string): string | null => {
-    const { compositeRasterizationCache } = this.stateApi.getInpaintMasksState();
-    return compositeRasterizationCache[hash] ?? null;
+  getCompositeRasterLayerHash = (extra: SerializableObject): string => {
+    const data: Record<string, SerializableObject> = {
+      extra,
+    };
+    for (const id of this.getCompositeRasterLayerEntityIds()) {
+      const adapter = this.rasterLayerAdapters.get(id);
+      if (!adapter) {
+        this.log.warn({ id }, 'Raster layer adapter not found');
+        continue;
+      }
+      data[id] = adapter.getHashableState();
+    }
+    return stableHash(data);
   };
 
-  getCompositeRasterLayerImageCache = (hash: string): string | null => {
-    const { compositeRasterizationCache } = this.stateApi.getRasterLayersState();
-    return compositeRasterizationCache[hash] ?? null;
+  getCompositeInpaintMaskHash = (extra: SerializableObject): string => {
+    const data: Record<string, SerializableObject> = {
+      extra,
+    };
+    for (const id of this.getCompositeInpaintMaskEntityIds()) {
+      const adapter = this.inpaintMaskAdapters.get(id);
+      if (!adapter) {
+        this.log.warn({ id }, 'Inpaint mask adapter not found');
+        continue;
+      }
+      data[id] = adapter.getHashableState();
+    }
+    return stableHash(data);
   };
 
   getCompositeRasterLayerImageDTO = async (rect: Rect): Promise<ImageDTO> => {
     let imageDTO: ImageDTO | null = null;
-    const hash = getHash(rect);
-    const cachedImageName = this.getCompositeRasterLayerImageCache(hash);
+
+    const hash = this.getCompositeRasterLayerHash({ rect });
+    const cachedImageName = this.imageNameCache.get(hash);
 
     if (cachedImageName) {
       imageDTO = await getImageDTO(cachedImageName);
@@ -661,14 +727,15 @@ export class CanvasManager {
     }
 
     imageDTO = await uploadImage(blob, 'composite-raster-layer.png', 'general', true);
-    this.stateApi.compositeRasterLayerRasterized({ imageName: imageDTO.image_name, hash });
+    this.imageNameCache.set(hash, imageDTO.image_name);
     return imageDTO;
   };
 
   getCompositeInpaintMaskImageDTO = async (rect: Rect): Promise<ImageDTO> => {
     let imageDTO: ImageDTO | null = null;
-    const hash = getHash(rect);
-    const cachedImageName = this.getCompositeInpaintMaskImageCache(hash);
+
+    const hash = this.getCompositeInpaintMaskHash({ rect });
+    const cachedImageName = this.imageNameCache.get(hash);
 
     if (cachedImageName) {
       imageDTO = await getImageDTO(cachedImageName);
@@ -687,30 +754,46 @@ export class CanvasManager {
     }
 
     imageDTO = await uploadImage(blob, 'composite-inpaint-mask.png', 'general', true);
-    this.stateApi.compositeInpaintMaskRasterized({ imageName: imageDTO.image_name, hash });
+    this.imageNameCache.set(hash, imageDTO.image_name);
     return imageDTO;
   };
 
   getGenerationMode(): GenerationMode {
     const { rect } = this.stateApi.getBbox();
+
+    const compositeInpaintMaskHash = this.getCompositeInpaintMaskHash({ rect });
+    const compositeRasterLayerHash = this.getCompositeRasterLayerHash({ rect });
+    const hash = stableHash({ rect, compositeInpaintMaskHash, compositeRasterLayerHash });
+    const cachedGenerationMode = this.generationModeCache.get(hash);
+
+    if (cachedGenerationMode) {
+      this.log.trace({ rect, cachedGenerationMode }, 'Using cached generation mode');
+      return cachedGenerationMode;
+    }
+
     const inpaintMaskImageData = canvasToImageData(this.getCompositeInpaintMaskCanvas(rect));
     const inpaintMaskTransparency = getImageDataTransparency(inpaintMaskImageData);
     const compositeLayerImageData = canvasToImageData(this.getCompositeRasterLayerCanvas(rect));
     const compositeLayerTransparency = getImageDataTransparency(compositeLayerImageData);
+
+    let generationMode: GenerationMode;
     if (compositeLayerTransparency === 'FULLY_TRANSPARENT') {
       // When the initial image is fully transparent, we are always doing txt2img
-      return 'txt2img';
+      generationMode = 'txt2img';
     } else if (compositeLayerTransparency === 'PARTIALLY_TRANSPARENT') {
       // When the initial image is partially transparent, we are always outpainting
-      return 'outpaint';
+      generationMode = 'outpaint';
     } else if (inpaintMaskTransparency === 'FULLY_TRANSPARENT') {
       // compositeLayerTransparency === 'OPAQUE'
       // When the inpaint mask is fully transparent, we are doing img2img
-      return 'img2img';
+      generationMode = 'img2img';
     } else {
       // Else at least some of the inpaint mask is opaque, so we are inpainting
-      return 'inpaint';
+      generationMode = 'inpaint';
     }
+
+    this.generationModeCache.set(hash, generationMode);
+    return generationMode;
   }
 
   getLoggingContext = (): JSONObject => {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasMaskAdapter.ts

@@ -1,4 +1,4 @@
-import type { JSONObject } from 'common/types';
+import type { JSONObject, SerializableObject } from 'common/types';
 import { deepClone } from 'common/util/deepClone';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer';
@@ -13,8 +13,9 @@ import type {
 import { getEntityIdentifier } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 import type { GroupConfig } from 'konva/lib/Group';
-import { get } from 'lodash-es';
+import { get, omit } from 'lodash-es';
 import type { Logger } from 'roarr';
+import stableHash from 'stable-hash';
 
 export class CanvasMaskAdapter {
   readonly type = 'mask_adapter';
@@ -143,9 +144,23 @@ export class CanvasMaskAdapter {
     };
   };
 
-  getCanvas = (rect: Rect): HTMLCanvasElement => {
+  getHashableState = (): SerializableObject => {
+    const keysToOmit: (keyof CanvasMaskAdapter['state'])[] = ['fill', 'name', 'opacity'];
+    return omit(this.state, keysToOmit);
+  };
+
+  hash = (extra?: SerializableObject): string => {
+    const arg = {
+      state: this.getHashableState(),
+      extra,
+    };
+    return stableHash(arg);
+  };
+
+  getCanvas = (rect?: Rect): HTMLCanvasElement => {
     // TODO(psyche): Cache this?
-    // Backend expects masks to be fully opaque
+    // The opacity may have been changed in response to user selecting a different entity category, and the mask regions
+    // should be fully opaque - set opacity to 1 before rendering the canvas
     const attrs: GroupConfig = { opacity: 1 };
     const canvas = this.renderer.getCanvas(rect, attrs);
     return canvas;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts

@@ -10,7 +10,6 @@ import { CanvasRectRenderer } from 'features/controlLayers/konva/CanvasRect';
 import { LightnessToAlphaFilter } from 'features/controlLayers/konva/filters';
 import { getPatternSVG } from 'features/controlLayers/konva/patterns/getPatternSVG';
 import {
-  getHash,
   getPrefixedId,
   konvaNodeToBlob,
   konvaNodeToCanvas,
@@ -484,10 +483,6 @@ export class CanvasObjectRenderer {
     return this.renderers.size > 0 || this.bufferState !== null || this.bufferRenderer !== null;
   };
 
-  getRasterizedImageCache = (hash: string): string | null => {
-    return this.parent.state.rasterizationCache[hash] ?? null;
-  };
-
   /**
    * Rasterizes the parent entity. If the entity has a rasterization cache for the given rect, the cached image is
    * returned. Otherwise, the entity is rasterized and the image is uploaded to the server.
@@ -498,10 +493,11 @@ export class CanvasObjectRenderer {
    * @param rect The rect to rasterize. If omitted, the entity's full rect will be used.
    * @returns A promise that resolves to the rasterized image DTO.
    */
-  rasterize = async (rect: Rect, replaceObjects: boolean = false, attrs?: GroupConfig): Promise<ImageDTO> => {
+  rasterize = async (options: { rect: Rect; replaceObjects?: boolean; attrs?: GroupConfig }): Promise<ImageDTO> => {
+    const { rect, replaceObjects, attrs } = { replaceObjects: false, attrs: {}, ...options };
     let imageDTO: ImageDTO | null = null;
-    const hash = getHash({ rect, attrs });
-    const cachedImageName = this.getRasterizedImageCache(hash);
+    const hash = this.parent.hash({ rect, attrs });
+    const cachedImageName = this.manager.imageNameCache.get(hash);
 
     if (cachedImageName) {
       imageDTO = await getImageDTO(cachedImageName);
@@ -513,7 +509,7 @@ export class CanvasObjectRenderer {
 
     this.log.trace({ rect }, 'Rasterizing entity');
 
-    const blob = await this.getBlob(rect);
+    const blob = await this.getBlob(rect, attrs);
     if (this.manager._isDebugging) {
       previewBlob(blob, 'Rasterized entity');
     }
@@ -526,10 +522,10 @@ export class CanvasObjectRenderer {
     this.manager.stateApi.rasterizeEntity({
       entityIdentifier: this.parent.getEntityIdentifier(),
       imageObject,
-      hash,
       rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: imageDTO.width, height: imageDTO.height },
       replaceObjects,
     });
+    this.manager.imageNameCache.set(hash, imageDTO.image_name);
 
     return imageDTO;
   };
@@ -545,17 +541,23 @@ export class CanvasObjectRenderer {
 
   getCanvas = (rect?: Rect, attrs?: GroupConfig): HTMLCanvasElement => {
     const clone = this.cloneObjectGroup(attrs);
-    return konvaNodeToCanvas(clone, rect);
+    const canvas = konvaNodeToCanvas(clone, rect);
+    clone.destroy();
+    return canvas;
   };
 
-  getBlob = (rect?: Rect, attrs?: GroupConfig): Promise<Blob> => {
+  getBlob = async (rect?: Rect, attrs?: GroupConfig): Promise<Blob> => {
     const clone = this.cloneObjectGroup(attrs);
-    return konvaNodeToBlob(clone, rect);
+    const blob = await konvaNodeToBlob(clone, rect);
+    clone.destroy();
+    return blob;
   };
 
   getImageData = (rect?: Rect, attrs?: GroupConfig): ImageData => {
     const clone = this.cloneObjectGroup(attrs);
-    return konvaNodeToImageData(clone, rect);
+    const imageData = konvaNodeToImageData(clone, rect);
+    clone.destroy();
+    return imageData;
   };
 
   /**

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts

@@ -24,8 +24,6 @@ import {
   entitySelected,
   eraserWidthChanged,
   fillChanged,
-  inpaintMaskCompositeRasterized,
-  rasterLayerCompositeRasterized,
   toolBufferChanged,
   toolChanged,
 } from 'features/controlLayers/store/canvasV2Slice';
@@ -108,12 +106,6 @@ export class CanvasStateApi {
   rasterizeEntity = (arg: EntityRasterizedPayload) => {
     this._store.dispatch(entityRasterized(arg));
   };
-  compositeRasterLayerRasterized = (arg: { hash: string; imageName: string }) => {
-    this._store.dispatch(rasterLayerCompositeRasterized(arg));
-  };
-  compositeInpaintMaskRasterized = (arg: { hash: string; imageName: string }) => {
-    this._store.dispatch(inpaintMaskCompositeRasterized(arg));
-  };
   setSelectedEntity = (arg: EntityIdentifierPayload) => {
     this._store.dispatch(entitySelected(arg));
   };

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts

@@ -2,7 +2,7 @@ import type { JSONObject } from 'common/types';
 import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
-import { getEmptyRect, getPrefixedId } from 'features/controlLayers/konva/util';
+import { canvasToImageData, getEmptyRect, getPrefixedId } from 'features/controlLayers/konva/util';
 import type { Coordinate, Rect } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 import type { GroupConfig } from 'konva/lib/Group';
@@ -508,7 +508,7 @@ export class CanvasTransformer {
   applyTransform = async () => {
     this.log.debug('Applying transform');
     const rect = this.getRelativeRect();
-    await this.parent.renderer.rasterize(rect, true);
+    await this.parent.renderer.rasterize({ rect, replaceObjects: true });
     this.requestRectCalculation();
     this.stopTransform();
   };
@@ -649,14 +649,8 @@ export class CanvasTransformer {
     }
 
     // We have eraser strokes - we must calculate the bbox using pixel data
-
-    const clone = this.parent.renderer.konva.objectGroup.clone();
-    const canvas = clone.toCanvas();
-    const ctx = canvas.getContext('2d');
-    if (!ctx) {
-      return;
-    }
-    const imageData = ctx.getImageData(0, 0, rect.width, rect.height);
+    const canvas = this.parent.renderer.getCanvas(undefined, { opacity: 1 });
+    const imageData = canvasToImageData(canvas);
     this.manager.requestBbox(
       { buffer: imageData.data.buffer, width: imageData.width, height: imageData.height },
       (extents) => {
@@ -676,7 +670,6 @@ export class CanvasTransformer {
         this.isPendingRectCalculation = false;
         this.log.trace({ nodeRect: this.nodeRect, pixelRect: this.pixelRect, extents }, `Got bbox from worker`);
         this.updateBbox();
-        clone.destroy();
       }
     );
   }, CanvasTransformer.RECT_CALC_DEBOUNCE_MS);

invokeai/frontend/web/src/features/controlLayers/konva/util.ts

@@ -3,7 +3,6 @@ import Konva from 'konva';
 import type { KonvaEventObject } from 'konva/lib/Node';
 import type { Vector2d } from 'konva/lib/types';
 import { customAlphabet } from 'nanoid';
-import stableHash from 'stable-hash';
 import { assert } from 'tsafe';
 
 /**
@@ -399,5 +398,3 @@ export const getRectUnion = (...rects: Rect[]): Rect => {
   }, getEmptyRect());
   return rect;
 };
-
-export const getHash = stableHash;

invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts

@@ -19,7 +19,7 @@ import { getScaledBoundingBoxDimensions } from 'features/controlLayers/util/getS
 import { simplifyFlatNumbersArray } from 'features/controlLayers/util/simplify';
 import { initialAspectRatioState } from 'features/parameters/components/DocumentSize/constants';
 import { getOptimalDimension } from 'features/parameters/util/optimalDimension';
-import { isEqual, pick } from 'lodash-es';
+import { pick } from 'lodash-es';
 import { atom } from 'nanostores';
 import { assert } from 'tsafe';
 
@@ -47,14 +47,12 @@ const initialState: CanvasV2State = {
   selectedEntityIdentifier: null,
   rasterLayers: {
     entities: [],
-    compositeRasterizationCache: {},
   },
   controlLayers: {
     entities: [],
   },
   inpaintMasks: {
     entities: [],
-    compositeRasterizationCache: {},
   },
   regions: {
     entities: [],
@@ -172,27 +170,6 @@ export function selectAllEntitiesOfType(state: CanvasV2State, type: CanvasEntity
   }
 }
 
-const invalidateRasterizationCaches = (
-  entity: CanvasRasterLayerState | CanvasControlLayerState | CanvasInpaintMaskState | CanvasRegionalGuidanceState,
-  state: CanvasV2State
-) => {
-  // TODO(psyche): We can be more efficient and only invalidate caches when the entity's changes intersect with the
-  // cached rect.
-
-  // Reset the entity's rasterization cache
-  entity.rasterizationCache = {};
-
-  // When an individual layer has its cache reset, we must also reset the composite rasterization cache because the
-  // layer's image data will contribute to the composite layer's image data.
-  // If the layer is used as a control layer, it will not contribute to the composite layer, so we do not need to reset
-  // its cache.
-  if (entity.type === 'raster_layer') {
-    state.rasterLayers.compositeRasterizationCache = {};
-  } else if (entity.type === 'inpaint_mask') {
-    state.inpaintMasks.compositeRasterizationCache = {};
-  }
-};
-
 export const canvasV2Slice = createSlice({
   name: 'canvasV2',
   initialState,
@@ -230,7 +207,6 @@ export const canvasV2Slice = createSlice({
         entity.isEnabled = true;
         entity.objects = [];
         entity.position = { x: 0, y: 0 };
-        invalidateRasterizationCaches(entity, state);
       } else {
         assert(false, 'Not implemented');
       }
@@ -252,22 +228,16 @@ export const canvasV2Slice = createSlice({
 
       if (isDrawableEntity(entity)) {
         entity.position = position;
-        // When an entity is moved, we need to invalidate the rasterization caches.
-        invalidateRasterizationCaches(entity, state);
       }
     },
     entityRasterized: (state, action: PayloadAction<EntityRasterizedPayload>) => {
-      const { entityIdentifier, imageObject, hash, rect, replaceObjects } = action.payload;
+      const { entityIdentifier, imageObject, rect, replaceObjects } = action.payload;
       const entity = selectEntity(state, entityIdentifier);
       if (!entity) {
         return;
       }
 
       if (isDrawableEntity(entity)) {
-        // Remove the cache for the given rect. This should never happen, because we should never rasterize the same
-        // rect twice. Just in case, we remove the old cache.
-        entity.rasterizationCache[hash] = imageObject.image.image_name;
-
         if (replaceObjects) {
           entity.objects = [imageObject];
           entity.position = { x: rect.x, y: rect.y };
@@ -286,8 +256,6 @@ export const canvasV2Slice = createSlice({
       }
 
       entity.objects.push({ ...brushLine, points: simplifyFlatNumbersArray(brushLine.points) });
-      // When adding a brush line, we need to invalidate the rasterization caches.
-      invalidateRasterizationCaches(entity, state);
     },
     entityEraserLineAdded: (state, action: PayloadAction<EntityEraserLineAddedPayload>) => {
       const { entityIdentifier, eraserLine } = action.payload;
@@ -301,8 +269,6 @@ export const canvasV2Slice = createSlice({
       }
 
       entity.objects.push({ ...eraserLine, points: simplifyFlatNumbersArray(eraserLine.points) });
-      // When adding an eraser line, we need to invalidate the rasterization caches.
-      invalidateRasterizationCaches(entity, state);
     },
     entityRectAdded: (state, action: PayloadAction<EntityRectAddedPayload>) => {
       const { entityIdentifier, rect } = action.payload;
@@ -316,8 +282,6 @@ export const canvasV2Slice = createSlice({
       }
 
       entity.objects.push(rect);
-      // When adding an eraser line, we need to invalidate the rasterization caches.
-      invalidateRasterizationCaches(entity, state);
     },
     entityDeleted: (state, action: PayloadAction<EntityIdentifierPayload>) => {
       const { entityIdentifier } = action.payload;
@@ -331,8 +295,6 @@ export const canvasV2Slice = createSlice({
       if (entityIdentifier.type === 'raster_layer') {
         const index = state.rasterLayers.entities.findIndex((layer) => layer.id === entityIdentifier.id);
         state.rasterLayers.entities = state.rasterLayers.entities.filter((layer) => layer.id !== entityIdentifier.id);
-        // When deleting a raster layer, we need to invalidate the composite rasterization cache.
-        state.rasterLayers.compositeRasterizationCache = {};
         const nextRasterLayer = state.rasterLayers.entities[index];
         if (nextRasterLayer) {
           selectedEntityIdentifier = { type: nextRasterLayer.type, id: nextRasterLayer.id };
@@ -361,8 +323,6 @@ export const canvasV2Slice = createSlice({
       } else if (entityIdentifier.type === 'inpaint_mask') {
         const index = state.inpaintMasks.entities.findIndex((layer) => layer.id === entityIdentifier.id);
         state.inpaintMasks.entities = state.inpaintMasks.entities.filter((rg) => rg.id !== entityIdentifier.id);
-        // When deleting a inpaint mask, we need to invalidate the composite rasterization cache.
-        state.inpaintMasks.compositeRasterizationCache = {};
         const entity = state.inpaintMasks.entities[index];
         if (entity) {
           selectedEntityIdentifier = { type: entity.type, id: entity.id };
@@ -381,16 +341,12 @@ export const canvasV2Slice = createSlice({
       }
       if (entity.type === 'raster_layer') {
         moveOneToEnd(state.rasterLayers.entities, entity);
-        // When arranging a raster layer, we need to invalidate the composite rasterization cache.
-        state.rasterLayers.compositeRasterizationCache = {};
       } else if (entity.type === 'control_layer') {
         moveOneToEnd(state.controlLayers.entities, entity);
       } else if (entity.type === 'regional_guidance') {
         moveOneToEnd(state.regions.entities, entity);
       } else if (entity.type === 'inpaint_mask') {
         moveOneToEnd(state.inpaintMasks.entities, entity);
-        // When arranging a inpaint mask, we need to invalidate the composite rasterization cache.
-        state.inpaintMasks.compositeRasterizationCache = {};
       }
     },
     entityArrangedToFront: (state, action: PayloadAction<EntityIdentifierPayload>) => {
@@ -401,16 +357,12 @@ export const canvasV2Slice = createSlice({
       }
       if (entity.type === 'raster_layer') {
         moveToEnd(state.rasterLayers.entities, entity);
-        // When arranging a raster layer, we need to invalidate the composite rasterization cache.
-        state.rasterLayers.compositeRasterizationCache = {};
       } else if (entity.type === 'control_layer') {
         moveToEnd(state.controlLayers.entities, entity);
       } else if (entity.type === 'regional_guidance') {
         moveToEnd(state.regions.entities, entity);
       } else if (entity.type === 'inpaint_mask') {
         moveToEnd(state.inpaintMasks.entities, entity);
-        // When arranging a inpaint mask, we need to invalidate the composite rasterization cache.
-        state.inpaintMasks.compositeRasterizationCache = {};
       }
     },
     entityArrangedBackwardOne: (state, action: PayloadAction<EntityIdentifierPayload>) => {
@@ -421,16 +373,12 @@ export const canvasV2Slice = createSlice({
       }
       if (entity.type === 'raster_layer') {
         moveOneToStart(state.rasterLayers.entities, entity);
-        // When arranging a raster layer, we need to invalidate the composite rasterization cache.
-        state.rasterLayers.compositeRasterizationCache = {};
       } else if (entity.type === 'control_layer') {
         moveOneToStart(state.controlLayers.entities, entity);
       } else if (entity.type === 'regional_guidance') {
         moveOneToStart(state.regions.entities, entity);
       } else if (entity.type === 'inpaint_mask') {
         moveOneToStart(state.inpaintMasks.entities, entity);
-        // When arranging a inpaint mask, we need to invalidate the composite rasterization cache.
-        state.inpaintMasks.compositeRasterizationCache = {};
       }
     },
     entityArrangedToBack: (state, action: PayloadAction<EntityIdentifierPayload>) => {
@@ -441,16 +389,12 @@ export const canvasV2Slice = createSlice({
       }
       if (entity.type === 'raster_layer') {
         moveToStart(state.rasterLayers.entities, entity);
-        // When arranging a raster layer, we need to invalidate the composite rasterization cache.
-        state.rasterLayers.compositeRasterizationCache = {};
       } else if (entity.type === 'control_layer') {
         moveToStart(state.controlLayers.entities, entity);
       } else if (entity.type === 'regional_guidance') {
         moveToStart(state.regions.entities, entity);
       } else if (entity.type === 'inpaint_mask') {
         moveToStart(state.inpaintMasks.entities, entity);
-        // When arranging a inpaint mask, we need to invalidate the composite rasterization cache.
-        state.inpaintMasks.compositeRasterizationCache = {};
       }
     },
     entityOpacityChanged: (state, action: PayloadAction<EntityIdentifierPayload<{ opacity: number }>>) => {
@@ -463,7 +407,6 @@ export const canvasV2Slice = createSlice({
         return;
       }
       entity.opacity = opacity;
-      invalidateRasterizationCaches(entity, state);
     },
     allEntitiesOfTypeToggled: (state, action: PayloadAction<{ type: CanvasEntityIdentifier['type'] }>) => {
       const { type } = action.payload;
@@ -505,23 +448,6 @@ export const canvasV2Slice = createSlice({
 
       state.selectedEntityIdentifier = deepClone(initialState.selectedEntityIdentifier);
     },
-    rasterizationCachesInvalidated: (state) => {
-      // Invalidate the rasterization caches for all entities.
-      const allEntities = [
-        ...state.rasterLayers.entities,
-        ...state.controlLayers.entities,
-        ...state.regions.entities,
-        ...state.inpaintMasks.entities,
-      ];
-
-      for (const entity of allEntities) {
-        entity.rasterizationCache = {};
-      }
-
-      // Also invalidate the composite rasterization caches.
-      state.rasterLayers.compositeRasterizationCache = {};
-      state.inpaintMasks.compositeRasterizationCache = {};
-    },
     canvasReset: (state) => {
       state.bbox = deepClone(initialState.bbox);
       const optimalDimension = getOptimalDimension(state.params.model);
@@ -553,7 +479,6 @@ export const {
   allEntitiesDeleted,
   clipToBboxChanged,
   canvasReset,
-  rasterizationCachesInvalidated,
   canvasBackgroundStyleChanged,
   // All entities
   entitySelected,
@@ -587,7 +512,6 @@ export const {
   rasterLayerRecalled,
   rasterLayerAllDeleted,
   rasterLayerConvertedToControlLayer,
-  rasterLayerCompositeRasterized,
   // Control layers
   controlLayerAdded,
   controlLayerRecalled,
@@ -676,7 +600,6 @@ export const {
   inpaintMaskRecalled,
   inpaintMaskFillColorChanged,
   inpaintMaskFillStyleChanged,
-  inpaintMaskCompositeRasterized,
   // Staging
   sessionStartedStaging,
   sessionImageStaged,

invokeai/frontend/web/src/features/controlLayers/store/controlLayersReducers.ts

@@ -40,7 +40,6 @@ export const controlLayersReducers = {
         objects: [],
         opacity: 1,
         position: { x: 0, y: 0 },
-        rasterizationCache: {},
         controlAdapter: deepClone(initialControlNetV2),
       };
       merge(layer, overrides);
@@ -82,9 +81,6 @@ export const controlLayersReducers = {
       // Add the new raster layer
       state.rasterLayers.entities.push(rasterLayerState);
 
-      // The composite layer's image data will change when the control layer is converted to raster layer.
-      state.rasterLayers.compositeRasterizationCache = {};
-
       state.selectedEntityIdentifier = { type: rasterLayerState.type, id: rasterLayerState.id };
     },
     prepare: (payload: { id: string }) => ({

invokeai/frontend/web/src/features/controlLayers/store/inpaintMaskReducers.ts

@@ -33,7 +33,6 @@ export const inpaintMaskReducers = {
         objects: [],
         opacity: 1,
         position: { x: 0, y: 0 },
-        rasterizationCache: {},
         fill: {
           style: 'diagonal',
           color: { r: 255, g: 122, b: 0 }, // some orange color
@@ -70,8 +69,4 @@ export const inpaintMaskReducers = {
     }
     entity.fill.style = style;
   },
-  inpaintMaskCompositeRasterized: (state, action: PayloadAction<{ hash: string; imageName: string }>) => {
-    const { hash, imageName } = action.payload;
-    state.inpaintMasks.compositeRasterizationCache[hash] = imageName;
-  },
 } satisfies SliceCaseReducers<CanvasV2State>;

invokeai/frontend/web/src/features/controlLayers/store/rasterLayersReducers.ts

@@ -30,18 +30,12 @@ export const rasterLayersReducers = {
         objects: [],
         opacity: 1,
         position: { x: 0, y: 0 },
-        rasterizationCache: {},
       };
       merge(layer, overrides);
       state.rasterLayers.entities.push(layer);
       if (isSelected) {
         state.selectedEntityIdentifier = { type: 'raster_layer', id };
       }
-
-      if (layer.objects.length > 0) {
-        // This new layer will change the composite layer's image data. Invalidate the cache.
-        state.rasterLayers.compositeRasterizationCache = {};
-      }
     },
     prepare: (payload: { overrides?: Partial<CanvasRasterLayerState>; isSelected?: boolean }) => ({
       payload: { ...payload, id: getPrefixedId('raster_layer') },
@@ -51,18 +45,9 @@ export const rasterLayersReducers = {
     const { data } = action.payload;
     state.rasterLayers.entities.push(data);
     state.selectedEntityIdentifier = { type: 'raster_layer', id: data.id };
-    if (data.objects.length > 0) {
-      // This new layer will change the composite layer's image data. Invalidate the cache.
-      state.rasterLayers.compositeRasterizationCache = {};
-    }
   },
   rasterLayerAllDeleted: (state) => {
     state.rasterLayers.entities = [];
-    state.rasterLayers.compositeRasterizationCache = {};
-  },
-  rasterLayerCompositeRasterized: (state, action: PayloadAction<{ hash: string; imageName: string }>) => {
-    const { hash, imageName } = action.payload;
-    state.rasterLayers.compositeRasterizationCache[hash] = imageName;
   },
   rasterLayerConvertedToControlLayer: {
     reducer: (state, action: PayloadAction<{ id: string; newId: string }>) => {
@@ -87,9 +72,6 @@ export const rasterLayersReducers = {
       // Add the converted control layer
       state.controlLayers.entities.push(controlLayerState);
 
-      // The composite layer's image data will change when the raster layer is converted to control layer.
-      state.rasterLayers.compositeRasterizationCache = {};
-
       state.selectedEntityIdentifier = { type: controlLayerState.type, id: controlLayerState.id };
     },
     prepare: (payload: { id: string }) => ({

invokeai/frontend/web/src/features/controlLayers/store/regionsReducers.ts

@@ -75,7 +75,6 @@ export const regionsReducers = {
         positivePrompt: '',
         negativePrompt: null,
         ipAdapters: [],
-        rasterizationCache: {},
       };
       state.regions.entities.push(rg);
       state.selectedEntityIdentifier = { type: 'regional_guidance', id };

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -648,9 +648,6 @@ export const isFillStyle = (v: unknown): v is FillStyle => zFillStyle.safeParse(
 const zFill = z.object({ style: zFillStyle, color: zRgbColor });
 export type Fill = z.infer<typeof zFill>;
 
-const zImageCache = z.record(z.string()).default({});
-export type ImageCache = z.infer<typeof zImageCache>;
-
 const zRegionalGuidanceIPAdapterConfig = z.object({
   id: zId,
   image: zImageWithDims.nullable(),
@@ -675,7 +672,6 @@ export const zCanvasRegionalGuidanceState = z.object({
   negativePrompt: zParameterNegativePrompt.nullable(),
   ipAdapters: z.array(zRegionalGuidanceIPAdapterConfig),
   autoNegative: zAutoNegative,
-  rasterizationCache: zImageCache,
 });
 export type CanvasRegionalGuidanceState = z.infer<typeof zCanvasRegionalGuidanceState>;
 
@@ -688,7 +684,6 @@ const zCanvasInpaintMaskState = z.object({
   fill: zFill,
   opacity: zOpacity,
   objects: z.array(zCanvasObjectState),
-  rasterizationCache: zImageCache,
 });
 export type CanvasInpaintMaskState = z.infer<typeof zCanvasInpaintMaskState>;
 
@@ -748,7 +743,6 @@ export const zCanvasRasterLayerState = z.object({
   position: zCoordinate,
   opacity: zOpacity,
   objects: z.array(zCanvasObjectState),
-  rasterizationCache: zImageCache,
 });
 export type CanvasRasterLayerState = z.infer<typeof zCanvasRasterLayerState>;
 
@@ -850,11 +844,9 @@ export type CanvasV2State = {
   selectedEntityIdentifier: CanvasEntityIdentifier | null;
   inpaintMasks: {
     entities: CanvasInpaintMaskState[];
-    compositeRasterizationCache: ImageCache;
   };
   rasterLayers: {
     entities: CanvasRasterLayerState[];
-    compositeRasterizationCache: ImageCache;
   };
   controlLayers: {
     entities: CanvasControlLayerState[];
@@ -961,9 +953,8 @@ export type EntityBrushLineAddedPayload = EntityIdentifierPayload<{ brushLine: C
 export type EntityEraserLineAddedPayload = EntityIdentifierPayload<{ eraserLine: CanvasEraserLineState }>;
 export type EntityRectAddedPayload = EntityIdentifierPayload<{ rect: CanvasRectState }>;
 export type EntityRasterizedPayload = EntityIdentifierPayload<{
-  hash: string;
   imageObject: CanvasImageState;
-  rect: Rect,
+  rect: Rect;
   replaceObjects: boolean;
 }>;
 export type ImageObjectAddedArg = { id: string; imageDTO: ImageDTO; position?: Coordinate };

invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts

@@ -25,7 +25,7 @@ export const addControlAdapters = async (
   for (const layer of validControlLayers) {
     const adapter = manager.controlLayerAdapters.get(layer.id);
     assert(adapter, 'Adapter not found');
-    const imageDTO = await adapter.renderer.rasterize(bbox);
+    const imageDTO = await adapter.renderer.rasterize({ rect: bbox, attrs: { opacity: 1, filters: [] } });
     if (layer.controlAdapter.type === 'controlnet') {
       await addControlNetToGraph(g, layer, imageDTO, denoise);
     } else {

invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts

@@ -49,7 +49,7 @@ export const addRegions = async (
   for (const region of validRegions) {
     const adapter = manager.regionalGuidanceAdapters.get(region.id);
     assert(adapter, 'Adapter not found');
-    const imageDTO = await adapter.renderer.rasterize(bbox);
+    const imageDTO = await adapter.renderer.rasterize({ rect: bbox });
 
     // The main mask-to-tensor node
     const maskToTensor = g.addNode({