feat(ui): canvas layer preview, revised reactivity for adapters

invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayer.tsx

@@ -3,11 +3,12 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/
 import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton';
 import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle';
 import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader';
+import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage';
 import { CanvasEntitySettingsWrapper } from 'features/controlLayers/components/common/CanvasEntitySettingsWrapper';
 import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit';
 import { ControlLayerControlAdapter } from 'features/controlLayers/components/ControlLayer/ControlLayerControlAdapter';
+import { EntityLayerAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext';
 import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
-import { EntityLayerAdapterProviderGate } from 'features/controlLayers/hooks/useEntityLayerAdapter';
 import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
 import { memo, useMemo } from 'react';
 
@@ -20,9 +21,10 @@ export const ControlLayer = memo(({ id }: Props) => {
 
   return (
     <EntityIdentifierContext.Provider value={entityIdentifier}>
-      <EntityLayerAdapterProviderGate>
+      <EntityLayerAdapterGate>
         <CanvasEntityContainer>
           <CanvasEntityHeader>
+            <CanvasEntityPreviewImage />
             <CanvasEntityEnabledToggle />
             <CanvasEntityEditableTitle />
             <Spacer />
@@ -32,7 +34,7 @@ export const ControlLayer = memo(({ id }: Props) => {
             <ControlLayerControlAdapter />
           </CanvasEntitySettingsWrapper>
         </CanvasEntityContainer>
-      </EntityLayerAdapterProviderGate>
+      </EntityLayerAdapterGate>
     </EntityIdentifierContext.Provider>
   );
 });

invokeai/frontend/web/src/features/controlLayers/components/ControlLayersToolbar.tsx

@@ -10,16 +10,17 @@ import { ToolChooser } from 'features/controlLayers/components/Tool/ToolChooser'
 import { ToolEraserWidth } from 'features/controlLayers/components/Tool/ToolEraserWidth';
 import { ToolFillColorPicker } from 'features/controlLayers/components/Tool/ToolFillColorPicker';
 import { UndoRedoButtonGroup } from 'features/controlLayers/components/UndoRedoButtonGroup';
-import { CanvasManagerProviderGate } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
+import { CanvasManagerProviderGate, useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
 import { ToggleProgressButton } from 'features/gallery/components/ImageViewer/ToggleProgressButton';
 import { ViewerToggleMenu } from 'features/gallery/components/ImageViewer/ViewerToggleMenu';
-import { memo } from 'react';
+import { memo, useSyncExternalStore } from 'react';
 
 export const ControlLayersToolbar = memo(() => {
   const tool = useAppSelector((s) => s.canvasV2.tool.selected);
   return (
     <CanvasManagerProviderGate>
       <Flex w="full" gap={2} alignItems="center">
+        <ReactiveTest />
         <ToggleProgressButton />
         <ToolChooser />
         <Spacer />
@@ -40,3 +41,15 @@ export const ControlLayersToolbar = memo(() => {
 });
 
 ControlLayersToolbar.displayName = 'ControlLayersToolbar';
+
+const ReactiveTest = () => {
+  const canvasManager = useCanvasManager();
+  const adapters = useSyncExternalStore(
+    canvasManager.adapters.rasterLayers.subscribe,
+    canvasManager.adapters.rasterLayers.getSnapshot
+  );
+
+  console.log(adapters);
+
+  return null;
+};

invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMask.tsx

@@ -3,9 +3,10 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/
 import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton';
 import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle';
 import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader';
+import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage';
 import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit';
+import { EntityMaskAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext';
 import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
-import { EntityMaskAdapterProviderGate } from 'features/controlLayers/hooks/useEntityMaskAdapter';
 import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
 import { memo, useMemo } from 'react';
 
@@ -20,9 +21,10 @@ export const InpaintMask = memo(({ id }: Props) => {
 
   return (
     <EntityIdentifierContext.Provider value={entityIdentifier}>
-      <EntityMaskAdapterProviderGate>
+      <EntityMaskAdapterGate>
         <CanvasEntityContainer>
           <CanvasEntityHeader>
+            <CanvasEntityPreviewImage />
             <CanvasEntityEnabledToggle />
             <CanvasEntityEditableTitle />
             <Spacer />
@@ -30,7 +32,7 @@ export const InpaintMask = memo(({ id }: Props) => {
             <CanvasEntityDeleteButton />
           </CanvasEntityHeader>
         </CanvasEntityContainer>
-      </EntityMaskAdapterProviderGate>
+      </EntityMaskAdapterGate>
     </EntityIdentifierContext.Provider>
   );
 });

invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayer.tsx

@@ -3,9 +3,10 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/
 import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton';
 import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle';
 import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader';
+import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage';
 import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit';
+import { EntityLayerAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext';
 import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
-import { EntityLayerAdapterProviderGate } from 'features/controlLayers/hooks/useEntityLayerAdapter';
 import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
 import { memo, useMemo } from 'react';
 
@@ -18,16 +19,17 @@ export const RasterLayer = memo(({ id }: Props) => {
 
   return (
     <EntityIdentifierContext.Provider value={entityIdentifier}>
-      <EntityLayerAdapterProviderGate>
+      <EntityLayerAdapterGate>
         <CanvasEntityContainer>
           <CanvasEntityHeader>
+            <CanvasEntityPreviewImage />
             <CanvasEntityEnabledToggle />
             <CanvasEntityEditableTitle />
             <Spacer />
             <CanvasEntityDeleteButton />
           </CanvasEntityHeader>
         </CanvasEntityContainer>
-      </EntityLayerAdapterProviderGate>
+      </EntityLayerAdapterGate>
     </EntityIdentifierContext.Provider>
   );
 });

invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidance.tsx

@@ -3,11 +3,12 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/
 import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton';
 import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle';
 import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader';
+import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage';
 import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit';
 import { RegionalGuidanceBadges } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceBadges';
 import { RegionalGuidanceSettings } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceSettings';
+import { EntityMaskAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext';
 import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
-import { EntityMaskAdapterProviderGate } from 'features/controlLayers/hooks/useEntityMaskAdapter';
 import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
 import { memo, useMemo } from 'react';
 
@@ -23,9 +24,10 @@ export const RegionalGuidance = memo(({ id }: Props) => {
 
   return (
     <EntityIdentifierContext.Provider value={entityIdentifier}>
-      <EntityMaskAdapterProviderGate>
+      <EntityMaskAdapterGate>
         <CanvasEntityContainer>
           <CanvasEntityHeader>
+            <CanvasEntityPreviewImage />
             <CanvasEntityEnabledToggle />
             <CanvasEntityEditableTitle />
             <Spacer />
@@ -36,7 +38,7 @@ export const RegionalGuidance = memo(({ id }: Props) => {
           </CanvasEntityHeader>
           <RegionalGuidanceSettings />
         </CanvasEntityContainer>
-      </EntityMaskAdapterProviderGate>
+      </EntityMaskAdapterGate>
     </EntityIdentifierContext.Provider>
   );
 });

invokeai/frontend/web/src/features/controlLayers/components/Settings/CanvasSettingsRecalculateRectsButton.tsx

@@ -7,13 +7,7 @@ export const CanvasSettingsRecalculateRectsButton = memo(() => {
   const { t } = useTranslation();
   const canvasManager = useCanvasManager();
   const onClick = useCallback(() => {
-    const adapters = [
+    for (const adapter of canvasManager.adapters.getAll()) {
-      ...canvasManager.rasterLayerAdapters.values(),
-      ...canvasManager.controlLayerAdapters.values(),
-      ...canvasManager.regionalGuidanceAdapters.values(),
-      ...canvasManager.inpaintMaskAdapters.values(),
-    ];
-    for (const adapter of adapters) {
       adapter.transformer.requestRectCalculation();
     }
   }, [canvasManager]);

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityContainer.tsx

@@ -21,7 +21,6 @@ export const CanvasEntityContainer = memo((props: PropsWithChildren) => {
 
   return (
     <Flex
-      position="relative" // necessary for drop overlay
       flexDir="column"
       w="full"
       bg={isSelected ? 'base.800' : 'base.850'}

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityHeader.tsx

@@ -53,7 +53,7 @@ export const CanvasEntityHeader = memo(({ children, ...rest }: FlexProps) => {
   return (
     <ContextMenu renderMenu={renderMenu}>
       {(ref) => (
-        <Flex ref={ref} gap={2} alignItems="center" p={2} {...rest}>
+        <Flex ref={ref} h={16} gap={2} alignItems="center" p={2} {...rest}>
           {children}
         </Flex>
       )}

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityPreviewImage.tsx

@@ -0,0 +1,59 @@
+import { Box, chakra, Flex } from '@invoke-ai/ui-library';
+import { useStore } from '@nanostores/react';
+import { useEntityAdapter } from 'features/controlLayers/contexts/EntityAdapterContext';
+import { TRANSPARENCY_CHECKER_PATTERN } from 'features/controlLayers/konva/constants';
+import { memo, useEffect, useRef } from 'react';
+
+const ChakraCanvas = chakra.canvas;
+
+export const CanvasEntityPreviewImage = memo(() => {
+  const adapter = useEntityAdapter();
+  const containerRef = useRef<HTMLDivElement>(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const cache = useStore(adapter.renderer.$canvasCache);
+  useEffect(() => {
+    if (!cache || !canvasRef.current || !containerRef.current) {
+      return;
+    }
+    const ctx = canvasRef.current.getContext('2d');
+    if (!ctx) {
+      return;
+    }
+    const { rect, canvas } = cache;
+
+    ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
+
+    canvasRef.current.width = rect.width;
+    canvasRef.current.height = rect.height;
+
+    ctx.drawImage(canvas, rect.x, rect.y, rect.width, rect.height, 0, 0, rect.width, rect.height);
+  }, [adapter.transformer, adapter.transformer.nodeRect, adapter.transformer.pixelRect, cache]);
+
+  return (
+    <Flex
+      position="relative"
+      ref={containerRef}
+      alignItems="center"
+      justifyContent="center"
+      w={12}
+      h={12}
+      borderRadius="sm"
+      borderWidth={1}
+      bg="base.900"
+    >
+      <Box
+        position="absolute"
+        top={0}
+        right={0}
+        bottom={0}
+        left={0}
+        bgImage={TRANSPARENCY_CHECKER_PATTERN}
+        bgSize="5px"
+        opacity={0.1}
+      />
+      <ChakraCanvas ref={canvasRef} objectFit="contain" maxW="full" maxH="full" />
+    </Flex>
+  );
+});
+
+CanvasEntityPreviewImage.displayName = 'CanvasEntityPreviewImage';

invokeai/frontend/web/src/features/controlLayers/contexts/EntityAdapterContext.tsx

@@ -0,0 +1,82 @@
+import type { SyncableMap } from 'common/util/SyncableMap/SyncableMap';
+import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
+import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
+import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
+import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
+import type { PropsWithChildren } from 'react';
+import { createContext, memo, useContext, useMemo, useSyncExternalStore } from 'react';
+import { assert } from 'tsafe';
+
+const EntityAdapterContext = createContext<CanvasLayerAdapter | CanvasMaskAdapter | null>(null);
+
+export const EntityLayerAdapterGate = memo(({ children }: PropsWithChildren) => {
+  const canvasManager = useCanvasManager();
+  const entityIdentifier = useEntityIdentifierContext();
+  const store = useMemo<SyncableMap<string, CanvasLayerAdapter>>(() => {
+    if (entityIdentifier.type === 'raster_layer') {
+      return canvasManager.adapters.rasterLayers;
+    }
+    if (entityIdentifier.type === 'control_layer') {
+      return canvasManager.adapters.controlLayers;
+    }
+    assert(false, 'Unknown entity type');
+  }, [canvasManager.adapters.controlLayers, canvasManager.adapters.rasterLayers, entityIdentifier.type]);
+  const adapters = useSyncExternalStore(store.subscribe, store.getSnapshot);
+  const adapter = useMemo(() => {
+    return adapters.get(entityIdentifier.id) ?? null;
+  }, [adapters, entityIdentifier.id]);
+
+  if (!adapter) {
+    return null;
+  }
+
+  return <EntityAdapterContext.Provider value={adapter}>{children}</EntityAdapterContext.Provider>;
+});
+
+EntityLayerAdapterGate.displayName = 'EntityLayerAdapterGate';
+
+export const useEntityLayerAdapter = (): CanvasLayerAdapter => {
+  const adapter = useContext(EntityAdapterContext);
+  assert(adapter, 'useEntityLayerAdapter must be used within a EntityLayerAdapterGate');
+  assert(adapter.type === 'layer_adapter', 'useEntityLayerAdapter must be used with a layer adapter');
+  return adapter;
+};
+
+export const EntityMaskAdapterGate = memo(({ children }: PropsWithChildren) => {
+  const canvasManager = useCanvasManager();
+  const entityIdentifier = useEntityIdentifierContext();
+  const store = useMemo<SyncableMap<string, CanvasMaskAdapter>>(() => {
+    if (entityIdentifier.type === 'inpaint_mask') {
+      return canvasManager.adapters.inpaintMasks;
+    }
+    if (entityIdentifier.type === 'regional_guidance') {
+      return canvasManager.adapters.regionMasks;
+    }
+    assert(false, 'Unknown entity type');
+  }, [canvasManager.adapters.inpaintMasks, canvasManager.adapters.regionMasks, entityIdentifier.type]);
+  const adapters = useSyncExternalStore(store.subscribe, store.getSnapshot);
+  const adapter = useMemo(() => {
+    return adapters.get(entityIdentifier.id) ?? null;
+  }, [adapters, entityIdentifier.id]);
+
+  if (!adapter) {
+    return null;
+  }
+
+  return <EntityAdapterContext.Provider value={adapter}>{children}</EntityAdapterContext.Provider>;
+});
+
+EntityMaskAdapterGate.displayName = 'EntityMaskAdapterGate';
+
+export const useEntityMaskAdapter = (): CanvasMaskAdapter => {
+  const adapter = useContext(EntityAdapterContext);
+  assert(adapter, 'useEntityMaskAdapter must be used within a CanvasMaskAdapterGate');
+  assert(adapter.type === 'mask_adapter', 'useEntityMaskAdapter must be used with a mask adapter');
+  return adapter;
+};
+
+export const useEntityAdapter = (): CanvasLayerAdapter | CanvasMaskAdapter => {
+  const adapter = useContext(EntityAdapterContext);
+  assert(adapter, 'useEntityAdapter must be used within a CanvasRasterLayerAdapterGate');
+  return adapter;
+};

invokeai/frontend/web/src/features/controlLayers/hooks/useEntityAdapter.ts

@@ -12,7 +12,7 @@ export const useEntityAdapter = (entityIdentifier: CanvasEntityIdentifier): Canv
     const entity = canvasManager.stateApi.getEntity(entityIdentifier);
     assert(entity, 'Entity adapter not found');
     return entity.adapter;
-  }, [canvasManager, entityIdentifier]);
+  }, [canvasManager.stateApi, entityIdentifier]);
 
   return adapter;
 };

invokeai/frontend/web/src/features/controlLayers/hooks/useEntityLayerAdapter.tsx

@@ -1,37 +0,0 @@
-import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
-import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
-import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
-import type { PropsWithChildren } from 'react';
-import { createContext, memo, useContext, useMemo } from 'react';
-import { assert } from 'tsafe';
-
-const EntityLayerAdapterContext = createContext<CanvasLayerAdapter | null>(null);
-
-export const EntityLayerAdapterProviderGate = memo(({ children }: PropsWithChildren) => {
-  const entityIdentifier = useEntityIdentifierContext();
-  const canvasManager = useCanvasManager();
-  const adapter = useMemo(() => {
-    if (entityIdentifier.type === 'raster_layer') {
-      return canvasManager.rasterLayerAdapters.get(entityIdentifier.id) ?? null;
-    } else if (entityIdentifier.type === 'control_layer') {
-      return canvasManager.controlLayerAdapters.get(entityIdentifier.id) ?? null;
-    }
-    assert(false, 'EntityLayerAdapterProviderGate must be used with a valid EntityIdentifierContext');
-  }, [canvasManager, entityIdentifier]);
-
-  if (!canvasManager) {
-    return null;
-  }
-
-  return <EntityLayerAdapterContext.Provider value={adapter}>{children}</EntityLayerAdapterContext.Provider>;
-});
-
-EntityLayerAdapterProviderGate.displayName = 'EntityLayerAdapterProviderGate';
-
-export const useEntityLayerAdapter = (): CanvasLayerAdapter => {
-  const adapter = useContext(EntityLayerAdapterContext);
-
-  assert(adapter, 'useEntityLayerAdapter must be used within a EntityLayerAdapterProviderGate');
-
-  return adapter;
-};

invokeai/frontend/web/src/features/controlLayers/hooks/useEntityMaskAdapter.tsx

@@ -1,37 +0,0 @@
-import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
-import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
-import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
-import type { PropsWithChildren } from 'react';
-import { createContext, memo, useContext, useMemo } from 'react';
-import { assert } from 'tsafe';
-
-const EntityMaskAdapterContext = createContext<CanvasMaskAdapter | null>(null);
-
-export const EntityMaskAdapterProviderGate = memo(({ children }: PropsWithChildren) => {
-  const entityIdentifier = useEntityIdentifierContext();
-  const canvasManager = useCanvasManager();
-  const adapter = useMemo(() => {
-    if (entityIdentifier.type === 'inpaint_mask') {
-      return canvasManager.inpaintMaskAdapters.get(entityIdentifier.id) ?? null;
-    } else if (entityIdentifier.type === 'regional_guidance') {
-      return canvasManager.regionalGuidanceAdapters.get(entityIdentifier.id) ?? null;
-    }
-    assert(false, 'EntityMaskAdapterProviderGate must be used with a valid EntityIdentifierContext');
-  }, [canvasManager, entityIdentifier]);
-
-  if (!canvasManager) {
-    return null;
-  }
-
-  return <EntityMaskAdapterContext.Provider value={adapter}>{children}</EntityMaskAdapterContext.Provider>;
-});
-
-EntityMaskAdapterProviderGate.displayName = 'EntityMaskAdapterProviderGate';
-
-export const useEntityMaskAdapter = (): CanvasMaskAdapter => {
-  const adapter = useContext(EntityMaskAdapterContext);
-
-  assert(adapter, 'useEntityMaskAdapter must be used within a EntityLayerAdapterProviderGate');
-
-  return adapter;
-};

invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts

@@ -30,7 +30,7 @@ export class CanvasCompositorModule {
 
   getCompositeRasterLayerEntityIds = (): string[] => {
     const ids = [];
-    for (const adapter of this.manager.rasterLayerAdapters.values()) {
+    for (const adapter of this.manager.adapters.rasterLayers.values()) {
       if (adapter.state.isEnabled && adapter.renderer.hasObjects()) {
         ids.push(adapter.id);
       }
@@ -40,7 +40,7 @@ export class CanvasCompositorModule {
 
   getCompositeInpaintMaskEntityIds = (): string[] => {
     const ids = [];
-    for (const adapter of this.manager.inpaintMaskAdapters.values()) {
+    for (const adapter of this.manager.adapters.inpaintMasks.values()) {
       if (adapter.state.isEnabled && adapter.renderer.hasObjects()) {
         ids.push(adapter.id);
       }
@@ -67,7 +67,7 @@ export class CanvasCompositorModule {
     assert(ctx !== null, 'Canvas 2D context is null');
 
     for (const id of this.getCompositeRasterLayerEntityIds()) {
-      const adapter = this.manager.rasterLayerAdapters.get(id);
+      const adapter = this.manager.adapters.rasterLayers.get(id);
       if (!adapter) {
         this.log.warn({ id }, 'Raster layer adapter not found');
         continue;
@@ -99,7 +99,7 @@ export class CanvasCompositorModule {
     assert(ctx !== null);
 
     for (const id of this.getCompositeInpaintMaskEntityIds()) {
-      const adapter = this.manager.inpaintMaskAdapters.get(id);
+      const adapter = this.manager.adapters.inpaintMasks.get(id);
       if (!adapter) {
         this.log.warn({ id }, 'Inpaint mask adapter not found');
         continue;
@@ -117,7 +117,7 @@ export class CanvasCompositorModule {
       extra,
     };
     for (const id of this.getCompositeRasterLayerEntityIds()) {
-      const adapter = this.manager.rasterLayerAdapters.get(id);
+      const adapter = this.manager.adapters.rasterLayers.get(id);
       if (!adapter) {
         this.log.warn({ id }, 'Raster layer adapter not found');
         continue;
@@ -132,7 +132,7 @@ export class CanvasCompositorModule {
       extra,
     };
     for (const id of this.getCompositeInpaintMaskEntityIds()) {
-      const adapter = this.manager.inpaintMaskAdapters.get(id);
+      const adapter = this.manager.adapters.inpaintMasks.get(id);
       if (!adapter) {
         this.log.warn({ id }, 'Inpaint mask adapter not found');
         continue;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts

@@ -9,6 +9,7 @@ import { CanvasRenderingModule } from 'features/controlLayers/konva/CanvasRender
 import { CanvasStageModule } from 'features/controlLayers/konva/CanvasStageModule';
 import { CanvasWorkerModule } from 'features/controlLayers/konva/CanvasWorkerModule.js';
 import { getPrefixedId } from 'features/controlLayers/konva/util';
+import { SyncableMap } from 'common/util/SyncableMap/SyncableMap';
 import type Konva from 'konva';
 import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
@@ -32,10 +33,20 @@ export class CanvasManager {
   store: AppStore;
   socket: AppSocket;
 
-  rasterLayerAdapters: Map<string, CanvasLayerAdapter> = new Map();
+  adapters = {
-  controlLayerAdapters: Map<string, CanvasLayerAdapter> = new Map();
+    rasterLayers: new SyncableMap<string, CanvasLayerAdapter>(),
-  regionalGuidanceAdapters: Map<string, CanvasMaskAdapter> = new Map();
+    controlLayers: new SyncableMap<string, CanvasLayerAdapter>(),
-  inpaintMaskAdapters: Map<string, CanvasMaskAdapter> = new Map();
+    regionMasks: new SyncableMap<string, CanvasMaskAdapter>(),
+    inpaintMasks: new SyncableMap<string, CanvasMaskAdapter>(),
+    getAll: (): (CanvasLayerAdapter | CanvasMaskAdapter)[] => {
+      return [
+        ...this.adapters.rasterLayers.values(),
+        ...this.adapters.controlLayers.values(),
+        ...this.adapters.regionMasks.values(),
+        ...this.adapters.inpaintMasks.values(),
+      ];
+    },
+  };
 
   stateApi: CanvasStateApiModule;
   preview: CanvasPreviewModule;
@@ -105,13 +116,7 @@ export class CanvasManager {
 
     return () => {
       this.log.debug('Cleaning up canvas manager');
-      const allAdapters = [
+      for (const adapter of this.adapters.getAll()) {
-        ...this.rasterLayerAdapters.values(),
-        ...this.controlLayerAdapters.values(),
-        ...this.inpaintMaskAdapters.values(),
-        ...this.regionalGuidanceAdapters.values(),
-      ];
-      for (const adapter of allAdapters) {
         adapter.destroy();
       }
       this.background.destroy();
@@ -148,9 +153,9 @@ export class CanvasManager {
   logDebugInfo() {
     // eslint-disable-next-line no-console
     console.log(this);
-    for (const layer of this.rasterLayerAdapters.values()) {
+    for (const adapter of this.adapters.getAll()) {
       // eslint-disable-next-line no-console
-      console.log(layer);
+      console.log(adapter);
     }
   }
 

invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts

@@ -27,6 +27,7 @@ import type {
 import { imageDTOToImageObject } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 import type { GroupConfig } from 'konva/lib/Group';
+import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
 import { getImageDTO, uploadImage } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
@@ -118,6 +119,8 @@ export class CanvasObjectRenderer {
     } | null;
   };
 
+  $canvasCache = atom<{ canvas: HTMLCanvasElement; rect: Rect } | null>(null);
+
   constructor(parent: CanvasLayerAdapter | CanvasMaskAdapter) {
     this.id = getPrefixedId(this.type);
     this.parent = parent;
@@ -205,7 +208,10 @@ export class CanvasObjectRenderer {
     } else if (force || !this.konva.objectGroup.isCached()) {
       this.log.trace('Caching object group');
       this.konva.objectGroup.clearCache();
-      this.konva.objectGroup.cache();
+      this.konva.objectGroup.cache({ pixelRatio: 1 });
+      if (!this.parent.transformer.isPendingRectCalculation) {
+        this.parent.renderer.updatePreviewCanvas();
+      }
     }
   };
 
@@ -530,6 +536,24 @@ export class CanvasObjectRenderer {
     return imageDTO;
   };
 
+  updatePreviewCanvas = () => {
+    if (this.parent.transformer.pixelRect.width === 0 || this.parent.transformer.pixelRect.height === 0) {
+      return;
+    }
+    const canvas = this.konva.objectGroup._getCachedSceneCanvas()._canvas as HTMLCanvasElement | undefined | null;
+    if (canvas) {
+      const nodeRect = this.parent.transformer.nodeRect;
+      const pixelRect = this.parent.transformer.pixelRect;
+      const rect = {
+        x: pixelRect.x - nodeRect.x,
+        y: pixelRect.y - nodeRect.y,
+        width: pixelRect.width,
+        height: pixelRect.height,
+      };
+      this.$canvasCache.set({ rect, canvas });
+    }
+  };
+
   cloneObjectGroup = (attrs?: GroupConfig): Konva.Group => {
     const clone = this.konva.objectGroup.clone();
     clone.cache();

invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts

@@ -68,25 +68,27 @@ export class CanvasRenderingModule {
   };
 
   renderRasterLayers = async (state: CanvasV2State, prevState: CanvasV2State | null) => {
+    const adapterMap = this.manager.adapters.rasterLayers;
+
     if (!prevState || state.rasterLayers.isHidden !== prevState.rasterLayers.isHidden) {
-      for (const adapter of this.manager.rasterLayerAdapters.values()) {
+      for (const adapter of adapterMap.values()) {
         adapter.renderer.updateOpacity(state.rasterLayers.isHidden ? 0 : adapter.state.opacity);
       }
     }
 
     if (!prevState || state.rasterLayers.entities !== prevState.rasterLayers.entities) {
-      for (const entityAdapter of this.manager.rasterLayerAdapters.values()) {
+      for (const entityAdapter of adapterMap.values()) {
         if (!state.rasterLayers.entities.find((l) => l.id === entityAdapter.id)) {
           await entityAdapter.destroy();
-          this.manager.rasterLayerAdapters.delete(entityAdapter.id);
+          adapterMap.delete(entityAdapter.id);
         }
       }
 
       for (const entityState of state.rasterLayers.entities) {
-        let adapter = this.manager.rasterLayerAdapters.get(entityState.id);
+        let adapter = adapterMap.get(entityState.id);
         if (!adapter) {
           adapter = new CanvasLayerAdapter(entityState, this.manager);
-          this.manager.rasterLayerAdapters.set(adapter.id, adapter);
+          adapterMap.set(adapter.id, adapter);
           this.manager.stage.addLayer(adapter.konva.layer);
         }
         await adapter.update({
@@ -99,25 +101,27 @@ export class CanvasRenderingModule {
   };
 
   renderControlLayers = async (prevState: CanvasV2State | null, state: CanvasV2State) => {
+    const adapterMap = this.manager.adapters.controlLayers;
+
     if (!prevState || state.controlLayers.isHidden !== prevState.controlLayers.isHidden) {
-      for (const adapter of this.manager.controlLayerAdapters.values()) {
+      for (const adapter of adapterMap.values()) {
         adapter.renderer.updateOpacity(state.controlLayers.isHidden ? 0 : adapter.state.opacity);
       }
     }
 
     if (!prevState || state.controlLayers.entities !== prevState.controlLayers.entities) {
-      for (const entityAdapter of this.manager.controlLayerAdapters.values()) {
+      for (const entityAdapter of adapterMap.values()) {
         if (!state.controlLayers.entities.find((l) => l.id === entityAdapter.id)) {
           await entityAdapter.destroy();
-          this.manager.controlLayerAdapters.delete(entityAdapter.id);
+          adapterMap.delete(entityAdapter.id);
         }
       }
 
       for (const entityState of state.controlLayers.entities) {
-        let adapter = this.manager.controlLayerAdapters.get(entityState.id);
+        let adapter = adapterMap.get(entityState.id);
         if (!adapter) {
           adapter = new CanvasLayerAdapter(entityState, this.manager);
-          this.manager.controlLayerAdapters.set(adapter.id, adapter);
+          adapterMap.set(adapter.id, adapter);
           this.manager.stage.addLayer(adapter.konva.layer);
         }
         await adapter.update({
@@ -130,8 +134,10 @@ export class CanvasRenderingModule {
   };
 
   renderRegionalGuidance = async (prevState: CanvasV2State | null, state: CanvasV2State) => {
+    const adapterMap = this.manager.adapters.regionMasks;
+
     if (!prevState || state.regions.isHidden !== prevState.regions.isHidden) {
-      for (const adapter of this.manager.regionalGuidanceAdapters.values()) {
+      for (const adapter of adapterMap.values()) {
         adapter.renderer.updateOpacity(state.regions.isHidden ? 0 : adapter.state.opacity);
       }
     }
@@ -143,18 +149,18 @@ export class CanvasRenderingModule {
       state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id
     ) {
       // Destroy the konva nodes for nonexistent entities
-      for (const canvasRegion of this.manager.regionalGuidanceAdapters.values()) {
+      for (const canvasRegion of adapterMap.values()) {
         if (!state.regions.entities.find((rg) => rg.id === canvasRegion.id)) {
           canvasRegion.destroy();
-          this.manager.regionalGuidanceAdapters.delete(canvasRegion.id);
+          adapterMap.delete(canvasRegion.id);
         }
       }
 
       for (const entityState of state.regions.entities) {
-        let adapter = this.manager.regionalGuidanceAdapters.get(entityState.id);
+        let adapter = adapterMap.get(entityState.id);
         if (!adapter) {
           adapter = new CanvasMaskAdapter(entityState, this.manager);
-          this.manager.regionalGuidanceAdapters.set(adapter.id, adapter);
+          adapterMap.set(adapter.id, adapter);
           this.manager.stage.addLayer(adapter.konva.layer);
         }
         await adapter.update({
@@ -167,8 +173,10 @@ export class CanvasRenderingModule {
   };
 
   renderInpaintMasks = async (state: CanvasV2State, prevState: CanvasV2State | null) => {
+    const adapterMap = this.manager.adapters.inpaintMasks;
+
     if (!prevState || state.inpaintMasks.isHidden !== prevState.inpaintMasks.isHidden) {
-      for (const adapter of this.manager.inpaintMaskAdapters.values()) {
+      for (const adapter of adapterMap.values()) {
         adapter.renderer.updateOpacity(state.inpaintMasks.isHidden ? 0 : adapter.state.opacity);
       }
     }
@@ -180,18 +188,18 @@ export class CanvasRenderingModule {
       state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id
     ) {
       // Destroy the konva nodes for nonexistent entities
-      for (const adapter of this.manager.inpaintMaskAdapters.values()) {
+      for (const adapter of adapterMap.values()) {
         if (!state.inpaintMasks.entities.find((rg) => rg.id === adapter.id)) {
           adapter.destroy();
-          this.manager.inpaintMaskAdapters.delete(adapter.id);
+          adapterMap.delete(adapter.id);
         }
       }
 
       for (const entityState of state.inpaintMasks.entities) {
-        let adapter = this.manager.inpaintMaskAdapters.get(entityState.id);
+        let adapter = adapterMap.get(entityState.id);
         if (!adapter) {
           adapter = new CanvasMaskAdapter(entityState, this.manager);
-          this.manager.inpaintMaskAdapters.set(adapter.id, adapter);
+          adapterMap.set(adapter.id, adapter);
           this.manager.stage.addLayer(adapter.konva.layer);
         }
         await adapter.update({
@@ -239,19 +247,19 @@ export class CanvasRenderingModule {
       this.manager.background.konva.layer.zIndex(++zIndex);
 
       for (const { id } of this.manager.stateApi.getRasterLayersState().entities) {
-        this.manager.rasterLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex);
+        this.manager.adapters.rasterLayers.get(id)?.konva.layer.zIndex(++zIndex);
       }
 
       for (const { id } of this.manager.stateApi.getControlLayersState().entities) {
-        this.manager.controlLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex);
+        this.manager.adapters.controlLayers.get(id)?.konva.layer.zIndex(++zIndex);
       }
 
       for (const { id } of this.manager.stateApi.getRegionsState().entities) {
-        this.manager.regionalGuidanceAdapters.get(id)?.konva.layer.zIndex(++zIndex);
+        this.manager.adapters.regionMasks.get(id)?.konva.layer.zIndex(++zIndex);
       }
 
       for (const { id } of this.manager.stateApi.getInpaintMasksState().entities) {
-        this.manager.inpaintMaskAdapters.get(id)?.konva.layer.zIndex(++zIndex);
+        this.manager.adapters.inpaintMasks.get(id)?.konva.layer.zIndex(++zIndex);
       }
 
       this.manager.preview.getLayer().zIndex(++zIndex);

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStageModule.ts

@@ -58,25 +58,7 @@ export class CanvasStageModule {
   getVisibleRect = (): Rect => {
     const rects = [];
 
-    for (const adapter of this.manager.inpaintMaskAdapters.values()) {
+    for (const adapter of this.manager.adapters.getAll()) {
-      if (adapter.state.isEnabled) {
-        rects.push(adapter.transformer.getRelativeRect());
-      }
-    }
-
-    for (const adapter of this.manager.rasterLayerAdapters.values()) {
-      if (adapter.state.isEnabled) {
-        rects.push(adapter.transformer.getRelativeRect());
-      }
-    }
-
-    for (const adapter of this.manager.controlLayerAdapters.values()) {
-      if (adapter.state.isEnabled) {
-        rects.push(adapter.transformer.getRelativeRect());
-      }
-    }
-
-    for (const adapter of this.manager.regionalGuidanceAdapters.values()) {
       if (adapter.state.isEnabled) {
         rects.push(adapter.transformer.getRelativeRect());
       }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApiModule.ts

@@ -174,16 +174,16 @@ export class CanvasStateApiModule {
 
     if (identifier.type === 'raster_layer') {
       entityState = state.rasterLayers.entities.find((i) => i.id === identifier.id) ?? null;
-      entityAdapter = this.manager.rasterLayerAdapters.get(identifier.id) ?? null;
+      entityAdapter = this.manager.adapters.rasterLayers.get(identifier.id) ?? null;
     } else if (identifier.type === 'control_layer') {
       entityState = state.controlLayers.entities.find((i) => i.id === identifier.id) ?? null;
-      entityAdapter = this.manager.controlLayerAdapters.get(identifier.id) ?? null;
+      entityAdapter = this.manager.adapters.controlLayers.get(identifier.id) ?? null;
     } else if (identifier.type === 'regional_guidance') {
       entityState = state.regions.entities.find((i) => i.id === identifier.id) ?? null;
-      entityAdapter = this.manager.regionalGuidanceAdapters.get(identifier.id) ?? null;
+      entityAdapter = this.manager.adapters.regionMasks.get(identifier.id) ?? null;
     } else if (identifier.type === 'inpaint_mask') {
       entityState = state.inpaintMasks.entities.find((i) => i.id === identifier.id) ?? null;
-      entityAdapter = this.manager.inpaintMaskAdapters.get(identifier.id) ?? null;
+      entityAdapter = this.manager.adapters.inpaintMasks.get(identifier.id) ?? null;
     }
 
     if (entityState && entityAdapter) {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts

@@ -496,7 +496,7 @@ export class CanvasTransformer {
   startTransform = () => {
     this.log.debug('Starting transform');
     this.isTransforming = true;
-    this.manager.stateApi.setTool('move')
+    this.manager.stateApi.setTool('move');
     // When transforming, we want the stage to still be movable if the view tool is selected. If the transformer or
     // interaction rect are listening, it will interrupt the stage's drag events. So we should disable listening
     // when the view tool is selected
@@ -605,6 +605,7 @@ export class CanvasTransformer {
 
     if (this.isPendingRectCalculation) {
       this.syncInteractionState();
+      this.parent.renderer.updatePreviewCanvas();
       return;
     }
 
@@ -615,6 +616,7 @@ export class CanvasTransformer {
       // The layer is fully transparent but has objects - reset it
       this.manager.stateApi.resetEntity({ entityIdentifier: this.parent.getEntityIdentifier() });
       this.syncInteractionState();
+      this.parent.renderer.updatePreviewCanvas();
       return;
     }
 
@@ -628,6 +630,7 @@ export class CanvasTransformer {
     };
     this.parent.renderer.konva.objectGroup.setAttrs(groupAttrs);
     this.parent.renderer.konva.bufferGroup.setAttrs(groupAttrs);
+    this.parent.renderer.updatePreviewCanvas();
   };
 
   calculateRect = debounce(() => {

invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts

@@ -23,7 +23,7 @@ export const addControlAdapters = async (
     .filter((layer) => isValidControlAdapter(layer.controlAdapter, base));
 
   for (const layer of validControlLayers) {
-    const adapter = manager.controlLayerAdapters.get(layer.id);
+    const adapter = manager.adapters.controlLayers.get(layer.id);
     assert(adapter, 'Adapter not found');
     const imageDTO = await adapter.renderer.rasterize({ rect: bbox, attrs: { opacity: 1, filters: [] } });
     if (layer.controlAdapter.type === 'controlnet') {

invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts

@@ -47,7 +47,7 @@ export const addRegions = async (
   const validRegions = regions.filter((rg) => isValidRegion(rg, base));
 
   for (const region of validRegions) {
-    const adapter = manager.regionalGuidanceAdapters.get(region.id);
+    const adapter = manager.adapters.regionMasks.get(region.id);
     assert(adapter, 'Adapter not found');
     const imageDTO = await adapter.renderer.rasterize({ rect: bbox });