From 78a59b5b7864ff24e623c8d6f4cbc0467af4a82c Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:36:21 +1000 Subject: [PATCH] feat(ui): canvas layer preview, revised reactivity for adapters --- .../components/ControlLayer/ControlLayer.tsx | 8 +- .../components/ControlLayersToolbar.tsx | 17 +++- .../components/InpaintMask/InpaintMask.tsx | 8 +- .../components/RasterLayer/RasterLayer.tsx | 8 +- .../RegionalGuidance/RegionalGuidance.tsx | 8 +- .../CanvasSettingsRecalculateRectsButton.tsx | 8 +- .../common/CanvasEntityContainer.tsx | 1 - .../components/common/CanvasEntityHeader.tsx | 2 +- .../common/CanvasEntityPreviewImage.tsx | 59 +++++++++++++ .../contexts/EntityAdapterContext.tsx | 82 +++++++++++++++++++ .../controlLayers/hooks/useEntityAdapter.ts | 2 +- .../hooks/useEntityLayerAdapter.tsx | 37 --------- .../hooks/useEntityMaskAdapter.tsx | 37 --------- .../konva/CanvasCompositorModule.ts | 12 +-- .../controlLayers/konva/CanvasManager.ts | 31 ++++--- .../konva/CanvasObjectRenderer.ts | 26 +++++- .../konva/CanvasRenderingModule.ts | 56 +++++++------ .../controlLayers/konva/CanvasStageModule.ts | 20 +---- .../konva/CanvasStateApiModule.ts | 8 +- .../controlLayers/konva/CanvasTransformer.ts | 5 +- .../graph/generation/addControlAdapters.ts | 2 +- .../nodes/util/graph/generation/addRegions.ts | 2 +- 22 files changed, 271 insertions(+), 168 deletions(-) create mode 100644 invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityPreviewImage.tsx create mode 100644 invokeai/frontend/web/src/features/controlLayers/contexts/EntityAdapterContext.tsx delete mode 100644 invokeai/frontend/web/src/features/controlLayers/hooks/useEntityLayerAdapter.tsx delete mode 100644 invokeai/frontend/web/src/features/controlLayers/hooks/useEntityMaskAdapter.tsx diff --git a/invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayer.tsx b/invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayer.tsx index 84a23e8346..7393cf080c 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayer.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayer.tsx @@ -3,11 +3,12 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/ import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton'; import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle'; import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader'; +import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage'; import { CanvasEntitySettingsWrapper } from 'features/controlLayers/components/common/CanvasEntitySettingsWrapper'; import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit'; import { ControlLayerControlAdapter } from 'features/controlLayers/components/ControlLayer/ControlLayerControlAdapter'; +import { EntityLayerAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext'; import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; -import { EntityLayerAdapterProviderGate } from 'features/controlLayers/hooks/useEntityLayerAdapter'; import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types'; import { memo, useMemo } from 'react'; @@ -20,9 +21,10 @@ export const ControlLayer = memo(({ id }: Props) => { return ( - + + @@ -32,7 +34,7 @@ export const ControlLayer = memo(({ id }: Props) => { - + ); }); diff --git a/invokeai/frontend/web/src/features/controlLayers/components/ControlLayersToolbar.tsx b/invokeai/frontend/web/src/features/controlLayers/components/ControlLayersToolbar.tsx index 26df1cf1d8..a82f2da707 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/ControlLayersToolbar.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/ControlLayersToolbar.tsx @@ -10,16 +10,17 @@ import { ToolChooser } from 'features/controlLayers/components/Tool/ToolChooser' import { ToolEraserWidth } from 'features/controlLayers/components/Tool/ToolEraserWidth'; import { ToolFillColorPicker } from 'features/controlLayers/components/Tool/ToolFillColorPicker'; import { UndoRedoButtonGroup } from 'features/controlLayers/components/UndoRedoButtonGroup'; -import { CanvasManagerProviderGate } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; +import { CanvasManagerProviderGate, useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; import { ToggleProgressButton } from 'features/gallery/components/ImageViewer/ToggleProgressButton'; import { ViewerToggleMenu } from 'features/gallery/components/ImageViewer/ViewerToggleMenu'; -import { memo } from 'react'; +import { memo, useSyncExternalStore } from 'react'; export const ControlLayersToolbar = memo(() => { const tool = useAppSelector((s) => s.canvasV2.tool.selected); return ( + @@ -40,3 +41,15 @@ export const ControlLayersToolbar = memo(() => { }); ControlLayersToolbar.displayName = 'ControlLayersToolbar'; + +const ReactiveTest = () => { + const canvasManager = useCanvasManager(); + const adapters = useSyncExternalStore( + canvasManager.adapters.rasterLayers.subscribe, + canvasManager.adapters.rasterLayers.getSnapshot + ); + + console.log(adapters); + + return null; +}; diff --git a/invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMask.tsx b/invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMask.tsx index 4908a438d5..817fddd1fa 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMask.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMask.tsx @@ -3,9 +3,10 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/ import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton'; import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle'; import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader'; +import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage'; import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit'; +import { EntityMaskAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext'; import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; -import { EntityMaskAdapterProviderGate } from 'features/controlLayers/hooks/useEntityMaskAdapter'; import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types'; import { memo, useMemo } from 'react'; @@ -20,9 +21,10 @@ export const InpaintMask = memo(({ id }: Props) => { return ( - + + @@ -30,7 +32,7 @@ export const InpaintMask = memo(({ id }: Props) => { - + ); }); diff --git a/invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayer.tsx b/invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayer.tsx index 400851b107..f031725dbc 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayer.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayer.tsx @@ -3,9 +3,10 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/ import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton'; import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle'; import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader'; +import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage'; import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit'; +import { EntityLayerAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext'; import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; -import { EntityLayerAdapterProviderGate } from 'features/controlLayers/hooks/useEntityLayerAdapter'; import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types'; import { memo, useMemo } from 'react'; @@ -18,16 +19,17 @@ export const RasterLayer = memo(({ id }: Props) => { return ( - + + - + ); }); diff --git a/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidance.tsx b/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidance.tsx index 12b545312a..f7ef19ee80 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidance.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidance.tsx @@ -3,11 +3,12 @@ import { CanvasEntityContainer } from 'features/controlLayers/components/common/ import { CanvasEntityDeleteButton } from 'features/controlLayers/components/common/CanvasEntityDeleteButton'; import { CanvasEntityEnabledToggle } from 'features/controlLayers/components/common/CanvasEntityEnabledToggle'; import { CanvasEntityHeader } from 'features/controlLayers/components/common/CanvasEntityHeader'; +import { CanvasEntityPreviewImage } from 'features/controlLayers/components/common/CanvasEntityPreviewImage'; import { CanvasEntityEditableTitle } from 'features/controlLayers/components/common/CanvasEntityTitleEdit'; import { RegionalGuidanceBadges } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceBadges'; import { RegionalGuidanceSettings } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceSettings'; +import { EntityMaskAdapterGate } from 'features/controlLayers/contexts/EntityAdapterContext'; import { EntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; -import { EntityMaskAdapterProviderGate } from 'features/controlLayers/hooks/useEntityMaskAdapter'; import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types'; import { memo, useMemo } from 'react'; @@ -23,9 +24,10 @@ export const RegionalGuidance = memo(({ id }: Props) => { return ( - + + @@ -36,7 +38,7 @@ export const RegionalGuidance = memo(({ id }: Props) => { - + ); }); diff --git a/invokeai/frontend/web/src/features/controlLayers/components/Settings/CanvasSettingsRecalculateRectsButton.tsx b/invokeai/frontend/web/src/features/controlLayers/components/Settings/CanvasSettingsRecalculateRectsButton.tsx index 7b3be93bdd..53394a3db5 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/Settings/CanvasSettingsRecalculateRectsButton.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/Settings/CanvasSettingsRecalculateRectsButton.tsx @@ -7,13 +7,7 @@ export const CanvasSettingsRecalculateRectsButton = memo(() => { const { t } = useTranslation(); const canvasManager = useCanvasManager(); const onClick = useCallback(() => { - const adapters = [ - ...canvasManager.rasterLayerAdapters.values(), - ...canvasManager.controlLayerAdapters.values(), - ...canvasManager.regionalGuidanceAdapters.values(), - ...canvasManager.inpaintMaskAdapters.values(), - ]; - for (const adapter of adapters) { + for (const adapter of canvasManager.adapters.getAll()) { adapter.transformer.requestRectCalculation(); } }, [canvasManager]); diff --git a/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityContainer.tsx b/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityContainer.tsx index 2e29a49817..f55283c1c0 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityContainer.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityContainer.tsx @@ -21,7 +21,6 @@ export const CanvasEntityContainer = memo((props: PropsWithChildren) => { return ( { return ( {(ref) => ( - + {children} )} diff --git a/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityPreviewImage.tsx b/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityPreviewImage.tsx new file mode 100644 index 0000000000..161eed1fda --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityPreviewImage.tsx @@ -0,0 +1,59 @@ +import { Box, chakra, Flex } from '@invoke-ai/ui-library'; +import { useStore } from '@nanostores/react'; +import { useEntityAdapter } from 'features/controlLayers/contexts/EntityAdapterContext'; +import { TRANSPARENCY_CHECKER_PATTERN } from 'features/controlLayers/konva/constants'; +import { memo, useEffect, useRef } from 'react'; + +const ChakraCanvas = chakra.canvas; + +export const CanvasEntityPreviewImage = memo(() => { + const adapter = useEntityAdapter(); + const containerRef = useRef(null); + const canvasRef = useRef(null); + const cache = useStore(adapter.renderer.$canvasCache); + useEffect(() => { + if (!cache || !canvasRef.current || !containerRef.current) { + return; + } + const ctx = canvasRef.current.getContext('2d'); + if (!ctx) { + return; + } + const { rect, canvas } = cache; + + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + + canvasRef.current.width = rect.width; + canvasRef.current.height = rect.height; + + ctx.drawImage(canvas, rect.x, rect.y, rect.width, rect.height, 0, 0, rect.width, rect.height); + }, [adapter.transformer, adapter.transformer.nodeRect, adapter.transformer.pixelRect, cache]); + + return ( + + + + + ); +}); + +CanvasEntityPreviewImage.displayName = 'CanvasEntityPreviewImage'; diff --git a/invokeai/frontend/web/src/features/controlLayers/contexts/EntityAdapterContext.tsx b/invokeai/frontend/web/src/features/controlLayers/contexts/EntityAdapterContext.tsx new file mode 100644 index 0000000000..1a091d284e --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/contexts/EntityAdapterContext.tsx @@ -0,0 +1,82 @@ +import type { SyncableMap } from 'common/util/SyncableMap/SyncableMap'; +import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; +import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; +import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter'; +import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter'; +import type { PropsWithChildren } from 'react'; +import { createContext, memo, useContext, useMemo, useSyncExternalStore } from 'react'; +import { assert } from 'tsafe'; + +const EntityAdapterContext = createContext(null); + +export const EntityLayerAdapterGate = memo(({ children }: PropsWithChildren) => { + const canvasManager = useCanvasManager(); + const entityIdentifier = useEntityIdentifierContext(); + const store = useMemo>(() => { + if (entityIdentifier.type === 'raster_layer') { + return canvasManager.adapters.rasterLayers; + } + if (entityIdentifier.type === 'control_layer') { + return canvasManager.adapters.controlLayers; + } + assert(false, 'Unknown entity type'); + }, [canvasManager.adapters.controlLayers, canvasManager.adapters.rasterLayers, entityIdentifier.type]); + const adapters = useSyncExternalStore(store.subscribe, store.getSnapshot); + const adapter = useMemo(() => { + return adapters.get(entityIdentifier.id) ?? null; + }, [adapters, entityIdentifier.id]); + + if (!adapter) { + return null; + } + + return {children}; +}); + +EntityLayerAdapterGate.displayName = 'EntityLayerAdapterGate'; + +export const useEntityLayerAdapter = (): CanvasLayerAdapter => { + const adapter = useContext(EntityAdapterContext); + assert(adapter, 'useEntityLayerAdapter must be used within a EntityLayerAdapterGate'); + assert(adapter.type === 'layer_adapter', 'useEntityLayerAdapter must be used with a layer adapter'); + return adapter; +}; + +export const EntityMaskAdapterGate = memo(({ children }: PropsWithChildren) => { + const canvasManager = useCanvasManager(); + const entityIdentifier = useEntityIdentifierContext(); + const store = useMemo>(() => { + if (entityIdentifier.type === 'inpaint_mask') { + return canvasManager.adapters.inpaintMasks; + } + if (entityIdentifier.type === 'regional_guidance') { + return canvasManager.adapters.regionMasks; + } + assert(false, 'Unknown entity type'); + }, [canvasManager.adapters.inpaintMasks, canvasManager.adapters.regionMasks, entityIdentifier.type]); + const adapters = useSyncExternalStore(store.subscribe, store.getSnapshot); + const adapter = useMemo(() => { + return adapters.get(entityIdentifier.id) ?? null; + }, [adapters, entityIdentifier.id]); + + if (!adapter) { + return null; + } + + return {children}; +}); + +EntityMaskAdapterGate.displayName = 'EntityMaskAdapterGate'; + +export const useEntityMaskAdapter = (): CanvasMaskAdapter => { + const adapter = useContext(EntityAdapterContext); + assert(adapter, 'useEntityMaskAdapter must be used within a CanvasMaskAdapterGate'); + assert(adapter.type === 'mask_adapter', 'useEntityMaskAdapter must be used with a mask adapter'); + return adapter; +}; + +export const useEntityAdapter = (): CanvasLayerAdapter | CanvasMaskAdapter => { + const adapter = useContext(EntityAdapterContext); + assert(adapter, 'useEntityAdapter must be used within a CanvasRasterLayerAdapterGate'); + return adapter; +}; diff --git a/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityAdapter.ts b/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityAdapter.ts index 63d93b85ca..9913faa3a4 100644 --- a/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityAdapter.ts +++ b/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityAdapter.ts @@ -12,7 +12,7 @@ export const useEntityAdapter = (entityIdentifier: CanvasEntityIdentifier): Canv const entity = canvasManager.stateApi.getEntity(entityIdentifier); assert(entity, 'Entity adapter not found'); return entity.adapter; - }, [canvasManager, entityIdentifier]); + }, [canvasManager.stateApi, entityIdentifier]); return adapter; }; diff --git a/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityLayerAdapter.tsx b/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityLayerAdapter.tsx deleted file mode 100644 index b572aa804b..0000000000 --- a/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityLayerAdapter.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; -import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; -import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter'; -import type { PropsWithChildren } from 'react'; -import { createContext, memo, useContext, useMemo } from 'react'; -import { assert } from 'tsafe'; - -const EntityLayerAdapterContext = createContext(null); - -export const EntityLayerAdapterProviderGate = memo(({ children }: PropsWithChildren) => { - const entityIdentifier = useEntityIdentifierContext(); - const canvasManager = useCanvasManager(); - const adapter = useMemo(() => { - if (entityIdentifier.type === 'raster_layer') { - return canvasManager.rasterLayerAdapters.get(entityIdentifier.id) ?? null; - } else if (entityIdentifier.type === 'control_layer') { - return canvasManager.controlLayerAdapters.get(entityIdentifier.id) ?? null; - } - assert(false, 'EntityLayerAdapterProviderGate must be used with a valid EntityIdentifierContext'); - }, [canvasManager, entityIdentifier]); - - if (!canvasManager) { - return null; - } - - return {children}; -}); - -EntityLayerAdapterProviderGate.displayName = 'EntityLayerAdapterProviderGate'; - -export const useEntityLayerAdapter = (): CanvasLayerAdapter => { - const adapter = useContext(EntityLayerAdapterContext); - - assert(adapter, 'useEntityLayerAdapter must be used within a EntityLayerAdapterProviderGate'); - - return adapter; -}; diff --git a/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityMaskAdapter.tsx b/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityMaskAdapter.tsx deleted file mode 100644 index 697d434c3b..0000000000 --- a/invokeai/frontend/web/src/features/controlLayers/hooks/useEntityMaskAdapter.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; -import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; -import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter'; -import type { PropsWithChildren } from 'react'; -import { createContext, memo, useContext, useMemo } from 'react'; -import { assert } from 'tsafe'; - -const EntityMaskAdapterContext = createContext(null); - -export const EntityMaskAdapterProviderGate = memo(({ children }: PropsWithChildren) => { - const entityIdentifier = useEntityIdentifierContext(); - const canvasManager = useCanvasManager(); - const adapter = useMemo(() => { - if (entityIdentifier.type === 'inpaint_mask') { - return canvasManager.inpaintMaskAdapters.get(entityIdentifier.id) ?? null; - } else if (entityIdentifier.type === 'regional_guidance') { - return canvasManager.regionalGuidanceAdapters.get(entityIdentifier.id) ?? null; - } - assert(false, 'EntityMaskAdapterProviderGate must be used with a valid EntityIdentifierContext'); - }, [canvasManager, entityIdentifier]); - - if (!canvasManager) { - return null; - } - - return {children}; -}); - -EntityMaskAdapterProviderGate.displayName = 'EntityMaskAdapterProviderGate'; - -export const useEntityMaskAdapter = (): CanvasMaskAdapter => { - const adapter = useContext(EntityMaskAdapterContext); - - assert(adapter, 'useEntityMaskAdapter must be used within a EntityLayerAdapterProviderGate'); - - return adapter; -}; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts index c7df665a46..f2f20b0937 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts @@ -30,7 +30,7 @@ export class CanvasCompositorModule { getCompositeRasterLayerEntityIds = (): string[] => { const ids = []; - for (const adapter of this.manager.rasterLayerAdapters.values()) { + for (const adapter of this.manager.adapters.rasterLayers.values()) { if (adapter.state.isEnabled && adapter.renderer.hasObjects()) { ids.push(adapter.id); } @@ -40,7 +40,7 @@ export class CanvasCompositorModule { getCompositeInpaintMaskEntityIds = (): string[] => { const ids = []; - for (const adapter of this.manager.inpaintMaskAdapters.values()) { + for (const adapter of this.manager.adapters.inpaintMasks.values()) { if (adapter.state.isEnabled && adapter.renderer.hasObjects()) { ids.push(adapter.id); } @@ -67,7 +67,7 @@ export class CanvasCompositorModule { assert(ctx !== null, 'Canvas 2D context is null'); for (const id of this.getCompositeRasterLayerEntityIds()) { - const adapter = this.manager.rasterLayerAdapters.get(id); + const adapter = this.manager.adapters.rasterLayers.get(id); if (!adapter) { this.log.warn({ id }, 'Raster layer adapter not found'); continue; @@ -99,7 +99,7 @@ export class CanvasCompositorModule { assert(ctx !== null); for (const id of this.getCompositeInpaintMaskEntityIds()) { - const adapter = this.manager.inpaintMaskAdapters.get(id); + const adapter = this.manager.adapters.inpaintMasks.get(id); if (!adapter) { this.log.warn({ id }, 'Inpaint mask adapter not found'); continue; @@ -117,7 +117,7 @@ export class CanvasCompositorModule { extra, }; for (const id of this.getCompositeRasterLayerEntityIds()) { - const adapter = this.manager.rasterLayerAdapters.get(id); + const adapter = this.manager.adapters.rasterLayers.get(id); if (!adapter) { this.log.warn({ id }, 'Raster layer adapter not found'); continue; @@ -132,7 +132,7 @@ export class CanvasCompositorModule { extra, }; for (const id of this.getCompositeInpaintMaskEntityIds()) { - const adapter = this.manager.inpaintMaskAdapters.get(id); + const adapter = this.manager.adapters.inpaintMasks.get(id); if (!adapter) { this.log.warn({ id }, 'Inpaint mask adapter not found'); continue; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts index 05d94c00e6..d07730df75 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts @@ -9,6 +9,7 @@ import { CanvasRenderingModule } from 'features/controlLayers/konva/CanvasRender import { CanvasStageModule } from 'features/controlLayers/konva/CanvasStageModule'; import { CanvasWorkerModule } from 'features/controlLayers/konva/CanvasWorkerModule.js'; import { getPrefixedId } from 'features/controlLayers/konva/util'; +import { SyncableMap } from 'common/util/SyncableMap/SyncableMap'; import type Konva from 'konva'; import { atom } from 'nanostores'; import type { Logger } from 'roarr'; @@ -32,10 +33,20 @@ export class CanvasManager { store: AppStore; socket: AppSocket; - rasterLayerAdapters: Map = new Map(); - controlLayerAdapters: Map = new Map(); - regionalGuidanceAdapters: Map = new Map(); - inpaintMaskAdapters: Map = new Map(); + adapters = { + rasterLayers: new SyncableMap(), + controlLayers: new SyncableMap(), + regionMasks: new SyncableMap(), + inpaintMasks: new SyncableMap(), + getAll: (): (CanvasLayerAdapter | CanvasMaskAdapter)[] => { + return [ + ...this.adapters.rasterLayers.values(), + ...this.adapters.controlLayers.values(), + ...this.adapters.regionMasks.values(), + ...this.adapters.inpaintMasks.values(), + ]; + }, + }; stateApi: CanvasStateApiModule; preview: CanvasPreviewModule; @@ -105,13 +116,7 @@ export class CanvasManager { return () => { this.log.debug('Cleaning up canvas manager'); - const allAdapters = [ - ...this.rasterLayerAdapters.values(), - ...this.controlLayerAdapters.values(), - ...this.inpaintMaskAdapters.values(), - ...this.regionalGuidanceAdapters.values(), - ]; - for (const adapter of allAdapters) { + for (const adapter of this.adapters.getAll()) { adapter.destroy(); } this.background.destroy(); @@ -148,9 +153,9 @@ export class CanvasManager { logDebugInfo() { // eslint-disable-next-line no-console console.log(this); - for (const layer of this.rasterLayerAdapters.values()) { + for (const adapter of this.adapters.getAll()) { // eslint-disable-next-line no-console - console.log(layer); + console.log(adapter); } } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts index 3d53538f82..3e03435ede 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts @@ -27,6 +27,7 @@ import type { import { imageDTOToImageObject } from 'features/controlLayers/store/types'; import Konva from 'konva'; import type { GroupConfig } from 'konva/lib/Group'; +import { atom } from 'nanostores'; import type { Logger } from 'roarr'; import { getImageDTO, uploadImage } from 'services/api/endpoints/images'; import type { ImageDTO } from 'services/api/types'; @@ -118,6 +119,8 @@ export class CanvasObjectRenderer { } | null; }; + $canvasCache = atom<{ canvas: HTMLCanvasElement; rect: Rect } | null>(null); + constructor(parent: CanvasLayerAdapter | CanvasMaskAdapter) { this.id = getPrefixedId(this.type); this.parent = parent; @@ -205,7 +208,10 @@ export class CanvasObjectRenderer { } else if (force || !this.konva.objectGroup.isCached()) { this.log.trace('Caching object group'); this.konva.objectGroup.clearCache(); - this.konva.objectGroup.cache(); + this.konva.objectGroup.cache({ pixelRatio: 1 }); + if (!this.parent.transformer.isPendingRectCalculation) { + this.parent.renderer.updatePreviewCanvas(); + } } }; @@ -530,6 +536,24 @@ export class CanvasObjectRenderer { return imageDTO; }; + updatePreviewCanvas = () => { + if (this.parent.transformer.pixelRect.width === 0 || this.parent.transformer.pixelRect.height === 0) { + return; + } + const canvas = this.konva.objectGroup._getCachedSceneCanvas()._canvas as HTMLCanvasElement | undefined | null; + if (canvas) { + const nodeRect = this.parent.transformer.nodeRect; + const pixelRect = this.parent.transformer.pixelRect; + const rect = { + x: pixelRect.x - nodeRect.x, + y: pixelRect.y - nodeRect.y, + width: pixelRect.width, + height: pixelRect.height, + }; + this.$canvasCache.set({ rect, canvas }); + } + }; + cloneObjectGroup = (attrs?: GroupConfig): Konva.Group => { const clone = this.konva.objectGroup.clone(); clone.cache(); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts index 901737e210..bde46351f9 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts @@ -68,25 +68,27 @@ export class CanvasRenderingModule { }; renderRasterLayers = async (state: CanvasV2State, prevState: CanvasV2State | null) => { + const adapterMap = this.manager.adapters.rasterLayers; + if (!prevState || state.rasterLayers.isHidden !== prevState.rasterLayers.isHidden) { - for (const adapter of this.manager.rasterLayerAdapters.values()) { + for (const adapter of adapterMap.values()) { adapter.renderer.updateOpacity(state.rasterLayers.isHidden ? 0 : adapter.state.opacity); } } if (!prevState || state.rasterLayers.entities !== prevState.rasterLayers.entities) { - for (const entityAdapter of this.manager.rasterLayerAdapters.values()) { + for (const entityAdapter of adapterMap.values()) { if (!state.rasterLayers.entities.find((l) => l.id === entityAdapter.id)) { await entityAdapter.destroy(); - this.manager.rasterLayerAdapters.delete(entityAdapter.id); + adapterMap.delete(entityAdapter.id); } } for (const entityState of state.rasterLayers.entities) { - let adapter = this.manager.rasterLayerAdapters.get(entityState.id); + let adapter = adapterMap.get(entityState.id); if (!adapter) { adapter = new CanvasLayerAdapter(entityState, this.manager); - this.manager.rasterLayerAdapters.set(adapter.id, adapter); + adapterMap.set(adapter.id, adapter); this.manager.stage.addLayer(adapter.konva.layer); } await adapter.update({ @@ -99,25 +101,27 @@ export class CanvasRenderingModule { }; renderControlLayers = async (prevState: CanvasV2State | null, state: CanvasV2State) => { + const adapterMap = this.manager.adapters.controlLayers; + if (!prevState || state.controlLayers.isHidden !== prevState.controlLayers.isHidden) { - for (const adapter of this.manager.controlLayerAdapters.values()) { + for (const adapter of adapterMap.values()) { adapter.renderer.updateOpacity(state.controlLayers.isHidden ? 0 : adapter.state.opacity); } } if (!prevState || state.controlLayers.entities !== prevState.controlLayers.entities) { - for (const entityAdapter of this.manager.controlLayerAdapters.values()) { + for (const entityAdapter of adapterMap.values()) { if (!state.controlLayers.entities.find((l) => l.id === entityAdapter.id)) { await entityAdapter.destroy(); - this.manager.controlLayerAdapters.delete(entityAdapter.id); + adapterMap.delete(entityAdapter.id); } } for (const entityState of state.controlLayers.entities) { - let adapter = this.manager.controlLayerAdapters.get(entityState.id); + let adapter = adapterMap.get(entityState.id); if (!adapter) { adapter = new CanvasLayerAdapter(entityState, this.manager); - this.manager.controlLayerAdapters.set(adapter.id, adapter); + adapterMap.set(adapter.id, adapter); this.manager.stage.addLayer(adapter.konva.layer); } await adapter.update({ @@ -130,8 +134,10 @@ export class CanvasRenderingModule { }; renderRegionalGuidance = async (prevState: CanvasV2State | null, state: CanvasV2State) => { + const adapterMap = this.manager.adapters.regionMasks; + if (!prevState || state.regions.isHidden !== prevState.regions.isHidden) { - for (const adapter of this.manager.regionalGuidanceAdapters.values()) { + for (const adapter of adapterMap.values()) { adapter.renderer.updateOpacity(state.regions.isHidden ? 0 : adapter.state.opacity); } } @@ -143,18 +149,18 @@ export class CanvasRenderingModule { state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id ) { // Destroy the konva nodes for nonexistent entities - for (const canvasRegion of this.manager.regionalGuidanceAdapters.values()) { + for (const canvasRegion of adapterMap.values()) { if (!state.regions.entities.find((rg) => rg.id === canvasRegion.id)) { canvasRegion.destroy(); - this.manager.regionalGuidanceAdapters.delete(canvasRegion.id); + adapterMap.delete(canvasRegion.id); } } for (const entityState of state.regions.entities) { - let adapter = this.manager.regionalGuidanceAdapters.get(entityState.id); + let adapter = adapterMap.get(entityState.id); if (!adapter) { adapter = new CanvasMaskAdapter(entityState, this.manager); - this.manager.regionalGuidanceAdapters.set(adapter.id, adapter); + adapterMap.set(adapter.id, adapter); this.manager.stage.addLayer(adapter.konva.layer); } await adapter.update({ @@ -167,8 +173,10 @@ export class CanvasRenderingModule { }; renderInpaintMasks = async (state: CanvasV2State, prevState: CanvasV2State | null) => { + const adapterMap = this.manager.adapters.inpaintMasks; + if (!prevState || state.inpaintMasks.isHidden !== prevState.inpaintMasks.isHidden) { - for (const adapter of this.manager.inpaintMaskAdapters.values()) { + for (const adapter of adapterMap.values()) { adapter.renderer.updateOpacity(state.inpaintMasks.isHidden ? 0 : adapter.state.opacity); } } @@ -180,18 +188,18 @@ export class CanvasRenderingModule { state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id ) { // Destroy the konva nodes for nonexistent entities - for (const adapter of this.manager.inpaintMaskAdapters.values()) { + for (const adapter of adapterMap.values()) { if (!state.inpaintMasks.entities.find((rg) => rg.id === adapter.id)) { adapter.destroy(); - this.manager.inpaintMaskAdapters.delete(adapter.id); + adapterMap.delete(adapter.id); } } for (const entityState of state.inpaintMasks.entities) { - let adapter = this.manager.inpaintMaskAdapters.get(entityState.id); + let adapter = adapterMap.get(entityState.id); if (!adapter) { adapter = new CanvasMaskAdapter(entityState, this.manager); - this.manager.inpaintMaskAdapters.set(adapter.id, adapter); + adapterMap.set(adapter.id, adapter); this.manager.stage.addLayer(adapter.konva.layer); } await adapter.update({ @@ -239,19 +247,19 @@ export class CanvasRenderingModule { this.manager.background.konva.layer.zIndex(++zIndex); for (const { id } of this.manager.stateApi.getRasterLayersState().entities) { - this.manager.rasterLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex); + this.manager.adapters.rasterLayers.get(id)?.konva.layer.zIndex(++zIndex); } for (const { id } of this.manager.stateApi.getControlLayersState().entities) { - this.manager.controlLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex); + this.manager.adapters.controlLayers.get(id)?.konva.layer.zIndex(++zIndex); } for (const { id } of this.manager.stateApi.getRegionsState().entities) { - this.manager.regionalGuidanceAdapters.get(id)?.konva.layer.zIndex(++zIndex); + this.manager.adapters.regionMasks.get(id)?.konva.layer.zIndex(++zIndex); } for (const { id } of this.manager.stateApi.getInpaintMasksState().entities) { - this.manager.inpaintMaskAdapters.get(id)?.konva.layer.zIndex(++zIndex); + this.manager.adapters.inpaintMasks.get(id)?.konva.layer.zIndex(++zIndex); } this.manager.preview.getLayer().zIndex(++zIndex); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStageModule.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStageModule.ts index a88b48913b..624027d177 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStageModule.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStageModule.ts @@ -58,25 +58,7 @@ export class CanvasStageModule { getVisibleRect = (): Rect => { const rects = []; - for (const adapter of this.manager.inpaintMaskAdapters.values()) { - if (adapter.state.isEnabled) { - rects.push(adapter.transformer.getRelativeRect()); - } - } - - for (const adapter of this.manager.rasterLayerAdapters.values()) { - if (adapter.state.isEnabled) { - rects.push(adapter.transformer.getRelativeRect()); - } - } - - for (const adapter of this.manager.controlLayerAdapters.values()) { - if (adapter.state.isEnabled) { - rects.push(adapter.transformer.getRelativeRect()); - } - } - - for (const adapter of this.manager.regionalGuidanceAdapters.values()) { + for (const adapter of this.manager.adapters.getAll()) { if (adapter.state.isEnabled) { rects.push(adapter.transformer.getRelativeRect()); } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApiModule.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApiModule.ts index e5571965fd..0f77f60713 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApiModule.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApiModule.ts @@ -174,16 +174,16 @@ export class CanvasStateApiModule { if (identifier.type === 'raster_layer') { entityState = state.rasterLayers.entities.find((i) => i.id === identifier.id) ?? null; - entityAdapter = this.manager.rasterLayerAdapters.get(identifier.id) ?? null; + entityAdapter = this.manager.adapters.rasterLayers.get(identifier.id) ?? null; } else if (identifier.type === 'control_layer') { entityState = state.controlLayers.entities.find((i) => i.id === identifier.id) ?? null; - entityAdapter = this.manager.controlLayerAdapters.get(identifier.id) ?? null; + entityAdapter = this.manager.adapters.controlLayers.get(identifier.id) ?? null; } else if (identifier.type === 'regional_guidance') { entityState = state.regions.entities.find((i) => i.id === identifier.id) ?? null; - entityAdapter = this.manager.regionalGuidanceAdapters.get(identifier.id) ?? null; + entityAdapter = this.manager.adapters.regionMasks.get(identifier.id) ?? null; } else if (identifier.type === 'inpaint_mask') { entityState = state.inpaintMasks.entities.find((i) => i.id === identifier.id) ?? null; - entityAdapter = this.manager.inpaintMaskAdapters.get(identifier.id) ?? null; + entityAdapter = this.manager.adapters.inpaintMasks.get(identifier.id) ?? null; } if (entityState && entityAdapter) { diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts index 658c616500..37b4b42d2f 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts @@ -496,7 +496,7 @@ export class CanvasTransformer { startTransform = () => { this.log.debug('Starting transform'); this.isTransforming = true; - this.manager.stateApi.setTool('move') + this.manager.stateApi.setTool('move'); // When transforming, we want the stage to still be movable if the view tool is selected. If the transformer or // interaction rect are listening, it will interrupt the stage's drag events. So we should disable listening // when the view tool is selected @@ -605,6 +605,7 @@ export class CanvasTransformer { if (this.isPendingRectCalculation) { this.syncInteractionState(); + this.parent.renderer.updatePreviewCanvas(); return; } @@ -615,6 +616,7 @@ export class CanvasTransformer { // The layer is fully transparent but has objects - reset it this.manager.stateApi.resetEntity({ entityIdentifier: this.parent.getEntityIdentifier() }); this.syncInteractionState(); + this.parent.renderer.updatePreviewCanvas(); return; } @@ -628,6 +630,7 @@ export class CanvasTransformer { }; this.parent.renderer.konva.objectGroup.setAttrs(groupAttrs); this.parent.renderer.konva.bufferGroup.setAttrs(groupAttrs); + this.parent.renderer.updatePreviewCanvas(); }; calculateRect = debounce(() => { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts index 7ab87a74df..ba0be20c46 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts @@ -23,7 +23,7 @@ export const addControlAdapters = async ( .filter((layer) => isValidControlAdapter(layer.controlAdapter, base)); for (const layer of validControlLayers) { - const adapter = manager.controlLayerAdapters.get(layer.id); + const adapter = manager.adapters.controlLayers.get(layer.id); assert(adapter, 'Adapter not found'); const imageDTO = await adapter.renderer.rasterize({ rect: bbox, attrs: { opacity: 1, filters: [] } }); if (layer.controlAdapter.type === 'controlnet') { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts index 516db193de..ec4091a447 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addRegions.ts @@ -47,7 +47,7 @@ export const addRegions = async ( const validRegions = regions.filter((rg) => isValidRegion(rg, base)); for (const region of validRegions) { - const adapter = manager.regionalGuidanceAdapters.get(region.id); + const adapter = manager.adapters.regionMasks.get(region.id); assert(adapter, 'Adapter not found'); const imageDTO = await adapter.renderer.rasterize({ rect: bbox });