diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts index 60ce3f8c88..d454b69642 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts @@ -231,6 +231,8 @@ export class CanvasBbox { } render() { + this.log.trace('Rendering generation bbox'); + const bbox = this.manager.stateApi.getBbox(); const toolState = this.manager.stateApi.getToolState(); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts index 9878c63276..7f1873e4b8 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts @@ -4,6 +4,7 @@ import type { AppStore } from 'app/store/store'; import type { SerializableObject } from 'common/types'; import { CanvasCacheModule } from 'features/controlLayers/konva/CanvasCacheModule'; import { CanvasFilter } from 'features/controlLayers/konva/CanvasFilter'; +import { CanvasRenderingModule } from 'features/controlLayers/konva/CanvasRenderingModule'; import { CanvasStageModule } from 'features/controlLayers/konva/CanvasStageModule'; import { CanvasWorkerModule } from 'features/controlLayers/konva/CanvasWorkerModule.js'; import { @@ -23,8 +24,8 @@ import stableHash from 'stable-hash'; import { assert } from 'tsafe'; import { CanvasBackground } from './CanvasBackground'; -import { CanvasLayerAdapter } from './CanvasLayerAdapter'; -import { CanvasMaskAdapter } from './CanvasMaskAdapter'; +import type { CanvasLayerAdapter } from './CanvasLayerAdapter'; +import type { CanvasMaskAdapter } from './CanvasMaskAdapter'; import { CanvasPreview } from './CanvasPreview'; import { CanvasStateApi } from './CanvasStateApi'; import { setStageEventHandlers } from './events'; @@ -38,10 +39,12 @@ export class CanvasManager { id: string; path: string[]; container: HTMLDivElement; + rasterLayerAdapters: Map = new Map(); controlLayerAdapters: Map = new Map(); regionalGuidanceAdapters: Map = new Map(); inpaintMaskAdapters: Map = new Map(); + stateApi: CanvasStateApi; preview: CanvasPreview; background: CanvasBackground; @@ -49,6 +52,7 @@ export class CanvasManager { stage: CanvasStageModule; worker: CanvasWorkerModule; cache: CanvasCacheModule; + renderer: CanvasRenderingModule; log: Logger; socket: AppSocket; @@ -81,7 +85,7 @@ export class CanvasManager { this.stage = new CanvasStageModule(stage, container, this); this.worker = new CanvasWorkerModule(this); this.cache = new CanvasCacheModule(this); - + this.renderer = new CanvasRenderingModule(this); this.preview = new CanvasPreview(this); this.stage.addLayer(this.preview.getLayer()); @@ -89,12 +93,6 @@ export class CanvasManager { this.stage.addLayer(this.background.konva.layer); this.filter = new CanvasFilter(this); - - this.stateApi.$transformingEntity.set(null); - this.stateApi.$toolState.set(this.stateApi.getToolState()); - this.stateApi.$selectedEntityIdentifier.set(this.stateApi.getState().selectedEntityIdentifier); - this.stateApi.$currentFill.set(this.stateApi.getCurrentFill()); - this.stateApi.$selectedEntity.set(this.stateApi.getSelectedEntity()); } enableDebugging() { @@ -106,30 +104,6 @@ export class CanvasManager { this._isDebugging = false; } - arrangeEntities() { - let zIndex = 0; - - this.background.konva.layer.zIndex(++zIndex); - - for (const { id } of this.stateApi.getRasterLayersState().entities) { - this.rasterLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex); - } - - for (const { id } of this.stateApi.getControlLayersState().entities) { - this.controlLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex); - } - - for (const { id } of this.stateApi.getRegionsState().entities) { - this.regionalGuidanceAdapters.get(id)?.konva.layer.zIndex(++zIndex); - } - - for (const { id } of this.stateApi.getInpaintMasksState().entities) { - this.inpaintMaskAdapters.get(id)?.konva.layer.zIndex(++zIndex); - } - - this.preview.getLayer().zIndex(++zIndex); - } - getTransformingLayer = (): CanvasLayerAdapter | CanvasMaskAdapter | null => { const transformingEntity = this.stateApi.$transformingEntity.get(); if (!transformingEntity) { @@ -185,203 +159,19 @@ export class CanvasManager { this.stateApi.$transformingEntity.set(null); } - render = async () => { - const state = this.stateApi.getState(); - - const isFirstRender = this.isFirstRender; - this.isFirstRender = false; - - if (isFirstRender) { - this.log.trace('First render'); - } - - const prevState = this.prevState; - this.prevState = state; - - if (prevState === state && !isFirstRender) { - this.log.trace('No changes detected, skipping render'); - return; - } - - if (isFirstRender || state.settings.canvasBackgroundStyle !== prevState.settings.canvasBackgroundStyle) { - this.background.render(); - } - - if (isFirstRender || state.rasterLayers.isHidden !== prevState.rasterLayers.isHidden) { - for (const adapter of this.rasterLayerAdapters.values()) { - adapter.renderer.updateOpacity(state.rasterLayers.isHidden ? 0 : adapter.state.opacity); - } - } - - if (isFirstRender || state.rasterLayers.entities !== prevState.rasterLayers.entities) { - this.log.debug('Rendering raster layers'); - - for (const entityAdapter of this.rasterLayerAdapters.values()) { - if (!state.rasterLayers.entities.find((l) => l.id === entityAdapter.id)) { - await entityAdapter.destroy(); - this.rasterLayerAdapters.delete(entityAdapter.id); - } - } - - for (const entityState of state.rasterLayers.entities) { - let adapter = this.rasterLayerAdapters.get(entityState.id); - if (!adapter) { - adapter = new CanvasLayerAdapter(entityState, this); - this.rasterLayerAdapters.set(adapter.id, adapter); - this.stage.addLayer(adapter.konva.layer); - } - await adapter.update({ - state: entityState, - toolState: state.tool, - isSelected: state.selectedEntityIdentifier?.id === entityState.id, - }); - } - } - - if (isFirstRender || state.controlLayers.isHidden !== prevState.controlLayers.isHidden) { - for (const adapter of this.controlLayerAdapters.values()) { - adapter.renderer.updateOpacity(state.controlLayers.isHidden ? 0 : adapter.state.opacity); - } - } - - if (isFirstRender || state.controlLayers.entities !== prevState.controlLayers.entities) { - this.log.debug('Rendering control layers'); - - for (const entityAdapter of this.controlLayerAdapters.values()) { - if (!state.controlLayers.entities.find((l) => l.id === entityAdapter.id)) { - await entityAdapter.destroy(); - this.controlLayerAdapters.delete(entityAdapter.id); - } - } - - for (const entityState of state.controlLayers.entities) { - let adapter = this.controlLayerAdapters.get(entityState.id); - if (!adapter) { - adapter = new CanvasLayerAdapter(entityState, this); - this.controlLayerAdapters.set(adapter.id, adapter); - this.stage.addLayer(adapter.konva.layer); - } - await adapter.update({ - state: entityState, - toolState: state.tool, - isSelected: state.selectedEntityIdentifier?.id === entityState.id, - }); - } - } - - if (isFirstRender || state.regions.isHidden !== prevState.regions.isHidden) { - for (const adapter of this.regionalGuidanceAdapters.values()) { - adapter.renderer.updateOpacity(state.regions.isHidden ? 0 : adapter.state.opacity); - } - } - - if ( - isFirstRender || - state.regions.entities !== prevState.regions.entities || - state.tool.selected !== prevState.tool.selected || - state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id - ) { - this.log.debug('Rendering regions'); - - // Destroy the konva nodes for nonexistent entities - for (const canvasRegion of this.regionalGuidanceAdapters.values()) { - if (!state.regions.entities.find((rg) => rg.id === canvasRegion.id)) { - canvasRegion.destroy(); - this.regionalGuidanceAdapters.delete(canvasRegion.id); - } - } - - for (const entityState of state.regions.entities) { - let adapter = this.regionalGuidanceAdapters.get(entityState.id); - if (!adapter) { - adapter = new CanvasMaskAdapter(entityState, this); - this.regionalGuidanceAdapters.set(adapter.id, adapter); - this.stage.addLayer(adapter.konva.layer); - } - await adapter.update({ - state: entityState, - toolState: state.tool, - isSelected: state.selectedEntityIdentifier?.id === entityState.id, - }); - } - } - - if (isFirstRender || state.inpaintMasks.isHidden !== prevState.inpaintMasks.isHidden) { - for (const adapter of this.inpaintMaskAdapters.values()) { - adapter.renderer.updateOpacity(state.inpaintMasks.isHidden ? 0 : adapter.state.opacity); - } - } - - if ( - isFirstRender || - state.inpaintMasks.entities !== prevState.inpaintMasks.entities || - state.tool.selected !== prevState.tool.selected || - state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id - ) { - this.log.debug('Rendering inpaint masks'); - - // Destroy the konva nodes for nonexistent entities - for (const adapter of this.inpaintMaskAdapters.values()) { - if (!state.inpaintMasks.entities.find((rg) => rg.id === adapter.id)) { - adapter.destroy(); - this.inpaintMaskAdapters.delete(adapter.id); - } - } - - for (const entityState of state.inpaintMasks.entities) { - let adapter = this.inpaintMaskAdapters.get(entityState.id); - if (!adapter) { - adapter = new CanvasMaskAdapter(entityState, this); - this.inpaintMaskAdapters.set(adapter.id, adapter); - this.stage.addLayer(adapter.konva.layer); - } - await adapter.update({ - state: entityState, - toolState: state.tool, - isSelected: state.selectedEntityIdentifier?.id === entityState.id, - }); - } - } - - this.stateApi.$toolState.set(state.tool); - this.stateApi.$selectedEntityIdentifier.set(state.selectedEntityIdentifier); - this.stateApi.$selectedEntity.set(this.stateApi.getSelectedEntity()); - this.stateApi.$currentFill.set(this.stateApi.getCurrentFill()); - - if (isFirstRender || state.bbox !== prevState.bbox || state.tool.selected !== prevState.tool.selected) { - this.log.debug('Rendering generation bbox'); - await this.preview.bbox.render(); - } - - if (isFirstRender || state.session !== prevState.session) { - this.log.debug('Rendering staging area'); - await this.preview.stagingArea.render(); - } - - if ( - isFirstRender || - state.rasterLayers.entities !== prevState.rasterLayers.entities || - state.controlLayers.entities !== prevState.controlLayers.entities || - state.regions.entities !== prevState.regions.entities || - state.inpaintMasks.entities !== prevState.inpaintMasks.entities || - state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id - ) { - this.log.debug('Arranging entities'); - await this.arrangeEntities(); - } - - if (isFirstRender) { - $canvasManager.set(this); - } - }; - initialize = () => { this.log.debug('Initializing canvas manager'); - const unsubscribeListeners = setStageEventHandlers(this); + // These atoms require the canvas manager to be set up before we can provide their initial values + this.stateApi.$transformingEntity.set(null); + this.stateApi.$toolState.set(this.stateApi.getToolState()); + this.stateApi.$selectedEntityIdentifier.set(this.stateApi.getState().selectedEntityIdentifier); + this.stateApi.$currentFill.set(this.stateApi.getCurrentFill()); + this.stateApi.$selectedEntity.set(this.stateApi.getSelectedEntity()); + const cleanupEventHandlers = setStageEventHandlers(this); const cleanupStage = this.stage.initialize(); - const unsubscribeRenderer = this._store.subscribe(this.render); + const cleanupStore = this._store.subscribe(this.renderer.render); return () => { this.log.debug('Cleaning up canvas manager'); @@ -396,8 +186,8 @@ export class CanvasManager { } this.background.destroy(); this.preview.destroy(); - unsubscribeRenderer(); - unsubscribeListeners(); + cleanupStore(); + cleanupEventHandlers(); cleanupStage(); }; }; @@ -608,6 +398,11 @@ export class CanvasManager { return generationMode; } + setCanvasManager = () => { + this.log.debug('Setting canvas manager'); + $canvasManager.set(this); + }; + getLoggingContext = (): SerializableObject => { return { path: this.path.join('.'), diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts new file mode 100644 index 0000000000..ee1c7f9370 --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasRenderingModule.ts @@ -0,0 +1,260 @@ +import type { SerializableObject } from 'common/types'; +import { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter'; +import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; +import { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter'; +import { getPrefixedId } from 'features/controlLayers/konva/util'; +import type { CanvasV2State } from 'features/controlLayers/store/types'; +import type { Logger } from 'roarr'; + +export class CanvasRenderingModule { + id: string; + path: string[]; + log: Logger; + manager: CanvasManager; + + state: CanvasV2State | null = null; + + constructor(manager: CanvasManager) { + this.id = getPrefixedId('canvas_renderer'); + this.manager = manager; + this.path = this.manager.path.concat(this.id); + this.log = this.manager.buildLogger(this.getLoggingContext); + this.log.debug('Creating canvas renderer'); + } + + render = async () => { + const state = this.manager.stateApi.getState(); + + if (!this.state) { + this.log.trace('First render'); + } + + const prevState = this.state; + this.state = state; + + if (prevState === state) { + // No changes to state - no need to render + return; + } + + this.renderBackground(state, prevState); + await this.renderRasterLayers(state, prevState); + await this.renderControlLayers(prevState, state); + await this.renderRegionalGuidance(prevState, state); + await this.renderInpaintMasks(state, prevState); + await this.renderBbox(state, prevState); + await this.renderStagingArea(state, prevState); + this.arrangeEntities(state, prevState); + + this.manager.stateApi.$toolState.set(state.tool); + this.manager.stateApi.$selectedEntityIdentifier.set(state.selectedEntityIdentifier); + this.manager.stateApi.$selectedEntity.set(this.manager.stateApi.getSelectedEntity()); + this.manager.stateApi.$currentFill.set(this.manager.stateApi.getCurrentFill()); + + // We have no prev state for the first render + if (!prevState) { + this.manager.setCanvasManager(); + } + }; + + getLoggingContext = (): SerializableObject => { + return { ...this.manager.getLoggingContext(), path: this.manager.path.join('.') }; + }; + + renderBackground = (state: CanvasV2State, prevState: CanvasV2State | null) => { + if (!prevState || state.settings.canvasBackgroundStyle !== prevState.settings.canvasBackgroundStyle) { + this.manager.background.render(); + } + }; + + renderRasterLayers = async (state: CanvasV2State, prevState: CanvasV2State | null) => { + if (!prevState || state.rasterLayers.isHidden !== prevState.rasterLayers.isHidden) { + for (const adapter of this.manager.rasterLayerAdapters.values()) { + adapter.renderer.updateOpacity(state.rasterLayers.isHidden ? 0 : adapter.state.opacity); + } + } + + if (!prevState || state.rasterLayers.entities !== prevState.rasterLayers.entities) { + for (const entityAdapter of this.manager.rasterLayerAdapters.values()) { + if (!state.rasterLayers.entities.find((l) => l.id === entityAdapter.id)) { + await entityAdapter.destroy(); + this.manager.rasterLayerAdapters.delete(entityAdapter.id); + } + } + + for (const entityState of state.rasterLayers.entities) { + let adapter = this.manager.rasterLayerAdapters.get(entityState.id); + if (!adapter) { + adapter = new CanvasLayerAdapter(entityState, this.manager); + this.manager.rasterLayerAdapters.set(adapter.id, adapter); + this.manager.stage.addLayer(adapter.konva.layer); + } + await adapter.update({ + state: entityState, + toolState: state.tool, + isSelected: state.selectedEntityIdentifier?.id === entityState.id, + }); + } + } + }; + + renderControlLayers = async (prevState: CanvasV2State | null, state: CanvasV2State) => { + if (!prevState || state.controlLayers.isHidden !== prevState.controlLayers.isHidden) { + for (const adapter of this.manager.controlLayerAdapters.values()) { + adapter.renderer.updateOpacity(state.controlLayers.isHidden ? 0 : adapter.state.opacity); + } + } + + if (!prevState || state.controlLayers.entities !== prevState.controlLayers.entities) { + for (const entityAdapter of this.manager.controlLayerAdapters.values()) { + if (!state.controlLayers.entities.find((l) => l.id === entityAdapter.id)) { + await entityAdapter.destroy(); + this.manager.controlLayerAdapters.delete(entityAdapter.id); + } + } + + for (const entityState of state.controlLayers.entities) { + let adapter = this.manager.controlLayerAdapters.get(entityState.id); + if (!adapter) { + adapter = new CanvasLayerAdapter(entityState, this.manager); + this.manager.controlLayerAdapters.set(adapter.id, adapter); + this.manager.stage.addLayer(adapter.konva.layer); + } + await adapter.update({ + state: entityState, + toolState: state.tool, + isSelected: state.selectedEntityIdentifier?.id === entityState.id, + }); + } + } + }; + + renderRegionalGuidance = async (prevState: CanvasV2State | null, state: CanvasV2State) => { + if (!prevState || state.regions.isHidden !== prevState.regions.isHidden) { + for (const adapter of this.manager.regionalGuidanceAdapters.values()) { + adapter.renderer.updateOpacity(state.regions.isHidden ? 0 : adapter.state.opacity); + } + } + + if ( + !prevState || + state.regions.entities !== prevState.regions.entities || + state.tool.selected !== prevState.tool.selected || + state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id + ) { + // Destroy the konva nodes for nonexistent entities + for (const canvasRegion of this.manager.regionalGuidanceAdapters.values()) { + if (!state.regions.entities.find((rg) => rg.id === canvasRegion.id)) { + canvasRegion.destroy(); + this.manager.regionalGuidanceAdapters.delete(canvasRegion.id); + } + } + + for (const entityState of state.regions.entities) { + let adapter = this.manager.regionalGuidanceAdapters.get(entityState.id); + if (!adapter) { + adapter = new CanvasMaskAdapter(entityState, this.manager); + this.manager.regionalGuidanceAdapters.set(adapter.id, adapter); + this.manager.stage.addLayer(adapter.konva.layer); + } + await adapter.update({ + state: entityState, + toolState: state.tool, + isSelected: state.selectedEntityIdentifier?.id === entityState.id, + }); + } + } + }; + + renderInpaintMasks = async (state: CanvasV2State, prevState: CanvasV2State | null) => { + if (!prevState || state.inpaintMasks.isHidden !== prevState.inpaintMasks.isHidden) { + for (const adapter of this.manager.inpaintMaskAdapters.values()) { + adapter.renderer.updateOpacity(state.inpaintMasks.isHidden ? 0 : adapter.state.opacity); + } + } + + if ( + !prevState || + state.inpaintMasks.entities !== prevState.inpaintMasks.entities || + state.tool.selected !== prevState.tool.selected || + state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id + ) { + // Destroy the konva nodes for nonexistent entities + for (const adapter of this.manager.inpaintMaskAdapters.values()) { + if (!state.inpaintMasks.entities.find((rg) => rg.id === adapter.id)) { + adapter.destroy(); + this.manager.inpaintMaskAdapters.delete(adapter.id); + } + } + + for (const entityState of state.inpaintMasks.entities) { + let adapter = this.manager.inpaintMaskAdapters.get(entityState.id); + if (!adapter) { + adapter = new CanvasMaskAdapter(entityState, this.manager); + this.manager.inpaintMaskAdapters.set(adapter.id, adapter); + this.manager.stage.addLayer(adapter.konva.layer); + } + await adapter.update({ + state: entityState, + toolState: state.tool, + isSelected: state.selectedEntityIdentifier?.id === entityState.id, + }); + } + } + }; + + renderBbox = (state: CanvasV2State, prevState: CanvasV2State | null) => { + if (!prevState || state.bbox !== prevState.bbox || state.tool.selected !== prevState.tool.selected) { + this.manager.preview.bbox.render(); + } + }; + + renderStagingArea = async (state: CanvasV2State, prevState: CanvasV2State | null) => { + if (!prevState || state.session !== prevState.session) { + await this.manager.preview.stagingArea.render(); + } + }; + + arrangeEntities = (state: CanvasV2State, prevState: CanvasV2State | null) => { + if ( + !prevState || + state.rasterLayers.entities !== prevState.rasterLayers.entities || + state.controlLayers.entities !== prevState.controlLayers.entities || + state.regions.entities !== prevState.regions.entities || + state.inpaintMasks.entities !== prevState.inpaintMasks.entities || + state.selectedEntityIdentifier?.id !== prevState.selectedEntityIdentifier?.id + ) { + this.log.debug('Arranging entities'); + + let zIndex = 0; + + // Draw order: + // 1. Background + // 2. Raster layers + // 3. Control layers + // 4. Regions + // 5. Inpaint masks + // 6. Preview (bbox, staging area, progress image, tool) + + this.manager.background.konva.layer.zIndex(++zIndex); + + for (const { id } of this.manager.stateApi.getRasterLayersState().entities) { + this.manager.rasterLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex); + } + + for (const { id } of this.manager.stateApi.getControlLayersState().entities) { + this.manager.controlLayerAdapters.get(id)?.konva.layer.zIndex(++zIndex); + } + + for (const { id } of this.manager.stateApi.getRegionsState().entities) { + this.manager.regionalGuidanceAdapters.get(id)?.konva.layer.zIndex(++zIndex); + } + + for (const { id } of this.manager.stateApi.getInpaintMasksState().entities) { + this.manager.inpaintMaskAdapters.get(id)?.konva.layer.zIndex(++zIndex); + } + + this.manager.preview.getLayer().zIndex(++zIndex); + } + }; +} diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStagingArea.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStagingArea.ts index e07833ce07..e86a26391f 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStagingArea.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasStagingArea.ts @@ -42,6 +42,7 @@ export class CanvasStagingArea { } render = async () => { + this.log.trace('Rendering staging area'); const session = this.manager.stateApi.getSession(); const { x, y, width, height } = this.manager.stateApi.getBbox().rect; const shouldShowStagedImage = this.manager.stateApi.$shouldShowStagedImage.get(); @@ -75,7 +76,7 @@ export class CanvasStagingArea { } if (!this.image.isLoading && !this.image.isError) { - await this.image.update({...this.image.state, image: imageDTOToImageWithDims(imageDTO)}, true); + await this.image.update({ ...this.image.state, image: imageDTOToImageWithDims(imageDTO) }, true); this.manager.stateApi.$lastCanvasProgressEvent.set(null); } this.image.konva.group.visible(shouldShowStagedImage);