From b7f9c5e2210cf395a9786f5361517c6fdc4e0183 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:24:13 +1000 Subject: [PATCH] refactor(ui): decouple konva renderer from react Subscribe to redux store directly, skipping all the react overhead. With react in dev mode, a typical frame while using the brush tool on almost-empty canvas is reduced from ~7.5ms to ~3.5ms. All things considered, this still feels slow, but it's a massive improvement. --- .../frontend/web/src/app/logging/logger.ts | 3 +- .../components/StageComponent.tsx | 420 +------------ .../features/controlLayers/konva/events.ts | 550 ++++++++++-------- .../controlLayers/konva/renderers/bbox.ts | 38 +- .../controlLayers/konva/renderers/renderer.ts | 393 +++++++++++++ .../controlLayers/konva/renderers/stage.ts | 16 + .../src/features/controlLayers/store/types.ts | 2 +- 7 files changed, 774 insertions(+), 648 deletions(-) create mode 100644 invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts create mode 100644 invokeai/frontend/web/src/features/controlLayers/konva/renderers/stage.ts diff --git a/invokeai/frontend/web/src/app/logging/logger.ts b/invokeai/frontend/web/src/app/logging/logger.ts index c0a3089fe4..3af19af2ef 100644 --- a/invokeai/frontend/web/src/app/logging/logger.ts +++ b/invokeai/frontend/web/src/app/logging/logger.ts @@ -28,7 +28,8 @@ export type LoggerNamespace = | 'queue' | 'dnd' | 'controlLayers' - | 'metadata'; + | 'metadata' + | 'konva'; export const logger = (namespace: LoggerNamespace) => $logger.get().child({ namespace }); diff --git a/invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx b/invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx index 6d7ce825d6..0be12933a1 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx @@ -1,408 +1,27 @@ -import { $alt, $ctrl, $meta, $shift, Flex, Heading } from '@invoke-ai/ui-library'; -import { useStore } from '@nanostores/react'; -import { logger } from 'app/logging/logger'; -import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import { Flex } from '@invoke-ai/ui-library'; +import { useAppStore } from 'app/store/storeHooks'; import { HeadsUpDisplay } from 'features/controlLayers/components/HeadsUpDisplay'; -import { setStageEventHandlers } from 'features/controlLayers/konva/events'; -import { renderBackgroundLayer } from 'features/controlLayers/konva/renderers/background'; -import { renderControlAdapters } from 'features/controlLayers/konva/renderers/caLayer'; -import { - arrangeEntities, - debouncedRenderers, - renderers as normalRenderers, -} from 'features/controlLayers/konva/renderers/layers'; -import { renderDocumentBoundsOverlay } from 'features/controlLayers/konva/renderers/previewLayer'; -import { renderLayers } from 'features/controlLayers/konva/renderers/rasterLayer'; -import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer'; -import { - $bbox, - $currentFill, - $document, - $isDrawing, - $isMouseDown, - $lastAddedPoint, - $lastCursorPos, - $lastMouseDownPos, - $selectedEntity, - $spaceKey, - $stageAttrs, - $toolState, - bboxChanged, - brushWidthChanged, - caBboxChanged, - caTranslated, - eraserWidthChanged, - layerBboxChanged, - layerBrushLineAdded, - layerEraserLineAdded, - layerLinePointAdded, - layerRectAdded, - layerTranslated, - rgBboxChanged, - rgBrushLineAdded, - rgEraserLineAdded, - rgLinePointAdded, - rgRectAdded, - rgTranslated, - toolBufferChanged, - toolChanged, -} from 'features/controlLayers/store/canvasV2Slice'; -import { selectEntityCount } from 'features/controlLayers/store/selectors'; -import type { - BboxChangedArg, - BrushLineAddedArg, - CanvasEntity, - EraserLineAddedArg, - PointAddedToLineArg, - PosChangedArg, - RectShapeAddedArg, - Tool, -} from 'features/controlLayers/store/types'; +import { initializeRenderer } from 'features/controlLayers/konva/renderers/renderer'; import Konva from 'konva'; -import type { IRect } from 'konva/lib/types'; -import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useState } from 'react'; -import { useTranslation } from 'react-i18next'; -import { getImageDTO } from 'services/api/endpoints/images'; +import { memo, useCallback, useEffect, useLayoutEffect, useState } from 'react'; import { useDevicePixelRatio } from 'use-device-pixel-ratio'; import { v4 as uuidv4 } from 'uuid'; // This will log warnings when layers > 5 - maybe use `import.meta.env.MODE === 'development'` instead? Konva.showWarnings = false; -const log = logger('controlLayers'); - const useStageRenderer = (stage: Konva.Stage, container: HTMLDivElement | null, asPreview: boolean) => { - const dispatch = useAppDispatch(); - const controlAdapters = useAppSelector((s) => s.canvasV2.controlAdapters); - const ipAdapters = useAppSelector((s) => s.canvasV2.ipAdapters); - const layers = useAppSelector((s) => s.canvasV2.layers); - const regions = useAppSelector((s) => s.canvasV2.regions); - const tool = useAppSelector((s) => s.canvasV2.tool); - const selectedEntityIdentifier = useAppSelector((s) => s.canvasV2.selectedEntityIdentifier); - const maskOpacity = useAppSelector((s) => s.canvasV2.settings.maskOpacity); - const bbox = useAppSelector((s) => s.canvasV2.bbox); - const document = useAppSelector((s) => s.canvasV2.document); - const lastCursorPos = useStore($lastCursorPos); - const lastMouseDownPos = useStore($lastMouseDownPos); - const isMouseDown = useStore($isMouseDown); - const isDrawing = useStore($isDrawing); - const selectedEntity = useMemo(() => { - const identifier = selectedEntityIdentifier; - if (!identifier) { - return null; - } else if (identifier.type === 'layer') { - return layers.find((i) => i.id === identifier.id) ?? null; - } else if (identifier.type === 'control_adapter') { - return controlAdapters.find((i) => i.id === identifier.id) ?? null; - } else if (identifier.type === 'ip_adapter') { - return ipAdapters.find((i) => i.id === identifier.id) ?? null; - } else if (identifier.type === 'regional_guidance') { - return regions.find((i) => i.id === identifier.id) ?? null; - } else { - return null; - } - }, [controlAdapters, ipAdapters, layers, regions, selectedEntityIdentifier]); - - const currentFill = useMemo(() => { - if (selectedEntity && selectedEntity.type === 'regional_guidance') { - return { ...selectedEntity.fill, a: maskOpacity }; - } - return tool.fill; - }, [maskOpacity, selectedEntity, tool.fill]); - - const renderers = useMemo(() => (asPreview ? debouncedRenderers : normalRenderers), [asPreview]); + const store = useAppStore(); const dpr = useDevicePixelRatio({ round: false }); useLayoutEffect(() => { - $toolState.set(tool); - $selectedEntity.set(selectedEntity); - $bbox.set(bbox); - $currentFill.set(currentFill); - $document.set(document); - }, [selectedEntity, tool, bbox, currentFill, document]); - - const onPosChanged = useCallback( - (arg: PosChangedArg, entityType: CanvasEntity['type']) => { - if (entityType === 'layer') { - dispatch(layerTranslated(arg)); - } else if (entityType === 'control_adapter') { - dispatch(caTranslated(arg)); - } else if (entityType === 'regional_guidance') { - dispatch(rgTranslated(arg)); - } - }, - [dispatch] - ); - - const onBboxChanged = useCallback( - (arg: BboxChangedArg, entityType: CanvasEntity['type']) => { - if (entityType === 'layer') { - dispatch(layerBboxChanged(arg)); - } else if (entityType === 'control_adapter') { - dispatch(caBboxChanged(arg)); - } else if (entityType === 'regional_guidance') { - dispatch(rgBboxChanged(arg)); - } - }, - [dispatch] - ); - - const onBrushLineAdded = useCallback( - (arg: BrushLineAddedArg, entityType: CanvasEntity['type']) => { - if (entityType === 'layer') { - dispatch(layerBrushLineAdded(arg)); - } else if (entityType === 'regional_guidance') { - dispatch(rgBrushLineAdded(arg)); - } - }, - [dispatch] - ); - const onEraserLineAdded = useCallback( - (arg: EraserLineAddedArg, entityType: CanvasEntity['type']) => { - if (entityType === 'layer') { - dispatch(layerEraserLineAdded(arg)); - } else if (entityType === 'regional_guidance') { - dispatch(rgEraserLineAdded(arg)); - } - }, - [dispatch] - ); - const onPointAddedToLine = useCallback( - (arg: PointAddedToLineArg, entityType: CanvasEntity['type']) => { - if (entityType === 'layer') { - dispatch(layerLinePointAdded(arg)); - } else if (entityType === 'regional_guidance') { - dispatch(rgLinePointAdded(arg)); - } - }, - [dispatch] - ); - const onRectShapeAdded = useCallback( - (arg: RectShapeAddedArg, entityType: CanvasEntity['type']) => { - if (entityType === 'layer') { - dispatch(layerRectAdded(arg)); - } else if (entityType === 'regional_guidance') { - dispatch(rgRectAdded(arg)); - } - }, - [dispatch] - ); - const onBboxTransformed = useCallback( - (bbox: IRect) => { - dispatch(bboxChanged(bbox)); - }, - [dispatch] - ); - const onBrushWidthChanged = useCallback( - (width: number) => { - dispatch(brushWidthChanged(width)); - }, - [dispatch] - ); - const onEraserWidthChanged = useCallback( - (width: number) => { - dispatch(eraserWidthChanged(width)); - }, - [dispatch] - ); - const setTool = useCallback( - (tool: Tool) => { - dispatch(toolChanged(tool)); - }, - [dispatch] - ); - const setToolBuffer = useCallback( - (toolBuffer: Tool | null) => { - dispatch(toolBufferChanged(toolBuffer)); - }, - [dispatch] - ); - - useLayoutEffect(() => { - log.trace('Initializing stage'); - if (!container) { - return; - } - stage.container(container); - return () => { - log.trace('Cleaning up stage'); - stage.destroy(); - }; - }, [container, stage]); - - useLayoutEffect(() => { - log.trace('Adding stage listeners'); - if (asPreview || !container) { - return; - } - - const cleanup = setStageEventHandlers({ - stage, - getToolState: $toolState.get, - setTool, - setToolBuffer, - getIsDrawing: $isDrawing.get, - setIsDrawing: $isDrawing.set, - getIsMouseDown: $isMouseDown.get, - setIsMouseDown: $isMouseDown.set, - getSelectedEntity: $selectedEntity.get, - getLastAddedPoint: $lastAddedPoint.get, - setLastAddedPoint: $lastAddedPoint.set, - getLastCursorPos: $lastCursorPos.get, - setLastCursorPos: $lastCursorPos.set, - getLastMouseDownPos: $lastMouseDownPos.get, - setLastMouseDownPos: $lastMouseDownPos.set, - getSpaceKey: $spaceKey.get, - setStageAttrs: $stageAttrs.set, - getDocument: $document.get, - getBbox: $bbox.get, - onBrushLineAdded, - onEraserLineAdded, - onPointAddedToLine, - onRectShapeAdded, - onBrushWidthChanged, - onEraserWidthChanged, - getCurrentFill: $currentFill.get, - }); - - return () => { - log.trace('Removing stage listeners'); - cleanup(); - }; - }, [ - asPreview, - onBrushLineAdded, - onBrushWidthChanged, - onEraserLineAdded, - onPointAddedToLine, - onRectShapeAdded, - stage, - container, - onEraserWidthChanged, - setTool, - setToolBuffer, - ]); - - useLayoutEffect(() => { - log.trace('Updating stage dimensions'); - if (!container) { - return; - } - - const fitStageToContainer = () => { - stage.width(container.offsetWidth); - stage.height(container.offsetHeight); - $stageAttrs.set({ - x: stage.x(), - y: stage.y(), - width: stage.width(), - height: stage.height(), - scale: stage.scaleX(), - }); - renderBackgroundLayer(stage); - renderDocumentBoundsOverlay(stage, $document.get); - }; - - const resizeObserver = new ResizeObserver(fitStageToContainer); - resizeObserver.observe(container); - fitStageToContainer(); - - return () => { - resizeObserver.disconnect(); - }; - }, [stage, container]); - - useLayoutEffect(() => { - if (asPreview) { - // Preview should not display tool - return; - } - log.trace('Rendering tool preview'); - renderers.renderToolPreview( - stage, - tool, - currentFill, - selectedEntity, - lastCursorPos, - lastMouseDownPos, - isDrawing, - isMouseDown - ); - }, [ - asPreview, - currentFill, - document, - isDrawing, - isMouseDown, - lastCursorPos, - lastMouseDownPos, - renderers, - selectedEntity, - stage, - tool, - ]); - - useLayoutEffect(() => { - if (asPreview) { - // Preview should not display tool - return; - } - log.trace('Rendering bbox preview'); - renderers.renderBboxPreview( - stage, - bbox, - tool.selected, - $bbox.get, - onBboxTransformed, - $shift.get, - $ctrl.get, - $meta.get, - $alt.get - ); - }, [asPreview, bbox, onBboxTransformed, renderers, stage, tool.selected]); - - useLayoutEffect(() => { - log.trace('Rendering layers'); - renderLayers(stage, layers, tool.selected, onPosChanged); - }, [layers, onPosChanged, stage, tool.selected]); - - useLayoutEffect(() => { - log.trace('Rendering regions'); - renderRegions(stage, regions, maskOpacity, tool.selected, selectedEntity, onPosChanged); - }, [maskOpacity, onPosChanged, regions, selectedEntity, stage, tool.selected]); - - useLayoutEffect(() => { - log.trace('Rendering layers'); - renderControlAdapters(stage, controlAdapters, getImageDTO); - }, [controlAdapters, stage]); - - useLayoutEffect(() => { - renderDocumentBoundsOverlay(stage, $document.get); - }, [stage, document]); - - useLayoutEffect(() => { - arrangeEntities(stage, layers, controlAdapters, regions); - }, [layers, controlAdapters, regions, stage]); - - // useLayoutEffect(() => { - // if (asPreview) { - // // Preview should not check for transparency - // return; - // } - // log.trace('Updating bboxes'); - // debouncedRenderers.updateBboxes(stage, state.layers, onBboxChanged); - // }, [stage, asPreview, state.layers, onBboxChanged]); + const cleanup = initializeRenderer(store, stage, container); + return cleanup; + }, [asPreview, container, stage, store]); useLayoutEffect(() => { Konva.pixelRatio = dpr; }, [dpr]); - - useEffect( - () => () => { - stage.destroy(); - }, - [stage] - ); }; type Props = { @@ -426,9 +45,15 @@ export const StageComponent = memo(({ asPreview = false }: Props) => { useStageRenderer(stage, container, asPreview); + useEffect( + () => () => { + stage.destroy(); + }, + [stage] + ); + return ( - {!asPreview && } { }); StageComponent.displayName = 'StageComponent'; - -const NoEntitiesFallback = () => { - const { t } = useTranslation(); - const entityCount = useAppSelector(selectEntityCount); - - if (entityCount) { - return null; - } - - return ( - - {t('controlLayers.noLayersAdded')} - - ); -}; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/events.ts b/invokeai/frontend/web/src/features/controlLayers/konva/events.ts index cb1f83a334..6614d1c9dc 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/events.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/events.ts @@ -1,5 +1,10 @@ import { renderBackgroundLayer } from 'features/controlLayers/konva/renderers/background'; -import { renderDocumentBoundsOverlay, scaleToolPreview } from 'features/controlLayers/konva/renderers/previewLayer'; +import { + renderDocumentBoundsOverlay, + renderToolPreview, + scaleToolPreview, +} from 'features/controlLayers/konva/renderers/previewLayer'; +import { fitDocumentToStage } from 'features/controlLayers/konva/renderers/stage'; import { getScaledFlooredCursorPosition } from 'features/controlLayers/konva/util'; import type { BrushLineAddedArg, @@ -19,7 +24,6 @@ import { clamp } from 'lodash-es'; import { BRUSH_SPACING_TARGET_SCALE, CANVAS_SCALE_BY, - DOCUMENT_FIT_PADDING_PX, MAX_BRUSH_SPACING_PX, MAX_CANVAS_SCALE, MIN_BRUSH_SPACING_PX, @@ -164,6 +168,16 @@ export const setStageEventHandlers = ({ } const tool = getToolState().selected; stage.findOne(`#${PREVIEW_TOOL_GROUP_ID}`)?.visible(tool === 'brush' || tool === 'eraser'); + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region mousedown @@ -176,33 +190,50 @@ export const setStageEventHandlers = ({ const toolState = getToolState(); const pos = updateLastCursorPos(stage, setLastCursorPos); const selectedEntity = getSelectedEntity(); - if (!pos || !selectedEntity) { - return; - } - if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') { - return; - } - if (getSpaceKey()) { - // No drawing when space is down - we are panning the stage - return; - } + if ( + pos && + selectedEntity && + (selectedEntity?.type === 'regional_guidance' || selectedEntity?.type === 'layer') && + !getSpaceKey() + ) { + setIsDrawing(true); + setLastMouseDownPos(pos); - setIsDrawing(true); - setLastMouseDownPos(pos); - - if (toolState.selected === 'brush') { - const bbox = getBbox(); - if (e.evt.shiftKey) { - const lastAddedPoint = getLastAddedPoint(); - // Create a straight line if holding shift - if (lastAddedPoint) { + if (toolState.selected === 'brush') { + const bbox = getBbox(); + if (e.evt.shiftKey) { + const lastAddedPoint = getLastAddedPoint(); + // Create a straight line if holding shift + if (lastAddedPoint) { + onBrushLineAdded( + { + id: selectedEntity.id, + points: [ + lastAddedPoint.x - selectedEntity.x, + lastAddedPoint.y - selectedEntity.y, + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, + ], + color: getCurrentFill(), + width: toolState.brush.width, + clip: { + x: bbox.x, + y: bbox.y, + width: bbox.width, + height: bbox.height, + }, + }, + selectedEntity.type + ); + } + } else { onBrushLineAdded( { id: selectedEntity.id, points: [ - lastAddedPoint.x - selectedEntity.x, - lastAddedPoint.y - selectedEntity.y, + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, pos.x - selectedEntity.x, pos.y - selectedEntity.y, ], @@ -218,43 +249,42 @@ export const setStageEventHandlers = ({ selectedEntity.type ); } - } else { - onBrushLineAdded( - { - id: selectedEntity.id, - points: [ - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - ], - color: getCurrentFill(), - width: toolState.brush.width, - clip: { - x: bbox.x, - y: bbox.y, - width: bbox.width, - height: bbox.height, - }, - }, - selectedEntity.type - ); + setLastAddedPoint(pos); } - setLastAddedPoint(pos); - } - if (toolState.selected === 'eraser') { - const bbox = getBbox(); - if (e.evt.shiftKey) { - // Create a straight line if holding shift - const lastAddedPoint = getLastAddedPoint(); - if (lastAddedPoint) { + if (toolState.selected === 'eraser') { + const bbox = getBbox(); + if (e.evt.shiftKey) { + // Create a straight line if holding shift + const lastAddedPoint = getLastAddedPoint(); + if (lastAddedPoint) { + onEraserLineAdded( + { + id: selectedEntity.id, + points: [ + lastAddedPoint.x - selectedEntity.x, + lastAddedPoint.y - selectedEntity.y, + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, + ], + width: toolState.eraser.width, + clip: { + x: bbox.x, + y: bbox.y, + width: bbox.width, + height: bbox.height, + }, + }, + selectedEntity.type + ); + } + } else { onEraserLineAdded( { id: selectedEntity.id, points: [ - lastAddedPoint.x - selectedEntity.x, - lastAddedPoint.y - selectedEntity.y, + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, pos.x - selectedEntity.x, pos.y - selectedEntity.y, ], @@ -269,29 +299,19 @@ export const setStageEventHandlers = ({ selectedEntity.type ); } - } else { - onEraserLineAdded( - { - id: selectedEntity.id, - points: [ - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - ], - width: toolState.eraser.width, - clip: { - x: bbox.x, - y: bbox.y, - width: bbox.width, - height: bbox.height, - }, - }, - selectedEntity.type - ); + setLastAddedPoint(pos); } - setLastAddedPoint(pos); } + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region mouseup @@ -304,41 +324,47 @@ export const setStageEventHandlers = ({ const pos = getLastCursorPos(); const selectedEntity = getSelectedEntity(); - if (!pos || !selectedEntity) { - return; - } - if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') { - return; - } + if ( + pos && + selectedEntity && + (selectedEntity?.type === 'regional_guidance' || selectedEntity?.type === 'layer') && + !getSpaceKey() + ) { + const toolState = getToolState(); - if (getSpaceKey()) { - // No drawing when space is down - we are panning the stage - return; - } - - const toolState = getToolState(); - - if (toolState.selected === 'rect') { - const lastMouseDownPos = getLastMouseDownPos(); - if (lastMouseDownPos) { - onRectShapeAdded( - { - id: selectedEntity.id, - rect: { - x: Math.min(pos.x, lastMouseDownPos.x), - y: Math.min(pos.y, lastMouseDownPos.y), - width: Math.abs(pos.x - lastMouseDownPos.x), - height: Math.abs(pos.y - lastMouseDownPos.y), + if (toolState.selected === 'rect') { + const lastMouseDownPos = getLastMouseDownPos(); + if (lastMouseDownPos) { + onRectShapeAdded( + { + id: selectedEntity.id, + rect: { + x: Math.min(pos.x, lastMouseDownPos.x), + y: Math.min(pos.y, lastMouseDownPos.y), + width: Math.abs(pos.x - lastMouseDownPos.x), + height: Math.abs(pos.y - lastMouseDownPos.y), + }, + color: getCurrentFill(), }, - color: getCurrentFill(), - }, - selectedEntity.type - ); + selectedEntity.type + ); + } } + + setIsDrawing(false); + setLastMouseDownPos(null); } - setIsDrawing(false); - setLastMouseDownPos(null); + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region mousemove @@ -355,84 +381,101 @@ export const setStageEventHandlers = ({ .findOne(`#${PREVIEW_TOOL_GROUP_ID}`) ?.visible(toolState.selected === 'brush' || toolState.selected === 'eraser'); - if (!pos || !selectedEntity) { - return; - } - if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') { - return; - } - - if (getSpaceKey()) { - // No drawing when space is down - we are panning the stage - return; - } - - if (!getIsMouseDown()) { - return; - } - - if (toolState.selected === 'brush') { - if (getIsDrawing()) { - // Continue the last line - maybeAddNextPoint(selectedEntity, pos, getToolState, getLastAddedPoint, setLastAddedPoint, onPointAddedToLine); - } else { - const bbox = getBbox(); - // Start a new line - onBrushLineAdded( - { - id: selectedEntity.id, - points: [ - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - ], - width: toolState.brush.width, - color: getCurrentFill(), - clip: { - x: bbox.x, - y: bbox.y, - width: bbox.width, - height: bbox.height, + if ( + pos && + selectedEntity && + (selectedEntity.type === 'regional_guidance' || selectedEntity.type === 'layer') && + !getSpaceKey() && + getIsMouseDown() + ) { + if (toolState.selected === 'brush') { + if (getIsDrawing()) { + // Continue the last line + maybeAddNextPoint( + selectedEntity, + pos, + getToolState, + getLastAddedPoint, + setLastAddedPoint, + onPointAddedToLine + ); + } else { + const bbox = getBbox(); + // Start a new line + onBrushLineAdded( + { + id: selectedEntity.id, + points: [ + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, + ], + width: toolState.brush.width, + color: getCurrentFill(), + clip: { + x: bbox.x, + y: bbox.y, + width: bbox.width, + height: bbox.height, + }, }, - }, - selectedEntity.type - ); - setLastAddedPoint(pos); - setIsDrawing(true); + selectedEntity.type + ); + setLastAddedPoint(pos); + setIsDrawing(true); + } + } + + if (toolState.selected === 'eraser') { + if (getIsDrawing()) { + // Continue the last line + maybeAddNextPoint( + selectedEntity, + pos, + getToolState, + getLastAddedPoint, + setLastAddedPoint, + onPointAddedToLine + ); + } else { + const bbox = getBbox(); + // Start a new line + onEraserLineAdded( + { + id: selectedEntity.id, + points: [ + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, + pos.x - selectedEntity.x, + pos.y - selectedEntity.y, + ], + width: toolState.eraser.width, + clip: { + x: bbox.x, + y: bbox.y, + width: bbox.width, + height: bbox.height, + }, + }, + selectedEntity.type + ); + setLastAddedPoint(pos); + setIsDrawing(true); + } } } - if (toolState.selected === 'eraser') { - if (getIsDrawing()) { - // Continue the last line - maybeAddNextPoint(selectedEntity, pos, getToolState, getLastAddedPoint, setLastAddedPoint, onPointAddedToLine); - } else { - const bbox = getBbox(); - // Start a new line - onEraserLineAdded( - { - id: selectedEntity.id, - points: [ - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - pos.x - selectedEntity.x, - pos.y - selectedEntity.y, - ], - width: toolState.eraser.width, - clip: { - x: bbox.x, - y: bbox.y, - width: bbox.width, - height: bbox.height, - }, - }, - selectedEntity.type - ); - setLastAddedPoint(pos); - setIsDrawing(true); - } - } + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region mouseleave @@ -450,24 +493,33 @@ export const setStageEventHandlers = ({ stage.findOne(`#${PREVIEW_TOOL_GROUP_ID}`)?.visible(false); - if (!pos || !selectedEntity) { - return; - } - if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') { - return; - } - if (getSpaceKey()) { - // No drawing when space is down - we are panning the stage - return; - } - if (getIsMouseDown()) { - if (toolState.selected === 'brush') { - onPointAddedToLine({ id: selectedEntity.id, point: [pos.x, pos.y] }, selectedEntity.type); - } - if (toolState.selected === 'eraser') { - onPointAddedToLine({ id: selectedEntity.id, point: [pos.x, pos.y] }, selectedEntity.type); + if ( + pos && + selectedEntity && + (selectedEntity.type === 'regional_guidance' || selectedEntity.type === 'layer') && + !getSpaceKey() && + getIsMouseDown() + ) { + if (getIsMouseDown()) { + if (toolState.selected === 'brush') { + onPointAddedToLine({ id: selectedEntity.id, point: [pos.x, pos.y] }, selectedEntity.type); + } + if (toolState.selected === 'eraser') { + onPointAddedToLine({ id: selectedEntity.id, point: [pos.x, pos.y] }, selectedEntity.type); + } } } + + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region wheel @@ -489,31 +541,40 @@ export const setStageEventHandlers = ({ } else { // We need the absolute cursor position - not the scaled position const cursorPos = stage.getPointerPosition(); - if (!cursorPos) { - return; - } - // Stage's x and y scale are always the same - const stageScale = stage.scaleX(); - // When wheeling on trackpad, e.evt.ctrlKey is true - in that case, let's reverse the direction - const delta = e.evt.ctrlKey ? -e.evt.deltaY : e.evt.deltaY; - const mousePointTo = { - x: (cursorPos.x - stage.x()) / stageScale, - y: (cursorPos.y - stage.y()) / stageScale, - }; - const newScale = clamp(stageScale * CANVAS_SCALE_BY ** delta, MIN_CANVAS_SCALE, MAX_CANVAS_SCALE); - const newPos = { - x: cursorPos.x - mousePointTo.x * newScale, - y: cursorPos.y - mousePointTo.y * newScale, - }; + if (cursorPos) { + // Stage's x and y scale are always the same + const stageScale = stage.scaleX(); + // When wheeling on trackpad, e.evt.ctrlKey is true - in that case, let's reverse the direction + const delta = e.evt.ctrlKey ? -e.evt.deltaY : e.evt.deltaY; + const mousePointTo = { + x: (cursorPos.x - stage.x()) / stageScale, + y: (cursorPos.y - stage.y()) / stageScale, + }; + const newScale = clamp(stageScale * CANVAS_SCALE_BY ** delta, MIN_CANVAS_SCALE, MAX_CANVAS_SCALE); + const newPos = { + x: cursorPos.x - mousePointTo.x * newScale, + y: cursorPos.y - mousePointTo.y * newScale, + }; - stage.scaleX(newScale); - stage.scaleY(newScale); - stage.position(newPos); - setStageAttrs({ ...newPos, width: stage.width(), height: stage.height(), scale: newScale }); - renderBackgroundLayer(stage); - scaleToolPreview(stage, getToolState()); - renderDocumentBoundsOverlay(stage, getDocument); + stage.scaleX(newScale); + stage.scaleY(newScale); + stage.position(newPos); + setStageAttrs({ ...newPos, width: stage.width(), height: stage.height(), scale: newScale }); + renderBackgroundLayer(stage); + scaleToolPreview(stage, getToolState()); + renderDocumentBoundsOverlay(stage, getDocument); + } } + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region dragmove @@ -527,6 +588,16 @@ export const setStageEventHandlers = ({ }); renderBackgroundLayer(stage); renderDocumentBoundsOverlay(stage, getDocument); + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region dragend @@ -539,6 +610,16 @@ export const setStageEventHandlers = ({ height: stage.height(), scale: stage.scaleX(), }); + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }); //#region key @@ -558,21 +639,22 @@ export const setStageEventHandlers = ({ setToolBuffer(getToolState().selected); setTool('view'); } else if (e.key === 'r') { - // Fit & center the document on the stage - const width = stage.width(); - const height = stage.height(); - const document = getDocument(); - const docWidthWithBuffer = document.width + DOCUMENT_FIT_PADDING_PX * 2; - const docHeightWithBuffer = document.height + DOCUMENT_FIT_PADDING_PX * 2; - const scale = Math.min(Math.min(width / docWidthWithBuffer, height / docHeightWithBuffer), 1); - const x = (width - docWidthWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale; - const y = (height - docHeightWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale; - stage.setAttrs({ x, y, width, height, scaleX: scale, scaleY: scale }); - setStageAttrs({ x, y, width, height, scale }); + const stageAttrs = fitDocumentToStage(stage, getDocument()); + setStageAttrs(stageAttrs); scaleToolPreview(stage, getToolState()); renderBackgroundLayer(stage); renderDocumentBoundsOverlay(stage, getDocument); } + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }; window.addEventListener('keydown', onKeyDown); @@ -589,6 +671,16 @@ export const setStageEventHandlers = ({ setTool(toolBuffer ?? 'move'); setToolBuffer(null); } + renderToolPreview( + stage, + getToolState(), + getCurrentFill(), + getSelectedEntity(), + getLastCursorPos(), + getLastAddedPoint(), + getIsDrawing(), + getIsMouseDown() + ); }; window.addEventListener('keyup', onKeyUp); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts index cd17862f3a..c14d643657 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts @@ -6,8 +6,14 @@ import { RG_LAYER_OBJECT_GROUP_NAME, } from 'features/controlLayers/konva/naming'; import { createBboxRect } from 'features/controlLayers/konva/renderers/objects'; -import { imageDataToDataURL } from "features/controlLayers/konva/util"; -import type { ControlAdapterEntity, LayerEntity, RegionEntity } from 'features/controlLayers/store/types'; +import { imageDataToDataURL } from 'features/controlLayers/konva/util'; +import type { + BboxChangedArg, + CanvasEntity, + ControlAdapterEntity, + LayerEntity, + RegionEntity, +} from 'features/controlLayers/store/types'; import Konva from 'konva'; import type { IRect } from 'konva/lib/types'; import { assert } from 'tsafe'; @@ -186,10 +192,12 @@ const filterCAChildren = (node: Konva.Node): boolean => node.name() === CA_LAYER */ export const updateBboxes = ( stage: Konva.Stage, - entityStates: (ControlAdapterEntity | LayerEntity | RegionEntity)[], - onBboxChanged: (layerId: string, bbox: IRect | null) => void + layers: LayerEntity[], + controlAdapters: ControlAdapterEntity[], + regions: RegionEntity[], + onBboxChanged: (arg: BboxChangedArg, entityType: CanvasEntity['type']) => void ): void => { - for (const entityState of entityStates) { + for (const entityState of [...layers, ...controlAdapters, ...regions]) { const konvaLayer = stage.findOne(`#${entityState.id}`); assert(konvaLayer, `Layer ${entityState.id} not found in stage`); // We only need to recalculate the bbox if the layer has changed @@ -202,24 +210,30 @@ export const updateBboxes = ( if (entityState.type === 'layer') { if (entityState.objects.length === 0) { - // No objects - no bbox to calculate - onBboxChanged(entityState.id, null); + // No objects - no bbox to calculate + onBboxChanged({ id: entityState.id, bbox: null }, 'layer'); } else { - onBboxChanged(entityState.id, getLayerBboxPixels(konvaLayer, filterLayerChildren)); + onBboxChanged({ id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterLayerChildren) }, 'layer'); } } else if (entityState.type === 'control_adapter') { if (!entityState.image && !entityState.processedImage) { // No objects - no bbox to calculate - onBboxChanged(entityState.id, null); + onBboxChanged({ id: entityState.id, bbox: null }, 'control_adapter'); } else { - onBboxChanged(entityState.id, getLayerBboxPixels(konvaLayer, filterCAChildren)); + onBboxChanged( + { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterCAChildren) }, + 'control_adapter' + ); } } else if (entityState.type === 'regional_guidance') { if (entityState.objects.length === 0) { // No objects - no bbox to calculate - onBboxChanged(entityState.id, null); + onBboxChanged({ id: entityState.id, bbox: null }, 'regional_guidance'); } else { - onBboxChanged(entityState.id, getLayerBboxPixels(konvaLayer, filterRGChildren)); + onBboxChanged( + { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterRGChildren) }, + 'regional_guidance' + ); } } diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts new file mode 100644 index 0000000000..d339de9d5b --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts @@ -0,0 +1,393 @@ +import { $alt, $ctrl, $meta, $shift } from '@invoke-ai/ui-library'; +import type { Store } from '@reduxjs/toolkit'; +import { logger } from 'app/logging/logger'; +import { $isDebugging } from 'app/store/nanostores/isDebugging'; +import type { RootState } from 'app/store/store'; +import { setStageEventHandlers } from 'features/controlLayers/konva/events'; +import { renderBackgroundLayer } from 'features/controlLayers/konva/renderers/background'; +import { updateBboxes } from 'features/controlLayers/konva/renderers/bbox'; +import { renderControlAdapters } from 'features/controlLayers/konva/renderers/caLayer'; +import { arrangeEntities } from 'features/controlLayers/konva/renderers/layers'; +import { + renderBboxPreview, + renderDocumentBoundsOverlay, + scaleToolPreview, +} from 'features/controlLayers/konva/renderers/previewLayer'; +import { renderLayers } from 'features/controlLayers/konva/renderers/rasterLayer'; +import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer'; +import { fitDocumentToStage } from 'features/controlLayers/konva/renderers/stage'; +import { + $isDrawing, + $isMouseDown, + $lastAddedPoint, + $lastCursorPos, + $lastMouseDownPos, + $spaceKey, + $stageAttrs, + bboxChanged, + brushWidthChanged, + caBboxChanged, + caTranslated, + eraserWidthChanged, + layerBboxChanged, + layerBrushLineAdded, + layerEraserLineAdded, + layerLinePointAdded, + layerRectAdded, + layerTranslated, + rgBboxChanged, + rgBrushLineAdded, + rgEraserLineAdded, + rgLinePointAdded, + rgRectAdded, + rgTranslated, + toolBufferChanged, + toolChanged, +} from 'features/controlLayers/store/canvasV2Slice'; +import type { + BboxChangedArg, + BrushLineAddedArg, + CanvasEntity, + CanvasEntityIdentifier, + CanvasV2State, + EraserLineAddedArg, + PointAddedToLineArg, + PosChangedArg, + RectShapeAddedArg, + Tool, +} from 'features/controlLayers/store/types'; +import type Konva from 'konva'; +import type { IRect } from 'konva/lib/types'; +import { debounce } from 'lodash-es'; +import type { RgbaColor } from 'react-colorful'; +import { getImageDTO } from 'services/api/endpoints/images'; +/** + * Initializes the canvas renderer. It subscribes to the redux store and listens for changes directly, bypassing the + * react rendering cycle entirely, improving canvas performance. + * @param store The Redux store + * @param stage The Konva stage + * @param container The stage's target container element + * @returns A cleanup function + */ +export const initializeRenderer = ( + store: Store, + stage: Konva.Stage, + container: HTMLDivElement | null +): (() => void) => { + const _log = logger('konva'); + /** + * Logs a message to the console if debugging is enabled. + */ + const logIfDebugging = (message: string) => { + if ($isDebugging.get()) { + _log.trace(message); + } + }; + + logIfDebugging('Initializing renderer'); + if (!container) { + // Nothing to clean up + logIfDebugging('No stage container, skipping initialization'); + return () => {}; + } + + stage.container(container); + + // Set up callbacks for various events + const onPosChanged = (arg: PosChangedArg, entityType: CanvasEntity['type']) => { + logIfDebugging('Position changed'); + if (entityType === 'layer') { + dispatch(layerTranslated(arg)); + } else if (entityType === 'control_adapter') { + dispatch(caTranslated(arg)); + } else if (entityType === 'regional_guidance') { + dispatch(rgTranslated(arg)); + } + }; + const onBboxChanged = (arg: BboxChangedArg, entityType: CanvasEntity['type']) => { + logIfDebugging('Entity bbox changed'); + if (entityType === 'layer') { + dispatch(layerBboxChanged(arg)); + } else if (entityType === 'control_adapter') { + dispatch(caBboxChanged(arg)); + } else if (entityType === 'regional_guidance') { + dispatch(rgBboxChanged(arg)); + } + }; + const onBrushLineAdded = (arg: BrushLineAddedArg, entityType: CanvasEntity['type']) => { + logIfDebugging('Brush line added'); + if (entityType === 'layer') { + dispatch(layerBrushLineAdded(arg)); + } else if (entityType === 'regional_guidance') { + dispatch(rgBrushLineAdded(arg)); + } + }; + const onEraserLineAdded = (arg: EraserLineAddedArg, entityType: CanvasEntity['type']) => { + logIfDebugging('Eraser line added'); + if (entityType === 'layer') { + dispatch(layerEraserLineAdded(arg)); + } else if (entityType === 'regional_guidance') { + dispatch(rgEraserLineAdded(arg)); + } + }; + const onPointAddedToLine = (arg: PointAddedToLineArg, entityType: CanvasEntity['type']) => { + logIfDebugging('Point added to line'); + if (entityType === 'layer') { + dispatch(layerLinePointAdded(arg)); + } else if (entityType === 'regional_guidance') { + dispatch(rgLinePointAdded(arg)); + } + }; + const onRectShapeAdded = (arg: RectShapeAddedArg, entityType: CanvasEntity['type']) => { + logIfDebugging('Rect shape added'); + if (entityType === 'layer') { + dispatch(layerRectAdded(arg)); + } else if (entityType === 'regional_guidance') { + dispatch(rgRectAdded(arg)); + } + }; + const onBboxTransformed = (bbox: IRect) => { + logIfDebugging('Generation bbox transformed'); + dispatch(bboxChanged(bbox)); + }; + const onBrushWidthChanged = (width: number) => { + logIfDebugging('Brush width changed'); + dispatch(brushWidthChanged(width)); + }; + const onEraserWidthChanged = (width: number) => { + logIfDebugging('Eraser width changed'); + dispatch(eraserWidthChanged(width)); + }; + const setTool = (tool: Tool) => { + logIfDebugging('Tool selection changed'); + dispatch(toolChanged(tool)); + }; + const setToolBuffer = (toolBuffer: Tool | null) => { + logIfDebugging('Tool buffer changed'); + dispatch(toolBufferChanged(toolBuffer)); + }; + + const _getSelectedEntity = (canvasV2: CanvasV2State): CanvasEntity | null => { + const identifier = canvasV2.selectedEntityIdentifier; + let selectedEntity: CanvasEntity | null = null; + if (!identifier) { + selectedEntity = null; + } else if (identifier.type === 'layer') { + selectedEntity = canvasV2.layers.find((i) => i.id === identifier.id) ?? null; + } else if (identifier.type === 'control_adapter') { + selectedEntity = canvasV2.controlAdapters.find((i) => i.id === identifier.id) ?? null; + } else if (identifier.type === 'ip_adapter') { + selectedEntity = canvasV2.ipAdapters.find((i) => i.id === identifier.id) ?? null; + } else if (identifier.type === 'regional_guidance') { + selectedEntity = canvasV2.regions.find((i) => i.id === identifier.id) ?? null; + } else { + selectedEntity = null; + } + logIfDebugging('Selected entity changed'); + return selectedEntity; + }; + + const _getCurrentFill = (canvasV2: CanvasV2State, selectedEntity: CanvasEntity | null) => { + let currentFill: RgbaColor = canvasV2.tool.fill; + if (selectedEntity && selectedEntity.type === 'regional_guidance') { + currentFill = { ...selectedEntity.fill, a: canvasV2.settings.maskOpacity }; + } else { + currentFill = canvasV2.tool.fill; + } + logIfDebugging('Current fill changed'); + return currentFill; + }; + + const { getState, subscribe, dispatch } = store; + + // Create closures for the rendering functions, used to check if specific parts of state have changed so we only + // render what needs to be rendered. + let prevCanvasV2 = getState().canvasV2; + let selectedEntityIdentifier: CanvasEntityIdentifier | null = prevCanvasV2.selectedEntityIdentifier; + let selectedEntity: CanvasEntity | null = _getSelectedEntity(prevCanvasV2); + let currentFill: RgbaColor = _getCurrentFill(prevCanvasV2, selectedEntity); + let didSelectedEntityChange: boolean = false; + + // On the first render, we need to render everything. + let isFirstRender = true; + + // Stage event listeners use a fully imperative approach to event handling, using these helpers to get state. + const getBbox = () => getState().canvasV2.bbox; + const getDocument = () => getState().canvasV2.document; + const getToolState = () => getState().canvasV2.tool; + const getSelectedEntity = () => selectedEntity; + const getCurrentFill = () => currentFill; + + // Calculating bounding boxes is expensive, must be debounced to not block the UI thread. + // TODO(psyche): Figure out how to do this in a worker. Probably means running the renderer in a worker and sending + // the entire state over when needed. + const debouncedUpdateBboxes = debounce(updateBboxes, 300); + + const cleanupListeners = setStageEventHandlers({ + stage, + getToolState, + setTool, + setToolBuffer, + getIsDrawing: $isDrawing.get, + setIsDrawing: $isDrawing.set, + getIsMouseDown: $isMouseDown.get, + setIsMouseDown: $isMouseDown.set, + getSelectedEntity, + getLastAddedPoint: $lastAddedPoint.get, + setLastAddedPoint: $lastAddedPoint.set, + getLastCursorPos: $lastCursorPos.get, + setLastCursorPos: $lastCursorPos.set, + getLastMouseDownPos: $lastMouseDownPos.get, + setLastMouseDownPos: $lastMouseDownPos.set, + getSpaceKey: $spaceKey.get, + setStageAttrs: $stageAttrs.set, + getDocument, + getBbox, + onBrushLineAdded, + onEraserLineAdded, + onPointAddedToLine, + onRectShapeAdded, + onBrushWidthChanged, + onEraserWidthChanged, + getCurrentFill, + }); + + const renderCanvas = () => { + const { canvasV2 } = store.getState(); + + if (prevCanvasV2 === canvasV2 && !isFirstRender) { + logIfDebugging('No changes detected, skipping render'); + return; + } + + // We can save some cycles for specific renderers if we track whether the selected entity has changed. + if (canvasV2.selectedEntityIdentifier !== selectedEntityIdentifier) { + selectedEntityIdentifier = canvasV2.selectedEntityIdentifier; + selectedEntity = _getSelectedEntity(canvasV2); + didSelectedEntityChange = true; + } else { + didSelectedEntityChange = false; + } + + // The current fill is either the tool fill or, if a regional guidance region is selected, the mask fill for that + // region. We need to manually sync this state. + if (isFirstRender || canvasV2.tool.fill !== prevCanvasV2.tool.fill || didSelectedEntityChange) { + currentFill = _getCurrentFill(canvasV2, selectedEntity); + } + + if ( + isFirstRender || + canvasV2.layers !== prevCanvasV2.layers || + canvasV2.tool.selected !== prevCanvasV2.tool.selected + ) { + logIfDebugging('Rendering layers'); + renderLayers(stage, canvasV2.layers, canvasV2.tool.selected, onPosChanged); + } + + if ( + isFirstRender || + canvasV2.regions !== prevCanvasV2.regions || + canvasV2.settings.maskOpacity !== prevCanvasV2.settings.maskOpacity || + canvasV2.tool.selected !== prevCanvasV2.tool.selected || + didSelectedEntityChange + ) { + logIfDebugging('Rendering regions'); + renderRegions( + stage, + canvasV2.regions, + canvasV2.settings.maskOpacity, + canvasV2.tool.selected, + selectedEntity, + onPosChanged + ); + } + + if (isFirstRender || canvasV2.controlAdapters !== prevCanvasV2.controlAdapters) { + logIfDebugging('Rendering control adapters'); + renderControlAdapters(stage, canvasV2.controlAdapters, getImageDTO); + } + + if (isFirstRender || canvasV2.document !== prevCanvasV2.document) { + logIfDebugging('Rendering document bounds overlay'); + renderDocumentBoundsOverlay(stage, getDocument); + } + + if (isFirstRender || canvasV2.bbox !== prevCanvasV2.bbox || canvasV2.tool.selected !== prevCanvasV2.tool.selected) { + logIfDebugging('Rendering generation bbox'); + renderBboxPreview( + stage, + canvasV2.bbox, + canvasV2.tool.selected, + getBbox, + onBboxTransformed, + $shift.get, + $ctrl.get, + $meta.get, + $alt.get + ); + } + + if ( + isFirstRender || + canvasV2.layers !== prevCanvasV2.layers || + canvasV2.controlAdapters !== prevCanvasV2.controlAdapters || + canvasV2.regions !== prevCanvasV2.regions + ) { + logIfDebugging('Updating entity bboxes'); + debouncedUpdateBboxes(stage, canvasV2.layers, canvasV2.controlAdapters, canvasV2.regions, onBboxChanged); + } + + if ( + isFirstRender || + canvasV2.layers !== prevCanvasV2.layers || + canvasV2.controlAdapters !== prevCanvasV2.controlAdapters || + canvasV2.regions !== prevCanvasV2.regions + ) { + logIfDebugging('Arranging entities'); + arrangeEntities(stage, canvasV2.layers, canvasV2.controlAdapters, canvasV2.regions); + } + + prevCanvasV2 = canvasV2; + + if (isFirstRender) { + isFirstRender = false; + } + }; + + // We can use a resize observer to ensure the stage always fits the container. We also need to re-render the bg and + // document bounds overlay when the stage is resized. + const fitStageToContainer = () => { + stage.width(container.offsetWidth); + stage.height(container.offsetHeight); + $stageAttrs.set({ + x: stage.x(), + y: stage.y(), + width: stage.width(), + height: stage.height(), + scale: stage.scaleX(), + }); + renderBackgroundLayer(stage); + renderDocumentBoundsOverlay(stage, getDocument); + }; + + const resizeObserver = new ResizeObserver(fitStageToContainer); + resizeObserver.observe(container); + fitStageToContainer(); + + const unsubscribeRenderer = subscribe(renderCanvas); + + logIfDebugging('First render of konva stage'); + // On first render, the document should be fit to the stage. + const stageAttrs = fitDocumentToStage(stage, prevCanvasV2.document); + // The HUD displays some of the stage attributes, so we need to update it here. + $stageAttrs.set(stageAttrs); + scaleToolPreview(stage, getToolState()); + renderCanvas(); + + return () => { + logIfDebugging('Cleaning up konva renderer'); + unsubscribeRenderer(); + cleanupListeners(); + resizeObserver.disconnect(); + }; +}; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stage.ts b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stage.ts new file mode 100644 index 0000000000..3af86041ed --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/konva/renderers/stage.ts @@ -0,0 +1,16 @@ +import { DOCUMENT_FIT_PADDING_PX } from 'features/controlLayers/konva/constants'; +import type { CanvasV2State, StageAttrs } from 'features/controlLayers/store/types'; +import type Konva from 'konva'; + +export const fitDocumentToStage = (stage: Konva.Stage, document: CanvasV2State['document']): StageAttrs => { + // Fit & center the document on the stage + const width = stage.width(); + const height = stage.height(); + const docWidthWithBuffer = document.width + DOCUMENT_FIT_PADDING_PX * 2; + const docHeightWithBuffer = document.height + DOCUMENT_FIT_PADDING_PX * 2; + const scale = Math.min(Math.min(width / docWidthWithBuffer, height / docHeightWithBuffer), 1); + const x = (width - docWidthWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale; + const y = (height - docHeightWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale; + stage.setAttrs({ x, y, width, height, scaleX: scale, scaleY: scale }); + return { x, y, width, height, scale }; +}; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/types.ts b/invokeai/frontend/web/src/features/controlLayers/store/types.ts index 6841e23d83..16d8284540 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/types.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/types.ts @@ -881,7 +881,7 @@ export type CanvasV2State = { export type StageAttrs = { x: number; y: number; width: number; height: number; scale: number }; export type PosChangedArg = { id: string; x: number; y: number }; -export type BboxChangedArg = { id: string; bbox: IRect }; +export type BboxChangedArg = { id: string; bbox: Rect | null }; export type EraserLineAddedArg = { id: string; points: [number, number, number, number];