refactor(ui): decouple konva renderer from react

Subscribe to redux store directly, skipping all the react overhead.

With react in dev mode, a typical frame while using the brush tool on almost-empty canvas is reduced from ~7.5ms to ~3.5ms. All things considered, this still feels slow, but it's a massive improvement.

invokeai/frontend/web/src/app/logging/logger.ts

@@ -28,7 +28,8 @@ export type LoggerNamespace =
   | 'queue'
   | 'dnd'
   | 'controlLayers'
-  | 'metadata';
+  | 'metadata'
+  | 'konva';
 
 export const logger = (namespace: LoggerNamespace) => $logger.get().child({ namespace });
 

invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx

@@ -1,408 +1,27 @@
-import { $alt, $ctrl, $meta, $shift, Flex, Heading } from '@invoke-ai/ui-library';
+import { Flex } from '@invoke-ai/ui-library';
-import { useStore } from '@nanostores/react';
+import { useAppStore } from 'app/store/storeHooks';
-import { logger } from 'app/logging/logger';
-import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { HeadsUpDisplay } from 'features/controlLayers/components/HeadsUpDisplay';
-import { setStageEventHandlers } from 'features/controlLayers/konva/events';
+import { initializeRenderer } from 'features/controlLayers/konva/renderers/renderer';
-import { renderBackgroundLayer } from 'features/controlLayers/konva/renderers/background';
-import { renderControlAdapters } from 'features/controlLayers/konva/renderers/caLayer';
-import {
-  arrangeEntities,
-  debouncedRenderers,
-  renderers as normalRenderers,
-} from 'features/controlLayers/konva/renderers/layers';
-import { renderDocumentBoundsOverlay } from 'features/controlLayers/konva/renderers/previewLayer';
-import { renderLayers } from 'features/controlLayers/konva/renderers/rasterLayer';
-import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer';
-import {
-  $bbox,
-  $currentFill,
-  $document,
-  $isDrawing,
-  $isMouseDown,
-  $lastAddedPoint,
-  $lastCursorPos,
-  $lastMouseDownPos,
-  $selectedEntity,
-  $spaceKey,
-  $stageAttrs,
-  $toolState,
-  bboxChanged,
-  brushWidthChanged,
-  caBboxChanged,
-  caTranslated,
-  eraserWidthChanged,
-  layerBboxChanged,
-  layerBrushLineAdded,
-  layerEraserLineAdded,
-  layerLinePointAdded,
-  layerRectAdded,
-  layerTranslated,
-  rgBboxChanged,
-  rgBrushLineAdded,
-  rgEraserLineAdded,
-  rgLinePointAdded,
-  rgRectAdded,
-  rgTranslated,
-  toolBufferChanged,
-  toolChanged,
-} from 'features/controlLayers/store/canvasV2Slice';
-import { selectEntityCount } from 'features/controlLayers/store/selectors';
-import type {
-  BboxChangedArg,
-  BrushLineAddedArg,
-  CanvasEntity,
-  EraserLineAddedArg,
-  PointAddedToLineArg,
-  PosChangedArg,
-  RectShapeAddedArg,
-  Tool,
-} from 'features/controlLayers/store/types';
 import Konva from 'konva';
-import type { IRect } from 'konva/lib/types';
+import { memo, useCallback, useEffect, useLayoutEffect, useState } from 'react';
-import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useState } from 'react';
-import { useTranslation } from 'react-i18next';
-import { getImageDTO } from 'services/api/endpoints/images';
 import { useDevicePixelRatio } from 'use-device-pixel-ratio';
 import { v4 as uuidv4 } from 'uuid';
 
 // This will log warnings when layers > 5 - maybe use `import.meta.env.MODE === 'development'` instead?
 Konva.showWarnings = false;
 
-const log = logger('controlLayers');
-
 const useStageRenderer = (stage: Konva.Stage, container: HTMLDivElement | null, asPreview: boolean) => {
-  const dispatch = useAppDispatch();
+  const store = useAppStore();
-  const controlAdapters = useAppSelector((s) => s.canvasV2.controlAdapters);
-  const ipAdapters = useAppSelector((s) => s.canvasV2.ipAdapters);
-  const layers = useAppSelector((s) => s.canvasV2.layers);
-  const regions = useAppSelector((s) => s.canvasV2.regions);
-  const tool = useAppSelector((s) => s.canvasV2.tool);
-  const selectedEntityIdentifier = useAppSelector((s) => s.canvasV2.selectedEntityIdentifier);
-  const maskOpacity = useAppSelector((s) => s.canvasV2.settings.maskOpacity);
-  const bbox = useAppSelector((s) => s.canvasV2.bbox);
-  const document = useAppSelector((s) => s.canvasV2.document);
-  const lastCursorPos = useStore($lastCursorPos);
-  const lastMouseDownPos = useStore($lastMouseDownPos);
-  const isMouseDown = useStore($isMouseDown);
-  const isDrawing = useStore($isDrawing);
-  const selectedEntity = useMemo(() => {
-    const identifier = selectedEntityIdentifier;
-    if (!identifier) {
-      return null;
-    } else if (identifier.type === 'layer') {
-      return layers.find((i) => i.id === identifier.id) ?? null;
-    } else if (identifier.type === 'control_adapter') {
-      return controlAdapters.find((i) => i.id === identifier.id) ?? null;
-    } else if (identifier.type === 'ip_adapter') {
-      return ipAdapters.find((i) => i.id === identifier.id) ?? null;
-    } else if (identifier.type === 'regional_guidance') {
-      return regions.find((i) => i.id === identifier.id) ?? null;
-    } else {
-      return null;
-    }
-  }, [controlAdapters, ipAdapters, layers, regions, selectedEntityIdentifier]);
-
-  const currentFill = useMemo(() => {
-    if (selectedEntity && selectedEntity.type === 'regional_guidance') {
-      return { ...selectedEntity.fill, a: maskOpacity };
-    }
-    return tool.fill;
-  }, [maskOpacity, selectedEntity, tool.fill]);
-
-  const renderers = useMemo(() => (asPreview ? debouncedRenderers : normalRenderers), [asPreview]);
   const dpr = useDevicePixelRatio({ round: false });
 
   useLayoutEffect(() => {
-    $toolState.set(tool);
+    const cleanup = initializeRenderer(store, stage, container);
-    $selectedEntity.set(selectedEntity);
+    return cleanup;
-    $bbox.set(bbox);
+  }, [asPreview, container, stage, store]);
-    $currentFill.set(currentFill);
-    $document.set(document);
-  }, [selectedEntity, tool, bbox, currentFill, document]);
-
-  const onPosChanged = useCallback(
-    (arg: PosChangedArg, entityType: CanvasEntity['type']) => {
-      if (entityType === 'layer') {
-        dispatch(layerTranslated(arg));
-      } else if (entityType === 'control_adapter') {
-        dispatch(caTranslated(arg));
-      } else if (entityType === 'regional_guidance') {
-        dispatch(rgTranslated(arg));
-      }
-    },
-    [dispatch]
-  );
-
-  const onBboxChanged = useCallback(
-    (arg: BboxChangedArg, entityType: CanvasEntity['type']) => {
-      if (entityType === 'layer') {
-        dispatch(layerBboxChanged(arg));
-      } else if (entityType === 'control_adapter') {
-        dispatch(caBboxChanged(arg));
-      } else if (entityType === 'regional_guidance') {
-        dispatch(rgBboxChanged(arg));
-      }
-    },
-    [dispatch]
-  );
-
-  const onBrushLineAdded = useCallback(
-    (arg: BrushLineAddedArg, entityType: CanvasEntity['type']) => {
-      if (entityType === 'layer') {
-        dispatch(layerBrushLineAdded(arg));
-      } else if (entityType === 'regional_guidance') {
-        dispatch(rgBrushLineAdded(arg));
-      }
-    },
-    [dispatch]
-  );
-  const onEraserLineAdded = useCallback(
-    (arg: EraserLineAddedArg, entityType: CanvasEntity['type']) => {
-      if (entityType === 'layer') {
-        dispatch(layerEraserLineAdded(arg));
-      } else if (entityType === 'regional_guidance') {
-        dispatch(rgEraserLineAdded(arg));
-      }
-    },
-    [dispatch]
-  );
-  const onPointAddedToLine = useCallback(
-    (arg: PointAddedToLineArg, entityType: CanvasEntity['type']) => {
-      if (entityType === 'layer') {
-        dispatch(layerLinePointAdded(arg));
-      } else if (entityType === 'regional_guidance') {
-        dispatch(rgLinePointAdded(arg));
-      }
-    },
-    [dispatch]
-  );
-  const onRectShapeAdded = useCallback(
-    (arg: RectShapeAddedArg, entityType: CanvasEntity['type']) => {
-      if (entityType === 'layer') {
-        dispatch(layerRectAdded(arg));
-      } else if (entityType === 'regional_guidance') {
-        dispatch(rgRectAdded(arg));
-      }
-    },
-    [dispatch]
-  );
-  const onBboxTransformed = useCallback(
-    (bbox: IRect) => {
-      dispatch(bboxChanged(bbox));
-    },
-    [dispatch]
-  );
-  const onBrushWidthChanged = useCallback(
-    (width: number) => {
-      dispatch(brushWidthChanged(width));
-    },
-    [dispatch]
-  );
-  const onEraserWidthChanged = useCallback(
-    (width: number) => {
-      dispatch(eraserWidthChanged(width));
-    },
-    [dispatch]
-  );
-  const setTool = useCallback(
-    (tool: Tool) => {
-      dispatch(toolChanged(tool));
-    },
-    [dispatch]
-  );
-  const setToolBuffer = useCallback(
-    (toolBuffer: Tool | null) => {
-      dispatch(toolBufferChanged(toolBuffer));
-    },
-    [dispatch]
-  );
-
-  useLayoutEffect(() => {
-    log.trace('Initializing stage');
-    if (!container) {
-      return;
-    }
-    stage.container(container);
-    return () => {
-      log.trace('Cleaning up stage');
-      stage.destroy();
-    };
-  }, [container, stage]);
-
-  useLayoutEffect(() => {
-    log.trace('Adding stage listeners');
-    if (asPreview || !container) {
-      return;
-    }
-
-    const cleanup = setStageEventHandlers({
-      stage,
-      getToolState: $toolState.get,
-      setTool,
-      setToolBuffer,
-      getIsDrawing: $isDrawing.get,
-      setIsDrawing: $isDrawing.set,
-      getIsMouseDown: $isMouseDown.get,
-      setIsMouseDown: $isMouseDown.set,
-      getSelectedEntity: $selectedEntity.get,
-      getLastAddedPoint: $lastAddedPoint.get,
-      setLastAddedPoint: $lastAddedPoint.set,
-      getLastCursorPos: $lastCursorPos.get,
-      setLastCursorPos: $lastCursorPos.set,
-      getLastMouseDownPos: $lastMouseDownPos.get,
-      setLastMouseDownPos: $lastMouseDownPos.set,
-      getSpaceKey: $spaceKey.get,
-      setStageAttrs: $stageAttrs.set,
-      getDocument: $document.get,
-      getBbox: $bbox.get,
-      onBrushLineAdded,
-      onEraserLineAdded,
-      onPointAddedToLine,
-      onRectShapeAdded,
-      onBrushWidthChanged,
-      onEraserWidthChanged,
-      getCurrentFill: $currentFill.get,
-    });
-
-    return () => {
-      log.trace('Removing stage listeners');
-      cleanup();
-    };
-  }, [
-    asPreview,
-    onBrushLineAdded,
-    onBrushWidthChanged,
-    onEraserLineAdded,
-    onPointAddedToLine,
-    onRectShapeAdded,
-    stage,
-    container,
-    onEraserWidthChanged,
-    setTool,
-    setToolBuffer,
-  ]);
-
-  useLayoutEffect(() => {
-    log.trace('Updating stage dimensions');
-    if (!container) {
-      return;
-    }
-
-    const fitStageToContainer = () => {
-      stage.width(container.offsetWidth);
-      stage.height(container.offsetHeight);
-      $stageAttrs.set({
-        x: stage.x(),
-        y: stage.y(),
-        width: stage.width(),
-        height: stage.height(),
-        scale: stage.scaleX(),
-      });
-      renderBackgroundLayer(stage);
-      renderDocumentBoundsOverlay(stage, $document.get);
-    };
-
-    const resizeObserver = new ResizeObserver(fitStageToContainer);
-    resizeObserver.observe(container);
-    fitStageToContainer();
-
-    return () => {
-      resizeObserver.disconnect();
-    };
-  }, [stage, container]);
-
-  useLayoutEffect(() => {
-    if (asPreview) {
-      // Preview should not display tool
-      return;
-    }
-    log.trace('Rendering tool preview');
-    renderers.renderToolPreview(
-      stage,
-      tool,
-      currentFill,
-      selectedEntity,
-      lastCursorPos,
-      lastMouseDownPos,
-      isDrawing,
-      isMouseDown
-    );
-  }, [
-    asPreview,
-    currentFill,
-    document,
-    isDrawing,
-    isMouseDown,
-    lastCursorPos,
-    lastMouseDownPos,
-    renderers,
-    selectedEntity,
-    stage,
-    tool,
-  ]);
-
-  useLayoutEffect(() => {
-    if (asPreview) {
-      // Preview should not display tool
-      return;
-    }
-    log.trace('Rendering bbox preview');
-    renderers.renderBboxPreview(
-      stage,
-      bbox,
-      tool.selected,
-      $bbox.get,
-      onBboxTransformed,
-      $shift.get,
-      $ctrl.get,
-      $meta.get,
-      $alt.get
-    );
-  }, [asPreview, bbox, onBboxTransformed, renderers, stage, tool.selected]);
-
-  useLayoutEffect(() => {
-    log.trace('Rendering layers');
-    renderLayers(stage, layers, tool.selected, onPosChanged);
-  }, [layers, onPosChanged, stage, tool.selected]);
-
-  useLayoutEffect(() => {
-    log.trace('Rendering regions');
-    renderRegions(stage, regions, maskOpacity, tool.selected, selectedEntity, onPosChanged);
-  }, [maskOpacity, onPosChanged, regions, selectedEntity, stage, tool.selected]);
-
-  useLayoutEffect(() => {
-    log.trace('Rendering layers');
-    renderControlAdapters(stage, controlAdapters, getImageDTO);
-  }, [controlAdapters, stage]);
-
-  useLayoutEffect(() => {
-    renderDocumentBoundsOverlay(stage, $document.get);
-  }, [stage, document]);
-
-  useLayoutEffect(() => {
-    arrangeEntities(stage, layers, controlAdapters, regions);
-  }, [layers, controlAdapters, regions, stage]);
-
-  // useLayoutEffect(() => {
-  //   if (asPreview) {
-  //     // Preview should not check for transparency
-  //     return;
-  //   }
-  //   log.trace('Updating bboxes');
-  //   debouncedRenderers.updateBboxes(stage, state.layers, onBboxChanged);
-  // }, [stage, asPreview, state.layers, onBboxChanged]);
 
   useLayoutEffect(() => {
     Konva.pixelRatio = dpr;
   }, [dpr]);
-
-  useEffect(
-    () => () => {
-      stage.destroy();
-    },
-    [stage]
-  );
 };
 
 type Props = {
@@ -426,9 +45,15 @@ export const StageComponent = memo(({ asPreview = false }: Props) => {
 
   useStageRenderer(stage, container, asPreview);
 
+  useEffect(
+    () => () => {
+      stage.destroy();
+    },
+    [stage]
+  );
+
   return (
     <Flex position="relative" w="full" h="full">
-      {!asPreview && <NoEntitiesFallback />}
       <Flex
         position="absolute"
         top={0}
@@ -454,18 +79,3 @@ export const StageComponent = memo(({ asPreview = false }: Props) => {
 });
 
 StageComponent.displayName = 'StageComponent';
-
-const NoEntitiesFallback = () => {
-  const { t } = useTranslation();
-  const entityCount = useAppSelector(selectEntityCount);
-
-  if (entityCount) {
-    return null;
-  }
-
-  return (
-    <Flex position="absolute" w="full" h="full" alignItems="center" justifyContent="center">
-      <Heading color="base.200">{t('controlLayers.noLayersAdded')}</Heading>
-    </Flex>
-  );
-};

invokeai/frontend/web/src/features/controlLayers/konva/events.ts

@@ -1,5 +1,10 @@
 import { renderBackgroundLayer } from 'features/controlLayers/konva/renderers/background';
-import { renderDocumentBoundsOverlay, scaleToolPreview } from 'features/controlLayers/konva/renderers/previewLayer';
+import {
+  renderDocumentBoundsOverlay,
+  renderToolPreview,
+  scaleToolPreview,
+} from 'features/controlLayers/konva/renderers/previewLayer';
+import { fitDocumentToStage } from 'features/controlLayers/konva/renderers/stage';
 import { getScaledFlooredCursorPosition } from 'features/controlLayers/konva/util';
 import type {
   BrushLineAddedArg,
@@ -19,7 +24,6 @@ import { clamp } from 'lodash-es';
 import {
   BRUSH_SPACING_TARGET_SCALE,
   CANVAS_SCALE_BY,
-  DOCUMENT_FIT_PADDING_PX,
   MAX_BRUSH_SPACING_PX,
   MAX_CANVAS_SCALE,
   MIN_BRUSH_SPACING_PX,
@@ -164,6 +168,16 @@ export const setStageEventHandlers = ({
     }
     const tool = getToolState().selected;
     stage.findOne<Konva.Layer>(`#${PREVIEW_TOOL_GROUP_ID}`)?.visible(tool === 'brush' || tool === 'eraser');
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region mousedown
@@ -176,18 +190,13 @@ export const setStageEventHandlers = ({
     const toolState = getToolState();
     const pos = updateLastCursorPos(stage, setLastCursorPos);
     const selectedEntity = getSelectedEntity();
-    if (!pos || !selectedEntity) {
-      return;
-    }
-    if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') {
-      return;
-    }
-
-    if (getSpaceKey()) {
-      // No drawing when space is down - we are panning the stage
-      return;
-    }
 
+    if (
+      pos &&
+      selectedEntity &&
+      (selectedEntity?.type === 'regional_guidance' || selectedEntity?.type === 'layer') &&
+      !getSpaceKey()
+    ) {
       setIsDrawing(true);
       setLastMouseDownPos(pos);
 
@@ -292,6 +301,17 @@ export const setStageEventHandlers = ({
         }
         setLastAddedPoint(pos);
       }
+    }
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region mouseup
@@ -304,18 +324,12 @@ export const setStageEventHandlers = ({
     const pos = getLastCursorPos();
     const selectedEntity = getSelectedEntity();
 
-    if (!pos || !selectedEntity) {
+    if (
-      return;
+      pos &&
-    }
+      selectedEntity &&
-    if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') {
+      (selectedEntity?.type === 'regional_guidance' || selectedEntity?.type === 'layer') &&
-      return;
+      !getSpaceKey()
-    }
+    ) {
-
-    if (getSpaceKey()) {
-      // No drawing when space is down - we are panning the stage
-      return;
-    }
-
       const toolState = getToolState();
 
       if (toolState.selected === 'rect') {
@@ -339,6 +353,18 @@ export const setStageEventHandlers = ({
 
       setIsDrawing(false);
       setLastMouseDownPos(null);
+    }
+
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region mousemove
@@ -355,26 +381,24 @@ export const setStageEventHandlers = ({
       .findOne<Konva.Layer>(`#${PREVIEW_TOOL_GROUP_ID}`)
       ?.visible(toolState.selected === 'brush' || toolState.selected === 'eraser');
 
-    if (!pos || !selectedEntity) {
+    if (
-      return;
+      pos &&
-    }
+      selectedEntity &&
-    if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') {
+      (selectedEntity.type === 'regional_guidance' || selectedEntity.type === 'layer') &&
-      return;
+      !getSpaceKey() &&
-    }
+      getIsMouseDown()
-
+    ) {
-    if (getSpaceKey()) {
-      // No drawing when space is down - we are panning the stage
-      return;
-    }
-
-    if (!getIsMouseDown()) {
-      return;
-    }
-
       if (toolState.selected === 'brush') {
         if (getIsDrawing()) {
           // Continue the last line
-        maybeAddNextPoint(selectedEntity, pos, getToolState, getLastAddedPoint, setLastAddedPoint, onPointAddedToLine);
+          maybeAddNextPoint(
+            selectedEntity,
+            pos,
+            getToolState,
+            getLastAddedPoint,
+            setLastAddedPoint,
+            onPointAddedToLine
+          );
         } else {
           const bbox = getBbox();
           // Start a new line
@@ -406,7 +430,14 @@ export const setStageEventHandlers = ({
       if (toolState.selected === 'eraser') {
         if (getIsDrawing()) {
           // Continue the last line
-        maybeAddNextPoint(selectedEntity, pos, getToolState, getLastAddedPoint, setLastAddedPoint, onPointAddedToLine);
+          maybeAddNextPoint(
+            selectedEntity,
+            pos,
+            getToolState,
+            getLastAddedPoint,
+            setLastAddedPoint,
+            onPointAddedToLine
+          );
         } else {
           const bbox = getBbox();
           // Start a new line
@@ -433,6 +464,18 @@ export const setStageEventHandlers = ({
           setIsDrawing(true);
         }
       }
+    }
+
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region mouseleave
@@ -450,16 +493,13 @@ export const setStageEventHandlers = ({
 
     stage.findOne<Konva.Layer>(`#${PREVIEW_TOOL_GROUP_ID}`)?.visible(false);
 
-    if (!pos || !selectedEntity) {
+    if (
-      return;
+      pos &&
-    }
+      selectedEntity &&
-    if (selectedEntity.type !== 'regional_guidance' && selectedEntity.type !== 'layer') {
+      (selectedEntity.type === 'regional_guidance' || selectedEntity.type === 'layer') &&
-      return;
+      !getSpaceKey() &&
-    }
+      getIsMouseDown()
-    if (getSpaceKey()) {
+    ) {
-      // No drawing when space is down - we are panning the stage
-      return;
-    }
       if (getIsMouseDown()) {
         if (toolState.selected === 'brush') {
           onPointAddedToLine({ id: selectedEntity.id, point: [pos.x, pos.y] }, selectedEntity.type);
@@ -468,6 +508,18 @@ export const setStageEventHandlers = ({
           onPointAddedToLine({ id: selectedEntity.id, point: [pos.x, pos.y] }, selectedEntity.type);
         }
       }
+    }
+
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region wheel
@@ -489,9 +541,7 @@ export const setStageEventHandlers = ({
     } else {
       // We need the absolute cursor position - not the scaled position
       const cursorPos = stage.getPointerPosition();
-      if (!cursorPos) {
+      if (cursorPos) {
-        return;
-      }
         // Stage's x and y scale are always the same
         const stageScale = stage.scaleX();
         // When wheeling on trackpad, e.evt.ctrlKey is true - in that case, let's reverse the direction
@@ -514,6 +564,17 @@ export const setStageEventHandlers = ({
         scaleToolPreview(stage, getToolState());
         renderDocumentBoundsOverlay(stage, getDocument);
       }
+    }
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region dragmove
@@ -527,6 +588,16 @@ export const setStageEventHandlers = ({
     });
     renderBackgroundLayer(stage);
     renderDocumentBoundsOverlay(stage, getDocument);
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region dragend
@@ -539,6 +610,16 @@ export const setStageEventHandlers = ({
       height: stage.height(),
       scale: stage.scaleX(),
     });
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   });
 
   //#region key
@@ -558,21 +639,22 @@ export const setStageEventHandlers = ({
       setToolBuffer(getToolState().selected);
       setTool('view');
     } else if (e.key === 'r') {
-      // Fit & center the document on the stage
+      const stageAttrs = fitDocumentToStage(stage, getDocument());
-      const width = stage.width();
+      setStageAttrs(stageAttrs);
-      const height = stage.height();
-      const document = getDocument();
-      const docWidthWithBuffer = document.width + DOCUMENT_FIT_PADDING_PX * 2;
-      const docHeightWithBuffer = document.height + DOCUMENT_FIT_PADDING_PX * 2;
-      const scale = Math.min(Math.min(width / docWidthWithBuffer, height / docHeightWithBuffer), 1);
-      const x = (width - docWidthWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale;
-      const y = (height - docHeightWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale;
-      stage.setAttrs({ x, y, width, height, scaleX: scale, scaleY: scale });
-      setStageAttrs({ x, y, width, height, scale });
       scaleToolPreview(stage, getToolState());
       renderBackgroundLayer(stage);
       renderDocumentBoundsOverlay(stage, getDocument);
     }
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   };
   window.addEventListener('keydown', onKeyDown);
 
@@ -589,6 +671,16 @@ export const setStageEventHandlers = ({
       setTool(toolBuffer ?? 'move');
       setToolBuffer(null);
     }
+    renderToolPreview(
+      stage,
+      getToolState(),
+      getCurrentFill(),
+      getSelectedEntity(),
+      getLastCursorPos(),
+      getLastAddedPoint(),
+      getIsDrawing(),
+      getIsMouseDown()
+    );
   };
   window.addEventListener('keyup', onKeyUp);
 

invokeai/frontend/web/src/features/controlLayers/konva/renderers/bbox.ts

@@ -6,8 +6,14 @@ import {
   RG_LAYER_OBJECT_GROUP_NAME,
 } from 'features/controlLayers/konva/naming';
 import { createBboxRect } from 'features/controlLayers/konva/renderers/objects';
-import { imageDataToDataURL } from "features/controlLayers/konva/util";
+import { imageDataToDataURL } from 'features/controlLayers/konva/util';
-import type { ControlAdapterEntity, LayerEntity, RegionEntity } from 'features/controlLayers/store/types';
+import type {
+  BboxChangedArg,
+  CanvasEntity,
+  ControlAdapterEntity,
+  LayerEntity,
+  RegionEntity,
+} from 'features/controlLayers/store/types';
 import Konva from 'konva';
 import type { IRect } from 'konva/lib/types';
 import { assert } from 'tsafe';
@@ -186,10 +192,12 @@ const filterCAChildren = (node: Konva.Node): boolean => node.name() === CA_LAYER
  */
 export const updateBboxes = (
   stage: Konva.Stage,
-  entityStates: (ControlAdapterEntity | LayerEntity | RegionEntity)[],
+  layers: LayerEntity[],
-  onBboxChanged: (layerId: string, bbox: IRect | null) => void
+  controlAdapters: ControlAdapterEntity[],
+  regions: RegionEntity[],
+  onBboxChanged: (arg: BboxChangedArg, entityType: CanvasEntity['type']) => void
 ): void => {
-  for (const entityState of entityStates) {
+  for (const entityState of [...layers, ...controlAdapters, ...regions]) {
     const konvaLayer = stage.findOne<Konva.Layer>(`#${entityState.id}`);
     assert(konvaLayer, `Layer ${entityState.id} not found in stage`);
     // We only need to recalculate the bbox if the layer has changed
@@ -203,23 +211,29 @@ export const updateBboxes = (
       if (entityState.type === 'layer') {
         if (entityState.objects.length === 0) {
           // No objects - no bbox to calculate  
-          onBboxChanged(entityState.id, null);
+          onBboxChanged({ id: entityState.id, bbox: null }, 'layer');
         } else {
-          onBboxChanged(entityState.id, getLayerBboxPixels(konvaLayer, filterLayerChildren));
+          onBboxChanged({ id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterLayerChildren) }, 'layer');
         }
       } else if (entityState.type === 'control_adapter') {
         if (!entityState.image && !entityState.processedImage) {
           // No objects - no bbox to calculate
-          onBboxChanged(entityState.id, null);
+          onBboxChanged({ id: entityState.id, bbox: null }, 'control_adapter');
         } else {
-          onBboxChanged(entityState.id, getLayerBboxPixels(konvaLayer, filterCAChildren));
+          onBboxChanged(
+            { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterCAChildren) },
+            'control_adapter'
+          );
         }
       } else if (entityState.type === 'regional_guidance') {
         if (entityState.objects.length === 0) {
           // No objects - no bbox to calculate
-          onBboxChanged(entityState.id, null);
+          onBboxChanged({ id: entityState.id, bbox: null }, 'regional_guidance');
         } else {
-          onBboxChanged(entityState.id, getLayerBboxPixels(konvaLayer, filterRGChildren));
+          onBboxChanged(
+            { id: entityState.id, bbox: getLayerBboxPixels(konvaLayer, filterRGChildren) },
+            'regional_guidance'
+          );
         }
       }
 

invokeai/frontend/web/src/features/controlLayers/konva/renderers/renderer.ts

@@ -0,0 +1,393 @@
+import { $alt, $ctrl, $meta, $shift } from '@invoke-ai/ui-library';
+import type { Store } from '@reduxjs/toolkit';
+import { logger } from 'app/logging/logger';
+import { $isDebugging } from 'app/store/nanostores/isDebugging';
+import type { RootState } from 'app/store/store';
+import { setStageEventHandlers } from 'features/controlLayers/konva/events';
+import { renderBackgroundLayer } from 'features/controlLayers/konva/renderers/background';
+import { updateBboxes } from 'features/controlLayers/konva/renderers/bbox';
+import { renderControlAdapters } from 'features/controlLayers/konva/renderers/caLayer';
+import { arrangeEntities } from 'features/controlLayers/konva/renderers/layers';
+import {
+  renderBboxPreview,
+  renderDocumentBoundsOverlay,
+  scaleToolPreview,
+} from 'features/controlLayers/konva/renderers/previewLayer';
+import { renderLayers } from 'features/controlLayers/konva/renderers/rasterLayer';
+import { renderRegions } from 'features/controlLayers/konva/renderers/rgLayer';
+import { fitDocumentToStage } from 'features/controlLayers/konva/renderers/stage';
+import {
+  $isDrawing,
+  $isMouseDown,
+  $lastAddedPoint,
+  $lastCursorPos,
+  $lastMouseDownPos,
+  $spaceKey,
+  $stageAttrs,
+  bboxChanged,
+  brushWidthChanged,
+  caBboxChanged,
+  caTranslated,
+  eraserWidthChanged,
+  layerBboxChanged,
+  layerBrushLineAdded,
+  layerEraserLineAdded,
+  layerLinePointAdded,
+  layerRectAdded,
+  layerTranslated,
+  rgBboxChanged,
+  rgBrushLineAdded,
+  rgEraserLineAdded,
+  rgLinePointAdded,
+  rgRectAdded,
+  rgTranslated,
+  toolBufferChanged,
+  toolChanged,
+} from 'features/controlLayers/store/canvasV2Slice';
+import type {
+  BboxChangedArg,
+  BrushLineAddedArg,
+  CanvasEntity,
+  CanvasEntityIdentifier,
+  CanvasV2State,
+  EraserLineAddedArg,
+  PointAddedToLineArg,
+  PosChangedArg,
+  RectShapeAddedArg,
+  Tool,
+} from 'features/controlLayers/store/types';
+import type Konva from 'konva';
+import type { IRect } from 'konva/lib/types';
+import { debounce } from 'lodash-es';
+import type { RgbaColor } from 'react-colorful';
+import { getImageDTO } from 'services/api/endpoints/images';
+/**
+ * Initializes the canvas renderer. It subscribes to the redux store and listens for changes directly, bypassing the
+ * react rendering cycle entirely, improving canvas performance.
+ * @param store The Redux store
+ * @param stage The Konva stage
+ * @param container The stage's target container element
+ * @returns A cleanup function
+ */
+export const initializeRenderer = (
+  store: Store<RootState>,
+  stage: Konva.Stage,
+  container: HTMLDivElement | null
+): (() => void) => {
+  const _log = logger('konva');
+  /**
+   * Logs a message to the console if debugging is enabled.
+   */
+  const logIfDebugging = (message: string) => {
+    if ($isDebugging.get()) {
+      _log.trace(message);
+    }
+  };
+
+  logIfDebugging('Initializing renderer');
+  if (!container) {
+    // Nothing to clean up
+    logIfDebugging('No stage container, skipping initialization');
+    return () => {};
+  }
+
+  stage.container(container);
+
+  // Set up callbacks for various events
+  const onPosChanged = (arg: PosChangedArg, entityType: CanvasEntity['type']) => {
+    logIfDebugging('Position changed');
+    if (entityType === 'layer') {
+      dispatch(layerTranslated(arg));
+    } else if (entityType === 'control_adapter') {
+      dispatch(caTranslated(arg));
+    } else if (entityType === 'regional_guidance') {
+      dispatch(rgTranslated(arg));
+    }
+  };
+  const onBboxChanged = (arg: BboxChangedArg, entityType: CanvasEntity['type']) => {
+    logIfDebugging('Entity bbox changed');
+    if (entityType === 'layer') {
+      dispatch(layerBboxChanged(arg));
+    } else if (entityType === 'control_adapter') {
+      dispatch(caBboxChanged(arg));
+    } else if (entityType === 'regional_guidance') {
+      dispatch(rgBboxChanged(arg));
+    }
+  };
+  const onBrushLineAdded = (arg: BrushLineAddedArg, entityType: CanvasEntity['type']) => {
+    logIfDebugging('Brush line added');
+    if (entityType === 'layer') {
+      dispatch(layerBrushLineAdded(arg));
+    } else if (entityType === 'regional_guidance') {
+      dispatch(rgBrushLineAdded(arg));
+    }
+  };
+  const onEraserLineAdded = (arg: EraserLineAddedArg, entityType: CanvasEntity['type']) => {
+    logIfDebugging('Eraser line added');
+    if (entityType === 'layer') {
+      dispatch(layerEraserLineAdded(arg));
+    } else if (entityType === 'regional_guidance') {
+      dispatch(rgEraserLineAdded(arg));
+    }
+  };
+  const onPointAddedToLine = (arg: PointAddedToLineArg, entityType: CanvasEntity['type']) => {
+    logIfDebugging('Point added to line');
+    if (entityType === 'layer') {
+      dispatch(layerLinePointAdded(arg));
+    } else if (entityType === 'regional_guidance') {
+      dispatch(rgLinePointAdded(arg));
+    }
+  };
+  const onRectShapeAdded = (arg: RectShapeAddedArg, entityType: CanvasEntity['type']) => {
+    logIfDebugging('Rect shape added');
+    if (entityType === 'layer') {
+      dispatch(layerRectAdded(arg));
+    } else if (entityType === 'regional_guidance') {
+      dispatch(rgRectAdded(arg));
+    }
+  };
+  const onBboxTransformed = (bbox: IRect) => {
+    logIfDebugging('Generation bbox transformed');
+    dispatch(bboxChanged(bbox));
+  };
+  const onBrushWidthChanged = (width: number) => {
+    logIfDebugging('Brush width changed');
+    dispatch(brushWidthChanged(width));
+  };
+  const onEraserWidthChanged = (width: number) => {
+    logIfDebugging('Eraser width changed');
+    dispatch(eraserWidthChanged(width));
+  };
+  const setTool = (tool: Tool) => {
+    logIfDebugging('Tool selection changed');
+    dispatch(toolChanged(tool));
+  };
+  const setToolBuffer = (toolBuffer: Tool | null) => {
+    logIfDebugging('Tool buffer changed');
+    dispatch(toolBufferChanged(toolBuffer));
+  };
+
+  const _getSelectedEntity = (canvasV2: CanvasV2State): CanvasEntity | null => {
+    const identifier = canvasV2.selectedEntityIdentifier;
+    let selectedEntity: CanvasEntity | null = null;
+    if (!identifier) {
+      selectedEntity = null;
+    } else if (identifier.type === 'layer') {
+      selectedEntity = canvasV2.layers.find((i) => i.id === identifier.id) ?? null;
+    } else if (identifier.type === 'control_adapter') {
+      selectedEntity = canvasV2.controlAdapters.find((i) => i.id === identifier.id) ?? null;
+    } else if (identifier.type === 'ip_adapter') {
+      selectedEntity = canvasV2.ipAdapters.find((i) => i.id === identifier.id) ?? null;
+    } else if (identifier.type === 'regional_guidance') {
+      selectedEntity = canvasV2.regions.find((i) => i.id === identifier.id) ?? null;
+    } else {
+      selectedEntity = null;
+    }
+    logIfDebugging('Selected entity changed');
+    return selectedEntity;
+  };
+
+  const _getCurrentFill = (canvasV2: CanvasV2State, selectedEntity: CanvasEntity | null) => {
+    let currentFill: RgbaColor = canvasV2.tool.fill;
+    if (selectedEntity && selectedEntity.type === 'regional_guidance') {
+      currentFill = { ...selectedEntity.fill, a: canvasV2.settings.maskOpacity };
+    } else {
+      currentFill = canvasV2.tool.fill;
+    }
+    logIfDebugging('Current fill changed');
+    return currentFill;
+  };
+
+  const { getState, subscribe, dispatch } = store;
+
+  // Create closures for the rendering functions, used to check if specific parts of state have changed so we only
+  // render what needs to be rendered.
+  let prevCanvasV2 = getState().canvasV2;
+  let selectedEntityIdentifier: CanvasEntityIdentifier | null = prevCanvasV2.selectedEntityIdentifier;
+  let selectedEntity: CanvasEntity | null = _getSelectedEntity(prevCanvasV2);
+  let currentFill: RgbaColor = _getCurrentFill(prevCanvasV2, selectedEntity);
+  let didSelectedEntityChange: boolean = false;
+
+  // On the first render, we need to render everything.
+  let isFirstRender = true;
+
+  // Stage event listeners use a fully imperative approach to event handling, using these helpers to get state.
+  const getBbox = () => getState().canvasV2.bbox;
+  const getDocument = () => getState().canvasV2.document;
+  const getToolState = () => getState().canvasV2.tool;
+  const getSelectedEntity = () => selectedEntity;
+  const getCurrentFill = () => currentFill;
+
+  // Calculating bounding boxes is expensive, must be debounced to not block the UI thread.
+  // TODO(psyche): Figure out how to do this in a worker. Probably means running the renderer in a worker and sending
+  // the entire state over when needed.
+  const debouncedUpdateBboxes = debounce(updateBboxes, 300);
+
+  const cleanupListeners = setStageEventHandlers({
+    stage,
+    getToolState,
+    setTool,
+    setToolBuffer,
+    getIsDrawing: $isDrawing.get,
+    setIsDrawing: $isDrawing.set,
+    getIsMouseDown: $isMouseDown.get,
+    setIsMouseDown: $isMouseDown.set,
+    getSelectedEntity,
+    getLastAddedPoint: $lastAddedPoint.get,
+    setLastAddedPoint: $lastAddedPoint.set,
+    getLastCursorPos: $lastCursorPos.get,
+    setLastCursorPos: $lastCursorPos.set,
+    getLastMouseDownPos: $lastMouseDownPos.get,
+    setLastMouseDownPos: $lastMouseDownPos.set,
+    getSpaceKey: $spaceKey.get,
+    setStageAttrs: $stageAttrs.set,
+    getDocument,
+    getBbox,
+    onBrushLineAdded,
+    onEraserLineAdded,
+    onPointAddedToLine,
+    onRectShapeAdded,
+    onBrushWidthChanged,
+    onEraserWidthChanged,
+    getCurrentFill,
+  });
+
+  const renderCanvas = () => {
+    const { canvasV2 } = store.getState();
+
+    if (prevCanvasV2 === canvasV2 && !isFirstRender) {
+      logIfDebugging('No changes detected, skipping render');
+      return;
+    }
+
+    // We can save some cycles for specific renderers if we track whether the selected entity has changed.
+    if (canvasV2.selectedEntityIdentifier !== selectedEntityIdentifier) {
+      selectedEntityIdentifier = canvasV2.selectedEntityIdentifier;
+      selectedEntity = _getSelectedEntity(canvasV2);
+      didSelectedEntityChange = true;
+    } else {
+      didSelectedEntityChange = false;
+    }
+
+    // The current fill is either the tool fill or, if a regional guidance region is selected, the mask fill for that
+    // region. We need to manually sync this state.
+    if (isFirstRender || canvasV2.tool.fill !== prevCanvasV2.tool.fill || didSelectedEntityChange) {
+      currentFill = _getCurrentFill(canvasV2, selectedEntity);
+    }
+
+    if (
+      isFirstRender ||
+      canvasV2.layers !== prevCanvasV2.layers ||
+      canvasV2.tool.selected !== prevCanvasV2.tool.selected
+    ) {
+      logIfDebugging('Rendering layers');
+      renderLayers(stage, canvasV2.layers, canvasV2.tool.selected, onPosChanged);
+    }
+
+    if (
+      isFirstRender ||
+      canvasV2.regions !== prevCanvasV2.regions ||
+      canvasV2.settings.maskOpacity !== prevCanvasV2.settings.maskOpacity ||
+      canvasV2.tool.selected !== prevCanvasV2.tool.selected ||
+      didSelectedEntityChange
+    ) {
+      logIfDebugging('Rendering regions');
+      renderRegions(
+        stage,
+        canvasV2.regions,
+        canvasV2.settings.maskOpacity,
+        canvasV2.tool.selected,
+        selectedEntity,
+        onPosChanged
+      );
+    }
+
+    if (isFirstRender || canvasV2.controlAdapters !== prevCanvasV2.controlAdapters) {
+      logIfDebugging('Rendering control adapters');
+      renderControlAdapters(stage, canvasV2.controlAdapters, getImageDTO);
+    }
+
+    if (isFirstRender || canvasV2.document !== prevCanvasV2.document) {
+      logIfDebugging('Rendering document bounds overlay');
+      renderDocumentBoundsOverlay(stage, getDocument);
+    }
+
+    if (isFirstRender || canvasV2.bbox !== prevCanvasV2.bbox || canvasV2.tool.selected !== prevCanvasV2.tool.selected) {
+      logIfDebugging('Rendering generation bbox');
+      renderBboxPreview(
+        stage,
+        canvasV2.bbox,
+        canvasV2.tool.selected,
+        getBbox,
+        onBboxTransformed,
+        $shift.get,
+        $ctrl.get,
+        $meta.get,
+        $alt.get
+      );
+    }
+
+    if (
+      isFirstRender ||
+      canvasV2.layers !== prevCanvasV2.layers ||
+      canvasV2.controlAdapters !== prevCanvasV2.controlAdapters ||
+      canvasV2.regions !== prevCanvasV2.regions
+    ) {
+      logIfDebugging('Updating entity bboxes');
+      debouncedUpdateBboxes(stage, canvasV2.layers, canvasV2.controlAdapters, canvasV2.regions, onBboxChanged);
+    }
+
+    if (
+      isFirstRender ||
+      canvasV2.layers !== prevCanvasV2.layers ||
+      canvasV2.controlAdapters !== prevCanvasV2.controlAdapters ||
+      canvasV2.regions !== prevCanvasV2.regions
+    ) {
+      logIfDebugging('Arranging entities');
+      arrangeEntities(stage, canvasV2.layers, canvasV2.controlAdapters, canvasV2.regions);
+    }
+
+    prevCanvasV2 = canvasV2;
+
+    if (isFirstRender) {
+      isFirstRender = false;
+    }
+  };
+
+  // We can use a resize observer to ensure the stage always fits the container. We also need to re-render the bg and
+  // document bounds overlay when the stage is resized.
+  const fitStageToContainer = () => {
+    stage.width(container.offsetWidth);
+    stage.height(container.offsetHeight);
+    $stageAttrs.set({
+      x: stage.x(),
+      y: stage.y(),
+      width: stage.width(),
+      height: stage.height(),
+      scale: stage.scaleX(),
+    });
+    renderBackgroundLayer(stage);
+    renderDocumentBoundsOverlay(stage, getDocument);
+  };
+
+  const resizeObserver = new ResizeObserver(fitStageToContainer);
+  resizeObserver.observe(container);
+  fitStageToContainer();
+
+  const unsubscribeRenderer = subscribe(renderCanvas);
+
+  logIfDebugging('First render of konva stage');
+  // On first render, the document should be fit to the stage.
+  const stageAttrs = fitDocumentToStage(stage, prevCanvasV2.document);
+  // The HUD displays some of the stage attributes, so we need to update it here.
+  $stageAttrs.set(stageAttrs);
+  scaleToolPreview(stage, getToolState());
+  renderCanvas();
+
+  return () => {
+    logIfDebugging('Cleaning up konva renderer');
+    unsubscribeRenderer();
+    cleanupListeners();
+    resizeObserver.disconnect();
+  };
+};

invokeai/frontend/web/src/features/controlLayers/konva/renderers/stage.ts

@@ -0,0 +1,16 @@
+import { DOCUMENT_FIT_PADDING_PX } from 'features/controlLayers/konva/constants';
+import type { CanvasV2State, StageAttrs } from 'features/controlLayers/store/types';
+import type Konva from 'konva';
+
+export const fitDocumentToStage = (stage: Konva.Stage, document: CanvasV2State['document']): StageAttrs => {
+  // Fit & center the document on the stage
+  const width = stage.width();
+  const height = stage.height();
+  const docWidthWithBuffer = document.width + DOCUMENT_FIT_PADDING_PX * 2;
+  const docHeightWithBuffer = document.height + DOCUMENT_FIT_PADDING_PX * 2;
+  const scale = Math.min(Math.min(width / docWidthWithBuffer, height / docHeightWithBuffer), 1);
+  const x = (width - docWidthWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale;
+  const y = (height - docHeightWithBuffer * scale) / 2 + DOCUMENT_FIT_PADDING_PX * scale;
+  stage.setAttrs({ x, y, width, height, scaleX: scale, scaleY: scale });
+  return { x, y, width, height, scale };
+};

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -881,7 +881,7 @@ export type CanvasV2State = {
 
 export type StageAttrs = { x: number; y: number; width: number; height: number; scale: number };
 export type PosChangedArg = { id: string; x: number; y: number };
-export type BboxChangedArg = { id: string; bbox: IRect };
+export type BboxChangedArg = { id: string; bbox: Rect | null };
 export type EraserLineAddedArg = {
   id: string;
   points: [number, number, number, number];