fix(ui): firefox drawing lag

Firefox v125.0.3 and below has a bug where `mouseenter` events are fired continually during mouse moves. The issue isn't present on FF v126.0b6 Developer Edition. It's not clear if the issue is present on FF nightly, and we're not sure if it will actually be fixed in the stable v126 release.

The control layers drawing logic relied on on `mouseenter` events to create new lines, and `mousemove` to extend existing lines. On the affected version of FF, all line extensions are turned into new lines, resulting in very poor performance, noncontiguous lines, and way-too-big internal state.

To resolve this, the drawing handling was updated to not use `mouseenter` at all. As a bonus, resolving this issue has resulted in simpler logic for drawing on the canvas.

invokeai/frontend/web/src/features/controlLayers/components/StageComponent.tsx

@@ -7,7 +7,6 @@ import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { useMouseEvents } from 'features/controlLayers/hooks/mouseEventHooks';
 import {
   $cursorPosition,
-  $isMouseOver,
   $lastMouseDownPos,
   $tool,
   isRegionalGuidanceLayer,
@@ -48,10 +47,9 @@ const useStageRenderer = (
   const dispatch = useAppDispatch();
   const state = useAppSelector((s) => s.controlLayers.present);
   const tool = useStore($tool);
-  const { onMouseDown, onMouseUp, onMouseMove, onMouseEnter, onMouseLeave, onMouseWheel } = useMouseEvents();
+  const mouseEventHandlers = useMouseEvents();
   const cursorPosition = useStore($cursorPosition);
   const lastMouseDownPos = useStore($lastMouseDownPos);
-  const isMouseOver = useStore($isMouseOver);
   const selectedLayerIdColor = useAppSelector(selectSelectedLayerColor);
   const selectedLayerType = useAppSelector(selectSelectedLayerType);
   const layerIds = useMemo(() => state.layers.map((l) => l.id), [state.layers]);
@@ -90,23 +88,21 @@ const useStageRenderer = (
     if (asPreview) {
       return;
     }
-    stage.on('mousedown', onMouseDown);
+    stage.on('mousedown', mouseEventHandlers.onMouseDown);
-    stage.on('mouseup', onMouseUp);
+    stage.on('mouseup', mouseEventHandlers.onMouseUp);
-    stage.on('mousemove', onMouseMove);
+    stage.on('mousemove', mouseEventHandlers.onMouseMove);
-    stage.on('mouseenter', onMouseEnter);
+    stage.on('mouseleave', mouseEventHandlers.onMouseLeave);
-    stage.on('mouseleave', onMouseLeave);
+    stage.on('wheel', mouseEventHandlers.onMouseWheel);
-    stage.on('wheel', onMouseWheel);
 
     return () => {
       log.trace('Cleaning up stage listeners');
-      stage.off('mousedown', onMouseDown);
+      stage.off('mousedown', mouseEventHandlers.onMouseDown);
-      stage.off('mouseup', onMouseUp);
+      stage.off('mouseup', mouseEventHandlers.onMouseUp);
-      stage.off('mousemove', onMouseMove);
+      stage.off('mousemove', mouseEventHandlers.onMouseMove);
-      stage.off('mouseenter', onMouseEnter);
+      stage.off('mouseleave', mouseEventHandlers.onMouseLeave);
-      stage.off('mouseleave', onMouseLeave);
+      stage.off('wheel', mouseEventHandlers.onMouseWheel);
-      stage.off('wheel', onMouseWheel);
     };
-  }, [stage, asPreview, onMouseDown, onMouseUp, onMouseMove, onMouseEnter, onMouseLeave, onMouseWheel]);
+  }, [stage, asPreview, mouseEventHandlers]);
 
   useLayoutEffect(() => {
     log.trace('Updating stage dimensions');
@@ -147,7 +143,6 @@ const useStageRenderer = (
       state.globalMaskLayerOpacity,
       cursorPosition,
       lastMouseDownPos,
-      isMouseOver,
       state.brushSize
     );
   }, [
@@ -159,7 +154,6 @@ const useStageRenderer = (
     state.globalMaskLayerOpacity,
     cursorPosition,
     lastMouseDownPos,
-    isMouseOver,
     state.brushSize,
     renderers,
   ]);

invokeai/frontend/web/src/features/controlLayers/hooks/mouseEventHooks.ts

@@ -4,8 +4,7 @@ import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { calculateNewBrushSize } from 'features/canvas/hooks/useCanvasZoom';
 import {
   $cursorPosition,
-  $isMouseDown,
+  $isDrawing,
-  $isMouseOver,
   $lastMouseDownPos,
   $tool,
   brushSizeChanged,
@@ -21,6 +20,7 @@ import { useCallback, useRef } from 'react';
 const getIsFocused = (stage: Konva.Stage) => {
   return stage.container().contains(document.activeElement);
 };
+const getIsMouseDown = (e: KonvaEventObject<MouseEvent>) => e.evt.buttons === 1;
 
 export const getScaledFlooredCursorPosition = (stage: Konva.Stage) => {
   const pointerPosition = stage.getPointerPosition();
@@ -55,7 +55,7 @@ export const useMouseEvents = () => {
   const brushSize = useAppSelector((s) => s.controlLayers.present.brushSize);
 
   const onMouseDown = useCallback(
-    (e: KonvaEventObject<MouseEvent | TouchEvent>) => {
+    (e: KonvaEventObject<MouseEvent>) => {
       const stage = e.target.getStage();
       if (!stage) {
         return;
@@ -64,7 +64,6 @@ export const useMouseEvents = () => {
       if (!pos) {
         return;
       }
-      $isMouseDown.set(true);
       $lastMouseDownPos.set(pos);
       if (!selectedLayerId) {
         return;
@@ -77,18 +76,18 @@ export const useMouseEvents = () => {
             tool,
           })
         );
+        $isDrawing.set(true);
       }
     },
     [dispatch, selectedLayerId, tool]
   );
 
   const onMouseUp = useCallback(
-    (e: KonvaEventObject<MouseEvent | TouchEvent>) => {
+    (e: KonvaEventObject<MouseEvent>) => {
       const stage = e.target.getStage();
       if (!stage) {
         return;
       }
-      $isMouseDown.set(false);
       const pos = $cursorPosition.get();
       const lastPos = $lastMouseDownPos.get();
       const tool = $tool.get();
@@ -105,13 +104,14 @@ export const useMouseEvents = () => {
           })
         );
       }
+      $isDrawing.set(false);
       $lastMouseDownPos.set(null);
     },
     [dispatch, selectedLayerId]
   );
 
   const onMouseMove = useCallback(
-    (e: KonvaEventObject<MouseEvent | TouchEvent>) => {
+    (e: KonvaEventObject<MouseEvent>) => {
       const stage = e.target.getStage();
       if (!stage) {
         return;
@@ -120,7 +120,9 @@ export const useMouseEvents = () => {
       if (!pos || !selectedLayerId) {
         return;
       }
-      if (getIsFocused(stage) && $isMouseOver.get() && $isMouseDown.get() && (tool === 'brush' || tool === 'eraser')) {
+      if (getIsFocused(stage) && getIsMouseDown(e) && (tool === 'brush' || tool === 'eraser')) {
+        if ($isDrawing.get()) {
+          // Continue the last line
           if (lastCursorPosRef.current) {
             // Dispatching redux events impacts perf substantially - using brush spacing keeps dispatches to a reasonable number
             if (Math.hypot(lastCursorPosRef.current[0] - pos.x, lastCursorPosRef.current[1] - pos.y) < BRUSH_SPACING) {
@@ -129,13 +131,18 @@ export const useMouseEvents = () => {
           }
           lastCursorPosRef.current = [pos.x, pos.y];
           dispatch(rgLayerPointsAdded({ layerId: selectedLayerId, point: lastCursorPosRef.current }));
+        } else {
+          // Start a new line
+          dispatch(rgLayerLineAdded({ layerId: selectedLayerId, points: [pos.x, pos.y, pos.x, pos.y], tool }));
+        }
+        $isDrawing.set(true);
       }
     },
     [dispatch, selectedLayerId, tool]
   );
 
   const onMouseLeave = useCallback(
-    (e: KonvaEventObject<MouseEvent | TouchEvent>) => {
+    (e: KonvaEventObject<MouseEvent>) => {
       const stage = e.target.getStage();
       if (!stage) {
         return;
@@ -145,54 +152,17 @@ export const useMouseEvents = () => {
         pos &&
         selectedLayerId &&
         getIsFocused(stage) &&
-        $isMouseOver.get() &&
+        getIsMouseDown(e) &&
-        $isMouseDown.get() &&
         (tool === 'brush' || tool === 'eraser')
       ) {
         dispatch(rgLayerPointsAdded({ layerId: selectedLayerId, point: [pos.x, pos.y] }));
       }
-      $isMouseOver.set(false);
+      $isDrawing.set(false);
-      $isMouseDown.set(false);
       $cursorPosition.set(null);
     },
     [selectedLayerId, tool, dispatch]
   );
 
-  const onMouseEnter = useCallback(
-    (e: KonvaEventObject<MouseEvent>) => {
-      const stage = e.target.getStage();
-      if (!stage) {
-        return;
-      }
-      $isMouseOver.set(true);
-      const pos = syncCursorPos(stage);
-      if (!pos) {
-        return;
-      }
-      if (!getIsFocused(stage)) {
-        return;
-      }
-      if (e.evt.buttons !== 1) {
-        $isMouseDown.set(false);
-      } else {
-        $isMouseDown.set(true);
-        if (!selectedLayerId) {
-          return;
-        }
-        if (tool === 'brush' || tool === 'eraser') {
-          dispatch(
-            rgLayerLineAdded({
-              layerId: selectedLayerId,
-              points: [pos.x, pos.y, pos.x, pos.y],
-              tool,
-            })
-          );
-        }
-      }
-    },
-    [dispatch, selectedLayerId, tool]
-  );
-
   const onMouseWheel = useCallback(
     (e: KonvaEventObject<WheelEvent>) => {
       e.evt.preventDefault();
@@ -213,5 +183,5 @@ export const useMouseEvents = () => {
     [shouldInvertBrushSizeScrollDirection, brushSize, dispatch]
   );
 
-  return { onMouseDown, onMouseUp, onMouseMove, onMouseEnter, onMouseLeave, onMouseWheel };
+  return { onMouseDown, onMouseUp, onMouseMove, onMouseLeave, onMouseWheel };
 };

invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts

@@ -801,8 +801,7 @@ const migrateControlLayersState = (state: any): any => {
   return state;
 };
 
-export const $isMouseDown = atom(false);
+export const $isDrawing = atom(false);
-export const $isMouseOver = atom(false);
 export const $lastMouseDownPos = atom<Vector2d | null>(null);
 export const $tool = atom<Tool>('brush');
 export const $cursorPosition = atom<Vector2d | null>(null);

invokeai/frontend/web/src/features/controlLayers/util/renderers.ts

@@ -137,7 +137,6 @@ const renderToolPreview = (
   globalMaskLayerOpacity: number,
   cursorPos: Vector2d | null,
   lastMouseDownPos: Vector2d | null,
-  isMouseOver: boolean,
   brushSize: number
 ) => {
   const layerCount = stage.find(`.${RG_LAYER_NAME}`).length;
@@ -161,7 +160,7 @@ const renderToolPreview = (
 
   const toolPreviewLayer = stage.findOne<Konva.Layer>(`#${TOOL_PREVIEW_LAYER_ID}`) ?? createToolPreviewLayer(stage);
 
-  if (!isMouseOver || layerCount === 0) {
+  if (!cursorPos || layerCount === 0) {
     // We can bail early if the mouse isn't over the stage or there are no layers
     toolPreviewLayer.visible(false);
     return;