diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts
index 983e574b63..60ce3f8c88 100644
--- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasBbox.ts
@@ -235,6 +235,7 @@ export class CanvasBbox {
     const toolState = this.manager.stateApi.getToolState();
 
     this.konva.group.visible(true);
+    this.parent.getLayer().listening(toolState.selected === 'bbox');
     this.konva.group.listening(toolState.selected === 'bbox');
     this.konva.rect.setAttrs({
       x: bbox.rect.x,
diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasPreview.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasPreview.ts
index b5969299cc..7d59aadb17 100644
--- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasPreview.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasPreview.ts
@@ -21,7 +21,7 @@ export class CanvasPreview {
   constructor(manager: CanvasManager) {
     this.manager = manager;
     this.konva = {
-      layer: new Konva.Layer({ listening: true, imageSmoothingEnabled: false }),
+      layer: new Konva.Layer({ listening: false, imageSmoothingEnabled: false }),
     };
 
     this.stagingArea = new CanvasStagingArea(this);
diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool.ts
index 37020570f8..49afdca9c6 100644
--- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool.ts
@@ -55,9 +55,9 @@ export class CanvasTool {
     this.path = this.manager.path.concat(this.id);
     this.log = this.manager.buildLogger(this.getLoggingContext);
     this.konva = {
-      group: new Konva.Group({ name: `${this.type}:group` }),
+      group: new Konva.Group({ name: `${this.type}:group`, listening: false }),
       brush: {
-        group: new Konva.Group({ name: `${this.type}:brush_group` }),
+        group: new Konva.Group({ name: `${this.type}:brush_group`, listening: false }),
         fillCircle: new Konva.Circle({
           name: `${this.type}:brush_fill_circle`,
           listening: false,
@@ -79,7 +79,7 @@ export class CanvasTool {
         }),
       },
       eraser: {
-        group: new Konva.Group({ name: `${this.type}:eraser_group` }),
+        group: new Konva.Group({ name: `${this.type}:eraser_group`, listening: false }),
         fillCircle: new Konva.Circle({
           name: `${this.type}:eraser_fill_circle`,
           listening: false,
@@ -103,7 +103,7 @@ export class CanvasTool {
         }),
       },
       eyeDropper: {
-        group: new Konva.Group({ name: `${this.type}:eyeDropper_group` }),
+        group: new Konva.Group({ name: `${this.type}:eyeDropper_group`, listening: false }),
         fillCircle: new Konva.Circle({
           name: `${this.type}:eyeDropper_fill_circle`,
           listening: false,