diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts
index 9fd475ffa4..37ce33cca9 100644
--- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts
@@ -420,7 +420,9 @@ export class CanvasObjectRenderer {
     }
     imageDTO = await uploadImage(blob, `${this.id}_rasterized.png`, 'other', true);
     const imageObject = imageDTOToImageObject(imageDTO);
+    if (replaceObjects) {
     await this.renderObject(imageObject, true);
+    }
     this.manager.stateApi.rasterizeEntity({
       entityIdentifier: this.parent.getEntityIdentifier(),
       imageObject,