diff --git a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts
index 6b3dfc5e6c..a1b5e0ebc8 100644
--- a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts
@@ -627,7 +627,7 @@ export const controlLayersSlice = createSlice({
       reducer: (state, action: PayloadAction<{ layerId: string; imageDTO: ImageDTO | null }>) => {
         const { layerId, imageDTO } = action.payload;
         // Highlander! There can be only one!
-        state.layers = state.layers.filter((l) => isInitialImageLayer(l));
+        state.layers = state.layers.filter((l) => (isInitialImageLayer(l) ? false : true));
         const layer: InitialImageLayer = {
           id: layerId,
           type: 'initial_image_layer',