diff --git a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts
index 5ab7b87ba0..c27a98c826 100644
--- a/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts
@@ -12,7 +12,12 @@ import type {
   ProcessorConfig,
   T2IAdapterConfig,
 } from 'features/controlLayers/util/controlAdapters';
-import { buildControlAdapterProcessor, imageDTOToImageWithDims } from 'features/controlLayers/util/controlAdapters';
+import {
+  buildControlAdapterProcessor,
+  controlNetToT2IAdapter,
+  imageDTOToImageWithDims,
+  t2iAdapterToControlNet,
+} from 'features/controlLayers/util/controlAdapters';
 import { zModelIdentifierField } from 'features/nodes/types/common';
 import { calculateNewSize } from 'features/parameters/components/ImageSize/calculateNewSize';
 import { initialAspectRatioState } from 'features/parameters/components/ImageSize/constants';
@@ -284,6 +289,14 @@ export const controlLayersSlice = createSlice({
         return;
       }
       layer.controlAdapter.model = zModelIdentifierField.parse(modelConfig);
+
+      // We may need to convert the CA to match the model
+      if (layer.controlAdapter.type === 't2i_adapter' && layer.controlAdapter.model.type === 'controlnet') {
+        layer.controlAdapter = t2iAdapterToControlNet(layer.controlAdapter);
+      } else if (layer.controlAdapter.type === 'controlnet' && layer.controlAdapter.model.type === 't2i_adapter') {
+        layer.controlAdapter = controlNetToT2IAdapter(layer.controlAdapter);
+      }
+
       const candidateProcessorConfig = buildControlAdapterProcessor(modelConfig);
       if (candidateProcessorConfig?.type !== layer.controlAdapter.processorConfig?.type) {
         // The processor has changed. For example, the previous model was a Canny model and the new model is a Depth
diff --git a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts
index 4b756b33e7..6cedc81a0b 100644
--- a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts
+++ b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts
@@ -4,7 +4,7 @@ import type {
   ParameterIPAdapterModel,
   ParameterT2IAdapterModel,
 } from 'features/parameters/types/parameterSchemas';
-import { merge } from 'lodash-es';
+import { merge, omit } from 'lodash-es';
 import type {
   BaseModelType,
   CannyImageProcessorInvocation,
@@ -466,3 +466,18 @@ export const imageDTOToImageWithDims = ({ image_name, width, height }: ImageDTO)
   width,
   height,
 });
+
+export const t2iAdapterToControlNet = (t2iAdapter: T2IAdapterConfig): ControlNetConfig => {
+  return {
+    ...deepClone(t2iAdapter),
+    type: 'controlnet',
+    controlMode: initialControlNet.controlMode,
+  };
+};
+
+export const controlNetToT2IAdapter = (controlNet: ControlNetConfig): T2IAdapterConfig => {
+  return {
+    ...omit(deepClone(controlNet), 'controlMode'),
+    type: 't2i_adapter',
+  };
+};