refactor(ui): continue wiring up CA logic across (wip)

It works!

invokeai/frontend/web/public/locales/en.json

@@ -917,6 +917,7 @@
             "missingInputForField": "{{nodeLabel}} -> {{fieldLabel}} missing input",
             "missingNodeTemplate": "Missing node template",
             "noControlImageForControlAdapter": "Control Adapter #{{number}} has no control image",
+            "imageNotProcessedForControlAdapter": "Control Adapter #{{number}}'s image is not processed",
             "noInitialImageSelected": "No initial image selected",
             "noModelForControlAdapter": "Control Adapter #{{number}} has no model selected.",
             "incompatibleBaseModelForControlAdapter": "Control Adapter #{{number}} model is incompatible with main model.",

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageDropped.ts

@@ -7,6 +7,11 @@ import {
   controlAdapterImageChanged,
   controlAdapterIsEnabledChanged,
 } from 'features/controlAdapters/store/controlAdaptersSlice';
+import {
+  caLayerImageChanged,
+  ipaLayerImageChanged,
+  rgLayerIPAdapterImageChanged,
+} from 'features/controlLayers/store/controlLayersSlice';
 import type { TypesafeDraggableData, TypesafeDroppableData } from 'features/dnd/types';
 import { imageSelected } from 'features/gallery/store/gallerySlice';
 import { fieldImageValueChanged } from 'features/nodes/store/nodesSlice';
@@ -83,6 +88,61 @@ export const addImageDroppedListener = (startAppListening: AppStartListening) =>
         return;
       }
 
+      /**
+       * Image dropped on Control Adapter Layer
+       */
+      if (
+        overData.actionType === 'SET_CA_LAYER_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        const { layerId } = overData.context;
+        dispatch(
+          caLayerImageChanged({
+            layerId,
+            imageDTO: activeData.payload.imageDTO,
+          })
+        );
+        return;
+      }
+
+      /**
+       * Image dropped on IP Adapter Layer
+       */
+      if (
+        overData.actionType === 'SET_IPA_LAYER_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        const { layerId } = overData.context;
+        dispatch(
+          ipaLayerImageChanged({
+            layerId,
+            imageDTO: activeData.payload.imageDTO,
+          })
+        );
+        return;
+      }
+
+      /**
+       * Image dropped on RG Layer IP Adapter
+       */
+      if (
+        overData.actionType === 'SET_RG_LAYER_IP_ADAPTER_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        const { layerId, ipAdapterId } = overData.context;
+        dispatch(
+          rgLayerIPAdapterImageChanged({
+            layerId,
+            ipAdapterId,
+            imageDTO: activeData.payload.imageDTO,
+          })
+        );
+        return;
+      }
+
       /**
        * Image dropped on Canvas
        */

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts

@@ -6,6 +6,11 @@ import {
   controlAdapterImageChanged,
   controlAdapterIsEnabledChanged,
 } from 'features/controlAdapters/store/controlAdaptersSlice';
+import {
+  caLayerImageChanged,
+  ipaLayerImageChanged,
+  rgLayerIPAdapterImageChanged,
+} from 'features/controlLayers/store/controlLayersSlice';
 import { fieldImageValueChanged } from 'features/nodes/store/nodesSlice';
 import { initialImageChanged, selectOptimalDimension } from 'features/parameters/store/generationSlice';
 import { addToast } from 'features/system/store/systemSlice';
@@ -108,6 +113,39 @@ export const addImageUploadedFulfilledListener = (startAppListening: AppStartLis
         return;
       }
 
+      if (postUploadAction?.type === 'SET_CA_LAYER_IMAGE') {
+        const { layerId } = postUploadAction;
+        dispatch(caLayerImageChanged({ layerId, imageDTO }));
+        dispatch(
+          addToast({
+            ...DEFAULT_UPLOADED_TOAST,
+            description: t('toast.setControlImage'),
+          })
+        );
+      }
+
+      if (postUploadAction?.type === 'SET_IPA_LAYER_IMAGE') {
+        const { layerId } = postUploadAction;
+        dispatch(ipaLayerImageChanged({ layerId, imageDTO }));
+        dispatch(
+          addToast({
+            ...DEFAULT_UPLOADED_TOAST,
+            description: t('toast.setControlImage'),
+          })
+        );
+      }
+
+      if (postUploadAction?.type === 'SET_RG_LAYER_IP_ADAPTER_IMAGE') {
+        const { layerId, ipAdapterId } = postUploadAction;
+        dispatch(rgLayerIPAdapterImageChanged({ layerId, ipAdapterId, imageDTO }));
+        dispatch(
+          addToast({
+            ...DEFAULT_UPLOADED_TOAST,
+            description: t('toast.setControlImage'),
+          })
+        );
+      }
+
       if (postUploadAction?.type === 'SET_INITIAL_IMAGE') {
         dispatch(initialImageChanged(imageDTO));
         dispatch(

invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts

@@ -16,6 +16,7 @@ import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
 import i18n from 'i18next';
 import { forEach } from 'lodash-es';
 import { getConnectedEdges } from 'reactflow';
+import { assert } from 'tsafe';
 
 const selector = createMemoizedSelector(
   [
@@ -97,73 +98,93 @@ const selector = createMemoizedSelector(
         reasons.push(i18n.t('parameters.invoke.noModelSelected'));
       }
 
-      let enabledControlAdapters = selectControlAdapterAll(controlAdapters).filter((ca) => ca.isEnabled);
-
       if (activeTabName === 'txt2img') {
-        // Special handling for control layers on txt2img
+        // Handling for Control Layers - only exists on txt2img tab now
-        const enabledControlLayersAdapterIds = []
+        controlLayers.present.layers
-        // const enabledControlLayersAdapterIds = controlLayers.present.layers
+          .filter((l) => l.isEnabled)
-        //   .filter((l) => l.isEnabled)
+          .flatMap((l) => {
-        //   .flatMap((layer) => {
+            if (l.type === 'control_adapter_layer') {
-        //     if (layer.type === 'regional_guidance_layer') {
+              return l.controlAdapter;
-        //       return layer.ipAdapterIds;
+            } else if (l.type === 'ip_adapter_layer') {
-        //     }
+              return l.ipAdapter;
-        //     if (layer.type === 'control_adapter_layer') {
+            } else if (l.type === 'regional_guidance_layer') {
-        //       return [layer.controlNetId];
+              return l.ipAdapters;
-        //     }
+            }
-        //     if (layer.type === 'ip_adapter_layer') {
+            assert(false);
-        //       return [layer.ipAdapterId];
+          })
-        //     }
+          .forEach((ca, i) => {
-        //   });
+            const hasNoModel = !ca.model;
+            const mismatchedModelBase = ca.model?.base !== model?.base;
+            const hasNoImage = !ca.image;
+            const imageNotProcessed =
+              (ca.type === 'controlnet' || ca.type === 't2i_adapter') && !ca.processedImage && ca.processorConfig;
 
-        enabledControlAdapters = enabledControlAdapters.filter((ca) => enabledControlLayersAdapterIds.includes(ca.id));
+            if (hasNoModel) {
+              reasons.push(
+                i18n.t('parameters.invoke.noModelForControlAdapter', {
+                  number: i + 1,
+                })
+              );
+            }
+            if (mismatchedModelBase) {
+              // This should never happen, just a sanity check
+              reasons.push(
+                i18n.t('parameters.invoke.incompatibleBaseModelForControlAdapter', {
+                  number: i + 1,
+                })
+              );
+            }
+            if (hasNoImage) {
+              reasons.push(
+                i18n.t('parameters.invoke.noControlImageForControlAdapter', {
+                  number: i + 1,
+                })
+              );
+            }
+            if (imageNotProcessed) {
+              reasons.push(
+                i18n.t('parameters.invoke.imageNotProcessedForControlAdapter', {
+                  number: i + 1,
+                })
+              );
+            }
+          });
       } else {
-        const allControlLayerAdapterIds = []
+        // Handling for all other tabs
-        // const allControlLayerAdapterIds = controlLayers.present.layers.flatMap((layer) => {
+        selectControlAdapterAll(controlAdapters)
-        //   if (layer.type === 'regional_guidance_layer') {
+          .filter((ca) => ca.isEnabled)
-        //     return layer.ipAdapterIds;
+          .forEach((ca, i) => {
-        //   }
+            if (!ca.isEnabled) {
-        //   if (layer.type === 'control_adapter_layer') {
+              return;
-        //     return [layer.controlNetId];
+            }
-        //   }
+
-        //   if (layer.type === 'ip_adapter_layer') {
+            if (!ca.model) {
-        //     return [layer.ipAdapterId];
+              reasons.push(
-        //   }
+                i18n.t('parameters.invoke.noModelForControlAdapter', {
-        // });
+                  number: i + 1,
-        enabledControlAdapters = enabledControlAdapters.filter((ca) => !allControlLayerAdapterIds.includes(ca.id));
+                })
+              );
+            } else if (ca.model.base !== model?.base) {
+              // This should never happen, just a sanity check
+              reasons.push(
+                i18n.t('parameters.invoke.incompatibleBaseModelForControlAdapter', {
+                  number: i + 1,
+                })
+              );
+            }
+
+            if (
+              !ca.controlImage ||
+              (isControlNetOrT2IAdapter(ca) && !ca.processedControlImage && ca.processorType !== 'none')
+            ) {
+              reasons.push(
+                i18n.t('parameters.invoke.noControlImageForControlAdapter', {
+                  number: i + 1,
+                })
+              );
+            }
+          });
       }
-
-      enabledControlAdapters.forEach((ca, i) => {
-        if (!ca.isEnabled) {
-          return;
-        }
-
-        if (!ca.model) {
-          reasons.push(
-            i18n.t('parameters.invoke.noModelForControlAdapter', {
-              number: i + 1,
-            })
-          );
-        } else if (ca.model.base !== model?.base) {
-          // This should never happen, just a sanity check
-          reasons.push(
-            i18n.t('parameters.invoke.incompatibleBaseModelForControlAdapter', {
-              number: i + 1,
-            })
-          );
-        }
-
-        if (
-          !ca.controlImage ||
-          (isControlNetOrT2IAdapter(ca) && !ca.processedControlImage && ca.processorType !== 'none')
-        ) {
-          reasons.push(
-            i18n.t('parameters.invoke.noControlImageForControlAdapter', {
-              number: i + 1,
-            })
-          );
-        }
-      });
     }
 
     return { isReady: !reasons.length, reasons };

invokeai/frontend/web/src/features/controlLayers/components/CALayer/CALayer.tsx

@@ -1,6 +1,6 @@
 import { Flex, Spacer, useDisclosure } from '@invoke-ai/ui-library';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import { CALayerConfig } from 'features/controlLayers/components/CALayer/CALayerConfig';
+import { CALayerControlAdapterWrapper } from 'features/controlLayers/components/CALayer/CALayerControlAdapterWrapper';
 import { LayerDeleteButton } from 'features/controlLayers/components/LayerCommon/LayerDeleteButton';
 import { LayerMenu } from 'features/controlLayers/components/LayerCommon/LayerMenu';
 import { LayerTitle } from 'features/controlLayers/components/LayerCommon/LayerTitle';
@@ -43,7 +43,7 @@ export const CALayer = memo(({ layerId }: Props) => {
         </Flex>
         {isOpen && (
           <Flex flexDir="column" gap={3} px={3} pb={3}>
-            <CALayerConfig layerId={layerId} />
+            <CALayerControlAdapterWrapper layerId={layerId} />
           </Flex>
         )}
       </Flex>

invokeai/frontend/web/src/features/controlLayers/components/CALayer/CALayerConfig.tsx

@@ -1,149 +0,0 @@
-import { Box, Flex, Icon, IconButton } from '@invoke-ai/ui-library';
-import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import { ControlAdapterModelCombobox } from 'features/controlLayers/components/CALayer/ControlAdapterModelCombobox';
-import {
-  caLayerControlModeChanged,
-  caLayerImageChanged,
-  caLayerModelChanged,
-  caLayerProcessorConfigChanged,
-  caOrIPALayerBeginEndStepPctChanged,
-  caOrIPALayerWeightChanged,
-  selectCALayer,
-} from 'features/controlLayers/store/controlLayersSlice';
-import type { ControlMode, ProcessorConfig } from 'features/controlLayers/util/controlAdapters';
-import { memo, useCallback } from 'react';
-import { useTranslation } from 'react-i18next';
-import { PiCaretUpBold } from 'react-icons/pi';
-import { useToggle } from 'react-use';
-import type { ControlNetModelConfig, ImageDTO, T2IAdapterModelConfig } from 'services/api/types';
-
-import { CALayerImagePreview } from './CALayerImagePreview';
-import { CALayerProcessor } from './CALayerProcessor';
-import { CALayerProcessorCombobox } from './CALayerProcessorCombobox';
-import { ControlAdapterBeginEndStepPct } from './ControlAdapterBeginEndStepPct';
-import { ControlAdapterControlModeSelect } from './ControlAdapterControlModeSelect';
-import { ControlAdapterWeight } from './ControlAdapterWeight';
-
-type Props = {
-  layerId: string;
-};
-
-export const CALayerConfig = memo(({ layerId }: Props) => {
-  const dispatch = useAppDispatch();
-  const controlAdapter = useAppSelector((s) => selectCALayer(s.controlLayers.present, layerId).controlAdapter);
-  const { t } = useTranslation();
-  const [isExpanded, toggleIsExpanded] = useToggle(false);
-
-  const onChangeBeginEndStepPct = useCallback(
-    (beginEndStepPct: [number, number]) => {
-      dispatch(
-        caOrIPALayerBeginEndStepPctChanged({
-          layerId,
-          beginEndStepPct,
-        })
-      );
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeControlMode = useCallback(
-    (controlMode: ControlMode) => {
-      dispatch(
-        caLayerControlModeChanged({
-          layerId,
-          controlMode,
-        })
-      );
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeWeight = useCallback(
-    (weight: number) => {
-      dispatch(caOrIPALayerWeightChanged({ layerId, weight }));
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeProcessorConfig = useCallback(
-    (processorConfig: ProcessorConfig | null) => {
-      dispatch(caLayerProcessorConfigChanged({ layerId, processorConfig }));
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeModel = useCallback(
-    (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig) => {
-      dispatch(
-        caLayerModelChanged({
-          layerId,
-          modelConfig,
-        })
-      );
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeImage = useCallback(
-    (imageDTO: ImageDTO | null) => {
-      dispatch(caLayerImageChanged({ layerId, imageDTO }));
-    },
-    [dispatch, layerId]
-  );
-
-  return (
-    <Flex flexDir="column" gap={4} position="relative" w="full">
-      <Flex gap={3} alignItems="center" w="full">
-        <Box minW={0} w="full" transitionProperty="common" transitionDuration="0.1s">
-          <ControlAdapterModelCombobox modelKey={controlAdapter.model?.key ?? null} onChange={onChangeModel} />
-        </Box>
-
-        <IconButton
-          size="sm"
-          tooltip={isExpanded ? t('controlnet.hideAdvanced') : t('controlnet.showAdvanced')}
-          aria-label={isExpanded ? t('controlnet.hideAdvanced') : t('controlnet.showAdvanced')}
-          onClick={toggleIsExpanded}
-          variant="ghost"
-          icon={
-            <Icon
-              boxSize={4}
-              as={PiCaretUpBold}
-              transform={isExpanded ? 'rotate(0deg)' : 'rotate(180deg)'}
-              transitionProperty="common"
-              transitionDuration="normal"
-            />
-          }
-        />
-      </Flex>
-      <Flex gap={4} w="full" alignItems="center">
-        <Flex flexDir="column" gap={3} w="full">
-          {controlAdapter.type === 'controlnet' && (
-            <ControlAdapterControlModeSelect controlMode={controlAdapter.controlMode} onChange={onChangeControlMode} />
-          )}
-          <ControlAdapterWeight weight={controlAdapter.weight} onChange={onChangeWeight} />
-          <ControlAdapterBeginEndStepPct
-            beginEndStepPct={controlAdapter.beginEndStepPct}
-            onChange={onChangeBeginEndStepPct}
-          />
-        </Flex>
-        <Flex alignItems="center" justifyContent="center" h={36} w={36} aspectRatio="1/1">
-          <CALayerImagePreview
-            image={controlAdapter.image}
-            processedImage={controlAdapter.processedImage}
-            onChangeImage={onChangeImage}
-            layerId={layerId}
-            hasProcessor={Boolean(controlAdapter.processorConfig)}
-          />
-        </Flex>
-      </Flex>
-      {isExpanded && (
-        <>
-          <CALayerProcessorCombobox config={controlAdapter.processorConfig} onChange={onChangeProcessorConfig} />
-          <CALayerProcessor config={controlAdapter.processorConfig} onChange={onChangeProcessorConfig} />
-        </>
-      )}
-    </Flex>
-  );
-});
-
-CALayerConfig.displayName = 'CALayerConfig';

invokeai/frontend/web/src/features/controlLayers/components/CALayer/CALayerControlAdapterWrapper.tsx

@@ -0,0 +1,121 @@
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { ControlAdapter } from 'features/controlLayers/components/ControlAndIPAdapter/ControlAdapter';
+import {
+  caLayerControlModeChanged,
+  caLayerImageChanged,
+  caLayerModelChanged,
+  caLayerProcessorConfigChanged,
+  caOrIPALayerBeginEndStepPctChanged,
+  caOrIPALayerWeightChanged,
+  selectCALayer,
+} from 'features/controlLayers/store/controlLayersSlice';
+import type { ControlMode, ProcessorConfig } from 'features/controlLayers/util/controlAdapters';
+import type { CALayerImageDropData } from 'features/dnd/types';
+import { memo, useCallback, useMemo } from 'react';
+import type {
+  CALayerImagePostUploadAction,
+  ControlNetModelConfig,
+  ImageDTO,
+  T2IAdapterModelConfig,
+} from 'services/api/types';
+
+type Props = {
+  layerId: string;
+};
+
+export const CALayerControlAdapterWrapper = memo(({ layerId }: Props) => {
+  const dispatch = useAppDispatch();
+  const controlAdapter = useAppSelector((s) => selectCALayer(s.controlLayers.present, layerId).controlAdapter);
+
+  const onChangeBeginEndStepPct = useCallback(
+    (beginEndStepPct: [number, number]) => {
+      dispatch(
+        caOrIPALayerBeginEndStepPctChanged({
+          layerId,
+          beginEndStepPct,
+        })
+      );
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeControlMode = useCallback(
+    (controlMode: ControlMode) => {
+      dispatch(
+        caLayerControlModeChanged({
+          layerId,
+          controlMode,
+        })
+      );
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeWeight = useCallback(
+    (weight: number) => {
+      dispatch(caOrIPALayerWeightChanged({ layerId, weight }));
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeProcessorConfig = useCallback(
+    (processorConfig: ProcessorConfig | null) => {
+      dispatch(caLayerProcessorConfigChanged({ layerId, processorConfig }));
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeModel = useCallback(
+    (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig) => {
+      dispatch(
+        caLayerModelChanged({
+          layerId,
+          modelConfig,
+        })
+      );
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeImage = useCallback(
+    (imageDTO: ImageDTO | null) => {
+      dispatch(caLayerImageChanged({ layerId, imageDTO }));
+    },
+    [dispatch, layerId]
+  );
+
+  const droppableData = useMemo<CALayerImageDropData>(
+    () => ({
+      actionType: 'SET_CA_LAYER_IMAGE',
+      context: {
+        layerId,
+      },
+      id: layerId,
+    }),
+    [layerId]
+  );
+
+  const postUploadAction = useMemo<CALayerImagePostUploadAction>(
+    () => ({
+      layerId,
+      type: 'SET_CA_LAYER_IMAGE',
+    }),
+    [layerId]
+  );
+
+  return (
+    <ControlAdapter
+      controlAdapter={controlAdapter}
+      onChangeBeginEndStepPct={onChangeBeginEndStepPct}
+      onChangeControlMode={onChangeControlMode}
+      onChangeWeight={onChangeWeight}
+      onChangeProcessorConfig={onChangeProcessorConfig}
+      onChangeModel={onChangeModel}
+      onChangeImage={onChangeImage}
+      droppableData={droppableData}
+      postUploadAction={postUploadAction}
+    />
+  );
+});
+
+CALayerControlAdapterWrapper.displayName = 'CALayerControlAdapterWrapper';

invokeai/frontend/web/src/features/controlLayers/components/CALayer/CALayerImagePreview.tsx

@@ -1,231 +0,0 @@
-import type { SystemStyleObject } from '@invoke-ai/ui-library';
-import { Box, Flex, Spinner, useShiftModifier } from '@invoke-ai/ui-library';
-import { skipToken } from '@reduxjs/toolkit/query';
-import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
-import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import IAIDndImage from 'common/components/IAIDndImage';
-import IAIDndImageIcon from 'common/components/IAIDndImageIcon';
-import { setBoundingBoxDimensions } from 'features/canvas/store/canvasSlice';
-import { selectControlAdaptersSlice } from 'features/controlAdapters/store/controlAdaptersSlice';
-import { heightChanged, widthChanged } from 'features/controlLayers/store/controlLayersSlice';
-import type { ImageWithDims } from 'features/controlLayers/util/controlAdapters';
-import type { ControlLayerDropData, ImageDraggableData } from 'features/dnd/types';
-import { calculateNewSize } from 'features/parameters/components/ImageSize/calculateNewSize';
-import { selectOptimalDimension } from 'features/parameters/store/generationSlice';
-import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
-import { memo, useCallback, useEffect, useMemo, useState } from 'react';
-import { useTranslation } from 'react-i18next';
-import { PiArrowCounterClockwiseBold, PiFloppyDiskBold, PiRulerBold } from 'react-icons/pi';
-import {
-  useAddImageToBoardMutation,
-  useChangeImageIsIntermediateMutation,
-  useGetImageDTOQuery,
-  useRemoveImageFromBoardMutation,
-} from 'services/api/endpoints/images';
-import type { ControlLayerAction, ImageDTO } from 'services/api/types';
-
-type Props = {
-  image: ImageWithDims | null;
-  processedImage: ImageWithDims | null;
-  onChangeImage: (imageDTO: ImageDTO | null) => void;
-  hasProcessor: boolean;
-  layerId: string; // required for the dnd/upload interactions
-};
-
-const selectPendingControlImages = createMemoizedSelector(
-  selectControlAdaptersSlice,
-  (controlAdapters) => controlAdapters.pendingControlImages
-);
-
-export const CALayerImagePreview = memo(({ image, processedImage, onChangeImage, hasProcessor, layerId }: Props) => {
-  const { t } = useTranslation();
-  const dispatch = useAppDispatch();
-  const autoAddBoardId = useAppSelector((s) => s.gallery.autoAddBoardId);
-  const isConnected = useAppSelector((s) => s.system.isConnected);
-  const activeTabName = useAppSelector(activeTabNameSelector);
-  const optimalDimension = useAppSelector(selectOptimalDimension);
-  const pendingControlImages = useAppSelector(selectPendingControlImages);
-  const shift = useShiftModifier();
-
-  const [isMouseOverImage, setIsMouseOverImage] = useState(false);
-
-  const { currentData: controlImage, isError: isErrorControlImage } = useGetImageDTOQuery(
-    image?.imageName ?? skipToken
-  );
-  const { currentData: processedControlImage, isError: isErrorProcessedControlImage } = useGetImageDTOQuery(
-    processedImage?.imageName ?? skipToken
-  );
-
-  const [changeIsIntermediate] = useChangeImageIsIntermediateMutation();
-  const [addToBoard] = useAddImageToBoardMutation();
-  const [removeFromBoard] = useRemoveImageFromBoardMutation();
-  const handleResetControlImage = useCallback(() => {
-    onChangeImage(null);
-  }, [onChangeImage]);
-
-  const handleSaveControlImage = useCallback(async () => {
-    if (!processedControlImage) {
-      return;
-    }
-
-    await changeIsIntermediate({
-      imageDTO: processedControlImage,
-      is_intermediate: false,
-    }).unwrap();
-
-    if (autoAddBoardId !== 'none') {
-      addToBoard({
-        imageDTO: processedControlImage,
-        board_id: autoAddBoardId,
-      });
-    } else {
-      removeFromBoard({ imageDTO: processedControlImage });
-    }
-  }, [processedControlImage, changeIsIntermediate, autoAddBoardId, addToBoard, removeFromBoard]);
-
-  const handleSetControlImageToDimensions = useCallback(() => {
-    if (!controlImage) {
-      return;
-    }
-
-    if (activeTabName === 'unifiedCanvas') {
-      dispatch(setBoundingBoxDimensions({ width: controlImage.width, height: controlImage.height }, optimalDimension));
-    } else {
-      if (shift) {
-        const { width, height } = controlImage;
-        dispatch(widthChanged({ width, updateAspectRatio: true }));
-        dispatch(heightChanged({ height, updateAspectRatio: true }));
-      } else {
-        const { width, height } = calculateNewSize(
-          controlImage.width / controlImage.height,
-          optimalDimension * optimalDimension
-        );
-        dispatch(widthChanged({ width, updateAspectRatio: true }));
-        dispatch(heightChanged({ height, updateAspectRatio: true }));
-      }
-    }
-  }, [controlImage, activeTabName, dispatch, optimalDimension, shift]);
-
-  const handleMouseEnter = useCallback(() => {
-    setIsMouseOverImage(true);
-  }, []);
-
-  const handleMouseLeave = useCallback(() => {
-    setIsMouseOverImage(false);
-  }, []);
-
-  const draggableData = useMemo<ImageDraggableData | undefined>(() => {
-    if (controlImage) {
-      return {
-        id: layerId,
-        payloadType: 'IMAGE_DTO',
-        payload: { imageDTO: controlImage },
-      };
-    }
-  }, [controlImage, layerId]);
-
-  const droppableData = useMemo<ControlLayerDropData>(
-    () => ({
-      id: layerId,
-      actionType: 'SET_CONTROL_LAYER_IMAGE',
-      context: { layerId },
-    }),
-    [layerId]
-  );
-
-  const postUploadAction = useMemo<ControlLayerAction>(() => ({ type: 'SET_CONTROL_LAYER_IMAGE', layerId }), [layerId]);
-
-  const shouldShowProcessedImage =
-    controlImage &&
-    processedControlImage &&
-    !isMouseOverImage &&
-    !pendingControlImages.includes(layerId) &&
-    hasProcessor;
-
-  useEffect(() => {
-    if (isConnected && (isErrorControlImage || isErrorProcessedControlImage)) {
-      handleResetControlImage();
-    }
-  }, [handleResetControlImage, isConnected, isErrorControlImage, isErrorProcessedControlImage]);
-
-  return (
-    <Flex
-      onMouseEnter={handleMouseEnter}
-      onMouseLeave={handleMouseLeave}
-      position="relative"
-      w="full"
-      h={36}
-      alignItems="center"
-      justifyContent="center"
-    >
-      <IAIDndImage
-        draggableData={draggableData}
-        droppableData={droppableData}
-        imageDTO={controlImage}
-        isDropDisabled={shouldShowProcessedImage}
-        postUploadAction={postUploadAction}
-      />
-
-      <Box
-        position="absolute"
-        top={0}
-        insetInlineStart={0}
-        w="full"
-        h="full"
-        opacity={shouldShowProcessedImage ? 1 : 0}
-        transitionProperty="common"
-        transitionDuration="normal"
-        pointerEvents="none"
-      >
-        <IAIDndImage
-          draggableData={draggableData}
-          droppableData={droppableData}
-          imageDTO={processedControlImage}
-          isUploadDisabled={true}
-        />
-      </Box>
-
-      <>
-        <IAIDndImageIcon
-          onClick={handleResetControlImage}
-          icon={controlImage ? <PiArrowCounterClockwiseBold size={16} /> : undefined}
-          tooltip={t('controlnet.resetControlImage')}
-        />
-        <IAIDndImageIcon
-          onClick={handleSaveControlImage}
-          icon={controlImage ? <PiFloppyDiskBold size={16} /> : undefined}
-          tooltip={t('controlnet.saveControlImage')}
-          styleOverrides={saveControlImageStyleOverrides}
-        />
-        <IAIDndImageIcon
-          onClick={handleSetControlImageToDimensions}
-          icon={controlImage ? <PiRulerBold size={16} /> : undefined}
-          tooltip={shift ? t('controlnet.setControlImageDimensionsForce') : t('controlnet.setControlImageDimensions')}
-          styleOverrides={setControlImageDimensionsStyleOverrides}
-        />
-      </>
-
-      {pendingControlImages.includes(layerId) && (
-        <Flex
-          position="absolute"
-          top={0}
-          insetInlineStart={0}
-          w="full"
-          h="full"
-          alignItems="center"
-          justifyContent="center"
-          opacity={0.8}
-          borderRadius="base"
-          bg="base.900"
-        >
-          <Spinner size="xl" color="base.400" />
-        </Flex>
-      )}
-    </Flex>
-  );
-});
-
-CALayerImagePreview.displayName = 'CALayerImagePreview';
-
-const saveControlImageStyleOverrides: SystemStyleObject = { mt: 6 };
-const setControlImageDimensionsStyleOverrides: SystemStyleObject = { mt: 12 };

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapter.tsx

@@ -0,0 +1,111 @@
+import { Box, Flex, Icon, IconButton } from '@invoke-ai/ui-library';
+import { ControlAdapterModelCombobox } from 'features/controlLayers/components/ControlAndIPAdapter/ControlAdapterModelCombobox';
+import type {
+  ControlMode,
+  ControlNetConfig,
+  ProcessorConfig,
+  T2IAdapterConfig,
+} from 'features/controlLayers/util/controlAdapters';
+import type { TypesafeDroppableData } from 'features/dnd/types';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiCaretUpBold } from 'react-icons/pi';
+import { useToggle } from 'react-use';
+import type { ControlNetModelConfig, ImageDTO, PostUploadAction, T2IAdapterModelConfig } from 'services/api/types';
+
+import { ControlAdapterBeginEndStepPct } from './ControlAdapterBeginEndStepPct';
+import { ControlAdapterControlModeSelect } from './ControlAdapterControlModeSelect';
+import { ControlAdapterImagePreview } from './ControlAdapterImagePreview';
+import { ControlAdapterProcessorConfig } from './ControlAdapterProcessorConfig';
+import { ControlAdapterProcessorTypeSelect } from './ControlAdapterProcessorTypeSelect';
+import { ControlAdapterWeight } from './ControlAdapterWeight';
+
+type Props = {
+  controlAdapter: ControlNetConfig | T2IAdapterConfig;
+  onChangeBeginEndStepPct: (beginEndStepPct: [number, number]) => void;
+  onChangeControlMode: (controlMode: ControlMode) => void;
+  onChangeWeight: (weight: number) => void;
+  onChangeProcessorConfig: (processorConfig: ProcessorConfig | null) => void;
+  onChangeModel: (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig) => void;
+  onChangeImage: (imageDTO: ImageDTO | null) => void;
+  droppableData: TypesafeDroppableData;
+  postUploadAction: PostUploadAction;
+};
+
+export const ControlAdapter = memo(
+  ({
+    controlAdapter,
+    onChangeBeginEndStepPct,
+    onChangeControlMode,
+    onChangeWeight,
+    onChangeProcessorConfig,
+    onChangeModel,
+    onChangeImage,
+    droppableData,
+    postUploadAction,
+  }: Props) => {
+    const { t } = useTranslation();
+    const [isExpanded, toggleIsExpanded] = useToggle(false);
+
+    return (
+      <Flex flexDir="column" gap={4} position="relative" w="full">
+        <Flex gap={3} alignItems="center" w="full">
+          <Box minW={0} w="full" transitionProperty="common" transitionDuration="0.1s">
+            <ControlAdapterModelCombobox modelKey={controlAdapter.model?.key ?? null} onChange={onChangeModel} />
+          </Box>
+
+          <IconButton
+            size="sm"
+            tooltip={isExpanded ? t('controlnet.hideAdvanced') : t('controlnet.showAdvanced')}
+            aria-label={isExpanded ? t('controlnet.hideAdvanced') : t('controlnet.showAdvanced')}
+            onClick={toggleIsExpanded}
+            variant="ghost"
+            icon={
+              <Icon
+                boxSize={4}
+                as={PiCaretUpBold}
+                transform={isExpanded ? 'rotate(0deg)' : 'rotate(180deg)'}
+                transitionProperty="common"
+                transitionDuration="normal"
+              />
+            }
+          />
+        </Flex>
+        <Flex gap={4} w="full" alignItems="center">
+          <Flex flexDir="column" gap={3} w="full">
+            {controlAdapter.type === 'controlnet' && (
+              <ControlAdapterControlModeSelect
+                controlMode={controlAdapter.controlMode}
+                onChange={onChangeControlMode}
+              />
+            )}
+            <ControlAdapterWeight weight={controlAdapter.weight} onChange={onChangeWeight} />
+            <ControlAdapterBeginEndStepPct
+              beginEndStepPct={controlAdapter.beginEndStepPct}
+              onChange={onChangeBeginEndStepPct}
+            />
+          </Flex>
+          <Flex alignItems="center" justifyContent="center" h={36} w={36} aspectRatio="1/1">
+            <ControlAdapterImagePreview
+              image={controlAdapter.image}
+              processedImage={controlAdapter.processedImage}
+              onChangeImage={onChangeImage}
+              hasProcessor={Boolean(controlAdapter.processorConfig)}
+              controlAdapterId={controlAdapter.id}
+              droppableData={droppableData}
+              postUploadAction={postUploadAction}
+            />
+          </Flex>
+        </Flex>
+        {isExpanded && (
+          <>
+            <ControlAdapterProcessorTypeSelect config={controlAdapter.processorConfig} onChange={onChangeProcessorConfig} />
+            <ControlAdapterProcessorConfig config={controlAdapter.processorConfig} onChange={onChangeProcessorConfig} />
+          </>
+        )}
+      </Flex>
+    );
+  }
+);
+
+ControlAdapter.displayName = 'ControlAdapter';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterImagePreview.tsx

@@ -0,0 +1,234 @@
+import type { SystemStyleObject } from '@invoke-ai/ui-library';
+import { Box, Flex, Spinner, useShiftModifier } from '@invoke-ai/ui-library';
+import { skipToken } from '@reduxjs/toolkit/query';
+import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import IAIDndImage from 'common/components/IAIDndImage';
+import IAIDndImageIcon from 'common/components/IAIDndImageIcon';
+import { setBoundingBoxDimensions } from 'features/canvas/store/canvasSlice';
+import { selectControlAdaptersSlice } from 'features/controlAdapters/store/controlAdaptersSlice';
+import { heightChanged, widthChanged } from 'features/controlLayers/store/controlLayersSlice';
+import type { ImageWithDims } from 'features/controlLayers/util/controlAdapters';
+import type { ImageDraggableData, TypesafeDroppableData } from 'features/dnd/types';
+import { calculateNewSize } from 'features/parameters/components/ImageSize/calculateNewSize';
+import { selectOptimalDimension } from 'features/parameters/store/generationSlice';
+import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
+import { memo, useCallback, useEffect, useMemo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiArrowCounterClockwiseBold, PiFloppyDiskBold, PiRulerBold } from 'react-icons/pi';
+import {
+  useAddImageToBoardMutation,
+  useChangeImageIsIntermediateMutation,
+  useGetImageDTOQuery,
+  useRemoveImageFromBoardMutation,
+} from 'services/api/endpoints/images';
+import type { ImageDTO, PostUploadAction } from 'services/api/types';
+
+type Props = {
+  controlAdapterId: string;
+  image: ImageWithDims | null;
+  processedImage: ImageWithDims | null;
+  onChangeImage: (imageDTO: ImageDTO | null) => void;
+  hasProcessor: boolean;
+  droppableData: TypesafeDroppableData;
+  postUploadAction: PostUploadAction;
+};
+
+const selectPendingControlImages = createMemoizedSelector(
+  selectControlAdaptersSlice,
+  (controlAdapters) => controlAdapters.pendingControlImages
+);
+
+export const ControlAdapterImagePreview = memo(
+  ({
+    image,
+    processedImage,
+    onChangeImage,
+    hasProcessor,
+    controlAdapterId,
+    droppableData,
+    postUploadAction,
+  }: Props) => {
+    const { t } = useTranslation();
+    const dispatch = useAppDispatch();
+    const autoAddBoardId = useAppSelector((s) => s.gallery.autoAddBoardId);
+    const isConnected = useAppSelector((s) => s.system.isConnected);
+    const activeTabName = useAppSelector(activeTabNameSelector);
+    const optimalDimension = useAppSelector(selectOptimalDimension);
+    const pendingControlImages = useAppSelector(selectPendingControlImages);
+    const shift = useShiftModifier();
+
+    const [isMouseOverImage, setIsMouseOverImage] = useState(false);
+
+    const { currentData: controlImage, isError: isErrorControlImage } = useGetImageDTOQuery(
+      image?.imageName ?? skipToken
+    );
+    const { currentData: processedControlImage, isError: isErrorProcessedControlImage } = useGetImageDTOQuery(
+      processedImage?.imageName ?? skipToken
+    );
+
+    const [changeIsIntermediate] = useChangeImageIsIntermediateMutation();
+    const [addToBoard] = useAddImageToBoardMutation();
+    const [removeFromBoard] = useRemoveImageFromBoardMutation();
+    const handleResetControlImage = useCallback(() => {
+      onChangeImage(null);
+    }, [onChangeImage]);
+
+    const handleSaveControlImage = useCallback(async () => {
+      if (!processedControlImage) {
+        return;
+      }
+
+      await changeIsIntermediate({
+        imageDTO: processedControlImage,
+        is_intermediate: false,
+      }).unwrap();
+
+      if (autoAddBoardId !== 'none') {
+        addToBoard({
+          imageDTO: processedControlImage,
+          board_id: autoAddBoardId,
+        });
+      } else {
+        removeFromBoard({ imageDTO: processedControlImage });
+      }
+    }, [processedControlImage, changeIsIntermediate, autoAddBoardId, addToBoard, removeFromBoard]);
+
+    const handleSetControlImageToDimensions = useCallback(() => {
+      if (!controlImage) {
+        return;
+      }
+
+      if (activeTabName === 'unifiedCanvas') {
+        dispatch(
+          setBoundingBoxDimensions({ width: controlImage.width, height: controlImage.height }, optimalDimension)
+        );
+      } else {
+        if (shift) {
+          const { width, height } = controlImage;
+          dispatch(widthChanged({ width, updateAspectRatio: true }));
+          dispatch(heightChanged({ height, updateAspectRatio: true }));
+        } else {
+          const { width, height } = calculateNewSize(
+            controlImage.width / controlImage.height,
+            optimalDimension * optimalDimension
+          );
+          dispatch(widthChanged({ width, updateAspectRatio: true }));
+          dispatch(heightChanged({ height, updateAspectRatio: true }));
+        }
+      }
+    }, [controlImage, activeTabName, dispatch, optimalDimension, shift]);
+
+    const handleMouseEnter = useCallback(() => {
+      setIsMouseOverImage(true);
+    }, []);
+
+    const handleMouseLeave = useCallback(() => {
+      setIsMouseOverImage(false);
+    }, []);
+
+    const draggableData = useMemo<ImageDraggableData | undefined>(() => {
+      if (controlImage) {
+        return {
+          id: controlAdapterId,
+          payloadType: 'IMAGE_DTO',
+          payload: { imageDTO: controlImage },
+        };
+      }
+    }, [controlImage, controlAdapterId]);
+
+    const shouldShowProcessedImage =
+      controlImage &&
+      processedControlImage &&
+      !isMouseOverImage &&
+      !pendingControlImages.includes(controlAdapterId) &&
+      hasProcessor;
+
+    useEffect(() => {
+      if (isConnected && (isErrorControlImage || isErrorProcessedControlImage)) {
+        handleResetControlImage();
+      }
+    }, [handleResetControlImage, isConnected, isErrorControlImage, isErrorProcessedControlImage]);
+
+    return (
+      <Flex
+        onMouseEnter={handleMouseEnter}
+        onMouseLeave={handleMouseLeave}
+        position="relative"
+        w="full"
+        h={36}
+        alignItems="center"
+        justifyContent="center"
+      >
+        <IAIDndImage
+          draggableData={draggableData}
+          droppableData={droppableData}
+          imageDTO={controlImage}
+          isDropDisabled={shouldShowProcessedImage}
+          postUploadAction={postUploadAction}
+        />
+
+        <Box
+          position="absolute"
+          top={0}
+          insetInlineStart={0}
+          w="full"
+          h="full"
+          opacity={shouldShowProcessedImage ? 1 : 0}
+          transitionProperty="common"
+          transitionDuration="normal"
+          pointerEvents="none"
+        >
+          <IAIDndImage
+            draggableData={draggableData}
+            droppableData={droppableData}
+            imageDTO={processedControlImage}
+            isUploadDisabled={true}
+          />
+        </Box>
+
+        <>
+          <IAIDndImageIcon
+            onClick={handleResetControlImage}
+            icon={controlImage ? <PiArrowCounterClockwiseBold size={16} /> : undefined}
+            tooltip={t('controlnet.resetControlImage')}
+          />
+          <IAIDndImageIcon
+            onClick={handleSaveControlImage}
+            icon={controlImage ? <PiFloppyDiskBold size={16} /> : undefined}
+            tooltip={t('controlnet.saveControlImage')}
+            styleOverrides={saveControlImageStyleOverrides}
+          />
+          <IAIDndImageIcon
+            onClick={handleSetControlImageToDimensions}
+            icon={controlImage ? <PiRulerBold size={16} /> : undefined}
+            tooltip={shift ? t('controlnet.setControlImageDimensionsForce') : t('controlnet.setControlImageDimensions')}
+            styleOverrides={setControlImageDimensionsStyleOverrides}
+          />
+        </>
+
+        {pendingControlImages.includes(controlAdapterId) && (
+          <Flex
+            position="absolute"
+            top={0}
+            insetInlineStart={0}
+            w="full"
+            h="full"
+            alignItems="center"
+            justifyContent="center"
+            opacity={0.8}
+            borderRadius="base"
+            bg="base.900"
+          >
+            <Spinner size="xl" color="base.400" />
+          </Flex>
+        )}
+      </Flex>
+    );
+  }
+);
+
+ControlAdapterImagePreview.displayName = 'ControlAdapterImagePreview';
+
+const saveControlImageStyleOverrides: SystemStyleObject = { mt: 6 };
+const setControlImageDimensionsStyleOverrides: SystemStyleObject = { mt: 12 };

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterProcessorConfig.tsx

@@ -18,7 +18,7 @@ type Props = {
   onChange: (config: ProcessorConfig | null) => void;
 };
 
-export const CALayerProcessor = memo(({ config, onChange }: Props) => {
+export const ControlAdapterProcessorConfig = memo(({ config, onChange }: Props) => {
   if (!config) {
     return null;
   }
@@ -82,4 +82,4 @@ export const CALayerProcessor = memo(({ config, onChange }: Props) => {
   }
 });
 
-CALayerProcessor.displayName = 'CALayerProcessor';
+ControlAdapterProcessorConfig.displayName = 'ControlAdapterProcessorConfig';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterProcessorTypeSelect.tsx

@@ -22,7 +22,7 @@ const selectDisabledProcessors = createMemoizedSelector(
   (config) => config.sd.disabledControlNetProcessors
 );
 
-export const CALayerProcessorCombobox = memo(({ config, onChange }: Props) => {
+export const ControlAdapterProcessorTypeSelect = memo(({ config, onChange }: Props) => {
   const { t } = useTranslation();
   const disabledProcessors = useAppSelector(selectDisabledProcessors);
   const options = useMemo(() => {
@@ -53,7 +53,7 @@ export const CALayerProcessorCombobox = memo(({ config, onChange }: Props) => {
         <FormLabel>{t('controlnet.processor')}</FormLabel>
       </InformationalPopover>
       <Flex gap={4}>
-        <Combobox value={value} options={options} onChange={_onChange} />
+        <Combobox value={value} options={options} onChange={_onChange} isSearchable={false} isClearable={false} />
         <IconButton
           aria-label={t('controlnet.processor')}
           onClick={clearProcessor}
@@ -65,4 +65,4 @@ export const CALayerProcessorCombobox = memo(({ config, onChange }: Props) => {
   );
 });
 
-CALayerProcessorCombobox.displayName = 'CALayerProcessorCombobox';
+ControlAdapterProcessorTypeSelect.displayName = 'ControlAdapterProcessorTypeSelect';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/IPAdapter.tsx

@@ -0,0 +1,72 @@
+import { Box, Flex } from '@invoke-ai/ui-library';
+import { ControlAdapterBeginEndStepPct } from 'features/controlLayers/components/ControlAndIPAdapter/ControlAdapterBeginEndStepPct';
+import { ControlAdapterWeight } from 'features/controlLayers/components/ControlAndIPAdapter/ControlAdapterWeight';
+import { IPAdapterImagePreview } from 'features/controlLayers/components/ControlAndIPAdapter/IPAdapterImagePreview';
+import { IPAdapterMethod } from 'features/controlLayers/components/ControlAndIPAdapter/IPAdapterMethod';
+import { IPAdapterModelSelect } from 'features/controlLayers/components/ControlAndIPAdapter/IPAdapterModelSelect';
+import type { CLIPVisionModel, IPAdapterConfig, IPMethod } from 'features/controlLayers/util/controlAdapters';
+import type { TypesafeDroppableData } from 'features/dnd/types';
+import { memo } from 'react';
+import type { ImageDTO, IPAdapterModelConfig, PostUploadAction } from 'services/api/types';
+
+type Props = {
+  ipAdapter: IPAdapterConfig;
+  onChangeBeginEndStepPct: (beginEndStepPct: [number, number]) => void;
+  onChangeWeight: (weight: number) => void;
+  onChangeIPMethod: (method: IPMethod) => void;
+  onChangeModel: (modelConfig: IPAdapterModelConfig) => void;
+  onChangeCLIPVisionModel: (clipVisionModel: CLIPVisionModel) => void;
+  onChangeImage: (imageDTO: ImageDTO | null) => void;
+  droppableData: TypesafeDroppableData;
+  postUploadAction: PostUploadAction;
+};
+
+export const IPAdapter = memo(
+  ({
+    ipAdapter,
+    onChangeBeginEndStepPct,
+    onChangeWeight,
+    onChangeIPMethod,
+    onChangeModel,
+    onChangeCLIPVisionModel,
+    onChangeImage,
+    droppableData,
+    postUploadAction,
+  }: Props) => {
+    return (
+      <Flex flexDir="column" gap={4} position="relative" w="full">
+        <Flex gap={3} alignItems="center" w="full">
+          <Box minW={0} w="full" transitionProperty="common" transitionDuration="0.1s">
+            <IPAdapterModelSelect
+              modelKey={ipAdapter.model?.key ?? null}
+              onChangeModel={onChangeModel}
+              clipVisionModel={ipAdapter.clipVisionModel}
+              onChangeCLIPVisionModel={onChangeCLIPVisionModel}
+            />
+          </Box>
+        </Flex>
+        <Flex gap={4} w="full" alignItems="center">
+          <Flex flexDir="column" gap={3} w="full">
+            <IPAdapterMethod method={ipAdapter.method} onChange={onChangeIPMethod} />
+            <ControlAdapterWeight weight={ipAdapter.weight} onChange={onChangeWeight} />
+            <ControlAdapterBeginEndStepPct
+              beginEndStepPct={ipAdapter.beginEndStepPct}
+              onChange={onChangeBeginEndStepPct}
+            />
+          </Flex>
+          <Flex alignItems="center" justifyContent="center" h={36} w={36} aspectRatio="1/1">
+            <IPAdapterImagePreview
+              image={ipAdapter.image}
+              onChangeImage={onChangeImage}
+              ipAdapterId={ipAdapter.id}
+              droppableData={droppableData}
+              postUploadAction={postUploadAction}
+            />
+          </Flex>
+        </Flex>
+      </Flex>
+    );
+  }
+);
+
+IPAdapter.displayName = 'IPAdapter';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/IPAdapterImagePreview.tsx

@@ -0,0 +1,114 @@
+import type { SystemStyleObject } from '@invoke-ai/ui-library';
+import { Flex, useShiftModifier } from '@invoke-ai/ui-library';
+import { skipToken } from '@reduxjs/toolkit/query';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import IAIDndImage from 'common/components/IAIDndImage';
+import IAIDndImageIcon from 'common/components/IAIDndImageIcon';
+import { setBoundingBoxDimensions } from 'features/canvas/store/canvasSlice';
+import { heightChanged, widthChanged } from 'features/controlLayers/store/controlLayersSlice';
+import type { ImageWithDims } from 'features/controlLayers/util/controlAdapters';
+import type { ImageDraggableData, TypesafeDroppableData } from 'features/dnd/types';
+import { calculateNewSize } from 'features/parameters/components/ImageSize/calculateNewSize';
+import { selectOptimalDimension } from 'features/parameters/store/generationSlice';
+import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
+import { memo, useCallback, useEffect, useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiArrowCounterClockwiseBold, PiRulerBold } from 'react-icons/pi';
+import { useGetImageDTOQuery } from 'services/api/endpoints/images';
+import type { ImageDTO, PostUploadAction } from 'services/api/types';
+
+type Props = {
+  image: ImageWithDims | null;
+  onChangeImage: (imageDTO: ImageDTO | null) => void;
+  ipAdapterId: string; // required for the dnd/upload interactions
+  droppableData: TypesafeDroppableData;
+  postUploadAction: PostUploadAction;
+};
+
+export const IPAdapterImagePreview = memo(
+  ({ image, onChangeImage, ipAdapterId, droppableData, postUploadAction }: Props) => {
+    const { t } = useTranslation();
+    const dispatch = useAppDispatch();
+    const isConnected = useAppSelector((s) => s.system.isConnected);
+    const activeTabName = useAppSelector(activeTabNameSelector);
+    const optimalDimension = useAppSelector(selectOptimalDimension);
+    const shift = useShiftModifier();
+
+    const { currentData: controlImage, isError: isErrorControlImage } = useGetImageDTOQuery(
+      image?.imageName ?? skipToken
+    );
+    const handleResetControlImage = useCallback(() => {
+      onChangeImage(null);
+    }, [onChangeImage]);
+
+    const handleSetControlImageToDimensions = useCallback(() => {
+      if (!controlImage) {
+        return;
+      }
+
+      if (activeTabName === 'unifiedCanvas') {
+        dispatch(
+          setBoundingBoxDimensions({ width: controlImage.width, height: controlImage.height }, optimalDimension)
+        );
+      } else {
+        if (shift) {
+          const { width, height } = controlImage;
+          dispatch(widthChanged({ width, updateAspectRatio: true }));
+          dispatch(heightChanged({ height, updateAspectRatio: true }));
+        } else {
+          const { width, height } = calculateNewSize(
+            controlImage.width / controlImage.height,
+            optimalDimension * optimalDimension
+          );
+          dispatch(widthChanged({ width, updateAspectRatio: true }));
+          dispatch(heightChanged({ height, updateAspectRatio: true }));
+        }
+      }
+    }, [controlImage, activeTabName, dispatch, optimalDimension, shift]);
+
+    const draggableData = useMemo<ImageDraggableData | undefined>(() => {
+      if (controlImage) {
+        return {
+          id: ipAdapterId,
+          payloadType: 'IMAGE_DTO',
+          payload: { imageDTO: controlImage },
+        };
+      }
+    }, [controlImage, ipAdapterId]);
+
+    useEffect(() => {
+      if (isConnected && isErrorControlImage) {
+        handleResetControlImage();
+      }
+    }, [handleResetControlImage, isConnected, isErrorControlImage]);
+
+    return (
+      <Flex position="relative" w="full" h={36} alignItems="center" justifyContent="center">
+        <IAIDndImage
+          draggableData={draggableData}
+          droppableData={droppableData}
+          imageDTO={controlImage}
+          postUploadAction={postUploadAction}
+        />
+
+        <>
+          <IAIDndImageIcon
+            onClick={handleResetControlImage}
+            icon={controlImage ? <PiArrowCounterClockwiseBold size={16} /> : undefined}
+            tooltip={t('controlnet.resetControlImage')}
+          />
+          <IAIDndImageIcon
+            onClick={handleSetControlImageToDimensions}
+            icon={controlImage ? <PiRulerBold size={16} /> : undefined}
+            tooltip={shift ? t('controlnet.setControlImageDimensionsForce') : t('controlnet.setControlImageDimensions')}
+            styleOverrides={setControlImageDimensionsStyleOverrides}
+          />
+        </>
+      </Flex>
+    );
+  }
+);
+
+IPAdapterImagePreview.displayName = 'IPAdapterImagePreview';
+
+const setControlImageDimensionsStyleOverrides: SystemStyleObject = { mt: 6 };

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/IPAdapterModelSelect.tsx

@@ -22,7 +22,7 @@ type Props = {
   onChangeCLIPVisionModel: (clipVisionModel: CLIPVisionModel) => void;
 };
 
-export const IPAdapterModelCombobox = memo(
+export const IPAdapterModelSelect = memo(
   ({ modelKey, onChangeModel, clipVisionModel, onChangeCLIPVisionModel }: Props) => {
     const { t } = useTranslation();
     const currentBaseModel = useAppSelector((s) => s.generation.model?.base);
@@ -97,4 +97,4 @@ export const IPAdapterModelCombobox = memo(
   }
 );
 
-IPAdapterModelCombobox.displayName = 'IPALayerModelCombobox';
+IPAdapterModelSelect.displayName = 'IPAdapterModelSelect';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/CannyProcessor.tsx

@@ -1,5 +1,5 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import { type CannyProcessorConfig, CONTROLNET_PROCESSORS } from 'features/controlLayers/util/controlAdapters';
 import { useCallback } from 'react';
 import { useTranslation } from 'react-i18next';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/ColorMapProcessor.tsx

@@ -1,5 +1,5 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import { type ColorMapProcessorConfig, CONTROLNET_PROCESSORS } from 'features/controlLayers/util/controlAdapters';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/ContentShuffleProcessor.tsx

@@ -1,5 +1,5 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { ContentShuffleProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import { CONTROLNET_PROCESSORS } from 'features/controlLayers/util/controlAdapters';
 import { memo, useCallback } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/DWOpenposeProcessor.tsx

@@ -1,5 +1,5 @@
 import { Flex, FormControl, FormLabel, Switch } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { DWOpenposeProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import { CONTROLNET_PROCESSORS } from 'features/controlLayers/util/controlAdapters';
 import type { ChangeEvent } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/DepthAnythingProcessor.tsx

@@ -1,6 +1,6 @@
 import type { ComboboxOnChange } from '@invoke-ai/ui-library';
 import { Combobox, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { DepthAnythingModelSize, DepthAnythingProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import { CONTROLNET_PROCESSORS, isDepthAnythingModelSize } from 'features/controlLayers/util/controlAdapters';
 import { memo, useCallback, useMemo } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/HedProcessor.tsx

@@ -1,5 +1,5 @@
 import { FormControl, FormLabel, Switch } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { HedProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import type { ChangeEvent } from 'react';
 import { memo, useCallback } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/LineartProcessor.tsx

@@ -1,5 +1,5 @@
 import { FormControl, FormLabel, Switch } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { LineartProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import type { ChangeEvent } from 'react';
 import { memo, useCallback } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/MediapipeFaceProcessor.tsx

@@ -1,5 +1,5 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import { CONTROLNET_PROCESSORS, type MediapipeFaceProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/MidasDepthProcessor.tsx

@@ -1,5 +1,5 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { MidasDepthProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import { CONTROLNET_PROCESSORS } from 'features/controlLayers/util/controlAdapters';
 import { memo, useCallback } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/MlsdImageProcessor.tsx

@@ -1,5 +1,5 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { MlsdProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import { CONTROLNET_PROCESSORS } from 'features/controlLayers/util/controlAdapters';
 import { memo, useCallback } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/PidiProcessor.tsx

@@ -1,5 +1,5 @@
 import { FormControl, FormLabel, Switch } from '@invoke-ai/ui-library';
-import type { ProcessorComponentProps } from 'features/controlLayers/components/CALayer/processors/types';
+import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types';
 import type { PidiProcessorConfig } from 'features/controlLayers/util/controlAdapters';
 import type { ChangeEvent } from 'react';
 import { useCallback } from 'react';

invokeai/frontend/web/src/features/controlLayers/components/IPALayer/IPALayer.tsx

@@ -1,5 +1,5 @@
 import { Flex, Spacer, useDisclosure } from '@invoke-ai/ui-library';
-import { IPALayerConfig } from 'features/controlLayers/components/IPALayer/IPALayerConfig';
+import { IPALayerIPAdapterWrapper } from 'features/controlLayers/components/IPALayer/IPALayerIPAdapterWrapper';
 import { LayerDeleteButton } from 'features/controlLayers/components/LayerCommon/LayerDeleteButton';
 import { LayerTitle } from 'features/controlLayers/components/LayerCommon/LayerTitle';
 import { LayerVisibilityToggle } from 'features/controlLayers/components/LayerCommon/LayerVisibilityToggle';
@@ -22,7 +22,7 @@ export const IPALayer = memo(({ layerId }: Props) => {
         </Flex>
         {isOpen && (
           <Flex flexDir="column" gap={3} px={3} pb={3}>
-            <IPALayerConfig layerId={layerId} />
+            <IPALayerIPAdapterWrapper layerId={layerId} />
           </Flex>
         )}
       </Flex>

invokeai/frontend/web/src/features/controlLayers/components/IPALayer/IPALayerConfig.tsx

@@ -1,105 +0,0 @@
-import { Box, Flex } from '@invoke-ai/ui-library';
-import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import { ControlAdapterBeginEndStepPct } from 'features/controlLayers/components/CALayer/ControlAdapterBeginEndStepPct';
-import { ControlAdapterWeight } from 'features/controlLayers/components/CALayer/ControlAdapterWeight';
-import { IPAdapterImagePreview } from 'features/controlLayers/components/IPALayer/IPAdapterImagePreview';
-import { IPAdapterMethod } from 'features/controlLayers/components/IPALayer/IPAdapterMethod';
-import { IPAdapterModelCombobox } from 'features/controlLayers/components/IPALayer/IPALayerModelCombobox';
-import {
-  caOrIPALayerBeginEndStepPctChanged,
-  caOrIPALayerWeightChanged,
-  ipaLayerCLIPVisionModelChanged,
-  ipaLayerImageChanged,
-  ipaLayerMethodChanged,
-  ipaLayerModelChanged,
-  selectIPALayer,
-} from 'features/controlLayers/store/controlLayersSlice';
-import type { CLIPVisionModel, IPMethod } from 'features/controlLayers/util/controlAdapters';
-import { memo, useCallback } from 'react';
-import type { ImageDTO, IPAdapterModelConfig } from 'services/api/types';
-
-type Props = {
-  layerId: string;
-};
-
-export const IPALayerConfig = memo(({ layerId }: Props) => {
-  const dispatch = useAppDispatch();
-  const ipAdapter = useAppSelector((s) => selectIPALayer(s.controlLayers.present, layerId).ipAdapter);
-
-  const onChangeBeginEndStepPct = useCallback(
-    (beginEndStepPct: [number, number]) => {
-      dispatch(
-        caOrIPALayerBeginEndStepPctChanged({
-          layerId,
-          beginEndStepPct,
-        })
-      );
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeWeight = useCallback(
-    (weight: number) => {
-      dispatch(caOrIPALayerWeightChanged({ layerId, weight }));
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeIPMethod = useCallback(
-    (method: IPMethod) => {
-      dispatch(ipaLayerMethodChanged({ layerId, method }));
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeModel = useCallback(
-    (modelConfig: IPAdapterModelConfig) => {
-      dispatch(ipaLayerModelChanged({ layerId, modelConfig }));
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeCLIPVisionModel = useCallback(
-    (clipVisionModel: CLIPVisionModel) => {
-      dispatch(ipaLayerCLIPVisionModelChanged({ layerId, clipVisionModel }));
-    },
-    [dispatch, layerId]
-  );
-
-  const onChangeImage = useCallback(
-    (imageDTO: ImageDTO | null) => {
-      dispatch(ipaLayerImageChanged({ layerId, imageDTO }));
-    },
-    [dispatch, layerId]
-  );
-
-  return (
-    <Flex flexDir="column" gap={4} position="relative" w="full">
-      <Flex gap={3} alignItems="center" w="full">
-        <Box minW={0} w="full" transitionProperty="common" transitionDuration="0.1s">
-          <IPAdapterModelCombobox
-            modelKey={ipAdapter.model?.key ?? null}
-            onChangeModel={onChangeModel}
-            clipVisionModel={ipAdapter.clipVisionModel}
-            onChangeCLIPVisionModel={onChangeCLIPVisionModel}
-          />
-        </Box>
-      </Flex>
-      <Flex gap={4} w="full" alignItems="center">
-        <Flex flexDir="column" gap={3} w="full">
-          <IPAdapterMethod method={ipAdapter.method} onChange={onChangeIPMethod} />
-          <ControlAdapterWeight weight={ipAdapter.weight} onChange={onChangeWeight} />
-          <ControlAdapterBeginEndStepPct
-            beginEndStepPct={ipAdapter.beginEndStepPct}
-            onChange={onChangeBeginEndStepPct}
-          />
-        </Flex>
-        <Flex alignItems="center" justifyContent="center" h={36} w={36} aspectRatio="1/1">
-          <IPAdapterImagePreview image={ipAdapter.image} onChangeImage={onChangeImage} layerId={layerId} />
-        </Flex>
-      </Flex>
-    </Flex>
-  );
-});
-
-IPALayerConfig.displayName = 'IPALayerConfig';

invokeai/frontend/web/src/features/controlLayers/components/IPALayer/IPALayerIPAdapterWrapper.tsx

@@ -0,0 +1,106 @@
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { IPAdapter } from 'features/controlLayers/components/ControlAndIPAdapter/IPAdapter';
+import {
+  caOrIPALayerBeginEndStepPctChanged,
+  caOrIPALayerWeightChanged,
+  ipaLayerCLIPVisionModelChanged,
+  ipaLayerImageChanged,
+  ipaLayerMethodChanged,
+  ipaLayerModelChanged,
+  selectIPALayer,
+} from 'features/controlLayers/store/controlLayersSlice';
+import type { CLIPVisionModel, IPMethod } from 'features/controlLayers/util/controlAdapters';
+import type { IPALayerImageDropData } from 'features/dnd/types';
+import { memo, useCallback, useMemo } from 'react';
+import type { ImageDTO, IPAdapterModelConfig, IPALayerImagePostUploadAction } from 'services/api/types';
+
+type Props = {
+  layerId: string;
+};
+
+export const IPALayerIPAdapterWrapper = memo(({ layerId }: Props) => {
+  const dispatch = useAppDispatch();
+  const ipAdapter = useAppSelector((s) => selectIPALayer(s.controlLayers.present, layerId).ipAdapter);
+
+  const onChangeBeginEndStepPct = useCallback(
+    (beginEndStepPct: [number, number]) => {
+      dispatch(
+        caOrIPALayerBeginEndStepPctChanged({
+          layerId,
+          beginEndStepPct,
+        })
+      );
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeWeight = useCallback(
+    (weight: number) => {
+      dispatch(caOrIPALayerWeightChanged({ layerId, weight }));
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeIPMethod = useCallback(
+    (method: IPMethod) => {
+      dispatch(ipaLayerMethodChanged({ layerId, method }));
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeModel = useCallback(
+    (modelConfig: IPAdapterModelConfig) => {
+      dispatch(ipaLayerModelChanged({ layerId, modelConfig }));
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeCLIPVisionModel = useCallback(
+    (clipVisionModel: CLIPVisionModel) => {
+      dispatch(ipaLayerCLIPVisionModelChanged({ layerId, clipVisionModel }));
+    },
+    [dispatch, layerId]
+  );
+
+  const onChangeImage = useCallback(
+    (imageDTO: ImageDTO | null) => {
+      dispatch(ipaLayerImageChanged({ layerId, imageDTO }));
+    },
+    [dispatch, layerId]
+  );
+
+  const droppableData = useMemo<IPALayerImageDropData>(
+    () => ({
+      actionType: 'SET_IPA_LAYER_IMAGE',
+      context: {
+        layerId,
+      },
+      id: layerId,
+    }),
+    [layerId]
+  );
+
+  const postUploadAction = useMemo<IPALayerImagePostUploadAction>(
+    () => ({
+      type: 'SET_IPA_LAYER_IMAGE',
+      layerId,
+    }),
+    [layerId]
+  );
+
+  return (
+    <IPAdapter
+      ipAdapter={ipAdapter}
+      onChangeBeginEndStepPct={onChangeBeginEndStepPct}
+      onChangeWeight={onChangeWeight}
+      onChangeIPMethod={onChangeIPMethod}
+      onChangeModel={onChangeModel}
+      onChangeCLIPVisionModel={onChangeCLIPVisionModel}
+      onChangeImage={onChangeImage}
+      droppableData={droppableData}
+      postUploadAction={postUploadAction}
+    />
+  );
+});
+
+IPALayerIPAdapterWrapper.displayName = 'IPALayerIPAdapterWrapper';

invokeai/frontend/web/src/features/controlLayers/components/IPALayer/IPAdapterImagePreview.tsx

@@ -1,119 +0,0 @@
-import type { SystemStyleObject } from '@invoke-ai/ui-library';
-import { Flex, useShiftModifier } from '@invoke-ai/ui-library';
-import { skipToken } from '@reduxjs/toolkit/query';
-import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import IAIDndImage from 'common/components/IAIDndImage';
-import IAIDndImageIcon from 'common/components/IAIDndImageIcon';
-import { setBoundingBoxDimensions } from 'features/canvas/store/canvasSlice';
-import { heightChanged, widthChanged } from 'features/controlLayers/store/controlLayersSlice';
-import type { ImageWithDims } from 'features/controlLayers/util/controlAdapters';
-import type { ControlLayerDropData, ImageDraggableData } from 'features/dnd/types';
-import { calculateNewSize } from 'features/parameters/components/ImageSize/calculateNewSize';
-import { selectOptimalDimension } from 'features/parameters/store/generationSlice';
-import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
-import { memo, useCallback, useEffect, useMemo } from 'react';
-import { useTranslation } from 'react-i18next';
-import { PiArrowCounterClockwiseBold, PiRulerBold } from 'react-icons/pi';
-import { useGetImageDTOQuery } from 'services/api/endpoints/images';
-import type { ControlLayerAction, ImageDTO } from 'services/api/types';
-
-type Props = {
-  image: ImageWithDims | null;
-  onChangeImage: (imageDTO: ImageDTO | null) => void;
-  layerId: string; // required for the dnd/upload interactions
-};
-
-export const IPAdapterImagePreview = memo(({ image, onChangeImage, layerId }: Props) => {
-  const { t } = useTranslation();
-  const dispatch = useAppDispatch();
-  const isConnected = useAppSelector((s) => s.system.isConnected);
-  const activeTabName = useAppSelector(activeTabNameSelector);
-  const optimalDimension = useAppSelector(selectOptimalDimension);
-  const shift = useShiftModifier();
-
-  const { currentData: controlImage, isError: isErrorControlImage } = useGetImageDTOQuery(
-    image?.imageName ?? skipToken
-  );
-  const handleResetControlImage = useCallback(() => {
-    onChangeImage(null);
-  }, [onChangeImage]);
-
-  const handleSetControlImageToDimensions = useCallback(() => {
-    if (!controlImage) {
-      return;
-    }
-
-    if (activeTabName === 'unifiedCanvas') {
-      dispatch(setBoundingBoxDimensions({ width: controlImage.width, height: controlImage.height }, optimalDimension));
-    } else {
-      if (shift) {
-        const { width, height } = controlImage;
-        dispatch(widthChanged({ width, updateAspectRatio: true }));
-        dispatch(heightChanged({ height, updateAspectRatio: true }));
-      } else {
-        const { width, height } = calculateNewSize(
-          controlImage.width / controlImage.height,
-          optimalDimension * optimalDimension
-        );
-        dispatch(widthChanged({ width, updateAspectRatio: true }));
-        dispatch(heightChanged({ height, updateAspectRatio: true }));
-      }
-    }
-  }, [controlImage, activeTabName, dispatch, optimalDimension, shift]);
-
-  const draggableData = useMemo<ImageDraggableData | undefined>(() => {
-    if (controlImage) {
-      return {
-        id: layerId,
-        payloadType: 'IMAGE_DTO',
-        payload: { imageDTO: controlImage },
-      };
-    }
-  }, [controlImage, layerId]);
-
-  const droppableData = useMemo<ControlLayerDropData>(
-    () => ({
-      id: layerId,
-      actionType: 'SET_CONTROL_LAYER_IMAGE',
-      context: { layerId },
-    }),
-    [layerId]
-  );
-
-  const postUploadAction = useMemo<ControlLayerAction>(() => ({ type: 'SET_CONTROL_LAYER_IMAGE', layerId }), [layerId]);
-
-  useEffect(() => {
-    if (isConnected && isErrorControlImage) {
-      handleResetControlImage();
-    }
-  }, [handleResetControlImage, isConnected, isErrorControlImage]);
-
-  return (
-    <Flex position="relative" w="full" h={36} alignItems="center" justifyContent="center">
-      <IAIDndImage
-        draggableData={draggableData}
-        droppableData={droppableData}
-        imageDTO={controlImage}
-        postUploadAction={postUploadAction}
-      />
-
-      <>
-        <IAIDndImageIcon
-          onClick={handleResetControlImage}
-          icon={controlImage ? <PiArrowCounterClockwiseBold size={16} /> : undefined}
-          tooltip={t('controlnet.resetControlImage')}
-        />
-        <IAIDndImageIcon
-          onClick={handleSetControlImageToDimensions}
-          icon={controlImage ? <PiRulerBold size={16} /> : undefined}
-          tooltip={shift ? t('controlnet.setControlImageDimensionsForce') : t('controlnet.setControlImageDimensions')}
-          styleOverrides={setControlImageDimensionsStyleOverrides}
-        />
-      </>
-    </Flex>
-  );
-});
-
-IPAdapterImagePreview.displayName = 'IPAdapterImagePreview';
-
-const setControlImageDimensionsStyleOverrides: SystemStyleObject = { mt: 6 };

invokeai/frontend/web/src/features/controlLayers/components/RGLayer/RGLayerIPAdapterList.tsx

@@ -1,13 +1,9 @@
-import { Divider, Flex, IconButton, Spacer, Text } from '@invoke-ai/ui-library';
+import { Divider, Flex } from '@invoke-ai/ui-library';
 import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
-import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { useAppSelector } from 'app/store/storeHooks';
-import {
+import { RGLayerIPAdapterWrapper } from 'features/controlLayers/components/RGLayer/RGLayerIPAdapterWrapper';
-  isRegionalGuidanceLayer,
+import { isRegionalGuidanceLayer, selectControlLayersSlice } from 'features/controlLayers/store/controlLayersSlice';
-  rgLayerIPAdapterDeleted,
+import { memo, useMemo } from 'react';
-  selectControlLayersSlice,
-} from 'features/controlLayers/store/controlLayersSlice';
-import { memo, useCallback, useMemo } from 'react';
-import { PiTrashSimpleBold } from 'react-icons/pi';
 import { assert } from 'tsafe';
 
 type Props = {
@@ -39,7 +35,7 @@ export const RGLayerIPAdapterList = memo(({ layerId }: Props) => {
               <Divider />
             </Flex>
           )}
-          <RGLayerIPAdapterListItem layerId={layerId} ipAdapterId={id} ipAdapterNumber={index + 1} />
+          <RGLayerIPAdapterWrapper layerId={layerId} ipAdapterId={id} ipAdapterNumber={index + 1} />
         </Flex>
       ))}
     </>
@@ -47,36 +43,3 @@ export const RGLayerIPAdapterList = memo(({ layerId }: Props) => {
 });
 
 RGLayerIPAdapterList.displayName = 'RGLayerIPAdapterList';
-
-type IPAdapterListItemProps = {
-  layerId: string;
-  ipAdapterId: string;
-  ipAdapterNumber: number;
-};
-
-const RGLayerIPAdapterListItem = memo(({ layerId, ipAdapterId, ipAdapterNumber }: IPAdapterListItemProps) => {
-  const dispatch = useAppDispatch();
-  const onDeleteIPAdapter = useCallback(() => {
-    dispatch(rgLayerIPAdapterDeleted({ layerId, ipAdapterId }));
-  }, [dispatch, ipAdapterId, layerId]);
-
-  return (
-    <Flex flexDir="column" gap={3}>
-      <Flex alignItems="center" gap={3}>
-        <Text fontWeight="semibold" color="base.400">{`IP Adapter ${ipAdapterNumber}`}</Text>
-        <Spacer />
-        <IconButton
-          size="sm"
-          icon={<PiTrashSimpleBold />}
-          aria-label="Delete IP Adapter"
-          onClick={onDeleteIPAdapter}
-          variant="ghost"
-          colorScheme="error"
-        />
-      </Flex>
-      {/* <ControlAdapterLayerConfig id={ipAdapterId} /> */}
-    </Flex>
-  );
-});
-
-RGLayerIPAdapterListItem.displayName = 'RGLayerIPAdapterListItem';

invokeai/frontend/web/src/features/controlLayers/components/RGLayer/RGLayerIPAdapterWrapper.tsx

@@ -0,0 +1,131 @@
+import { Flex, IconButton, Spacer, Text } from '@invoke-ai/ui-library';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { IPAdapter } from 'features/controlLayers/components/ControlAndIPAdapter/IPAdapter';
+import {
+  rgLayerIPAdapterBeginEndStepPctChanged,
+  rgLayerIPAdapterCLIPVisionModelChanged,
+  rgLayerIPAdapterDeleted,
+  rgLayerIPAdapterImageChanged,
+  rgLayerIPAdapterMethodChanged,
+  rgLayerIPAdapterModelChanged,
+  rgLayerIPAdapterWeightChanged,
+  selectRGLayerIPAdapter,
+} from 'features/controlLayers/store/controlLayersSlice';
+import type { CLIPVisionModel, IPMethod } from 'features/controlLayers/util/controlAdapters';
+import type { RGLayerIPAdapterImageDropData } from 'features/dnd/types';
+import { memo, useCallback, useMemo } from 'react';
+import { PiTrashSimpleBold } from 'react-icons/pi';
+import type { ImageDTO, IPAdapterModelConfig, RGLayerIPAdapterImagePostUploadAction } from 'services/api/types';
+
+type Props = {
+  layerId: string;
+  ipAdapterId: string;
+  ipAdapterNumber: number;
+};
+
+export const RGLayerIPAdapterWrapper = memo(({ layerId, ipAdapterId, ipAdapterNumber }: Props) => {
+  const dispatch = useAppDispatch();
+  const onDeleteIPAdapter = useCallback(() => {
+    dispatch(rgLayerIPAdapterDeleted({ layerId, ipAdapterId }));
+  }, [dispatch, ipAdapterId, layerId]);
+  const ipAdapter = useAppSelector((s) => selectRGLayerIPAdapter(s.controlLayers.present, layerId, ipAdapterId));
+
+  const onChangeBeginEndStepPct = useCallback(
+    (beginEndStepPct: [number, number]) => {
+      dispatch(
+        rgLayerIPAdapterBeginEndStepPctChanged({
+          layerId,
+          ipAdapterId,
+          beginEndStepPct,
+        })
+      );
+    },
+    [dispatch, ipAdapterId, layerId]
+  );
+
+  const onChangeWeight = useCallback(
+    (weight: number) => {
+      dispatch(rgLayerIPAdapterWeightChanged({ layerId, ipAdapterId, weight }));
+    },
+    [dispatch, ipAdapterId, layerId]
+  );
+
+  const onChangeIPMethod = useCallback(
+    (method: IPMethod) => {
+      dispatch(rgLayerIPAdapterMethodChanged({ layerId, ipAdapterId, method }));
+    },
+    [dispatch, ipAdapterId, layerId]
+  );
+
+  const onChangeModel = useCallback(
+    (modelConfig: IPAdapterModelConfig) => {
+      dispatch(rgLayerIPAdapterModelChanged({ layerId, ipAdapterId, modelConfig }));
+    },
+    [dispatch, ipAdapterId, layerId]
+  );
+
+  const onChangeCLIPVisionModel = useCallback(
+    (clipVisionModel: CLIPVisionModel) => {
+      dispatch(rgLayerIPAdapterCLIPVisionModelChanged({ layerId, ipAdapterId, clipVisionModel }));
+    },
+    [dispatch, ipAdapterId, layerId]
+  );
+
+  const onChangeImage = useCallback(
+    (imageDTO: ImageDTO | null) => {
+      dispatch(rgLayerIPAdapterImageChanged({ layerId, ipAdapterId, imageDTO }));
+    },
+    [dispatch, ipAdapterId, layerId]
+  );
+
+  const droppableData = useMemo<RGLayerIPAdapterImageDropData>(
+    () => ({
+      actionType: 'SET_RG_LAYER_IP_ADAPTER_IMAGE',
+      context: {
+        layerId,
+        ipAdapterId,
+      },
+      id: layerId,
+    }),
+    [ipAdapterId, layerId]
+  );
+
+  const postUploadAction = useMemo<RGLayerIPAdapterImagePostUploadAction>(
+    () => ({
+      type: 'SET_RG_LAYER_IP_ADAPTER_IMAGE',
+      layerId,
+      ipAdapterId,
+    }),
+    [ipAdapterId, layerId]
+  );
+
+  return (
+    <Flex flexDir="column" gap={3}>
+      <Flex alignItems="center" gap={3}>
+        <Text fontWeight="semibold" color="base.400">{`IP Adapter ${ipAdapterNumber}`}</Text>
+        <Spacer />
+        <IconButton
+          size="sm"
+          icon={<PiTrashSimpleBold />}
+          aria-label="Delete IP Adapter"
+          onClick={onDeleteIPAdapter}
+          variant="ghost"
+          colorScheme="error"
+        />
+      </Flex>
+      <IPAdapter
+        ipAdapter={ipAdapter}
+        onChangeBeginEndStepPct={onChangeBeginEndStepPct}
+        onChangeWeight={onChangeWeight}
+        onChangeIPMethod={onChangeIPMethod}
+        onChangeModel={onChangeModel}
+        onChangeCLIPVisionModel={onChangeCLIPVisionModel}
+        onChangeImage={onChangeImage}
+        droppableData={droppableData}
+        postUploadAction={postUploadAction}
+      />
+    </Flex>
+  );
+});
+
+RGLayerIPAdapterWrapper.displayName = 'RGLayerIPAdapterWrapper';

invokeai/frontend/web/src/features/controlLayers/hooks/useControlLayersTitle.ts

@@ -5,9 +5,6 @@ import { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 
 const selectValidLayerCount = createSelector(selectControlLayersSlice, (controlLayers) => {
-  if (!controlLayers.present.isEnabled) {
-    return 0;
-  }
   const validLayers = controlLayers.present.layers
     .filter(isRegionalGuidanceLayer)
     .filter((l) => l.isEnabled)

invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts

@@ -47,7 +47,6 @@ export const initialControlLayersState: ControlLayersState = {
   brushSize: 100,
   layers: [],
   globalMaskLayerOpacity: 0.3, // this globally changes all mask layers' opacity
-  isEnabled: true,
   positivePrompt: '',
   negativePrompt: '',
   positivePrompt2: '',
@@ -77,10 +76,6 @@ const resetLayer = (layer: Layer) => {
     layer.bboxNeedsUpdate = false;
     return;
   }
-
-  if (layer.type === 'control_adapter_layer') {
-    // TODO
-  }
 };
 
 export const selectCALayer = (state: ControlLayersState, layerId: string): ControlAdapterLayer => {
@@ -101,12 +96,16 @@ export const selectCAOrIPALayer = (
   assert(isControlAdapterLayer(layer) || isIPAdapterLayer(layer));
   return layer;
 };
-const selectRGLayer = (state: ControlLayersState, layerId: string): RegionalGuidanceLayer => {
+export const selectRGLayer = (state: ControlLayersState, layerId: string): RegionalGuidanceLayer => {
   const layer = state.layers.find((l) => l.id === layerId);
   assert(isRegionalGuidanceLayer(layer));
   return layer;
 };
-const selectRGLayerIPAdapter = (state: ControlLayersState, layerId: string, ipAdapterId: string): IPAdapterConfig => {
+export const selectRGLayerIPAdapter = (
+  state: ControlLayersState,
+  layerId: string,
+  ipAdapterId: string
+): IPAdapterConfig => {
   const layer = state.layers.find((l) => l.id === layerId);
   assert(isRegionalGuidanceLayer(layer));
   const ipAdapter = layer.ipAdapters.find((ipAdapter) => ipAdapter.id === ipAdapterId);
@@ -556,6 +555,22 @@ export const controlLayersSlice = createSlice({
       const ipAdapter = selectRGLayerIPAdapter(state, layerId, ipAdapterId);
       ipAdapter.method = method;
     },
+    rgLayerIPAdapterModelChanged: (
+      state,
+      action: PayloadAction<{
+        layerId: string;
+        ipAdapterId: string;
+        modelConfig: IPAdapterModelConfig | null;
+      }>
+    ) => {
+      const { layerId, ipAdapterId, modelConfig } = action.payload;
+      const ipAdapter = selectRGLayerIPAdapter(state, layerId, ipAdapterId);
+      if (!modelConfig) {
+        ipAdapter.model = null;
+        return;
+      }
+      ipAdapter.model = zModelIdentifierField.parse(modelConfig);
+    },
     rgLayerIPAdapterCLIPVisionModelChanged: (
       state,
       action: PayloadAction<{ layerId: string; ipAdapterId: string; clipVisionModel: CLIPVisionModel }>
@@ -609,9 +624,6 @@ export const controlLayersSlice = createSlice({
     globalMaskLayerOpacityChanged: (state, action: PayloadAction<number>) => {
       state.globalMaskLayerOpacity = action.payload;
     },
-    isEnabledChanged: (state, action: PayloadAction<boolean>) => {
-      state.isEnabled = action.payload;
-    },
     undo: (state) => {
       // Invalidate the bbox for all layers to prevent stale bboxes
       for (const layer of state.layers.filter(isRenderableLayer)) {
@@ -734,6 +746,7 @@ export const {
   rgLayerIPAdapterWeightChanged,
   rgLayerIPAdapterBeginEndStepPctChanged,
   rgLayerIPAdapterMethodChanged,
+  rgLayerIPAdapterModelChanged,
   rgLayerIPAdapterCLIPVisionModelChanged,
   // Globals
   positivePromptChanged,
@@ -746,7 +759,6 @@ export const {
   aspectRatioChanged,
   brushSizeChanged,
   globalMaskLayerOpacityChanged,
-  isEnabledChanged,
   undo,
   redo,
 } = controlLayersSlice.actions;

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -77,7 +77,6 @@ export type ControlLayersState = {
   layers: Layer[];
   brushSize: number;
   globalMaskLayerOpacity: number;
-  isEnabled: boolean;
   positivePrompt: ParameterPositivePrompt;
   negativePrompt: ParameterNegativePrompt;
   positivePrompt2: ParameterPositiveStylePromptSDXL;

invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts

@@ -80,7 +80,6 @@ export type ImageWithDims = {
 
 type ControlAdapterBase = {
   id: string;
-  isEnabled: boolean;
   weight: number;
   image: ImageWithDims | null;
   processedImage: ImageWithDims | null;
@@ -97,11 +96,15 @@ export type ControlNetConfig = ControlAdapterBase & {
   model: ParameterControlNetModel | null;
   controlMode: ControlMode;
 };
+export const isControlNetConfig = (ca: ControlNetConfig | T2IAdapterConfig): ca is ControlNetConfig =>
+  ca.type === 'controlnet';
 
 export type T2IAdapterConfig = ControlAdapterBase & {
   type: 't2i_adapter';
   model: ParameterT2IAdapterModel | null;
 };
+export const isT2IAdapterConfig = (ca: ControlNetConfig | T2IAdapterConfig): ca is T2IAdapterConfig =>
+  ca.type === 't2i_adapter';
 
 const zCLIPVisionModel = z.enum(['ViT-H', 'ViT-G']);
 export type CLIPVisionModel = z.infer<typeof zCLIPVisionModel>;
@@ -114,7 +117,6 @@ export const isIPMethod = (v: unknown): v is IPMethod => zIPMethod.safeParse(v).
 export type IPAdapterConfig = {
   id: string;
   type: 'ip_adapter';
-  isEnabled: boolean;
   weight: number;
   method: IPMethod;
   image: ImageWithDims | null;
@@ -295,10 +297,9 @@ export const isProcessorType = (v: unknown): v is ProcessorType => zProcessorTyp
 
 export const initialControlNet: Omit<ControlNetConfig, 'id'> = {
   type: 'controlnet',
-  isEnabled: true,
   model: null,
   weight: 1,
-  beginEndStepPct: [0, 0],
+  beginEndStepPct: [0, 1],
   controlMode: 'balanced',
   image: null,
   processedImage: null,
@@ -307,10 +308,9 @@ export const initialControlNet: Omit<ControlNetConfig, 'id'> = {
 
 export const initialT2IAdapter: Omit<T2IAdapterConfig, 'id'> = {
   type: 't2i_adapter',
-  isEnabled: true,
   model: null,
   weight: 1,
-  beginEndStepPct: [0, 0],
+  beginEndStepPct: [0, 1],
   image: null,
   processedImage: null,
   processorConfig: CONTROLNET_PROCESSORS.canny_image_processor.buildDefaults(),
@@ -318,10 +318,9 @@ export const initialT2IAdapter: Omit<T2IAdapterConfig, 'id'> = {
 
 export const initialIPAdapter: Omit<IPAdapterConfig, 'id'> = {
   type: 'ip_adapter',
-  isEnabled: true,
   image: null,
   model: null,
-  beginEndStepPct: [0, 0],
+  beginEndStepPct: [0, 1],
   method: 'full',
   clipVisionModel: 'ViT-H',
   weight: 1,

invokeai/frontend/web/src/features/dnd/types/index.ts

@@ -33,13 +33,28 @@ type ControlAdapterDropData = BaseDropData & {
   };
 };
 
-export type ControlLayerDropData = BaseDropData & {
+export type CALayerImageDropData = BaseDropData & {
-  actionType: 'SET_CONTROL_LAYER_IMAGE';
+  actionType: 'SET_CA_LAYER_IMAGE';
   context: {
     layerId: string;
   };
 };
 
+export type IPALayerImageDropData = BaseDropData & {
+  actionType: 'SET_IPA_LAYER_IMAGE';
+  context: {
+    layerId: string;
+  };
+};
+
+export type RGLayerIPAdapterImageDropData = BaseDropData & {
+  actionType: 'SET_RG_LAYER_IP_ADAPTER_IMAGE';
+  context: {
+    layerId: string;
+    ipAdapterId: string;
+  };
+};
+
 export type CanvasInitialImageDropData = BaseDropData & {
   actionType: 'SET_CANVAS_INITIAL_IMAGE';
 };
@@ -69,7 +84,9 @@ export type TypesafeDroppableData =
   | NodesImageDropData
   | AddToBoardDropData
   | RemoveFromBoardDropData
-  | ControlLayerDropData;
+  | CALayerImageDropData
+  | IPALayerImageDropData
+  | RGLayerIPAdapterImageDropData;
 
 type BaseDragData = {
   id: string;

invokeai/frontend/web/src/features/dnd/util/isValidDrop.ts

@@ -19,6 +19,12 @@ export const isValidDrop = (overData: TypesafeDroppableData | undefined, active:
       return payloadType === 'IMAGE_DTO';
     case 'SET_CONTROL_ADAPTER_IMAGE':
       return payloadType === 'IMAGE_DTO';
+    case 'SET_CA_LAYER_IMAGE':
+      return payloadType === 'IMAGE_DTO';
+    case 'SET_IPA_LAYER_IMAGE':
+      return payloadType === 'IMAGE_DTO';
+    case 'SET_RG_LAYER_IP_ADAPTER_IMAGE':
+      return payloadType === 'IMAGE_DTO';
     case 'SET_CANVAS_INITIAL_IMAGE':
       return payloadType === 'IMAGE_DTO';
     case 'SET_NODES_IMAGE':

invokeai/frontend/web/src/features/nodes/util/graph/addControlLayersToGraph.ts

@@ -1,9 +1,23 @@
 import { getStore } from 'app/store/nanostores/store';
 import type { RootState } from 'app/store/store';
-import { selectAllIPAdapters } from 'features/controlAdapters/store/controlAdaptersSlice';
-import { isRegionalGuidanceLayer } from 'features/controlLayers/store/controlLayersSlice';
-import { getRegionalPromptLayerBlobs } from 'features/controlLayers/util/getLayerBlobs';
 import {
+  isControlAdapterLayer,
+  isIPAdapterLayer,
+  isRegionalGuidanceLayer,
+} from 'features/controlLayers/store/controlLayersSlice';
+import {
+  type ControlNetConfig,
+  type ImageWithDims,
+  type IPAdapterConfig,
+  isControlNetConfig,
+  isT2IAdapterConfig,
+  type ProcessorConfig,
+  type T2IAdapterConfig,
+} from 'features/controlLayers/util/controlAdapters';
+import { getRegionalPromptLayerBlobs } from 'features/controlLayers/util/getLayerBlobs';
+import type { ImageField } from 'features/nodes/types/common';
+import {
+  CONTROL_NET_COLLECT,
   IP_ADAPTER_COLLECT,
   NEGATIVE_CONDITIONING,
   NEGATIVE_CONDITIONING_COLLECT,
@@ -14,45 +28,383 @@ import {
   PROMPT_REGION_NEGATIVE_COND_PREFIX,
   PROMPT_REGION_POSITIVE_COND_INVERTED_PREFIX,
   PROMPT_REGION_POSITIVE_COND_PREFIX,
+  T2I_ADAPTER_COLLECT,
 } from 'features/nodes/util/graph/constants';
+import { upsertMetadata } from 'features/nodes/util/graph/metadata';
 import { size } from 'lodash-es';
 import { imagesApi } from 'services/api/endpoints/images';
-import type { CollectInvocation, Edge, IPAdapterInvocation, NonNullableGraph, S } from 'services/api/types';
+import type {
+  CollectInvocation,
+  ControlNetInvocation,
+  CoreMetadataInvocation,
+  Edge,
+  IPAdapterInvocation,
+  NonNullableGraph,
+  S,
+  T2IAdapterInvocation,
+} from 'services/api/types';
 import { assert } from 'tsafe';
 
-export const addControlLayersToGraph = async (state: RootState, graph: NonNullableGraph, denoiseNodeId: string) => {
+const buildControlImage = (
-  if (!state.controlLayers.present.isEnabled) {
+  image: ImageWithDims | null,
+  processedImage: ImageWithDims | null,
+  processorConfig: ProcessorConfig | null
+): ImageField => {
+  if (processedImage && processorConfig) {
+    // We've processed the image in the app - use it for the control image.
+    return {
+      image_name: processedImage.imageName,
+    };
+  } else if (image) {
+    // No processor selected, and we have an image - the user provided a processed image, use it for the control image.
+    return {
+      image_name: image.imageName,
+    };
+  }
+  assert(false, 'Attempted to add unprocessed control image');
+};
+
+const buildControlNetMetadata = (controlNet: ControlNetConfig): S['ControlNetMetadataField'] => {
+  const { beginEndStepPct, controlMode, image, model, processedImage, processorConfig, weight } = controlNet;
+
+  assert(model, 'ControlNet model is required');
+  assert(image, 'ControlNet image is required');
+
+  const processed_image =
+    processedImage && processorConfig
+      ? {
+          image_name: processedImage.imageName,
+        }
+      : null;
+
+  return {
+    control_model: model,
+    control_weight: weight,
+    control_mode: controlMode,
+    begin_step_percent: beginEndStepPct[0],
+    end_step_percent: beginEndStepPct[1],
+    resize_mode: 'just_resize',
+    image: {
+      image_name: image.imageName,
+    },
+    processed_image,
+  };
+};
+
+const addControlNetCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => {
+  if (graph.nodes[CONTROL_NET_COLLECT]) {
+    // You see, we've already got one!
     return;
   }
+  // Add the ControlNet collector
+  const controlNetIterateNode: CollectInvocation = {
+    id: CONTROL_NET_COLLECT,
+    type: 'collect',
+    is_intermediate: true,
+  };
+  graph.nodes[CONTROL_NET_COLLECT] = controlNetIterateNode;
+  graph.edges.push({
+    source: { node_id: CONTROL_NET_COLLECT, field: 'collection' },
+    destination: {
+      node_id: denoiseNodeId,
+      field: 'control',
+    },
+  });
+};
+
+const addGlobalControlNetsToGraph = async (
+  controlNets: ControlNetConfig[],
+  graph: NonNullableGraph,
+  denoiseNodeId: string
+) => {
+  if (controlNets.length === 0) {
+    return;
+  }
+  const controlNetMetadata: CoreMetadataInvocation['controlnets'] = [];
+  addControlNetCollectorSafe(graph, denoiseNodeId);
+
+  for (const controlNet of controlNets) {
+    if (!controlNet.model) {
+      return;
+    }
+    const { id, beginEndStepPct, controlMode, image, model, processedImage, processorConfig, weight } = controlNet;
+
+    const controlNetNode: ControlNetInvocation = {
+      id: `control_net_${id}`,
+      type: 'controlnet',
+      is_intermediate: true,
+      begin_step_percent: beginEndStepPct[0],
+      end_step_percent: beginEndStepPct[1],
+      control_mode: controlMode,
+      resize_mode: 'just_resize',
+      control_model: model,
+      control_weight: weight,
+      image: buildControlImage(image, processedImage, processorConfig),
+    };
+
+    graph.nodes[controlNetNode.id] = controlNetNode;
+
+    controlNetMetadata.push(buildControlNetMetadata(controlNet));
+
+    graph.edges.push({
+      source: { node_id: controlNetNode.id, field: 'control' },
+      destination: {
+        node_id: CONTROL_NET_COLLECT,
+        field: 'item',
+      },
+    });
+  }
+  upsertMetadata(graph, { controlnets: controlNetMetadata });
+};
+
+const buildT2IAdapterMetadata = (t2iAdapter: T2IAdapterConfig): S['T2IAdapterMetadataField'] => {
+  const { beginEndStepPct, image, model, processedImage, processorConfig, weight } = t2iAdapter;
+
+  assert(model, 'T2I Adapter model is required');
+  assert(image, 'T2I Adapter image is required');
+
+  const processed_image =
+    processedImage && processorConfig
+      ? {
+          image_name: processedImage.imageName,
+        }
+      : null;
+
+  return {
+    t2i_adapter_model: model,
+    weight,
+    begin_step_percent: beginEndStepPct[0],
+    end_step_percent: beginEndStepPct[1],
+    resize_mode: 'just_resize',
+    image: {
+      image_name: image.imageName,
+    },
+    processed_image,
+  };
+};
+
+const addT2IAdapterCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => {
+  if (graph.nodes[T2I_ADAPTER_COLLECT]) {
+    // You see, we've already got one!
+    return;
+  }
+  // Even though denoise_latents' t2i adapter input is collection or scalar, keep it simple and always use a collect
+  const t2iAdapterCollectNode: CollectInvocation = {
+    id: T2I_ADAPTER_COLLECT,
+    type: 'collect',
+    is_intermediate: true,
+  };
+  graph.nodes[T2I_ADAPTER_COLLECT] = t2iAdapterCollectNode;
+  graph.edges.push({
+    source: { node_id: T2I_ADAPTER_COLLECT, field: 'collection' },
+    destination: {
+      node_id: denoiseNodeId,
+      field: 't2i_adapter',
+    },
+  });
+};
+
+const addGlobalT2IAdaptersToGraph = async (
+  t2iAdapters: T2IAdapterConfig[],
+  graph: NonNullableGraph,
+  denoiseNodeId: string
+) => {
+  if (t2iAdapters.length === 0) {
+    return;
+  }
+  const t2iAdapterMetadata: CoreMetadataInvocation['t2iAdapters'] = [];
+  addT2IAdapterCollectorSafe(graph, denoiseNodeId);
+
+  for (const t2iAdapter of t2iAdapters) {
+    if (!t2iAdapter.model) {
+      return;
+    }
+    const { id, beginEndStepPct, image, model, processedImage, processorConfig, weight } = t2iAdapter;
+
+    const t2iAdapterNode: T2IAdapterInvocation = {
+      id: `t2i_adapter_${id}`,
+      type: 't2i_adapter',
+      is_intermediate: true,
+      begin_step_percent: beginEndStepPct[0],
+      end_step_percent: beginEndStepPct[1],
+      resize_mode: 'just_resize',
+      t2i_adapter_model: model,
+      weight: weight,
+      image: buildControlImage(image, processedImage, processorConfig),
+    };
+
+    graph.nodes[t2iAdapterNode.id] = t2iAdapterNode;
+
+    t2iAdapterMetadata.push(buildT2IAdapterMetadata(t2iAdapter));
+
+    graph.edges.push({
+      source: { node_id: t2iAdapterNode.id, field: 't2i_adapter' },
+      destination: {
+        node_id: T2I_ADAPTER_COLLECT,
+        field: 'item',
+      },
+    });
+  }
+
+  upsertMetadata(graph, { t2iAdapters: t2iAdapterMetadata });
+};
+
+const buildIPAdapterMetadata = (ipAdapter: IPAdapterConfig): S['IPAdapterMetadataField'] => {
+  const { weight, model, clipVisionModel, method, beginEndStepPct, image } = ipAdapter;
+
+  assert(model, 'IP Adapter model is required');
+  assert(image, 'IP Adapter image is required');
+
+  return {
+    ip_adapter_model: model,
+    clip_vision_model: clipVisionModel,
+    weight,
+    method,
+    begin_step_percent: beginEndStepPct[0],
+    end_step_percent: beginEndStepPct[1],
+    image: {
+      image_name: image.imageName,
+    },
+  };
+};
+
+const addIPAdapterCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => {
+  if (graph.nodes[IP_ADAPTER_COLLECT]) {
+    // You see, we've already got one!
+    return;
+  }
+
+  const ipAdapterCollectNode: CollectInvocation = {
+    id: IP_ADAPTER_COLLECT,
+    type: 'collect',
+    is_intermediate: true,
+  };
+  graph.nodes[IP_ADAPTER_COLLECT] = ipAdapterCollectNode;
+  graph.edges.push({
+    source: { node_id: IP_ADAPTER_COLLECT, field: 'collection' },
+    destination: {
+      node_id: denoiseNodeId,
+      field: 'ip_adapter',
+    },
+  });
+};
+
+const addGlobalIPAdaptersToGraph = async (
+  ipAdapters: IPAdapterConfig[],
+  graph: NonNullableGraph,
+  denoiseNodeId: string
+) => {
+  if (ipAdapters.length === 0) {
+    return;
+  }
+  const ipAdapterMetdata: CoreMetadataInvocation['ipAdapters'] = [];
+  addIPAdapterCollectorSafe(graph, denoiseNodeId);
+
+  for (const ipAdapter of ipAdapters) {
+    const { id, weight, model, clipVisionModel, method, beginEndStepPct, image } = ipAdapter;
+    assert(image, 'IP Adapter image is required');
+    assert(model, 'IP Adapter model is required');
+
+    const ipAdapterNode: IPAdapterInvocation = {
+      id: `ip_adapter_${id}`,
+      type: 'ip_adapter',
+      is_intermediate: true,
+      weight,
+      method,
+      ip_adapter_model: model,
+      clip_vision_model: clipVisionModel,
+      begin_step_percent: beginEndStepPct[0],
+      end_step_percent: beginEndStepPct[1],
+      image: {
+        image_name: image.imageName,
+      },
+    };
+
+    graph.nodes[ipAdapterNode.id] = ipAdapterNode;
+
+    ipAdapterMetdata.push(buildIPAdapterMetadata(ipAdapter));
+
+    graph.edges.push({
+      source: { node_id: ipAdapterNode.id, field: 'ip_adapter' },
+      destination: {
+        node_id: IP_ADAPTER_COLLECT,
+        field: 'item',
+      },
+    });
+  }
+
+  upsertMetadata(graph, { ipAdapters: ipAdapterMetdata });
+};
+
+export const addControlLayersToGraph = async (state: RootState, graph: NonNullableGraph, denoiseNodeId: string) => {
   const { dispatch } = getStore();
-  const isSDXL = state.generation.model?.base === 'sdxl';
+  const mainModel = state.generation.model;
-  const layers = state.controlLayers.present.layers
+  assert(mainModel, 'Missing main model when building graph');
-    // Only support vector mask layers now
+  const isSDXL = mainModel.base === 'sdxl';
-    // TODO: Image masks
+
+  // Add global control adapters
+  const globalControlNets = state.controlLayers.present.layers
+    // Must be a CA layer
+    .filter(isControlAdapterLayer)
+    // Must be enabled
+    .filter((l) => l.isEnabled)
+    // We want the CAs themselves
+    .map((l) => l.controlAdapter)
+    // Must be a ControlNet
+    .filter(isControlNetConfig)
+    .filter((ca) => {
+      const hasModel = Boolean(ca.model);
+      const modelMatchesBase = ca.model?.base === mainModel.base;
+      const hasControlImage = ca.image || (ca.processedImage && ca.processorConfig);
+      return hasModel && modelMatchesBase && hasControlImage;
+    });
+  addGlobalControlNetsToGraph(globalControlNets, graph, denoiseNodeId);
+
+  const globalT2IAdapters = state.controlLayers.present.layers
+    // Must be a CA layer
+    .filter(isControlAdapterLayer)
+    // Must be enabled
+    .filter((l) => l.isEnabled)
+    // We want the CAs themselves
+    .map((l) => l.controlAdapter)
+    // Must have a ControlNet CA
+    .filter(isT2IAdapterConfig)
+    .filter((ca) => {
+      const hasModel = Boolean(ca.model);
+      const modelMatchesBase = ca.model?.base === mainModel.base;
+      const hasControlImage = ca.image || (ca.processedImage && ca.processorConfig);
+      return hasModel && modelMatchesBase && hasControlImage;
+    });
+  addGlobalT2IAdaptersToGraph(globalT2IAdapters, graph, denoiseNodeId);
+
+  const globalIPAdapters = state.controlLayers.present.layers
+    // Must be an IP Adapter layer
+    .filter(isIPAdapterLayer)
+    // Must be enabled
+    .filter((l) => l.isEnabled)
+    // We want the IP Adapters themselves
+    .map((l) => l.ipAdapter)
+    .filter((ca) => {
+      const hasModel = Boolean(ca.model);
+      const modelMatchesBase = ca.model?.base === mainModel.base;
+      const hasControlImage = Boolean(ca.image);
+      return hasModel && modelMatchesBase && hasControlImage;
+    });
+  addGlobalIPAdaptersToGraph(globalIPAdapters, graph, denoiseNodeId);
+
+  const rgLayers = state.controlLayers.present.layers
+    // Only RG layers are get masks
     .filter(isRegionalGuidanceLayer)
     // Only visible layers are rendered on the canvas
     .filter((l) => l.isEnabled)
     // Only layers with prompts get added to the graph
     .filter((l) => {
       const hasTextPrompt = Boolean(l.positivePrompt || l.negativePrompt);
-      const hasIPAdapter = l.ipAdapterIds.length !== 0;
+      const hasIPAdapter = l.ipAdapters.length !== 0;
       return hasTextPrompt || hasIPAdapter;
     });
 
-  // Collect all IP Adapter ids for IP adapter layers
+  const layerIds = rgLayers.map((l) => l.id);
-  const layerIPAdapterIds = layers.flatMap((l) => l.ipAdapterIds);
-
-  const regionalIPAdapters = selectAllIPAdapters(state.controlAdapters).filter(
-    ({ id, model, controlImage, isEnabled }) => {
-      const hasModel = Boolean(model);
-      const doesBaseMatch = model?.base === state.generation.model?.base;
-      const hasControlImage = controlImage;
-      const isRegional = layerIPAdapterIds.includes(id);
-      return isEnabled && hasModel && doesBaseMatch && hasControlImage && isRegional;
-    }
-  );
-
-  const layerIds = layers.map((l) => l.id);
   const blobs = await getRegionalPromptLayerBlobs(layerIds);
   assert(size(blobs) === size(layerIds), 'Mismatch between layer IDs and blobs');
 
@@ -118,27 +470,11 @@ export const addControlLayersToGraph = async (state: RootState, graph: NonNullab
     },
   });
 
-  if (!graph.nodes[IP_ADAPTER_COLLECT] && regionalIPAdapters.length > 0) {
-    const ipAdapterCollectNode: CollectInvocation = {
-      id: IP_ADAPTER_COLLECT,
-      type: 'collect',
-      is_intermediate: true,
-    };
-    graph.nodes[IP_ADAPTER_COLLECT] = ipAdapterCollectNode;
-    graph.edges.push({
-      source: { node_id: IP_ADAPTER_COLLECT, field: 'collection' },
-      destination: {
-        node_id: denoiseNodeId,
-        field: 'ip_adapter',
-      },
-    });
-  }
-
   // Upload the blobs to the backend, add each to graph
   // TODO: Store the uploaded image names in redux to reuse them, so long as the layer hasn't otherwise changed. This
   // would be a great perf win - not only would we skip re-uploading the same image, but we'd be able to use the node
   // cache (currently, when we re-use the same mask data, since it is a different image, the node cache is not used).
-  for (const layer of layers) {
+  for (const layer of rgLayers) {
     const blob = blobs[layer.id];
     assert(blob, `Blob for layer ${layer.id} not found`);
 
@@ -296,36 +632,32 @@ export const addControlLayersToGraph = async (state: RootState, graph: NonNullab
       }
     }
 
-    for (const ipAdapterId of layer.ipAdapterIds) {
+    // TODO(psyche): For some reason, I have to explicitly annotate regionalIPAdapters here. Not sure why.
-      const ipAdapter = selectAllIPAdapters(state.controlAdapters)
+    const regionalIPAdapters: IPAdapterConfig[] = layer.ipAdapters.filter((ipAdapter) => {
-        .filter(({ id, model, controlImage, isEnabled }) => {
+      const hasModel = Boolean(ipAdapter.model);
-          const hasModel = Boolean(model);
+      const modelMatchesBase = ipAdapter.model?.base === mainModel.base;
-          const doesBaseMatch = model?.base === state.generation.model?.base;
+      const hasControlImage = Boolean(ipAdapter.image);
-          const hasControlImage = controlImage;
+      return hasModel && modelMatchesBase && hasControlImage;
-          const isRegional = layers.some((l) => l.ipAdapterIds.includes(id));
+    });
-          return isEnabled && hasModel && doesBaseMatch && hasControlImage && isRegional;
-        })
-        .find((ca) => ca.id === ipAdapterId);
 
-      if (!ipAdapter?.model) {
+    for (const ipAdapter of regionalIPAdapters) {
-        return;
+      addIPAdapterCollectorSafe(graph, denoiseNodeId);
-      }
+      const { id, weight, model, clipVisionModel, method, beginEndStepPct, image } = ipAdapter;
-      const { id, weight, model, clipVisionModel, method, beginStepPct, endStepPct, controlImage } = ipAdapter;
+      assert(model, 'IP Adapter model is required');
-
+      assert(image, 'IP Adapter image is required');
-      assert(controlImage, 'IP Adapter image is required');
 
       const ipAdapterNode: IPAdapterInvocation = {
         id: `ip_adapter_${id}`,
         type: 'ip_adapter',
         is_intermediate: true,
-        weight: weight,
+        weight,
-        method: method,
+        method,
         ip_adapter_model: model,
         clip_vision_model: clipVisionModel,
-        begin_step_percent: beginStepPct,
+        begin_step_percent: beginEndStepPct[0],
-        end_step_percent: endStepPct,
+        end_step_percent: beginEndStepPct[1],
         image: {
-          image_name: controlImage,
+          image_name: image.imageName,
         },
       };
 

invokeai/frontend/web/src/features/nodes/util/graph/addControlNetToLinearGraph.ts

@@ -1,10 +1,8 @@
 import type { RootState } from 'app/store/store';
 import { selectValidControlNets } from 'features/controlAdapters/store/controlAdaptersSlice';
 import type { ControlAdapterProcessorType, ControlNetConfig } from 'features/controlAdapters/store/types';
-import { isControlAdapterLayer } from 'features/controlLayers/store/controlLayersSlice';
 import type { ImageField } from 'features/nodes/types/common';
 import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
-import { differenceWith, intersectionWith } from 'lodash-es';
 import type {
   CollectInvocation,
   ControlNetInvocation,
@@ -17,9 +15,13 @@ import { assert } from 'tsafe';
 import { CONTROL_NET_COLLECT } from './constants';
 import { upsertMetadata } from './metadata';
 
-const getControlNets = (state: RootState) => {
+export const addControlNetToLinearGraph = async (
-  // Start with the valid controlnets
+  state: RootState,
-  const validControlNets = selectValidControlNets(state.controlAdapters).filter(
+  graph: NonNullableGraph,
+  baseNodeId: string
+): Promise<void> => {
+  const controlNetMetadata: CoreMetadataInvocation['controlnets'] = [];
+  const controlNets = selectValidControlNets(state.controlAdapters).filter(
     ({ model, processedControlImage, processorType, controlImage, isEnabled }) => {
       const hasModel = Boolean(model);
       const doesBaseMatch = model?.base === state.generation.model?.base;
@@ -29,35 +31,9 @@ const getControlNets = (state: RootState) => {
     }
   );
 
-  // txt2img tab has special handling - it uses layers exclusively, while the other tabs use the older control adapters
+  // The txt2img tab has special handling - its control adapters are set up in the Control Layers graph helper.
-  // accordion. We need to filter the list of valid T2I adapters according to the tab.
   const activeTabName = activeTabNameSelector(state);
-
+  assert(activeTabName !== 'txt2img', 'Tried to use addControlNetToLinearGraph on txt2img tab');
-  if (activeTabName === 'txt2img') {
-    // Add only the cnets that are used in control layers
-    // Collect all ControlNet ids for enabled ControlNet layers
-    const layerControlNetIds = state.controlLayers.present.layers
-      .filter(isControlAdapterLayer)
-      .filter((l) => l.isEnabled)
-      .map((l) => l.controlNetId);
-    return intersectionWith(validControlNets, layerControlNetIds, (a, b) => a.id === b);
-  } else {
-    // Else, we want to exclude the cnets that are used in control layers
-    // Collect all ControlNet ids for all ControlNet layers
-    const layerControlNetIds = state.controlLayers.present.layers
-      .filter(isControlAdapterLayer)
-      .map((l) => l.controlNetId);
-    return differenceWith(validControlNets, layerControlNetIds, (a, b) => a.id === b);
-  }
-};
-
-export const addControlNetToLinearGraph = async (
-  state: RootState,
-  graph: NonNullableGraph,
-  baseNodeId: string
-): Promise<void> => {
-  const controlNets = getControlNets(state);
-  const controlNetMetadata: CoreMetadataInvocation['controlnets'] = [];
 
   if (controlNets.length) {
     // Even though denoise_latents' control input is collection or scalar, keep it simple and always use a collect

invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts

@@ -1,10 +1,8 @@
 import type { RootState } from 'app/store/store';
 import { selectValidIPAdapters } from 'features/controlAdapters/store/controlAdaptersSlice';
 import type { IPAdapterConfig } from 'features/controlAdapters/store/types';
-import { isIPAdapterLayer, isRegionalGuidanceLayer } from 'features/controlLayers/store/controlLayersSlice';
 import type { ImageField } from 'features/nodes/types/common';
 import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
-import { differenceWith, intersectionWith } from 'lodash-es';
 import type {
   CollectInvocation,
   CoreMetadataInvocation,
@@ -17,48 +15,21 @@ import { assert } from 'tsafe';
 import { IP_ADAPTER_COLLECT } from './constants';
 import { upsertMetadata } from './metadata';
 
-const getIPAdapters = (state: RootState) => {
-  // Start with the valid IP adapters
-  const validIPAdapters = selectValidIPAdapters(state.controlAdapters).filter(({ model, controlImage, isEnabled }) => {
-    const hasModel = Boolean(model);
-    const doesBaseMatch = model?.base === state.generation.model?.base;
-    const hasControlImage = controlImage;
-    return isEnabled && hasModel && doesBaseMatch && hasControlImage;
-  });
-
-  // Masked IP adapters are handled in the graph helper for regional control - skip them here
-  const maskedIPAdapterIds = state.controlLayers.present.layers
-    .filter(isRegionalGuidanceLayer)
-    .map((l) => l.ipAdapterIds)
-    .flat();
-  const nonMaskedIPAdapters = differenceWith(validIPAdapters, maskedIPAdapterIds, (a, b) => a.id === b);
-
-  // txt2img tab has special handling - it uses layers exclusively, while the other tabs use the older control adapters
-  // accordion. We need to filter the list of valid IP adapters according to the tab.
-  const activeTabName = activeTabNameSelector(state);
-
-  if (activeTabName === 'txt2img') {
-    // If we are on the t2i tab, we only want to add the IP adapters that are used in unmasked IP Adapter layers
-    // Collect all IP Adapter ids for enabled IP adapter layers
-    const layerIPAdapterIds = state.controlLayers.present.layers
-      .filter(isIPAdapterLayer)
-      .filter((l) => l.isEnabled)
-      .map((l) => l.ipAdapterId);
-    return intersectionWith(nonMaskedIPAdapters, layerIPAdapterIds, (a, b) => a.id === b);
-  } else {
-    // Else, we want to exclude the IP adapters that are used in IP Adapter layers
-    // Collect all IP Adapter ids for enabled IP adapter layers
-    const layerIPAdapterIds = state.controlLayers.present.layers.filter(isIPAdapterLayer).map((l) => l.ipAdapterId);
-    return differenceWith(nonMaskedIPAdapters, layerIPAdapterIds, (a, b) => a.id === b);
-  }
-};
-
 export const addIPAdapterToLinearGraph = async (
   state: RootState,
   graph: NonNullableGraph,
   baseNodeId: string
 ): Promise<void> => {
-  const ipAdapters = getIPAdapters(state);
+  // The txt2img tab has special handling - its control adapters are set up in the Control Layers graph helper.
+  const activeTabName = activeTabNameSelector(state);
+  assert(activeTabName !== 'txt2img', 'Tried to use addT2IAdaptersToLinearGraph on txt2img tab');
+
+  const ipAdapters = selectValidIPAdapters(state.controlAdapters).filter(({ model, controlImage, isEnabled }) => {
+    const hasModel = Boolean(model);
+    const doesBaseMatch = model?.base === state.generation.model?.base;
+    const hasControlImage = controlImage;
+    return isEnabled && hasModel && doesBaseMatch && hasControlImage;
+  });
 
   if (ipAdapters.length) {
     // Even though denoise_latents' ip adapter input is collection or scalar, keep it simple and always use a collect

invokeai/frontend/web/src/features/nodes/util/graph/addT2IAdapterToLinearGraph.ts

@@ -1,10 +1,8 @@
 import type { RootState } from 'app/store/store';
 import { selectValidT2IAdapters } from 'features/controlAdapters/store/controlAdaptersSlice';
 import type { ControlAdapterProcessorType, T2IAdapterConfig } from 'features/controlAdapters/store/types';
-import { isControlAdapterLayer } from 'features/controlLayers/store/controlLayersSlice';
 import type { ImageField } from 'features/nodes/types/common';
 import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
-import { differenceWith, intersectionWith } from 'lodash-es';
 import type {
   CollectInvocation,
   CoreMetadataInvocation,
@@ -17,9 +15,16 @@ import { assert } from 'tsafe';
 import { T2I_ADAPTER_COLLECT } from './constants';
 import { upsertMetadata } from './metadata';
 
-const getT2IAdapters = (state: RootState) => {
+export const addT2IAdaptersToLinearGraph = async (
-  // Start with the valid controlnets
+  state: RootState,
-  const validT2IAdapters = selectValidT2IAdapters(state.controlAdapters).filter(
+  graph: NonNullableGraph,
+  baseNodeId: string
+): Promise<void> => {
+  // The txt2img tab has special handling - its control adapters are set up in the Control Layers graph helper.
+  const activeTabName = activeTabNameSelector(state);
+  assert(activeTabName !== 'txt2img', 'Tried to use addT2IAdaptersToLinearGraph on txt2img tab');
+
+  const t2iAdapters = selectValidT2IAdapters(state.controlAdapters).filter(
     ({ model, processedControlImage, processorType, controlImage, isEnabled }) => {
       const hasModel = Boolean(model);
       const doesBaseMatch = model?.base === state.generation.model?.base;
@@ -29,34 +34,6 @@ const getT2IAdapters = (state: RootState) => {
     }
   );
 
-  // txt2img tab has special handling - it uses layers exclusively, while the other tabs use the older control adapters
-  // accordion. We need to filter the list of valid T2I adapters according to the tab.
-  const activeTabName = activeTabNameSelector(state);
-
-  if (activeTabName === 'txt2img') {
-    // Add only the T2Is that are used in control layers
-    // Collect all ids for enabled control adapter layers
-    const layerControlAdapterIds = state.controlLayers.present.layers
-      .filter(isControlAdapterLayer)
-      .filter((l) => l.isEnabled)
-      .map((l) => l.controlNetId);
-    return intersectionWith(validT2IAdapters, layerControlAdapterIds, (a, b) => a.id === b);
-  } else {
-    // Else, we want to exclude the T2Is that are used in control layers
-    const layerControlAdapterIds = state.controlLayers.present.layers
-      .filter(isControlAdapterLayer)
-      .map((l) => l.controlNetId);
-    return differenceWith(validT2IAdapters, layerControlAdapterIds, (a, b) => a.id === b);
-  }
-};
-
-export const addT2IAdaptersToLinearGraph = async (
-  state: RootState,
-  graph: NonNullableGraph,
-  baseNodeId: string
-): Promise<void> => {
-  const t2iAdapters = getT2IAdapters(state);
-
   if (t2iAdapters.length) {
     // Even though denoise_latents' t2i adapter input is collection or scalar, keep it simple and always use a collect
     const t2iAdapterCollectNode: CollectInvocation = {

invokeai/frontend/web/src/features/nodes/util/graph/buildLinearSDXLTextToImageGraph.ts

@@ -4,13 +4,10 @@ import { fetchModelConfigWithTypeGuard } from 'features/metadata/util/modelFetch
 import { addControlLayersToGraph } from 'features/nodes/util/graph/addControlLayersToGraph';
 import { isNonRefinerMainModelConfig, type NonNullableGraph } from 'services/api/types';
 
-import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
-import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSDXLLoRAsToGraph } from './addSDXLLoRAstoGraph';
 import { addSDXLRefinerToGraph } from './addSDXLRefinerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
-import { addT2IAdaptersToLinearGraph } from './addT2IAdapterToLinearGraph';
 import { addVAEToGraph } from './addVAEToGraph';
 import { addWatermarkerToGraph } from './addWatermarkerToGraph';
 import {
@@ -264,14 +261,6 @@ export const buildLinearSDXLTextToImageGraph = async (state: RootState): Promise
   // add LoRA support
   await addSDXLLoRAsToGraph(state, graph, SDXL_DENOISE_LATENTS, modelLoaderNodeId);
 
-  // add controlnet, mutating `graph`
-  await addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
-
-  // add IP Adapter
-  await addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
-
-  await addT2IAdaptersToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
-
   await addControlLayersToGraph(state, graph, SDXL_DENOISE_LATENTS);
 
   // NSFW & watermark - must be last thing added to graph

invokeai/frontend/web/src/features/nodes/util/graph/buildLinearTextToImageGraph.ts

@@ -5,13 +5,10 @@ import { addControlLayersToGraph } from 'features/nodes/util/graph/addControlLay
 import { getBoardField, getIsIntermediate } from 'features/nodes/util/graph/graphBuilderUtils';
 import { isNonRefinerMainModelConfig, type NonNullableGraph } from 'services/api/types';
 
-import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addHrfToGraph } from './addHrfToGraph';
-import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
-import { addT2IAdaptersToLinearGraph } from './addT2IAdapterToLinearGraph';
 import { addVAEToGraph } from './addVAEToGraph';
 import { addWatermarkerToGraph } from './addWatermarkerToGraph';
 import {
@@ -246,14 +243,6 @@ export const buildLinearTextToImageGraph = async (state: RootState): Promise<Non
   // add LoRA support
   await addLoRAsToGraph(state, graph, DENOISE_LATENTS, modelLoaderNodeId);
 
-  // add controlnet, mutating `graph`
-  await addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
-
-  // add IP Adapter
-  await addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
-
-  await addT2IAdaptersToLinearGraph(state, graph, DENOISE_LATENTS);
-
   await addControlLayersToGraph(state, graph, DENOISE_LATENTS);
 
   // High resolution fix.

invokeai/frontend/web/src/services/api/types.ts

@@ -177,11 +177,22 @@ type ControlAdapterAction = {
   id: string;
 };
 
-export type ControlLayerAction = {
+export type CALayerImagePostUploadAction = {
-  type: 'SET_CONTROL_LAYER_IMAGE';
+  type: 'SET_CA_LAYER_IMAGE';
   layerId: string;
 };
 
+export type IPALayerImagePostUploadAction = {
+  type: 'SET_IPA_LAYER_IMAGE';
+  layerId: string;
+};
+
+export type RGLayerIPAdapterImagePostUploadAction = {
+  type: 'SET_RG_LAYER_IP_ADAPTER_IMAGE';
+  layerId: string;
+  ipAdapterId: string;
+};
+
 type InitialImageAction = {
   type: 'SET_INITIAL_IMAGE';
 };
@@ -212,4 +223,6 @@ export type PostUploadAction =
   | CanvasInitialImageAction
   | ToastAction
   | AddToBatchAction
-  | ControlLayerAction;
+  | CALayerImagePostUploadAction
+  | IPALayerImagePostUploadAction
+  | RGLayerIPAdapterImagePostUploadAction;