wip: Add IP Adapter To Linear UI

invokeai/frontend/web/public/locales/en.json

@@ -191,7 +191,9 @@
         "showAdvanced": "Show Advanced",
         "toggleControlNet": "Toggle this ControlNet",
         "w": "W",
-        "weight": "Weight"
+        "weight": "Weight",
+        "enableIPAdapter": "Enable IP Adapter",
+        "ipAdapterModel": "Adapter Model"
     },
     "embedding": {
         "addEmbedding": "Add Embedding",
@@ -1036,6 +1038,7 @@
         "serverError": "Server Error",
         "setCanvasInitialImage": "Set as canvas initial image",
         "setControlImage": "Set as control image",
+        "setIPAdapterImage": "Set as IP Adapter Image",
         "setInitialImage": "Set as initial image",
         "setNodeField": "Set as node field",
         "tempFoldersEmptied": "Temp Folder Emptied",

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageDropped.ts

@@ -1,7 +1,11 @@
 import { createAction } from '@reduxjs/toolkit';
 import { logger } from 'app/logging/logger';
+import { parseify } from 'common/util/serialize';
 import { setInitialCanvasImage } from 'features/canvas/store/canvasSlice';
-import { controlNetImageChanged } from 'features/controlNet/store/controlNetSlice';
+import {
+  controlNetImageChanged,
+  ipAdapterImageChanged,
+} from 'features/controlNet/store/controlNetSlice';
 import {
   TypesafeDraggableData,
   TypesafeDroppableData,
@@ -14,7 +18,6 @@ import {
 import { initialImageChanged } from 'features/parameters/store/generationSlice';
 import { imagesApi } from 'services/api/endpoints/images';
 import { startAppListening } from '../';
-import { parseify } from 'common/util/serialize';
 
 export const dndDropped = createAction<{
   overData: TypesafeDroppableData;
@@ -99,6 +102,18 @@ export const addImageDroppedListener = () => {
         return;
       }
 
+      /**
+       * Image dropped on IP Adapter image
+       */
+      if (
+        overData.actionType === 'SET_IP_ADAPTER_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        dispatch(ipAdapterImageChanged(activeData.payload.imageDTO));
+        return;
+      }
+
       /**
        * Image dropped on Canvas
        */

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts

@@ -1,15 +1,18 @@
 import { UseToastOptions } from '@chakra-ui/react';
 import { logger } from 'app/logging/logger';
 import { setInitialCanvasImage } from 'features/canvas/store/canvasSlice';
-import { controlNetImageChanged } from 'features/controlNet/store/controlNetSlice';
+import {
+  controlNetImageChanged,
+  ipAdapterImageChanged,
+} from 'features/controlNet/store/controlNetSlice';
 import { fieldImageValueChanged } from 'features/nodes/store/nodesSlice';
 import { initialImageChanged } from 'features/parameters/store/generationSlice';
 import { addToast } from 'features/system/store/systemSlice';
+import { t } from 'i18next';
 import { omit } from 'lodash-es';
 import { boardsApi } from 'services/api/endpoints/boards';
 import { startAppListening } from '..';
 import { imagesApi } from '../../../../../services/api/endpoints/images';
-import { t } from 'i18next';
 
 const DEFAULT_UPLOADED_TOAST: UseToastOptions = {
   title: t('toast.imageUploaded'),
@@ -99,6 +102,17 @@ export const addImageUploadedFulfilledListener = () => {
         return;
       }
 
+      if (postUploadAction?.type === 'SET_IP_ADAPTER_IMAGE') {
+        dispatch(ipAdapterImageChanged(imageDTO));
+        dispatch(
+          addToast({
+            ...DEFAULT_UPLOADED_TOAST,
+            description: t('toast.setIPAdapterImage'),
+          })
+        );
+        return;
+      }
+
       if (postUploadAction?.type === 'SET_INITIAL_IMAGE') {
         dispatch(initialImageChanged(imageDTO));
         dispatch(

invokeai/frontend/web/src/features/controlNet/components/ControlNet.tsx

@@ -18,6 +18,7 @@ import { defaultSelectorOptions } from 'app/store/util/defaultMemoizeOptions';
 import IAIIconButton from 'common/components/IAIIconButton';
 import IAISwitch from 'common/components/IAISwitch';
 import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
+import { useTranslation } from 'react-i18next';
 import { useToggle } from 'react-use';
 import { v4 as uuidv4 } from 'uuid';
 import ControlNetImagePreview from './ControlNetImagePreview';
@@ -28,7 +29,6 @@ import ParamControlNetBeginEnd from './parameters/ParamControlNetBeginEnd';
 import ParamControlNetControlMode from './parameters/ParamControlNetControlMode';
 import ParamControlNetProcessorSelect from './parameters/ParamControlNetProcessorSelect';
 import ParamControlNetResizeMode from './parameters/ParamControlNetResizeMode';
-import { useTranslation } from 'react-i18next';
 
 type ControlNetProps = {
   controlNet: ControlNetConfig;

invokeai/frontend/web/src/features/controlNet/components/ipAdapter/IPAdapterPanel.tsx

@@ -0,0 +1,33 @@
+import { Flex } from '@chakra-ui/react';
+import { memo } from 'react';
+import ParamIPAdapterFeatureToggle from './ParamIPAdapterFeatureToggle';
+import ParamIPAdapterImage from './ParamIPAdapterImage';
+import ParamIPAdapterModelSelect from './ParamIPAdapterModelSelect';
+import ParamIPAdapterWeight from './ParamIPAdapterWeight';
+
+const IPAdapterPanel = () => {
+  return (
+    <Flex
+      sx={{
+        flexDir: 'column',
+        gap: 3,
+        paddingInline: 3,
+        paddingBlock: 2,
+        paddingBottom: 6,
+        borderRadius: 'base',
+        position: 'relative',
+        bg: 'base.250',
+        _dark: {
+          bg: 'base.750',
+        },
+      }}
+    >
+      <ParamIPAdapterFeatureToggle />
+      <ParamIPAdapterImage />
+      <ParamIPAdapterModelSelect />
+      <ParamIPAdapterWeight />
+    </Flex>
+  );
+};
+
+export default memo(IPAdapterPanel);

invokeai/frontend/web/src/features/controlNet/components/ipAdapter/ParamIPAdapterFeatureToggle.tsx

@@ -0,0 +1,41 @@
+import { createSelector } from '@reduxjs/toolkit';
+import { stateSelector } from 'app/store/store';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { defaultSelectorOptions } from 'app/store/util/defaultMemoizeOptions';
+import IAISwitch from 'common/components/IAISwitch';
+import { isIPAdapterEnableToggled } from 'features/controlNet/store/controlNetSlice';
+import { memo, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+
+const selector = createSelector(
+  stateSelector,
+  (state) => {
+    const { isIPAdapterEnabled } = state.controlNet;
+
+    return { isIPAdapterEnabled };
+  },
+  defaultSelectorOptions
+);
+
+const ParamIPAdapterFeatureToggle = () => {
+  const { isIPAdapterEnabled } = useAppSelector(selector);
+  const dispatch = useAppDispatch();
+  const { t } = useTranslation();
+
+  const handleChange = useCallback(() => {
+    dispatch(isIPAdapterEnableToggled());
+  }, [dispatch]);
+
+  return (
+    <IAISwitch
+      label={t('controlnet.enableIPAdapter')}
+      isChecked={isIPAdapterEnabled}
+      onChange={handleChange}
+      formControlProps={{
+        width: '100%',
+      }}
+    />
+  );
+};
+
+export default memo(ParamIPAdapterFeatureToggle);

invokeai/frontend/web/src/features/controlNet/components/ipAdapter/ParamIPAdapterImage.tsx

@@ -0,0 +1,69 @@
+import { Flex } from '@chakra-ui/react';
+import { skipToken } from '@reduxjs/toolkit/dist/query';
+import { RootState } from 'app/store/store';
+import { useAppSelector } from 'app/store/storeHooks';
+import IAIDndImage from 'common/components/IAIDndImage';
+import { IAINoContentFallback } from 'common/components/IAIImageFallback';
+import {
+  TypesafeDraggableData,
+  TypesafeDroppableData,
+} from 'features/dnd/types';
+import { memo, useMemo } from 'react';
+import { useGetImageDTOQuery } from 'services/api/endpoints/images';
+
+const ParamIPAdapterImage = () => {
+  const ipAdapterInfo = useAppSelector(
+    (state: RootState) => state.controlNet.ipAdapterInfo
+  );
+
+  const isIPAdapterEnabled = useAppSelector(
+    (state: RootState) => state.controlNet.isIPAdapterEnabled
+  );
+
+  const { currentData: imageDTO } = useGetImageDTOQuery(
+    ipAdapterInfo.adapterImage?.image_name ?? skipToken
+  );
+
+  const draggableData = useMemo<TypesafeDraggableData | undefined>(() => {
+    if (imageDTO) {
+      return {
+        id: 'ip-adapter-image',
+        payloadType: 'IMAGE_DTO',
+        payload: { imageDTO },
+      };
+    }
+  }, [imageDTO]);
+
+  const droppableData = useMemo<TypesafeDroppableData | undefined>(
+    () => ({
+      id: 'ip-adapter-image',
+      actionType: 'SET_IP_ADAPTER_IMAGE',
+    }),
+    []
+  );
+
+  return (
+    <Flex
+      sx={{
+        position: 'relative',
+        w: 'full',
+        alignItems: 'center',
+        justifyContent: 'center',
+      }}
+    >
+      <IAIDndImage
+        imageDTO={imageDTO}
+        droppableData={droppableData}
+        draggableData={draggableData}
+        isUploadDisabled={!isIPAdapterEnabled}
+        fitContainer
+        // dropLabel="Set as Initial Image"
+        noContentFallback={
+          <IAINoContentFallback label="No IP Adapter Image Selected" />
+        }
+      />
+    </Flex>
+  );
+};
+
+export default memo(ParamIPAdapterImage);

invokeai/frontend/web/src/features/controlNet/components/ipAdapter/ParamIPAdapterModelSelect.tsx

@@ -0,0 +1,97 @@
+import { SelectItem } from '@mantine/core';
+import { RootState } from 'app/store/store';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import IAIMantineSelect from 'common/components/IAIMantineSelect';
+import { ipAdapterModelChanged } from 'features/controlNet/store/controlNetSlice';
+import { MODEL_TYPE_MAP } from 'features/parameters/types/constants';
+import { modelIdToIPAdapterModelParam } from 'features/parameters/util/modelIdToIPAdapterModelParams';
+import { forEach } from 'lodash-es';
+import { memo, useCallback, useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { useGetIPAdapterModelsQuery } from 'services/api/endpoints/models';
+
+const ParamIPAdapterModelSelect = () => {
+  const ipAdapterModel = useAppSelector(
+    (state: RootState) => state.controlNet.ipAdapterInfo.model
+  );
+  const model = useAppSelector((state: RootState) => state.generation.model);
+
+  const dispatch = useAppDispatch();
+  const { t } = useTranslation();
+
+  const { data: ipAdapterModels } = useGetIPAdapterModelsQuery();
+
+  // grab the full model entity from the RTK Query cache
+  const selectedModel = useMemo(
+    () =>
+      ipAdapterModels?.entities[
+        `${ipAdapterModel?.base_model}/ip_adapter/${ipAdapterModel?.model_name}`
+      ] ?? null,
+    [
+      ipAdapterModel?.base_model,
+      ipAdapterModel?.model_name,
+      ipAdapterModels?.entities,
+    ]
+  );
+
+  const data = useMemo(() => {
+    if (!ipAdapterModels) {
+      return [];
+    }
+
+    const data: SelectItem[] = [];
+
+    forEach(ipAdapterModels.entities, (ipAdapterModel, id) => {
+      if (!ipAdapterModel) {
+        return;
+      }
+
+      const disabled = model?.base_model !== ipAdapterModel.base_model;
+
+      data.push({
+        value: id,
+        label: ipAdapterModel.model_name,
+        group: MODEL_TYPE_MAP[ipAdapterModel.base_model],
+        disabled,
+        tooltip: disabled
+          ? `Incompatible base model: ${ipAdapterModel.base_model}`
+          : undefined,
+      });
+    });
+
+    return data.sort((a, b) => (a.disabled && !b.disabled ? 1 : -1));
+  }, [ipAdapterModels, model?.base_model]);
+
+  const handleValueChanged = useCallback(
+    (v: string | null) => {
+      if (!v) {
+        return;
+      }
+
+      const newIPAdapterModel = modelIdToIPAdapterModelParam(v);
+
+      if (!newIPAdapterModel) {
+        return;
+      }
+
+      dispatch(ipAdapterModelChanged(newIPAdapterModel));
+    },
+    [dispatch]
+  );
+
+  return (
+    <IAIMantineSelect
+      label={t('controlnet.ipAdapterModel')}
+      className="nowheel nodrag"
+      tooltip={selectedModel?.description}
+      value={selectedModel?.id ?? null}
+      placeholder="Pick one"
+      error={!selectedModel}
+      data={data}
+      onChange={handleValueChanged}
+      sx={{ width: '100%' }}
+    />
+  );
+};
+
+export default memo(ParamIPAdapterModelSelect);

invokeai/frontend/web/src/features/controlNet/components/ipAdapter/ParamIPAdapterWeight.tsx

@@ -0,0 +1,40 @@
+import { RootState } from 'app/store/store';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import IAISlider from 'common/components/IAISlider';
+import { ipAdapterWeightChanged } from 'features/controlNet/store/controlNetSlice';
+import { memo, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+
+const ParamIPAdapterWeight = () => {
+  const isIpAdapterEnabled = useAppSelector(
+    (state: RootState) => state.controlNet.isIPAdapterEnabled
+  );
+  const ipAdapterWeight = useAppSelector(
+    (state: RootState) => state.controlNet.ipAdapterInfo.weight
+  );
+  const dispatch = useAppDispatch();
+  const { t } = useTranslation();
+
+  const handleWeightChanged = useCallback(
+    (weight: number) => {
+      dispatch(ipAdapterWeightChanged(weight));
+    },
+    [dispatch]
+  );
+
+  return (
+    <IAISlider
+      isDisabled={!isIpAdapterEnabled}
+      label={t('controlnet.weight')}
+      value={ipAdapterWeight}
+      onChange={handleWeightChanged}
+      min={0}
+      max={2}
+      step={0.01}
+      withSliderMarks
+      sliderMarks={[0, 1, 2]}
+    />
+  );
+};
+
+export default memo(ParamIPAdapterWeight);

invokeai/frontend/web/src/features/controlNet/store/controlNetSlice.ts

@@ -1,9 +1,13 @@
 import { PayloadAction, createSlice } from '@reduxjs/toolkit';
-import { ControlNetModelParam } from 'features/parameters/types/parameterSchemas';
+import {
+  ControlNetModelParam,
+  IPAdapterModelParam,
+} from 'features/parameters/types/parameterSchemas';
 import { cloneDeep, forEach } from 'lodash-es';
 import { imagesApi } from 'services/api/endpoints/images';
 import { components } from 'services/api/schema';
 import { isAnySessionRejected } from 'services/api/thunks/session';
+import { ImageDTO } from 'services/api/types';
 import { appSocketInvocationError } from 'services/events/actions';
 import { controlNetImageProcessed } from './actions';
 import {
@@ -56,16 +60,30 @@ export type ControlNetConfig = {
   shouldAutoConfig: boolean;
 };
 
+export type IPAdapterConfig = {
+  adapterImage: ImageDTO | null;
+  model: IPAdapterModelParam | null;
+  weight: number;
+};
+
 export type ControlNetState = {
   controlNets: Record<string, ControlNetConfig>;
   isEnabled: boolean;
   pendingControlImages: string[];
+  isIPAdapterEnabled: boolean;
+  ipAdapterInfo: IPAdapterConfig;
 };
 
 export const initialControlNetState: ControlNetState = {
   controlNets: {},
   isEnabled: false,
   pendingControlImages: [],
+  isIPAdapterEnabled: false,
+  ipAdapterInfo: {
+    adapterImage: null,
+    model: null,
+    weight: 1,
+  },
 };
 
 export const controlNetSlice = createSlice({
@@ -353,6 +371,21 @@ export const controlNetSlice = createSlice({
     controlNetReset: () => {
       return { ...initialControlNetState };
     },
+    isIPAdapterEnableToggled: (state) => {
+      state.isIPAdapterEnabled = !state.isIPAdapterEnabled;
+    },
+    ipAdapterImageChanged: (state, action: PayloadAction<ImageDTO | null>) => {
+      state.ipAdapterInfo.adapterImage = action.payload;
+    },
+    ipAdapterWeightChanged: (state, action: PayloadAction<number>) => {
+      state.ipAdapterInfo.weight = action.payload;
+    },
+    ipAdapterModelChanged: (
+      state,
+      action: PayloadAction<IPAdapterModelParam | null>
+    ) => {
+      state.ipAdapterInfo.model = action.payload;
+    },
   },
   extraReducers: (builder) => {
     builder.addCase(controlNetImageProcessed, (state, action) => {
@@ -412,6 +445,10 @@ export const {
   controlNetProcessorTypeChanged,
   controlNetReset,
   controlNetAutoConfigToggled,
+  isIPAdapterEnableToggled,
+  ipAdapterImageChanged,
+  ipAdapterWeightChanged,
+  ipAdapterModelChanged,
 } = controlNetSlice.actions;
 
 export default controlNetSlice.reducer;

invokeai/frontend/web/src/features/dnd/types/index.ts

@@ -35,6 +35,10 @@ export type ControlNetDropData = BaseDropData & {
   };
 };
 
+export type IPAdapterImageDropData = BaseDropData & {
+  actionType: 'SET_IP_ADAPTER_IMAGE';
+};
+
 export type CanvasInitialImageDropData = BaseDropData & {
   actionType: 'SET_CANVAS_INITIAL_IMAGE';
 };
@@ -73,6 +77,7 @@ export type TypesafeDroppableData =
   | CurrentImageDropData
   | InitialImageDropData
   | ControlNetDropData
+  | IPAdapterImageDropData
   | CanvasInitialImageDropData
   | NodesImageDropData
   | AddToBatchDropData

invokeai/frontend/web/src/features/dnd/util/isValidDrop.ts

@@ -24,6 +24,8 @@ export const isValidDrop = (
       return payloadType === 'IMAGE_DTO';
     case 'SET_CONTROLNET_IMAGE':
       return payloadType === 'IMAGE_DTO';
+    case 'SET_IP_ADAPTER_IMAGE':
+      return payloadType === 'IMAGE_DTO';
     case 'SET_CANVAS_INITIAL_IMAGE':
       return payloadType === 'IMAGE_DTO';
     case 'SET_NODES_IMAGE':

invokeai/frontend/web/src/features/nodes/util/graphBuilders/addIPAdapterToLinearGraph.ts

@@ -0,0 +1,57 @@
+import { RootState } from 'app/store/store';
+import { IPAdapterInvocation } from 'services/api/types';
+import { NonNullableGraph } from '../../types/types';
+import { IP_ADAPTER } from './constants';
+
+export const addIPAdapterToLinearGraph = (
+  state: RootState,
+  graph: NonNullableGraph,
+  baseNodeId: string
+): void => {
+  const { isIPAdapterEnabled, ipAdapterInfo } = state.controlNet;
+
+  // const metadataAccumulator = graph.nodes[METADATA_ACCUMULATOR] as
+  //   | MetadataAccumulatorInvocation
+  //   | undefined;
+
+  if (isIPAdapterEnabled) {
+    const ipAdapterNode: IPAdapterInvocation = {
+      id: IP_ADAPTER,
+      type: 'ip_adapter',
+      is_intermediate: true,
+      weight: ipAdapterInfo.weight,
+      ip_adapter_model: {
+        base_model: 'sd-1',
+        model_name: 'ip_adapter_plus_sd15',
+      },
+    };
+
+    if (ipAdapterInfo.adapterImage) {
+      ipAdapterNode.image = {
+        image_name: ipAdapterInfo.adapterImage.image_name,
+      };
+    } else {
+      return;
+    }
+
+    graph.nodes[ipAdapterNode.id] = ipAdapterNode as IPAdapterInvocation;
+
+    // if (metadataAccumulator?.ip_adapters) {
+    //   // metadata accumulator only needs the ip_adapter field - not the whole node
+    //   // extract what we need and add to the accumulator
+    //   const ipAdapterField = omit(ipAdapterNode, [
+    //     'id',
+    //     'type',
+    //   ]) as IPAdapterField;
+    //   metadataAccumulator.ip_adapters.push(ipAdapterField);
+    // }
+
+    graph.edges.push({
+      source: { node_id: ipAdapterNode.id, field: 'ip_adapter' },
+      destination: {
+        node_id: baseNodeId,
+        field: 'ip_adapter',
+      },
+    });
+  }
+};

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts

@@ -5,6 +5,7 @@ import { initialGenerationState } from 'features/parameters/store/generationSlic
 import { ImageDTO, ImageToLatentsInvocation } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
@@ -366,6 +367,9 @@ export const buildCanvasImageToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts

@@ -12,6 +12,7 @@ import {
   RangeOfSizeInvocation,
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
@@ -736,6 +737,9 @@ export const buildCanvasInpaintGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasOutpaintGraph.ts

@@ -11,6 +11,7 @@ import {
   RangeOfSizeInvocation,
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
@@ -838,6 +839,9 @@ export const buildCanvasOutpaintGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLImageToImageGraph.ts

@@ -5,6 +5,7 @@ import { initialGenerationState } from 'features/parameters/store/generationSlic
 import { ImageDTO, ImageToLatentsInvocation } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSDXLLoRAsToGraph } from './addSDXLLoRAstoGraph';
 import { addSDXLRefinerToGraph } from './addSDXLRefinerToGraph';
@@ -392,6 +393,9 @@ export const buildCanvasSDXLImageToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLInpaintGraph.ts

@@ -46,6 +46,7 @@ import {
   SEAMLESS,
 } from './constants';
 import { craftSDXLStylePrompt } from './helpers/craftSDXLStylePrompt';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 
 /**
  * Builds the Canvas tab's Inpaint graph.
@@ -765,6 +766,9 @@ export const buildCanvasSDXLInpaintGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLOutpaintGraph.ts

@@ -11,6 +11,7 @@ import {
   RangeOfSizeInvocation,
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSDXLLoRAsToGraph } from './addSDXLLoRAstoGraph';
 import { addSDXLRefinerToGraph } from './addSDXLRefinerToGraph';
@@ -868,6 +869,9 @@ export const buildCanvasSDXLOutpaintGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasSDXLTextToImageGraph.ts

@@ -8,6 +8,7 @@ import {
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSDXLLoRAsToGraph } from './addSDXLLoRAstoGraph';
 import { addSDXLRefinerToGraph } from './addSDXLRefinerToGraph';
@@ -372,6 +373,9 @@ export const buildCanvasSDXLTextToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts

@@ -8,6 +8,7 @@ import {
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
@@ -345,6 +346,9 @@ export const buildCanvasTextToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts

@@ -8,6 +8,7 @@ import {
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
@@ -364,6 +365,9 @@ export const buildLinearImageToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearSDXLImageToImageGraph.ts

@@ -8,6 +8,7 @@ import {
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSDXLLoRAsToGraph } from './addSDXLLoRAstoGraph';
 import { addSDXLRefinerToGraph } from './addSDXLRefinerToGraph';
@@ -384,6 +385,9 @@ export const buildLinearSDXLImageToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
 
+  // Add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
+
   // add dynamic prompts - also sets up core iteration and seed
   addDynamicPromptsToGraph(state, graph);
 

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearSDXLTextToImageGraph.ts

@@ -4,6 +4,7 @@ import { NonNullableGraph } from 'features/nodes/types/types';
 import { initialGenerationState } from 'features/parameters/store/generationSlice';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSDXLLoRAsToGraph } from './addSDXLLoRAstoGraph';
 import { addSDXLRefinerToGraph } from './addSDXLRefinerToGraph';
@@ -277,6 +278,9 @@ export const buildLinearSDXLTextToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
 
+  // add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, SDXL_DENOISE_LATENTS);
+
   // add dynamic prompts - also sets up core iteration and seed
   addDynamicPromptsToGraph(state, graph);
 

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts

@@ -8,6 +8,7 @@ import {
 } from 'services/api/types';
 import { addControlNetToLinearGraph } from './addControlNetToLinearGraph';
 import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
+import { addIPAdapterToLinearGraph } from './addIPAdapterToLinearGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addNSFWCheckerToGraph } from './addNSFWCheckerToGraph';
 import { addSeamlessToLinearGraph } from './addSeamlessToLinearGraph';
@@ -282,6 +283,9 @@ export const buildLinearTextToImageGraph = (
   // add controlnet, mutating `graph`
   addControlNetToLinearGraph(state, graph, DENOISE_LATENTS);
 
+  // add IP Adapter
+  addIPAdapterToLinearGraph(state, graph, DENOISE_LATENTS);
+
   // NSFW & watermark - must be last thing added to graph
   if (state.system.shouldUseNSFWChecker) {
     // must add before watermarker!

invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts

@@ -45,6 +45,7 @@ export const MASK_RESIZE_DOWN = 'mask_resize_down';
 export const COLOR_CORRECT = 'color_correct';
 export const PASTE_IMAGE = 'img_paste';
 export const CONTROL_NET_COLLECT = 'control_net_collect';
+export const IP_ADAPTER = 'ip_adapter';
 export const DYNAMIC_PROMPT = 'dynamic_prompt';
 export const IMAGE_COLLECTION = 'image_collection';
 export const IMAGE_COLLECTION_ITERATE = 'image_collection_iterate';

invokeai/frontend/web/src/features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse.tsx

@@ -6,6 +6,7 @@ import { defaultSelectorOptions } from 'app/store/util/defaultMemoizeOptions';
 import IAICollapse from 'common/components/IAICollapse';
 import IAIIconButton from 'common/components/IAIIconButton';
 import ControlNet from 'features/controlNet/components/ControlNet';
+import IPAdapterPanel from 'features/controlNet/components/ipAdapter/IPAdapterPanel';
 import ParamControlNetFeatureToggle from 'features/controlNet/components/parameters/ParamControlNetFeatureToggle';
 import {
   controlNetAdded,
@@ -101,6 +102,7 @@ const ParamControlNetCollapse = () => {
             <ControlNet controlNet={c} />
           </Fragment>
         ))}
+        <IPAdapterPanel />
       </Flex>
     </IAICollapse>
   );

invokeai/frontend/web/src/features/parameters/types/parameterSchemas.ts

@@ -339,7 +339,7 @@ export const zIPAdapterModel = z.object({
 /**
  * Type alias for model parameter, inferred from its zod schema
  */
-export type zIPAdapterModelParam = z.infer<typeof zIPAdapterModel>;
+export type IPAdapterModelParam = z.infer<typeof zIPAdapterModel>;
 /**
  * Zod schema for l2l strength parameter
  */

invokeai/frontend/web/src/services/api/types.ts

@@ -63,6 +63,7 @@ export type ControlNetModelField = s['ControlNetModelField'];
 export type IPAdapterModelField = s['IPAdapterModelField'];
 export type ModelsList = s['ModelsList'];
 export type ControlField = s['ControlField'];
+export type IPAdapterField = s['IPAdapterField'];
 
 // Model Configs
 export type LoRAModelConfig = s['LoRAModelConfig'];
@@ -139,6 +140,7 @@ export type SeamlessModeInvocation = s['SeamlessModeInvocation'];
 
 // ControlNet Nodes
 export type ControlNetInvocation = s['ControlNetInvocation'];
+export type IPAdapterInvocation = s['IPAdapterInvocation'];
 export type CannyImageProcessorInvocation = s['CannyImageProcessorInvocation'];
 export type ContentShuffleImageProcessorInvocation =
   s['ContentShuffleImageProcessorInvocation'];
@@ -177,6 +179,10 @@ export type ControlNetAction = {
   controlNetId: string;
 };
 
+export type IPAdapterAction = {
+  type: 'SET_IP_ADAPTER_IMAGE';
+};
+
 export type InitialImageAction = {
   type: 'SET_INITIAL_IMAGE';
 };
@@ -202,6 +208,7 @@ export type AddToBatchAction = {
 
 export type PostUploadAction =
   | ControlNetAction
+  | IPAdapterAction
   | InitialImageAction
   | NodesAction
   | CanvasInitialImageAction