feat(ui): add initial image layer to CL

invokeai/frontend/web/public/locales/en.json

@@ -1543,6 +1543,8 @@
         "globalControlAdapterLayer": "Global $t(controlnet.controlAdapter_one) $t(unifiedCanvas.layer)",
         "globalIPAdapter": "Global $t(common.ipAdapter)",
         "globalIPAdapterLayer": "Global $t(common.ipAdapter) $t(unifiedCanvas.layer)",
+        "globalInitialImage": "Global Initial Image",
+        "globalInitialImageLayer": "$t(controlLayers.globalInitialImage) $t(unifiedCanvas.layer)",
         "opacityFilter": "Opacity Filter",
         "clearProcessor": "Clear Processor",
         "resetProcessor": "Reset Processor to Defaults"

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageDropped.ts

@@ -9,6 +9,7 @@ import {
 } from 'features/controlAdapters/store/controlAdaptersSlice';
 import {
   caLayerImageChanged,
+  iiLayerImageChanged,
   ipaLayerImageChanged,
   rgLayerIPAdapterImageChanged,
 } from 'features/controlLayers/store/controlLayersSlice';
@@ -143,6 +144,24 @@ export const addImageDroppedListener = (startAppListening: AppStartListening) =>
         return;
       }
 
+      /**
+       * Image dropped on II Layer Image
+       */
+      if (
+        overData.actionType === 'SET_II_LAYER_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        const { layerId } = overData.context;
+        dispatch(
+          iiLayerImageChanged({
+            layerId,
+            imageDTO: activeData.payload.imageDTO,
+          })
+        );
+        return;
+      }
+
       /**
        * Image dropped on Canvas
        */

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts

@@ -8,6 +8,7 @@ import {
 } from 'features/controlAdapters/store/controlAdaptersSlice';
 import {
   caLayerImageChanged,
+  iiLayerImageChanged,
   ipaLayerImageChanged,
   rgLayerIPAdapterImageChanged,
 } from 'features/controlLayers/store/controlLayersSlice';
@@ -146,6 +147,17 @@ export const addImageUploadedFulfilledListener = (startAppListening: AppStartLis
         );
       }
 
+      if (postUploadAction?.type === 'SET_II_LAYER_IMAGE') {
+        const { layerId } = postUploadAction;
+        dispatch(iiLayerImageChanged({ layerId, imageDTO }));
+        dispatch(
+          addToast({
+            ...DEFAULT_UPLOADED_TOAST,
+            description: t('toast.setControlImage'),
+          })
+        );
+      }
+
       if (postUploadAction?.type === 'SET_INITIAL_IMAGE') {
         dispatch(initialImageChanged(imageDTO));
         dispatch(

invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts

@@ -16,7 +16,6 @@ import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
 import i18n from 'i18next';
 import { forEach } from 'lodash-es';
 import { getConnectedEdges } from 'reactflow';
-import { assert } from 'tsafe';
 
 const selector = createMemoizedSelector(
   [
@@ -110,7 +109,7 @@ const selector = createMemoizedSelector(
             } else if (l.type === 'regional_guidance_layer') {
               return l.ipAdapters;
             }
-            assert(false);
+            return [];
           })
           .forEach((ca, i) => {
             const hasNoModel = !ca.model;

invokeai/frontend/web/src/features/controlLayers/components/AddLayerButton.tsx

@@ -1,6 +1,6 @@
 import { Button, Menu, MenuButton, MenuItem, MenuList } from '@invoke-ai/ui-library';
 import { useAppDispatch } from 'app/store/storeHooks';
-import { useAddCALayer, useAddIPALayer } from 'features/controlLayers/hooks/addLayerHooks';
+import { useAddCALayer, useAddIILayer, useAddIPALayer } from 'features/controlLayers/hooks/addLayerHooks';
 import { rgLayerAdded } from 'features/controlLayers/store/controlLayersSlice';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
@@ -11,6 +11,7 @@ export const AddLayerButton = memo(() => {
   const dispatch = useAppDispatch();
   const [addCALayer, isAddCALayerDisabled] = useAddCALayer();
   const [addIPALayer, isAddIPALayerDisabled] = useAddIPALayer();
+  const [addIILayer, isAddIILayerDisabled] = useAddIILayer();
   const addRGLayer = useCallback(() => {
     dispatch(rgLayerAdded());
   }, [dispatch]);
@@ -30,6 +31,9 @@ export const AddLayerButton = memo(() => {
         <MenuItem icon={<PiPlusBold />} onClick={addIPALayer} isDisabled={isAddIPALayerDisabled}>
           {t('controlLayers.globalIPAdapterLayer')}
         </MenuItem>
+        <MenuItem icon={<PiPlusBold />} onClick={addIILayer} isDisabled={isAddIILayerDisabled}>
+          {t('controlLayers.globalInitialImageLayer')}
+        </MenuItem>
       </MenuList>
     </Menu>
   );

invokeai/frontend/web/src/features/controlLayers/components/ControlLayersPanelContent.tsx

@@ -6,6 +6,7 @@ import ScrollableContent from 'common/components/OverlayScrollbars/ScrollableCon
 import { AddLayerButton } from 'features/controlLayers/components/AddLayerButton';
 import { CALayer } from 'features/controlLayers/components/CALayer/CALayer';
 import { DeleteAllLayersButton } from 'features/controlLayers/components/DeleteAllLayersButton';
+import { IILayer } from 'features/controlLayers/components/IILayer/IILayer';
 import { IPALayer } from 'features/controlLayers/components/IPALayer/IPALayer';
 import { RGLayer } from 'features/controlLayers/components/RGLayer/RGLayer';
 import { isRenderableLayer, selectControlLayersSlice } from 'features/controlLayers/store/controlLayersSlice';
@@ -54,6 +55,9 @@ const LayerWrapper = memo(({ id, type }: LayerWrapperProps) => {
   if (type === 'ip_adapter_layer') {
     return <IPALayer key={id} layerId={id} />;
   }
+  if (type === 'initial_image_layer') {
+    return <IILayer key={id} layerId={id} />;
+  }
 });
 
 LayerWrapper.displayName = 'LayerWrapper';

invokeai/frontend/web/src/features/controlLayers/components/IILayer/IILayer.tsx

@@ -0,0 +1,81 @@
+import { Flex, Spacer, useDisclosure } from '@invoke-ai/ui-library';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { InitialImagePreview } from 'features/controlLayers/components/IILayer/InitialImagePreview';
+import { LayerDeleteButton } from 'features/controlLayers/components/LayerCommon/LayerDeleteButton';
+import { LayerMenu } from 'features/controlLayers/components/LayerCommon/LayerMenu';
+import { LayerTitle } from 'features/controlLayers/components/LayerCommon/LayerTitle';
+import { LayerVisibilityToggle } from 'features/controlLayers/components/LayerCommon/LayerVisibilityToggle';
+import { LayerWrapper } from 'features/controlLayers/components/LayerCommon/LayerWrapper';
+import {
+  iiLayerImageChanged,
+  layerSelected,
+  selectIILayerOrThrow,
+} from 'features/controlLayers/store/controlLayersSlice';
+import type { IILayerImageDropData } from 'features/dnd/types';
+import ImageToImageStrength from 'features/parameters/components/ImageToImage/ImageToImageStrength';
+import { memo, useCallback, useMemo } from 'react';
+import type { IILayerImagePostUploadAction, ImageDTO } from 'services/api/types';
+
+type Props = {
+  layerId: string;
+};
+
+export const IILayer = memo(({ layerId }: Props) => {
+  const dispatch = useAppDispatch();
+  const layer = useAppSelector((s) => selectIILayerOrThrow(s.controlLayers.present, layerId));
+  const onClick = useCallback(() => {
+    dispatch(layerSelected(layerId));
+  }, [dispatch, layerId]);
+  const { isOpen, onToggle } = useDisclosure({ defaultIsOpen: true });
+
+  const onChangeImage = useCallback(
+    (imageDTO: ImageDTO | null) => {
+      dispatch(iiLayerImageChanged({ layerId, imageDTO }));
+    },
+    [dispatch, layerId]
+  );
+
+  const droppableData = useMemo<IILayerImageDropData>(
+    () => ({
+      actionType: 'SET_II_LAYER_IMAGE',
+      context: {
+        layerId,
+      },
+      id: layerId,
+    }),
+    [layerId]
+  );
+
+  const postUploadAction = useMemo<IILayerImagePostUploadAction>(
+    () => ({
+      layerId,
+      type: 'SET_II_LAYER_IMAGE',
+    }),
+    [layerId]
+  );
+
+  return (
+    <LayerWrapper onClick={onClick} borderColor={layer.isSelected ? 'base.400' : 'base.800'}>
+      <Flex gap={3} alignItems="center" p={3} cursor="pointer" onDoubleClick={onToggle}>
+        <LayerVisibilityToggle layerId={layerId} />
+        <LayerTitle type="initial_image_layer" />
+        <Spacer />
+        <LayerMenu layerId={layerId} />
+        <LayerDeleteButton layerId={layerId} />
+      </Flex>
+      {isOpen && (
+        <Flex flexDir="column" gap={3} px={3} pb={3}>
+          <ImageToImageStrength />
+          <InitialImagePreview
+            image={layer.image}
+            onChangeImage={onChangeImage}
+            droppableData={droppableData}
+            postUploadAction={postUploadAction}
+          />
+        </Flex>
+      )}
+    </LayerWrapper>
+  );
+});
+
+IILayer.displayName = 'IILayer';

invokeai/frontend/web/src/features/controlLayers/components/IILayer/InitialImagePreview.tsx

@@ -0,0 +1,109 @@
+import type { SystemStyleObject } from '@invoke-ai/ui-library';
+import { Flex, useShiftModifier } from '@invoke-ai/ui-library';
+import { skipToken } from '@reduxjs/toolkit/query';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import IAIDndImage from 'common/components/IAIDndImage';
+import IAIDndImageIcon from 'common/components/IAIDndImageIcon';
+import { setBoundingBoxDimensions } from 'features/canvas/store/canvasSlice';
+import { heightChanged, widthChanged } from 'features/controlLayers/store/controlLayersSlice';
+import type { ImageWithDims } from 'features/controlLayers/util/controlAdapters';
+import type { ImageDraggableData, TypesafeDroppableData } from 'features/dnd/types';
+import { calculateNewSize } from 'features/parameters/components/ImageSize/calculateNewSize';
+import { selectOptimalDimension } from 'features/parameters/store/generationSlice';
+import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
+import { memo, useCallback, useEffect, useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiArrowCounterClockwiseBold, PiRulerBold } from 'react-icons/pi';
+import { useGetImageDTOQuery } from 'services/api/endpoints/images';
+import type { ImageDTO, PostUploadAction } from 'services/api/types';
+
+type Props = {
+  image: ImageWithDims | null;
+  onChangeImage: (imageDTO: ImageDTO | null) => void;
+  droppableData: TypesafeDroppableData;
+  postUploadAction: PostUploadAction;
+};
+
+export const InitialImagePreview = memo(({ image, onChangeImage, droppableData, postUploadAction }: Props) => {
+  const { t } = useTranslation();
+  const dispatch = useAppDispatch();
+  const isConnected = useAppSelector((s) => s.system.isConnected);
+  const activeTabName = useAppSelector(activeTabNameSelector);
+  const optimalDimension = useAppSelector(selectOptimalDimension);
+  const shift = useShiftModifier();
+
+  const { currentData: imageDTO, isError: isErrorControlImage } = useGetImageDTOQuery(image?.imageName ?? skipToken);
+
+  const onReset = useCallback(() => {
+    onChangeImage(null);
+  }, [onChangeImage]);
+
+  const onUseSize = useCallback(() => {
+    if (!imageDTO) {
+      return;
+    }
+
+    if (activeTabName === 'unifiedCanvas') {
+      dispatch(setBoundingBoxDimensions({ width: imageDTO.width, height: imageDTO.height }, optimalDimension));
+    } else {
+      const options = { updateAspectRatio: true, clamp: true };
+      if (shift) {
+        const { width, height } = imageDTO;
+        dispatch(widthChanged({ width, ...options }));
+        dispatch(heightChanged({ height, ...options }));
+      } else {
+        const { width, height } = calculateNewSize(
+          imageDTO.width / imageDTO.height,
+          optimalDimension * optimalDimension
+        );
+        dispatch(widthChanged({ width, ...options }));
+        dispatch(heightChanged({ height, ...options }));
+      }
+    }
+  }, [imageDTO, activeTabName, dispatch, optimalDimension, shift]);
+
+  const draggableData = useMemo<ImageDraggableData | undefined>(() => {
+    if (imageDTO) {
+      return {
+        id: 'initial_image_layer',
+        payloadType: 'IMAGE_DTO',
+        payload: { imageDTO: imageDTO },
+      };
+    }
+  }, [imageDTO]);
+
+  useEffect(() => {
+    if (isConnected && isErrorControlImage) {
+      onReset();
+    }
+  }, [onReset, isConnected, isErrorControlImage]);
+
+  return (
+    <Flex position="relative" w="full" h={36} alignItems="center" justifyContent="center">
+      <IAIDndImage
+        draggableData={draggableData}
+        droppableData={droppableData}
+        imageDTO={imageDTO}
+        postUploadAction={postUploadAction}
+      />
+
+      <>
+        <IAIDndImageIcon
+          onClick={onReset}
+          icon={imageDTO ? <PiArrowCounterClockwiseBold size={16} /> : undefined}
+          tooltip={t('controlnet.resetControlImage')}
+        />
+        <IAIDndImageIcon
+          onClick={onUseSize}
+          icon={imageDTO ? <PiRulerBold size={16} /> : undefined}
+          tooltip={shift ? t('controlnet.setControlImageDimensionsForce') : t('controlnet.setControlImageDimensions')}
+          styleOverrides={useSizeStyleOverrides}
+        />
+      </>
+    </Flex>
+  );
+});
+
+InitialImagePreview.displayName = 'InitialImagePreview';
+
+const useSizeStyleOverrides: SystemStyleObject = { mt: 6 };

invokeai/frontend/web/src/features/controlLayers/components/LayerCommon/LayerMenu.tsx

@@ -37,7 +37,9 @@ export const LayerMenu = memo(({ layerId }: Props) => {
             <MenuDivider />
           </>
         )}
-        {(layerType === 'regional_guidance_layer' || layerType === 'control_adapter_layer') && (
+        {(layerType === 'regional_guidance_layer' ||
+          layerType === 'control_adapter_layer' ||
+          layerType === 'initial_image_layer') && (
           <>
             <LayerMenuArrangeActions layerId={layerId} />
             <MenuDivider />

invokeai/frontend/web/src/features/controlLayers/components/LayerCommon/LayerTitle.tsx

@@ -16,6 +16,8 @@ export const LayerTitle = memo(({ type }: Props) => {
       return t('controlLayers.globalControlAdapter');
     } else if (type === 'ip_adapter_layer') {
       return t('controlLayers.globalIPAdapter');
+    } else if (type === 'initial_image_layer') {
+      return t('controlLayers.globalInitialImage');
     }
   }, [t, type]);
 

invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts

@@ -1,5 +1,11 @@
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
-import { caLayerAdded, ipaLayerAdded, rgLayerIPAdapterAdded } from 'features/controlLayers/store/controlLayersSlice';
+import {
+  caLayerAdded,
+  iiLayerAdded,
+  ipaLayerAdded,
+  isInitialImageLayer,
+  rgLayerIPAdapterAdded,
+} from 'features/controlLayers/store/controlLayersSlice';
 import {
   buildControlNet,
   buildIPAdapter,
@@ -93,3 +99,13 @@ export const useAddIPAdapterToIPALayer = (layerId: string) => {
 
   return [addIPAdapter, isDisabled] as const;
 };
+
+export const useAddIILayer = () => {
+  const dispatch = useAppDispatch();
+  const isDisabled = useAppSelector((s) => Boolean(s.controlLayers.present.layers.find(isInitialImageLayer)));
+  const addIILayer = useCallback(() => {
+    dispatch(iiLayerAdded(null));
+  }, [dispatch]);
+
+  return [addIILayer, isDisabled] as const;
+};

invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts

@@ -39,6 +39,7 @@ import type {
   ControlAdapterLayer,
   ControlLayersState,
   DrawingTool,
+  InitialImageLayer,
   IPAdapterLayer,
   Layer,
   RegionalGuidanceLayer,
@@ -71,8 +72,13 @@ export const isRegionalGuidanceLayer = (layer?: Layer): layer is RegionalGuidanc
 export const isControlAdapterLayer = (layer?: Layer): layer is ControlAdapterLayer =>
   layer?.type === 'control_adapter_layer';
 export const isIPAdapterLayer = (layer?: Layer): layer is IPAdapterLayer => layer?.type === 'ip_adapter_layer';
-export const isRenderableLayer = (layer?: Layer): layer is RegionalGuidanceLayer | ControlAdapterLayer =>
+export const isInitialImageLayer = (layer?: Layer): layer is InitialImageLayer => layer?.type === 'initial_image_layer';
-  layer?.type === 'regional_guidance_layer' || layer?.type === 'control_adapter_layer';
+export const isRenderableLayer = (
+  layer?: Layer
+): layer is RegionalGuidanceLayer | ControlAdapterLayer | InitialImageLayer =>
+  layer?.type === 'regional_guidance_layer' ||
+  layer?.type === 'control_adapter_layer' ||
+  layer?.type === 'initial_image_layer';
 const resetLayer = (layer: Layer) => {
   if (layer.type === 'regional_guidance_layer') {
     layer.maskObjects = [];
@@ -94,6 +100,11 @@ export const selectIPALayerOrThrow = (state: ControlLayersState, layerId: string
   assert(isIPAdapterLayer(layer));
   return layer;
 };
+export const selectIILayerOrThrow = (state: ControlLayersState, layerId: string): InitialImageLayer => {
+  const layer = state.layers.find((l) => l.id === layerId);
+  assert(isInitialImageLayer(layer));
+  return layer;
+};
 const selectCAOrIPALayerOrThrow = (
   state: ControlLayersState,
   layerId: string
@@ -611,6 +622,45 @@ export const controlLayersSlice = createSlice({
     },
     //#endregion
 
+    //#region Initial Image Layer
+    iiLayerAdded: {
+      reducer: (state, action: PayloadAction<{ layerId: string; imageDTO: ImageDTO | null }>) => {
+        const { layerId, imageDTO } = action.payload;
+        // Highlander! There can be only one!
+        assert(!state.layers.find(isInitialImageLayer));
+        const layer: InitialImageLayer = {
+          id: layerId,
+          type: 'initial_image_layer',
+          x: 0,
+          y: 0,
+          bbox: null,
+          bboxNeedsUpdate: false,
+          isEnabled: true,
+          image: imageDTO ? imageDTOToImageWithDims(imageDTO) : null,
+          isSelected: true,
+        };
+        state.layers.push(layer);
+        state.selectedLayerId = layer.id;
+        for (const layer of state.layers.filter(isRenderableLayer)) {
+          if (layer.id !== layerId) {
+            layer.isSelected = false;
+          }
+        }
+      },
+      prepare: (imageDTO: ImageDTO | null) => ({ payload: { layerId: 'initial_image_layer', imageDTO } }),
+    },
+    iiLayerImageChanged: (state, action: PayloadAction<{ layerId: string; imageDTO: ImageDTO | null }>) => {
+      const { layerId, imageDTO } = action.payload;
+      const layer = selectIILayerOrThrow(state, layerId);
+      if (layer) {
+        layer.bbox = null;
+        layer.bboxNeedsUpdate = true;
+        layer.isEnabled = true;
+        layer.image = imageDTO ? imageDTOToImageWithDims(imageDTO) : null;
+      }
+    },
+    //#endregion
+
     //#region Globals
     positivePromptChanged: (state, action: PayloadAction<string>) => {
       state.positivePrompt = action.payload;
@@ -780,6 +830,9 @@ export const {
   rgLayerIPAdapterMethodChanged,
   rgLayerIPAdapterModelChanged,
   rgLayerIPAdapterCLIPVisionModelChanged,
+  // II Layer
+  iiLayerAdded,
+  iiLayerImageChanged,
   // Globals
   positivePromptChanged,
   negativePromptChanged,

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -1,5 +1,6 @@
 import type {
   ControlNetConfigV2,
+  ImageWithDims,
   IPAdapterConfigV2,
   T2IAdapterConfigV2,
 } from 'features/controlLayers/util/controlAdapters';
@@ -73,7 +74,12 @@ export type RegionalGuidanceLayer = RenderableLayerBase & {
   needsPixelBbox: boolean; // Needs the slower pixel-based bbox calculation - set to true when an there is an eraser object
 };
 
-export type Layer = RegionalGuidanceLayer | ControlAdapterLayer | IPAdapterLayer;
+export type InitialImageLayer = RenderableLayerBase & {
+  type: 'initial_image_layer';
+  image: ImageWithDims | null;
+};
+
+export type Layer = RegionalGuidanceLayer | ControlAdapterLayer | IPAdapterLayer | InitialImageLayer;
 
 export type ControlLayersState = {
   _version: 1;

invokeai/frontend/web/src/features/dnd/types/index.ts

@@ -55,6 +55,13 @@ export type RGLayerIPAdapterImageDropData = BaseDropData & {
   };
 };
 
+export type IILayerImageDropData = BaseDropData & {
+  actionType: 'SET_II_LAYER_IMAGE';
+  context: {
+    layerId: string;
+  };
+};
+
 export type CanvasInitialImageDropData = BaseDropData & {
   actionType: 'SET_CANVAS_INITIAL_IMAGE';
 };
@@ -86,7 +93,8 @@ export type TypesafeDroppableData =
   | RemoveFromBoardDropData
   | CALayerImageDropData
   | IPALayerImageDropData
-  | RGLayerIPAdapterImageDropData;
+  | RGLayerIPAdapterImageDropData
+  | IILayerImageDropData;
 
 type BaseDragData = {
   id: string;

invokeai/frontend/web/src/features/dnd/util/isValidDrop.ts

@@ -25,6 +25,8 @@ export const isValidDrop = (overData: TypesafeDroppableData | undefined, active:
       return payloadType === 'IMAGE_DTO';
     case 'SET_RG_LAYER_IP_ADAPTER_IMAGE':
       return payloadType === 'IMAGE_DTO';
+    case 'SET_II_LAYER_IMAGE':
+      return payloadType === 'IMAGE_DTO';
     case 'SET_CANVAS_INITIAL_IMAGE':
       return payloadType === 'IMAGE_DTO';
     case 'SET_NODES_IMAGE':

invokeai/frontend/web/src/services/api/types.ts

@@ -193,6 +193,11 @@ export type RGLayerIPAdapterImagePostUploadAction = {
   ipAdapterId: string;
 };
 
+export type IILayerImagePostUploadAction = {
+  type: 'SET_II_LAYER_IMAGE';
+  layerId: string;
+};
+
 type InitialImageAction = {
   type: 'SET_INITIAL_IMAGE';
 };
@@ -225,4 +230,5 @@ export type PostUploadAction =
   | AddToBatchAction
   | CALayerImagePostUploadAction
   | IPALayerImagePostUploadAction
-  | RGLayerIPAdapterImagePostUploadAction;
+  | RGLayerIPAdapterImagePostUploadAction
+  | IILayerImagePostUploadAction;