feat(ui): scaffold out raster layers

Raster layers may have images, lines and shapes. These will replace initial image layers and provide sketching functionality like we have on canvas.

invokeai/frontend/web/public/locales/en.json

@@ -1665,6 +1665,8 @@
         "addIPAdapter": "Add $t(common.ipAdapter)",
         "regionalGuidance": "Regional Guidance",
         "regionalGuidanceLayer": "$t(controlLayers.regionalGuidance) $t(unifiedCanvas.layer)",
+        "raster": "Raster",
+        "rasterLayer": "$t(controlLayers.raster) $t(unifiedCanvas.layer)",
         "opacity": "Opacity",
         "globalControlAdapter": "Global $t(controlnet.controlAdapter_one)",
         "globalControlAdapterLayer": "Global $t(controlnet.controlAdapter_one) $t(unifiedCanvas.layer)",

invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts

@@ -29,6 +29,7 @@ const LAYER_TYPE_TO_TKEY: Record<Layer['type'], string> = {
   control_adapter_layer: 'controlLayers.globalControlAdapter',
   ip_adapter_layer: 'controlLayers.globalIPAdapter',
   regional_guidance_layer: 'controlLayers.regionalGuidance',
+  raster_layer: 'controlLayers.raster',
 };
 
 const createSelector = (templates: Templates) =>

invokeai/frontend/web/src/features/controlLayers/components/AddLayerButton.tsx

@@ -1,7 +1,7 @@
 import { Button, Menu, MenuButton, MenuItem, MenuList } from '@invoke-ai/ui-library';
 import { useAppDispatch } from 'app/store/storeHooks';
 import { useAddCALayer, useAddIILayer, useAddIPALayer } from 'features/controlLayers/hooks/addLayerHooks';
-import { rgLayerAdded } from 'features/controlLayers/store/controlLayersSlice';
+import { rasterLayerAdded, rgLayerAdded } from 'features/controlLayers/store/controlLayersSlice';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 import { PiPlusBold } from 'react-icons/pi';
@@ -15,6 +15,9 @@ export const AddLayerButton = memo(() => {
   const addRGLayer = useCallback(() => {
     dispatch(rgLayerAdded());
   }, [dispatch]);
+  const addRasterLayer = useCallback(() => {
+    dispatch(rasterLayerAdded());
+  }, [dispatch]);
 
   return (
     <Menu>
@@ -30,6 +33,9 @@ export const AddLayerButton = memo(() => {
         <MenuItem icon={<PiPlusBold />} onClick={addRGLayer}>
           {t('controlLayers.regionalGuidanceLayer')}
         </MenuItem>
+        <MenuItem icon={<PiPlusBold />} onClick={addRasterLayer}>
+          {t('controlLayers.rasterLayer')}
+        </MenuItem>
         <MenuItem icon={<PiPlusBold />} onClick={addCALayer} isDisabled={isAddCALayerDisabled}>
           {t('controlLayers.globalControlAdapterLayer')}
         </MenuItem>

invokeai/frontend/web/src/features/controlLayers/components/CALayer/CALayerOpacity.tsx

@@ -14,7 +14,7 @@ import {
 } from '@invoke-ai/ui-library';
 import { useAppDispatch } from 'app/store/storeHooks';
 import { stopPropagation } from 'common/util/stopPropagation';
-import { useLayerOpacity } from 'features/controlLayers/hooks/layerStateHooks';
+import { useCALayerOpacity } from 'features/controlLayers/hooks/layerStateHooks';
 import { caLayerIsFilterEnabledChanged, caLayerOpacityChanged } from 'features/controlLayers/store/controlLayersSlice';
 import type { ChangeEvent } from 'react';
 import { memo, useCallback } from 'react';
@@ -31,7 +31,7 @@ const formatPct = (v: number | string) => `${v} %`;
 const CALayerOpacity = ({ layerId }: Props) => {
   const { t } = useTranslation();
   const dispatch = useAppDispatch();
-  const { opacity, isFilterEnabled } = useLayerOpacity(layerId);
+  const { opacity, isFilterEnabled } = useCALayerOpacity(layerId);
   const onChangeOpacity = useCallback(
     (v: number) => {
       dispatch(caLayerOpacityChanged({ layerId, opacity: v / 100 }));

invokeai/frontend/web/src/features/controlLayers/components/ControlLayersPanelContent.tsx

@@ -9,6 +9,7 @@ import { CALayer } from 'features/controlLayers/components/CALayer/CALayer';
 import { DeleteAllLayersButton } from 'features/controlLayers/components/DeleteAllLayersButton';
 import { IILayer } from 'features/controlLayers/components/IILayer/IILayer';
 import { IPALayer } from 'features/controlLayers/components/IPALayer/IPALayer';
+import { RasterLayer } from 'features/controlLayers/components/RasterLayer/RasterLayer';
 import { RGLayer } from 'features/controlLayers/components/RGLayer/RGLayer';
 import { isRenderableLayer, selectControlLayersSlice } from 'features/controlLayers/store/controlLayersSlice';
 import type { Layer } from 'features/controlLayers/store/types';
@@ -64,6 +65,9 @@ const LayerWrapper = memo(({ id, type }: LayerWrapperProps) => {
   if (type === 'initial_image_layer') {
     return <IILayer key={id} layerId={id} />;
   }
+  if (type === 'raster_layer') {
+    return <RasterLayer key={id} layerId={id} />;
+  }
 });
 
 LayerWrapper.displayName = 'LayerWrapper';

invokeai/frontend/web/src/features/controlLayers/components/LayerCommon/LayerMenu.tsx

@@ -5,7 +5,7 @@ import { LayerMenuArrangeActions } from 'features/controlLayers/components/Layer
 import { LayerMenuRGActions } from 'features/controlLayers/components/LayerCommon/LayerMenuRGActions';
 import { useLayerType } from 'features/controlLayers/hooks/layerStateHooks';
 import { layerDeleted, layerReset } from 'features/controlLayers/store/controlLayersSlice';
-import { memo, useCallback } from 'react';
+import { memo, useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { PiArrowCounterClockwiseBold, PiDotsThreeVerticalBold, PiTrashSimpleBold } from 'react-icons/pi';
 
@@ -21,6 +21,15 @@ export const LayerMenu = memo(({ layerId }: Props) => {
   const deleteLayer = useCallback(() => {
     dispatch(layerDeleted(layerId));
   }, [dispatch, layerId]);
+  const shouldShowArrangeActions = useMemo(() => {
+    return (
+      layerType === 'regional_guidance_layer' ||
+      layerType === 'control_adapter_layer' ||
+      layerType === 'initial_image_layer' ||
+      layerType === 'raster_layer'
+    );
+  }, [layerType]);
+
   return (
     <Menu>
       <MenuButton
@@ -37,9 +46,7 @@ export const LayerMenu = memo(({ layerId }: Props) => {
             <MenuDivider />
           </>
         )}
-        {(layerType === 'regional_guidance_layer' ||
-          layerType === 'control_adapter_layer' ||
-          layerType === 'initial_image_layer') && (
+        {shouldShowArrangeActions && (
           <>
             <LayerMenuArrangeActions layerId={layerId} />
             <MenuDivider />

invokeai/frontend/web/src/features/controlLayers/components/LayerCommon/LayerTitle.tsx

@@ -18,6 +18,8 @@ export const LayerTitle = memo(({ type }: Props) => {
       return t('controlLayers.globalIPAdapter');
     } else if (type === 'initial_image_layer') {
       return t('controlLayers.globalInitialImage');
+    } else if (type === 'raster_layer') {
+      return t('controlLayers.rasterLayer');
     }
   }, [t, type]);
 

invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayer.tsx

@@ -0,0 +1,44 @@
+import { Flex, Spacer, useDisclosure } from '@invoke-ai/ui-library';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import { LayerDeleteButton } from 'features/controlLayers/components/LayerCommon/LayerDeleteButton';
+import { LayerMenu } from 'features/controlLayers/components/LayerCommon/LayerMenu';
+import { LayerTitle } from 'features/controlLayers/components/LayerCommon/LayerTitle';
+import { LayerIsEnabledToggle } from 'features/controlLayers/components/LayerCommon/LayerVisibilityToggle';
+import { LayerWrapper } from 'features/controlLayers/components/LayerCommon/LayerWrapper';
+import { layerSelected, selectRasterLayerOrThrow } from 'features/controlLayers/store/controlLayersSlice';
+import { memo, useCallback } from 'react';
+
+import { RasterLayerOpacity } from './RasterLayerOpacity';
+
+type Props = {
+  layerId: string;
+};
+
+export const RasterLayer = memo(({ layerId }: Props) => {
+  const dispatch = useAppDispatch();
+  const isSelected = useAppSelector((s) => selectRasterLayerOrThrow(s.controlLayers.present, layerId).isSelected);
+  const onClick = useCallback(() => {
+    dispatch(layerSelected(layerId));
+  }, [dispatch, layerId]);
+  const { isOpen, onToggle } = useDisclosure({ defaultIsOpen: true });
+
+  return (
+    <LayerWrapper onClick={onClick} borderColor={isSelected ? 'base.400' : 'base.800'}>
+      <Flex gap={3} alignItems="center" p={3} cursor="pointer" onDoubleClick={onToggle}>
+        <LayerIsEnabledToggle layerId={layerId} />
+        <LayerTitle type="raster_layer" />
+        <Spacer />
+        <RasterLayerOpacity layerId={layerId} />
+        <LayerMenu layerId={layerId} />
+        <LayerDeleteButton layerId={layerId} />
+      </Flex>
+      {isOpen && (
+        <Flex flexDir="column" gap={3} px={3} pb={3}>
+          PLACEHOLDER
+        </Flex>
+      )}
+    </LayerWrapper>
+  );
+});
+
+RasterLayer.displayName = 'RasterLayer';

invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayerOpacity.tsx

@@ -0,0 +1,84 @@
+import {
+  CompositeNumberInput,
+  CompositeSlider,
+  Flex,
+  FormControl,
+  FormLabel,
+  IconButton,
+  Popover,
+  PopoverArrow,
+  PopoverBody,
+  PopoverContent,
+  PopoverTrigger,
+} from '@invoke-ai/ui-library';
+import { useAppDispatch } from 'app/store/storeHooks';
+import { stopPropagation } from 'common/util/stopPropagation';
+import { useRasterLayerOpacity } from 'features/controlLayers/hooks/layerStateHooks';
+import { rasterLayerOpacityChanged } from 'features/controlLayers/store/controlLayersSlice';
+import { memo, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiDropHalfFill } from 'react-icons/pi';
+
+type Props = {
+  layerId: string;
+};
+
+const marks = [0, 25, 50, 75, 100];
+const formatPct = (v: number | string) => `${v} %`;
+
+export const RasterLayerOpacity = memo(({ layerId }: Props) => {
+  const { t } = useTranslation();
+  const dispatch = useAppDispatch();
+  const opacity = useRasterLayerOpacity(layerId);
+  const onChangeOpacity = useCallback(
+    (v: number) => {
+      dispatch(rasterLayerOpacityChanged({ layerId, opacity: v / 100 }));
+    },
+    [dispatch, layerId]
+  );
+  return (
+    <Popover isLazy>
+      <PopoverTrigger>
+        <IconButton
+          aria-label={t('controlLayers.opacity')}
+          size="sm"
+          icon={<PiDropHalfFill size={16} />}
+          variant="ghost"
+          onDoubleClick={stopPropagation}
+        />
+      </PopoverTrigger>
+      <PopoverContent onDoubleClick={stopPropagation}>
+        <PopoverArrow />
+        <PopoverBody>
+          <Flex direction="column" gap={2}>
+            <FormControl orientation="horizontal">
+              <FormLabel m={0}>{t('controlLayers.opacity')}</FormLabel>
+              <CompositeSlider
+                min={0}
+                max={100}
+                step={1}
+                value={opacity}
+                defaultValue={100}
+                onChange={onChangeOpacity}
+                marks={marks}
+                w={48}
+              />
+              <CompositeNumberInput
+                min={0}
+                max={100}
+                step={1}
+                value={opacity}
+                defaultValue={100}
+                onChange={onChangeOpacity}
+                w={24}
+                format={formatPct}
+              />
+            </FormControl>
+          </Flex>
+        </PopoverBody>
+      </PopoverContent>
+    </Popover>
+  );
+});
+
+RasterLayerOpacity.displayName = 'RasterLayerOpacity';

invokeai/frontend/web/src/features/controlLayers/hooks/layerStateHooks.ts

@@ -3,6 +3,7 @@ import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
 import { useAppSelector } from 'app/store/storeHooks';
 import {
   isControlAdapterLayer,
+  isRasterLayer,
   isRegionalGuidanceLayer,
   selectControlLayersSlice,
 } from 'features/controlLayers/store/controlLayersSlice';
@@ -67,7 +68,7 @@ export const useLayerType = (layerId: string) => {
   return type;
 };
 
-export const useLayerOpacity = (layerId: string) => {
+export const useCALayerOpacity = (layerId: string) => {
   const selectLayer = useMemo(
     () =>
       createMemoizedSelector(selectControlLayersSlice, (controlLayers) => {
@@ -80,3 +81,17 @@ export const useLayerOpacity = (layerId: string) => {
   const opacity = useAppSelector(selectLayer);
   return opacity;
 };
+
+export const useRasterLayerOpacity = (layerId: string) => {
+  const selectLayer = useMemo(
+    () =>
+      createMemoizedSelector(selectControlLayersSlice, (controlLayers) => {
+        const layer = controlLayers.present.layers.filter(isRasterLayer).find((l) => l.id === layerId);
+        assert(layer, `Layer ${layerId} not found`);
+        return Math.round(layer.opacity * 100);
+      }),
+    [layerId]
+  );
+  const opacity = useAppSelector(selectLayer);
+  return opacity;
+};

invokeai/frontend/web/src/features/controlLayers/konva/naming.ts

@@ -25,9 +25,11 @@ export const INITIAL_IMAGE_LAYER_NAME = 'initial_image_layer';
 export const INITIAL_IMAGE_LAYER_IMAGE_NAME = 'initial_image_layer.image';
 export const LAYER_BBOX_NAME = 'layer.bbox';
 export const COMPOSITING_RECT_NAME = 'compositing-rect';
+export const RASTER_LAYER_NAME = 'raster_layer';
 
 // Getters for non-singleton layer and object IDs
 export const getRGLayerId = (layerId: string) => `${RG_LAYER_NAME}_${layerId}`;
+export const getRasterLayerId = (layerId: string) => `${RASTER_LAYER_NAME}_${layerId}`;
 export const getRGLayerLineId = (layerId: string, lineId: string) => `${layerId}.line_${lineId}`;
 export const getRGLayerRectId = (layerId: string, lineId: string) => `${layerId}.rect_${lineId}`;
 export const getRGLayerObjectGroupId = (layerId: string, groupId: string) => `${layerId}.objectGroup_${groupId}`;

invokeai/frontend/web/src/features/controlLayers/store/controlLayersSlice.ts

@@ -7,6 +7,7 @@ import { roundDownToMultiple } from 'common/util/roundDownToMultiple';
 import {
   getCALayerId,
   getIPALayerId,
+  getRasterLayerId,
   getRGLayerId,
   getRGLayerLineId,
   getRGLayerRectId,
@@ -55,6 +56,7 @@ import type {
   InitialImageLayer,
   IPAdapterLayer,
   Layer,
+  RasterLayer,
   RectShape,
   RegionalGuidanceLayer,
   Tool,
@@ -87,12 +89,14 @@ export const isControlAdapterLayer = (layer?: Layer): layer is ControlAdapterLay
   layer?.type === 'control_adapter_layer';
 export const isIPAdapterLayer = (layer?: Layer): layer is IPAdapterLayer => layer?.type === 'ip_adapter_layer';
 export const isInitialImageLayer = (layer?: Layer): layer is InitialImageLayer => layer?.type === 'initial_image_layer';
+export const isRasterLayer = (layer?: Layer): layer is RasterLayer => layer?.type === 'raster_layer';
 export const isRenderableLayer = (
   layer?: Layer
 ): layer is RegionalGuidanceLayer | ControlAdapterLayer | InitialImageLayer =>
   layer?.type === 'regional_guidance_layer' ||
   layer?.type === 'control_adapter_layer' ||
-  layer?.type === 'initial_image_layer';
+  layer?.type === 'initial_image_layer' ||
+  layer?.type === 'raster_layer';
 
 export const selectCALayerOrThrow = (state: ControlLayersState, layerId: string): ControlAdapterLayer => {
   const layer = state.layers.find((l) => l.id === layerId);
@@ -109,6 +113,11 @@ export const selectIILayerOrThrow = (state: ControlLayersState, layerId: string)
   assert(isInitialImageLayer(layer));
   return layer;
 };
+export const selectRasterLayerOrThrow = (state: ControlLayersState, layerId: string): RasterLayer => {
+  const layer = state.layers.find((l) => l.id === layerId);
+  assert(isRasterLayer(layer));
+  return layer;
+};
 const selectCAOrIPALayerOrThrow = (
   state: ControlLayersState,
   layerId: string
@@ -699,6 +708,34 @@ export const controlLayersSlice = createSlice({
     },
     //#endregion
 
+    //#region Raster Layers
+    rasterLayerAdded: {
+      reducer: (state, action: PayloadAction<{ layerId: string }>) => {
+        const { layerId } = action.payload;
+        const layer: RasterLayer = {
+          id: getRasterLayerId(layerId),
+          type: 'raster_layer',
+          isEnabled: true,
+          bbox: null,
+          bboxNeedsUpdate: false,
+          objects: [],
+          opacity: 1,
+          x: 0,
+          y: 0,
+          isSelected: true,
+        };
+        state.layers.push(layer);
+        exclusivelySelectLayer(state, layer.id);
+      },
+      prepare: () => ({ payload: { layerId: uuidv4() } }),
+    },
+    rasterLayerOpacityChanged: (state, action: PayloadAction<{ layerId: string; opacity: number }>) => {
+      const { layerId, opacity } = action.payload;
+      const layer = selectRasterLayerOrThrow(state, layerId);
+      layer.opacity = opacity;
+    },
+    //#endregion
+
     //#region Globals
     positivePromptChanged: (state, action: PayloadAction<string>) => {
       state.positivePrompt = action.payload;
@@ -874,6 +911,9 @@ export const {
   iiLayerImageChanged,
   iiLayerOpacityChanged,
   iiLayerDenoisingStrengthChanged,
+  // Raster layers
+  rasterLayerAdded,
+  rasterLayerOpacityChanged,
   // Globals
   positivePromptChanged,
   negativePromptChanged,

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -58,6 +58,8 @@ const zRgbaColor = zRgbColor.extend({
 type RgbaColor = z.infer<typeof zRgbaColor>;
 export const DEFAULT_RGBA_COLOR: RgbaColor = { r: 255, g: 255, b: 255, a: 1 };
 
+const zOpacity = z.number().gte(0).lte(1);
+
 const zBrushLine = z.object({
   id: z.string(),
   type: z.literal('brush_line'),
@@ -86,6 +88,36 @@ const zRectShape = z.object({
 });
 export type RectShape = z.infer<typeof zRectShape>;
 
+const zEllipseShape = z.object({
+  id: z.string(),
+  type: z.literal('ellipse_shape'),
+  x: z.number(),
+  y: z.number(),
+  width: z.number().min(1),
+  height: z.number().min(1),
+  color: zRgbaColor,
+});
+export type EllipseShape = z.infer<typeof zEllipseShape>;
+
+const zPolygonShape = z.object({
+  id: z.string(),
+  type: z.literal('polygon_shape'),
+  points: zPoints,
+  color: zRgbaColor,
+});
+export type PolygonShape = z.infer<typeof zPolygonShape>;
+
+const zImageObject = z.object({
+  id: z.string(),
+  type: z.literal('image'),
+  image: zImageWithDims,
+  x: z.number(),
+  y: z.number(),
+  width: z.number().min(1),
+  height: z.number().min(1),
+});
+export type ImageObject = z.infer<typeof zImageObject>;
+
 const zLayerBase = z.object({
   id: z.string(),
   isEnabled: z.boolean().default(true),
@@ -105,9 +137,18 @@ const zRenderableLayerBase = zLayerBase.extend({
   bboxNeedsUpdate: z.boolean(),
 });
 
+const zRasterLayer = zRenderableLayerBase.extend({
+  type: z.literal('raster_layer'),
+  opacity: zOpacity,
+  objects: z.array(
+    z.discriminatedUnion('type', [zImageObject, zBrushLine, zEraserline, zRectShape, zEllipseShape, zPolygonShape])
+  ),
+});
+export type RasterLayer = z.infer<typeof zRasterLayer>;
+
 const zControlAdapterLayer = zRenderableLayerBase.extend({
   type: z.literal('control_adapter_layer'),
-  opacity: z.number().gte(0).lte(1),
+  opacity: zOpacity,
   isFilterEnabled: z.boolean(),
   controlAdapter: z.discriminatedUnion('type', [zControlNetConfigV2, zT2IAdapterConfigV2]),
 });
@@ -166,7 +207,7 @@ export type RegionalGuidanceLayer = z.infer<typeof zRegionalGuidanceLayer>;
 
 const zInitialImageLayer = zRenderableLayerBase.extend({
   type: z.literal('initial_image_layer'),
-  opacity: z.number().gte(0).lte(1),
+  opacity: zOpacity,
   image: zImageWithDims.nullable(),
   denoisingStrength: zParameterStrength,
 });
@@ -177,6 +218,7 @@ export const zLayer = z.discriminatedUnion('type', [
   zControlAdapterLayer,
   zIPAdapterLayer,
   zInitialImageLayer,
+  zRasterLayer,
 ]);
 export type Layer = z.infer<typeof zLayer>;