feat(ui): handle control adapter processed images

- Add helper functions to build metadata for control adapters, including the processed images
- Update parses to parse the new metadata

invokeai/frontend/web/src/features/nodes/util/graph/addControlNetToLinearGraph.ts

@@ -1,11 +1,15 @@
 import type { RootState } from 'app/store/store';
 import { selectValidControlNets } from 'features/controlAdapters/store/controlAdaptersSlice';
+import type { ControlAdapterProcessorType, ControlNetConfig } from 'features/controlAdapters/store/types';
+import type { ImageField } from 'features/nodes/types/common';
 import type {
   CollectInvocation,
   ControlNetInvocation,
   CoreMetadataInvocation,
   NonNullableGraph,
+  S,
 } from 'services/api/types';
+import { assert } from 'tsafe';
 
 import { CONTROL_NET_COLLECT } from './constants';
 import { upsertMetadata } from './metadata';
@@ -70,34 +74,12 @@ export const addControlNetToLinearGraph = async (
         resize_mode: resizeMode,
         control_model: model,
         control_weight: weight,
+        image: buildControlImage(controlImage, processedControlImage, processorType),
       };
 
-      if (processedControlImage && processorType !== 'none') {
-        // We've already processed the image in the app, so we can just use the processed image
-        controlNetNode.image = {
-          image_name: processedControlImage,
-        };
-      } else if (controlImage) {
-        // The control image is preprocessed
-        controlNetNode.image = {
-          image_name: controlImage,
-        };
-      } else {
-        // Skip CAs without an unprocessed image - should never happen, we already filtered the list of valid CAs
-        return;
-      }
+      graph.nodes[controlNetNode.id] = controlNetNode;
 
-      graph.nodes[controlNetNode.id] = controlNetNode as ControlNetInvocation;
-
-      controlNetMetadata.push({
-        control_model: model,
-        control_weight: weight,
-        control_mode: controlMode,
-        begin_step_percent: beginStepPct,
-        end_step_percent: endStepPct,
-        resize_mode: resizeMode,
-        image: controlNetNode.image,
-      });
+      controlNetMetadata.push(buildControlNetMetadata(controlNet));
 
       graph.edges.push({
         source: { node_id: controlNetNode.id, field: 'control' },
@@ -110,3 +92,62 @@ export const addControlNetToLinearGraph = async (
     upsertMetadata(graph, { controlnets: controlNetMetadata });
   }
 };
+
+const buildControlImage = (
+  controlImage: string | null,
+  processedControlImage: string | null,
+  processorType: ControlAdapterProcessorType
+): ImageField => {
+  let image: ImageField | null = null;
+  if (processedControlImage && processorType !== 'none') {
+    // We've already processed the image in the app, so we can just use the processed image
+    image = {
+      image_name: processedControlImage,
+    };
+  } else if (controlImage) {
+    // The control image is preprocessed
+    image = {
+      image_name: controlImage,
+    };
+  }
+  assert(image, 'ControlNet image is required');
+  return image;
+};
+
+const buildControlNetMetadata = (controlNet: ControlNetConfig): S['ControlNetMetadataField'] => {
+  const {
+    controlImage,
+    processedControlImage,
+    beginStepPct,
+    endStepPct,
+    controlMode,
+    resizeMode,
+    model,
+    processorType,
+    weight,
+  } = controlNet;
+
+  assert(model, 'ControlNet model is required');
+
+  const processed_image =
+    processedControlImage && processorType !== 'none'
+      ? {
+          image_name: processedControlImage,
+        }
+      : null;
+
+  assert(controlImage, 'ControlNet image is required');
+
+  return {
+    control_model: model,
+    control_weight: weight,
+    control_mode: controlMode,
+    begin_step_percent: beginStepPct,
+    end_step_percent: endStepPct,
+    resize_mode: resizeMode,
+    image: {
+      image_name: controlImage,
+    },
+    processed_image,
+  };
+};

invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts

@@ -1,11 +1,15 @@
 import type { RootState } from 'app/store/store';
 import { selectValidIPAdapters } from 'features/controlAdapters/store/controlAdaptersSlice';
+import type { IPAdapterConfig } from 'features/controlAdapters/store/types';
+import type { ImageField } from 'features/nodes/types/common';
 import type {
   CollectInvocation,
   CoreMetadataInvocation,
   IPAdapterInvocation,
   NonNullableGraph,
+  S,
 } from 'services/api/types';
+import { assert } from 'tsafe';
 
 import { IP_ADAPTER_COLLECT } from './constants';
 import { upsertMetadata } from './metadata';
@@ -44,7 +48,10 @@ export const addIPAdapterToLinearGraph = async (
       if (!ipAdapter.model) {
         return;
       }
-      const { id, weight, model, beginStepPct, endStepPct } = ipAdapter;
+      const { id, weight, model, beginStepPct, endStepPct, controlImage } = ipAdapter;
+
+      assert(controlImage, 'IP Adapter image is required');
+
       const ipAdapterNode: IPAdapterInvocation = {
         id: `ip_adapter_${id}`,
         type: 'ip_adapter',
@@ -53,25 +60,14 @@ export const addIPAdapterToLinearGraph = async (
         ip_adapter_model: model,
         begin_step_percent: beginStepPct,
         end_step_percent: endStepPct,
+        image: {
+          image_name: controlImage,
+        },
       };
 
-      if (ipAdapter.controlImage) {
-        ipAdapterNode.image = {
-          image_name: ipAdapter.controlImage,
-        };
-      } else {
-        return;
-      }
-
       graph.nodes[ipAdapterNode.id] = ipAdapterNode;
 
-      ipAdapterMetdata.push({
-        weight: weight,
-        ip_adapter_model: model,
-        begin_step_percent: beginStepPct,
-        end_step_percent: endStepPct,
-        image: ipAdapterNode.image,
-      });
+      ipAdapterMetdata.push(buildIPAdapterMetadata(ipAdapter));
 
       graph.edges.push({
         source: { node_id: ipAdapterNode.id, field: 'ip_adapter' },
@@ -85,3 +81,27 @@ export const addIPAdapterToLinearGraph = async (
     upsertMetadata(graph, { ipAdapters: ipAdapterMetdata });
   }
 };
+
+const buildIPAdapterMetadata = (ipAdapter: IPAdapterConfig): S['IPAdapterMetadataField'] => {
+  const { controlImage, beginStepPct, endStepPct, model, weight } = ipAdapter;
+
+  assert(model, 'IP Adapter model is required');
+
+  let image: ImageField | null = null;
+
+  if (controlImage) {
+    image = {
+      image_name: controlImage,
+    };
+  }
+
+  assert(image, 'IP Adapter image is required');
+
+  return {
+    ip_adapter_model: model,
+    weight,
+    begin_step_percent: beginStepPct,
+    end_step_percent: endStepPct,
+    image,
+  };
+};

invokeai/frontend/web/src/features/nodes/util/graph/addT2IAdapterToLinearGraph.ts

@@ -1,11 +1,15 @@
 import type { RootState } from 'app/store/store';
 import { selectValidT2IAdapters } from 'features/controlAdapters/store/controlAdaptersSlice';
+import type { ControlAdapterProcessorType, T2IAdapterConfig } from 'features/controlAdapters/store/types';
+import type { ImageField } from 'features/nodes/types/common';
 import type {
   CollectInvocation,
   CoreMetadataInvocation,
   NonNullableGraph,
+  S,
   T2IAdapterInvocation,
 } from 'services/api/types';
+import { assert } from 'tsafe';
 
 import { T2I_ADAPTER_COLLECT } from './constants';
 import { upsertMetadata } from './metadata';
@@ -68,33 +72,12 @@ export const addT2IAdaptersToLinearGraph = async (
         resize_mode: resizeMode,
         t2i_adapter_model: model,
         weight: weight,
+        image: buildControlImage(controlImage, processedControlImage, processorType),
       };
 
-      if (processedControlImage && processorType !== 'none') {
-        // We've already processed the image in the app, so we can just use the processed image
-        t2iAdapterNode.image = {
-          image_name: processedControlImage,
-        };
-      } else if (controlImage) {
-        // The control image is preprocessed
-        t2iAdapterNode.image = {
-          image_name: controlImage,
-        };
-      } else {
-        // Skip CAs without an unprocessed image - should never happen, we already filtered the list of valid CAs
-        return;
-      }
-
       graph.nodes[t2iAdapterNode.id] = t2iAdapterNode;
 
-      t2iAdapterMetadata.push({
-        begin_step_percent: beginStepPct,
-        end_step_percent: endStepPct,
-        resize_mode: resizeMode,
-        t2i_adapter_model: t2iAdapter.model,
-        weight: weight,
-        image: t2iAdapterNode.image,
-      });
+      t2iAdapterMetadata.push(buildT2IAdapterMetadata(t2iAdapter));
 
       graph.edges.push({
         source: { node_id: t2iAdapterNode.id, field: 't2i_adapter' },
@@ -108,3 +91,52 @@ export const addT2IAdaptersToLinearGraph = async (
     upsertMetadata(graph, { t2iAdapters: t2iAdapterMetadata });
   }
 };
+
+const buildControlImage = (
+  controlImage: string | null,
+  processedControlImage: string | null,
+  processorType: ControlAdapterProcessorType
+): ImageField => {
+  let image: ImageField | null = null;
+  if (processedControlImage && processorType !== 'none') {
+    // We've already processed the image in the app, so we can just use the processed image
+    image = {
+      image_name: processedControlImage,
+    };
+  } else if (controlImage) {
+    // The control image is preprocessed
+    image = {
+      image_name: controlImage,
+    };
+  }
+  assert(image, 'T2I Adapter image is required');
+  return image;
+};
+
+const buildT2IAdapterMetadata = (t2iAdapter: T2IAdapterConfig): S['T2IAdapterMetadataField'] => {
+  const { controlImage, processedControlImage, beginStepPct, endStepPct, resizeMode, model, processorType, weight } =
+    t2iAdapter;
+
+  assert(model, 'T2I Adapter model is required');
+
+  const processed_image =
+    processedControlImage && processorType !== 'none'
+      ? {
+          image_name: processedControlImage,
+        }
+      : null;
+
+  assert(controlImage, 'T2I Adapter image is required');
+
+  return {
+    t2i_adapter_model: model,
+    weight,
+    begin_step_percent: beginStepPct,
+    end_step_percent: endStepPct,
+    resize_mode: resizeMode,
+    image: {
+      image_name: controlImage,
+    },
+    processed_image,
+  };
+};