feat(ui): convert canvas txt2img & img2img to latents

- Add graph builders for canvas txt2img & img2img - they are mostly copy and paste from the linear graph builders but different in a few ways that are very tricky to work around. Just made totally new functions for them.
- Canvas txt2img and img2img support ControlNet (not inpaint/outpaint). There's no way to determine in real-time which mode the canvas is in just yet, so we cannot disable the ControlNet UI when the mode will be inpaint/outpaint - it will always display. It's possible to determine this in near-real-time, will add this at some point.
- Canvas inpaint/outpaint migrated to use model loader, though inpaint/outpaint are still using the non-latents nodes.

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts

@@ -1,11 +1,10 @@
 import { startAppListening } from '..';
 import { sessionCreated } from 'services/thunks/session';
-import { buildCanvasGraphComponents } from 'features/nodes/util/graphBuilders/buildCanvasGraph';
+import { buildCanvasGraph } from 'features/nodes/util/graphBuilders/buildCanvasGraph';
 import { log } from 'app/logging/useLogger';
 import { canvasGraphBuilt } from 'features/nodes/store/actions';
 import { imageUpdated, imageUploaded } from 'services/thunks/image';
-import { v4 as uuidv4 } from 'uuid';
+import { ImageDTO } from 'services/api';
-import { Graph } from 'services/api';
 import {
   canvasSessionIdChanged,
   stagingAreaInitialized,
@@ -67,112 +66,106 @@ export const addUserInvokedCanvasListener = () => {
 
       moduleLog.debug(`Generation mode: ${generationMode}`);
 
-      // Build the canvas graph
+      // Temp placeholders for the init and mask images
-      const graphComponents = await buildCanvasGraphComponents(
+      let canvasInitImage: ImageDTO | undefined;
-        state,
+      let canvasMaskImage: ImageDTO | undefined;
-        generationMode
-      );
 
-      if (!graphComponents) {
+      // For img2img and inpaint/outpaint, we need to upload the init images
-        moduleLog.error('Problem building graph');
+      if (['img2img', 'inpaint', 'outpaint'].includes(generationMode)) {
-        return;
+        // upload the image, saving the request id
-      }
+        const { requestId: initImageUploadedRequestId } = dispatch(
-
-      const { rangeNode, iterateNode, baseNode, edges } = graphComponents;
-
-      // Assemble! Note that this graph *does not have the init or mask image set yet!*
-      const nodes: Graph['nodes'] = {
-        [rangeNode.id]: rangeNode,
-        [iterateNode.id]: iterateNode,
-        [baseNode.id]: baseNode,
-      };
-
-      const graph = { nodes, edges };
-
-      dispatch(canvasGraphBuilt(graph));
-
-      moduleLog.debug({ data: graph }, 'Canvas graph built');
-
-      // If we are generating img2img or inpaint, we need to upload the init images
-      if (baseNode.type === 'img2img' || baseNode.type === 'inpaint') {
-        const baseFilename = `${uuidv4()}.png`;
-        dispatch(
           imageUploaded({
             formData: {
-              file: new File([baseBlob], baseFilename, { type: 'image/png' }),
+              file: new File([baseBlob], 'canvasInitImage.png', {
+                type: 'image/png',
+              }),
             },
             imageCategory: 'general',
             isIntermediate: true,
           })
         );
 
-        // Wait for the image to be uploaded
+        // Wait for the image to be uploaded, matching by request id
-        const [{ payload: baseImageDTO }] = await take(
+        const [{ payload }] = await take(
           (action): action is ReturnType<typeof imageUploaded.fulfilled> =>
             imageUploaded.fulfilled.match(action) &&
-            action.meta.arg.formData.file.name === baseFilename
+            action.meta.requestId === initImageUploadedRequestId
         );
 
-        // Update the base node with the image name and type
+        canvasInitImage = payload;
-        baseNode.image = {
-          image_name: baseImageDTO.image_name,
-        };
       }
 
-      // For inpaint, we also need to upload the mask layer
+      // For inpaint/outpaint, we also need to upload the mask layer
-      if (baseNode.type === 'inpaint') {
+      if (['inpaint', 'outpaint'].includes(generationMode)) {
-        const maskFilename = `${uuidv4()}.png`;
+        // upload the image, saving the request id
-        dispatch(
+        const { requestId: maskImageUploadedRequestId } = dispatch(
           imageUploaded({
             formData: {
-              file: new File([maskBlob], maskFilename, { type: 'image/png' }),
+              file: new File([maskBlob], 'canvasMaskImage.png', {
+                type: 'image/png',
+              }),
             },
             imageCategory: 'mask',
             isIntermediate: true,
           })
         );
 
-        // Wait for the mask to be uploaded
+        // Wait for the image to be uploaded, matching by request id
-        const [{ payload: maskImageDTO }] = await take(
+        const [{ payload }] = await take(
           (action): action is ReturnType<typeof imageUploaded.fulfilled> =>
             imageUploaded.fulfilled.match(action) &&
-            action.meta.arg.formData.file.name === maskFilename
+            action.meta.requestId === maskImageUploadedRequestId
         );
 
-        // Update the base node with the image name and type
+        canvasMaskImage = payload;
-        baseNode.mask = {
-          image_name: maskImageDTO.image_name,
-        };
       }
 
-      // Create the session and wait for response
+      const graph = buildCanvasGraph(
-      dispatch(sessionCreated({ graph }));
+        state,
-      const [sessionCreatedAction] = await take(sessionCreated.fulfilled.match);
+        generationMode,
+        canvasInitImage,
+        canvasMaskImage
+      );
+
+      moduleLog.debug({ graph }, `Canvas graph built`);
+
+      // currently this action is just listened to for logging
+      dispatch(canvasGraphBuilt(graph));
+
+      // Create the session, store the request id
+      const { requestId: sessionCreatedRequestId } = dispatch(
+        sessionCreated({ graph })
+      );
+
+      // Take the session created action, matching by its request id
+      const [sessionCreatedAction] = await take(
+        (action): action is ReturnType<typeof sessionCreated.fulfilled> =>
+          sessionCreated.fulfilled.match(action) &&
+          action.meta.requestId === sessionCreatedRequestId
+      );
       const sessionId = sessionCreatedAction.payload.id;
 
       // Associate the init image with the session, now that we have the session ID
-      if (
+      if (['img2img', 'inpaint'].includes(generationMode) && canvasInitImage) {
-        (baseNode.type === 'img2img' || baseNode.type === 'inpaint') &&
-        baseNode.image
-      ) {
         dispatch(
           imageUpdated({
-            imageName: baseNode.image.image_name,
+            imageName: canvasInitImage.image_name,
             requestBody: { session_id: sessionId },
           })
         );
       }
 
       // Associate the mask image with the session, now that we have the session ID
-      if (baseNode.type === 'inpaint' && baseNode.mask) {
+      if (['inpaint'].includes(generationMode) && canvasMaskImage) {
         dispatch(
           imageUpdated({
-            imageName: baseNode.mask.image_name,
+            imageName: canvasMaskImage.image_name,
             requestBody: { session_id: sessionId },
           })
         );
       }
 
+      // Prep the canvas staging area if it is not yet initialized
       if (!state.canvas.layerState.stagingArea.boundingBox) {
         dispatch(
           stagingAreaInitialized({

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts

@@ -4,7 +4,7 @@ import { log } from 'app/logging/useLogger';
 import { imageToImageGraphBuilt } from 'features/nodes/store/actions';
 import { userInvoked } from 'app/store/actions';
 import { sessionReadyToInvoke } from 'features/system/store/actions';
-import { buildImageToImageGraph } from 'features/nodes/util/graphBuilders/buildImageToImageGraph';
+import { buildLinearImageToImageGraph } from 'features/nodes/util/graphBuilders/buildLinearImageToImageGraph';
 
 const moduleLog = log.child({ namespace: 'invoke' });
 
@@ -15,7 +15,7 @@ export const addUserInvokedImageToImageListener = () => {
     effect: async (action, { getState, dispatch, take }) => {
       const state = getState();
 
-      const graph = buildImageToImageGraph(state);
+      const graph = buildLinearImageToImageGraph(state);
       dispatch(imageToImageGraphBuilt(graph));
       moduleLog.debug({ data: graph }, 'Image to Image graph built');
 

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts

@@ -4,7 +4,7 @@ import { log } from 'app/logging/useLogger';
 import { textToImageGraphBuilt } from 'features/nodes/store/actions';
 import { userInvoked } from 'app/store/actions';
 import { sessionReadyToInvoke } from 'features/system/store/actions';
-import { buildTextToImageGraph } from 'features/nodes/util/graphBuilders/buildTextToImageGraph';
+import { buildLinearTextToImageGraph } from 'features/nodes/util/graphBuilders/buildLinearTextToImageGraph';
 
 const moduleLog = log.child({ namespace: 'invoke' });
 
@@ -15,7 +15,7 @@ export const addUserInvokedTextToImageListener = () => {
     effect: async (action, { getState, dispatch, take }) => {
       const state = getState();
 
-      const graph = buildTextToImageGraph(state);
+      const graph = buildLinearTextToImageGraph(state);
 
       dispatch(textToImageGraphBuilt(graph));
 

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts

@@ -1,116 +1,39 @@
 import { RootState } from 'app/store/store';
-import {
+import { ImageDTO } from 'services/api';
-  Edge,
-  ImageToImageInvocation,
-  InpaintInvocation,
-  IterateInvocation,
-  RandomRangeInvocation,
-  RangeInvocation,
-  TextToImageInvocation,
-} from 'services/api';
-import { buildImg2ImgNode } from '../nodeBuilders/buildImageToImageNode';
-import { buildTxt2ImgNode } from '../nodeBuilders/buildTextToImageNode';
-import { buildRangeNode } from '../nodeBuilders/buildRangeNode';
-import { buildIterateNode } from '../nodeBuilders/buildIterateNode';
-import { buildEdges } from '../edgeBuilders/buildEdges';
 import { log } from 'app/logging/useLogger';
-import { buildInpaintNode } from '../nodeBuilders/buildInpaintNode';
+import { forEach } from 'lodash-es';
+import { buildCanvasInpaintGraph } from './buildCanvasInpaintGraph';
+import { NonNullableGraph } from 'features/nodes/types/types';
+import { buildCanvasImageToImageGraph } from './buildCanvasImageToImageGraph';
+import { buildCanvasTextToImageGraph } from './buildCanvasTextToImageGraph';
 
 const moduleLog = log.child({ namespace: 'nodes' });
 
-const buildBaseNode = (
+export const buildCanvasGraph = (
-  nodeType: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint',
-  state: RootState
-):
-  | TextToImageInvocation
-  | ImageToImageInvocation
-  | InpaintInvocation
-  | undefined => {
-  const overrides = {
-    ...state.canvas.boundingBoxDimensions,
-    is_intermediate: true,
-  };
-
-  if (nodeType === 'txt2img') {
-    return buildTxt2ImgNode(state, overrides);
-  }
-
-  if (nodeType === 'img2img') {
-    return buildImg2ImgNode(state, overrides);
-  }
-
-  if (nodeType === 'inpaint' || nodeType === 'outpaint') {
-    return buildInpaintNode(state, overrides);
-  }
-};
-
-/**
- * Builds the Canvas workflow graph and image blobs.
- */
-export const buildCanvasGraphComponents = async (
   state: RootState,
-  generationMode: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint'
+  generationMode: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint',
-): Promise<
+  canvasInitImage: ImageDTO | undefined,
-  | {
+  canvasMaskImage: ImageDTO | undefined
-      rangeNode: RangeInvocation | RandomRangeInvocation;
+) => {
-      iterateNode: IterateInvocation;
+  let graph: NonNullableGraph;
-      baseNode:
-        | TextToImageInvocation
-        | ImageToImageInvocation
-        | InpaintInvocation;
-      edges: Edge[];
-    }
-  | undefined
-> => {
-  // The base node is a txt2img, img2img or inpaint node
-  const baseNode = buildBaseNode(generationMode, state);
 
-  if (!baseNode) {
+  if (generationMode === 'txt2img') {
-    moduleLog.error('Problem building base node');
+    graph = buildCanvasTextToImageGraph(state);
-    return;
+  } else if (generationMode === 'img2img') {
+    if (!canvasInitImage) {
+      throw new Error('Missing canvas init image');
+    }
+    graph = buildCanvasImageToImageGraph(state, canvasInitImage);
+  } else {
+    if (!canvasInitImage || !canvasMaskImage) {
+      throw new Error('Missing canvas init and mask images');
+    }
+    graph = buildCanvasInpaintGraph(state, canvasInitImage, canvasMaskImage);
   }
 
-  if (baseNode.type === 'inpaint') {
+  forEach(graph.nodes, (node) => {
-    const {
+    graph.nodes[node.id].is_intermediate = true;
-      seamSize,
+  });
-      seamBlur,
-      seamSteps,
-      seamStrength,
-      tileSize,
-      infillMethod,
-    } = state.generation;
 
-    const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } =
+  return graph;
-      state.canvas;
-
-    if (boundingBoxScaleMethod !== 'none') {
-      baseNode.inpaint_width = scaledBoundingBoxDimensions.width;
-      baseNode.inpaint_height = scaledBoundingBoxDimensions.height;
-    }
-
-    baseNode.seam_size = seamSize;
-    baseNode.seam_blur = seamBlur;
-    baseNode.seam_strength = seamStrength;
-    baseNode.seam_steps = seamSteps;
-    baseNode.infill_method = infillMethod as InpaintInvocation['infill_method'];
-
-    if (infillMethod === 'tile') {
-      baseNode.tile_size = tileSize;
-    }
-  }
-
-  // We always range and iterate nodes, no matter the iteration count
-  // This is required to provide the correct seeds to the backend engine
-  const rangeNode = buildRangeNode(state);
-  const iterateNode = buildIterateNode();
-
-  // Build the edges for the nodes selected.
-  const edges = buildEdges(baseNode, rangeNode, iterateNode);
-
-  return {
-    rangeNode,
-    iterateNode,
-    baseNode,
-    edges,
-  };
 };

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts

@@ -0,0 +1,331 @@
+import { RootState } from 'app/store/store';
+import {
+  ImageDTO,
+  ImageResizeInvocation,
+  RandomIntInvocation,
+  RangeOfSizeInvocation,
+} from 'services/api';
+import { NonNullableGraph } from 'features/nodes/types/types';
+import { log } from 'app/logging/useLogger';
+import {
+  ITERATE,
+  LATENTS_TO_IMAGE,
+  MODEL_LOADER,
+  NEGATIVE_CONDITIONING,
+  NOISE,
+  POSITIVE_CONDITIONING,
+  RANDOM_INT,
+  RANGE_OF_SIZE,
+  IMAGE_TO_IMAGE_GRAPH,
+  IMAGE_TO_LATENTS,
+  LATENTS_TO_LATENTS,
+  RESIZE,
+} from './constants';
+import { set } from 'lodash-es';
+import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph';
+
+const moduleLog = log.child({ namespace: 'nodes' });
+
+/**
+ * Builds the Canvas tab's Image to Image graph.
+ */
+export const buildCanvasImageToImageGraph = (
+  state: RootState,
+  initialImage: ImageDTO
+): NonNullableGraph => {
+  const {
+    positivePrompt,
+    negativePrompt,
+    model: model_name,
+    cfgScale: cfg_scale,
+    scheduler,
+    steps,
+    img2imgStrength: strength,
+    iterations,
+    seed,
+    shouldRandomizeSeed,
+  } = state.generation;
+
+  // The bounding box determines width and height, not the width and height params
+  const { width, height } = state.canvas.boundingBoxDimensions;
+
+  /**
+   * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the
+   * full graph here as a template. Then use the parameters from app state and set friendlier node
+   * ids.
+   *
+   * The only thing we need extra logic for is handling randomized seed, control net, and for img2img,
+   * the `fit` param. These are added to the graph at the end.
+   */
+
+  // copy-pasted graph from node editor, filled in with state values & friendly node ids
+  const graph: NonNullableGraph = {
+    id: IMAGE_TO_IMAGE_GRAPH,
+    nodes: {
+      [POSITIVE_CONDITIONING]: {
+        type: 'compel',
+        id: POSITIVE_CONDITIONING,
+        prompt: positivePrompt,
+      },
+      [NEGATIVE_CONDITIONING]: {
+        type: 'compel',
+        id: NEGATIVE_CONDITIONING,
+        prompt: negativePrompt,
+      },
+      [RANGE_OF_SIZE]: {
+        type: 'range_of_size',
+        id: RANGE_OF_SIZE,
+        // seed - must be connected manually
+        // start: 0,
+        size: iterations,
+        step: 1,
+      },
+      [NOISE]: {
+        type: 'noise',
+        id: NOISE,
+      },
+      [MODEL_LOADER]: {
+        type: 'sd1_model_loader',
+        id: MODEL_LOADER,
+        model_name,
+      },
+      [LATENTS_TO_IMAGE]: {
+        type: 'l2i',
+        id: LATENTS_TO_IMAGE,
+      },
+      [ITERATE]: {
+        type: 'iterate',
+        id: ITERATE,
+      },
+      [LATENTS_TO_LATENTS]: {
+        type: 'l2l',
+        id: LATENTS_TO_LATENTS,
+        cfg_scale,
+        scheduler,
+        steps,
+        strength,
+      },
+      [IMAGE_TO_LATENTS]: {
+        type: 'i2l',
+        id: IMAGE_TO_LATENTS,
+        // must be set manually later, bc `fit` parameter may require a resize node inserted
+        // image: {
+        //   image_name: initialImage.image_name,
+        // },
+      },
+    },
+    edges: [
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'clip',
+        },
+        destination: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'clip',
+        },
+        destination: {
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'vae',
+        },
+        destination: {
+          node_id: LATENTS_TO_IMAGE,
+          field: 'vae',
+        },
+      },
+      {
+        source: {
+          node_id: RANGE_OF_SIZE,
+          field: 'collection',
+        },
+        destination: {
+          node_id: ITERATE,
+          field: 'collection',
+        },
+      },
+      {
+        source: {
+          node_id: ITERATE,
+          field: 'item',
+        },
+        destination: {
+          node_id: NOISE,
+          field: 'seed',
+        },
+      },
+      {
+        source: {
+          node_id: LATENTS_TO_LATENTS,
+          field: 'latents',
+        },
+        destination: {
+          node_id: LATENTS_TO_IMAGE,
+          field: 'latents',
+        },
+      },
+      {
+        source: {
+          node_id: IMAGE_TO_LATENTS,
+          field: 'latents',
+        },
+        destination: {
+          node_id: LATENTS_TO_LATENTS,
+          field: 'latents',
+        },
+      },
+      {
+        source: {
+          node_id: NOISE,
+          field: 'noise',
+        },
+        destination: {
+          node_id: LATENTS_TO_LATENTS,
+          field: 'noise',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'vae',
+        },
+        destination: {
+          node_id: IMAGE_TO_LATENTS,
+          field: 'vae',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'unet',
+        },
+        destination: {
+          node_id: LATENTS_TO_LATENTS,
+          field: 'unet',
+        },
+      },
+      {
+        source: {
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: LATENTS_TO_LATENTS,
+          field: 'negative_conditioning',
+        },
+      },
+      {
+        source: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: LATENTS_TO_LATENTS,
+          field: 'positive_conditioning',
+        },
+      },
+    ],
+  };
+
+  // handle seed
+  if (shouldRandomizeSeed) {
+    // Random int node to generate the starting seed
+    const randomIntNode: RandomIntInvocation = {
+      id: RANDOM_INT,
+      type: 'rand_int',
+    };
+
+    graph.nodes[RANDOM_INT] = randomIntNode;
+
+    // Connect random int to the start of the range of size so the range starts on the random first seed
+    graph.edges.push({
+      source: { node_id: RANDOM_INT, field: 'a' },
+      destination: { node_id: RANGE_OF_SIZE, field: 'start' },
+    });
+  } else {
+    // User specified seed, so set the start of the range of size to the seed
+    (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed;
+  }
+
+  // handle `fit`
+  if (initialImage.width !== width || initialImage.height !== height) {
+    // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS`
+
+    // Create a resize node, explicitly setting its image
+    const resizeNode: ImageResizeInvocation = {
+      id: RESIZE,
+      type: 'img_resize',
+      image: {
+        image_name: initialImage.image_name,
+      },
+      is_intermediate: true,
+      width,
+      height,
+    };
+
+    graph.nodes[RESIZE] = resizeNode;
+
+    // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS`
+    graph.edges.push({
+      source: { node_id: RESIZE, field: 'image' },
+      destination: {
+        node_id: IMAGE_TO_LATENTS,
+        field: 'image',
+      },
+    });
+
+    // The `RESIZE` node also passes its width and height to `NOISE`
+    graph.edges.push({
+      source: { node_id: RESIZE, field: 'width' },
+      destination: {
+        node_id: NOISE,
+        field: 'width',
+      },
+    });
+
+    graph.edges.push({
+      source: { node_id: RESIZE, field: 'height' },
+      destination: {
+        node_id: NOISE,
+        field: 'height',
+      },
+    });
+  } else {
+    // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly
+    set(graph.nodes[IMAGE_TO_LATENTS], 'image', {
+      image_name: initialImage.image_name,
+    });
+
+    // Pass the image's dimensions to the `NOISE` node
+    graph.edges.push({
+      source: { node_id: IMAGE_TO_LATENTS, field: 'width' },
+      destination: {
+        node_id: NOISE,
+        field: 'width',
+      },
+    });
+    graph.edges.push({
+      source: { node_id: IMAGE_TO_LATENTS, field: 'height' },
+      destination: {
+        node_id: NOISE,
+        field: 'height',
+      },
+    });
+  }
+
+  // add controlnet
+  addControlNetToLinearGraph(graph, LATENTS_TO_LATENTS, state);
+
+  return graph;
+};

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts

@@ -0,0 +1,224 @@
+import { RootState } from 'app/store/store';
+import {
+  ImageDTO,
+  InpaintInvocation,
+  RandomIntInvocation,
+  RangeOfSizeInvocation,
+} from 'services/api';
+import { NonNullableGraph } from 'features/nodes/types/types';
+import { log } from 'app/logging/useLogger';
+import {
+  ITERATE,
+  MODEL_LOADER,
+  NEGATIVE_CONDITIONING,
+  POSITIVE_CONDITIONING,
+  RANDOM_INT,
+  RANGE_OF_SIZE,
+  INPAINT_GRAPH,
+  INPAINT,
+} from './constants';
+
+const moduleLog = log.child({ namespace: 'nodes' });
+
+/**
+ * Builds the Canvas tab's Inpaint graph.
+ */
+export const buildCanvasInpaintGraph = (
+  state: RootState,
+  canvasInitImage: ImageDTO,
+  canvasMaskImage: ImageDTO
+): NonNullableGraph => {
+  const {
+    positivePrompt,
+    negativePrompt,
+    model: model_name,
+    cfgScale: cfg_scale,
+    scheduler,
+    steps,
+    img2imgStrength: strength,
+    shouldFitToWidthHeight,
+    iterations,
+    seed,
+    shouldRandomizeSeed,
+    seamSize,
+    seamBlur,
+    seamSteps,
+    seamStrength,
+    tileSize,
+    infillMethod,
+  } = state.generation;
+
+  // The bounding box determines width and height, not the width and height params
+  const { width, height } = state.canvas.boundingBoxDimensions;
+
+  // We may need to set the inpaint width and height to scale the image
+  const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } = state.canvas;
+
+  const graph: NonNullableGraph = {
+    id: INPAINT_GRAPH,
+    nodes: {
+      [INPAINT]: {
+        type: 'inpaint',
+        id: INPAINT,
+        steps,
+        width,
+        height,
+        cfg_scale,
+        scheduler,
+        image: {
+          image_name: canvasInitImage.image_name,
+        },
+        strength,
+        fit: shouldFitToWidthHeight,
+        mask: {
+          image_name: canvasMaskImage.image_name,
+        },
+        seam_size: seamSize,
+        seam_blur: seamBlur,
+        seam_strength: seamStrength,
+        seam_steps: seamSteps,
+        tile_size: infillMethod === 'tile' ? tileSize : undefined,
+        infill_method: infillMethod as InpaintInvocation['infill_method'],
+        inpaint_width:
+          boundingBoxScaleMethod !== 'none'
+            ? scaledBoundingBoxDimensions.width
+            : undefined,
+        inpaint_height:
+          boundingBoxScaleMethod !== 'none'
+            ? scaledBoundingBoxDimensions.height
+            : undefined,
+      },
+      [POSITIVE_CONDITIONING]: {
+        type: 'compel',
+        id: POSITIVE_CONDITIONING,
+        prompt: positivePrompt,
+      },
+      [NEGATIVE_CONDITIONING]: {
+        type: 'compel',
+        id: NEGATIVE_CONDITIONING,
+        prompt: negativePrompt,
+      },
+      [MODEL_LOADER]: {
+        type: 'sd1_model_loader',
+        id: MODEL_LOADER,
+        model_name,
+      },
+      [RANGE_OF_SIZE]: {
+        type: 'range_of_size',
+        id: RANGE_OF_SIZE,
+        // seed - must be connected manually
+        // start: 0,
+        size: iterations,
+        step: 1,
+      },
+      [ITERATE]: {
+        type: 'iterate',
+        id: ITERATE,
+      },
+    },
+    edges: [
+      {
+        source: {
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'negative_conditioning',
+        },
+      },
+      {
+        source: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'positive_conditioning',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'clip',
+        },
+        destination: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'clip',
+        },
+        destination: {
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'unet',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'unet',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'vae',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'vae',
+        },
+      },
+      {
+        source: {
+          node_id: RANGE_OF_SIZE,
+          field: 'collection',
+        },
+        destination: {
+          node_id: ITERATE,
+          field: 'collection',
+        },
+      },
+      {
+        source: {
+          node_id: ITERATE,
+          field: 'item',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'seed',
+        },
+      },
+    ],
+  };
+
+  // handle seed
+  if (shouldRandomizeSeed) {
+    // Random int node to generate the starting seed
+    const randomIntNode: RandomIntInvocation = {
+      id: RANDOM_INT,
+      type: 'rand_int',
+    };
+
+    graph.nodes[RANDOM_INT] = randomIntNode;
+
+    // Connect random int to the start of the range of size so the range starts on the random first seed
+    graph.edges.push({
+      source: { node_id: RANDOM_INT, field: 'a' },
+      destination: { node_id: RANGE_OF_SIZE, field: 'start' },
+    });
+  } else {
+    // User specified seed, so set the start of the range of size to the seed
+    (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed;
+  }
+
+  return graph;
+};

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts

@@ -0,0 +1,224 @@
+import { RootState } from 'app/store/store';
+import { NonNullableGraph } from 'features/nodes/types/types';
+import { RandomIntInvocation, RangeOfSizeInvocation } from 'services/api';
+import {
+  ITERATE,
+  LATENTS_TO_IMAGE,
+  MODEL_LOADER,
+  NEGATIVE_CONDITIONING,
+  NOISE,
+  POSITIVE_CONDITIONING,
+  RANDOM_INT,
+  RANGE_OF_SIZE,
+  TEXT_TO_IMAGE_GRAPH,
+  TEXT_TO_LATENTS,
+} from './constants';
+import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph';
+
+/**
+ * Builds the Canvas tab's Text to Image graph.
+ */
+export const buildCanvasTextToImageGraph = (
+  state: RootState
+): NonNullableGraph => {
+  const {
+    positivePrompt,
+    negativePrompt,
+    model: model_name,
+    cfgScale: cfg_scale,
+    scheduler,
+    steps,
+    iterations,
+    seed,
+    shouldRandomizeSeed,
+  } = state.generation;
+
+  // The bounding box determines width and height, not the width and height params
+  const { width, height } = state.canvas.boundingBoxDimensions;
+
+  /**
+   * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the
+   * full graph here as a template. Then use the parameters from app state and set friendlier node
+   * ids.
+   *
+   * The only thing we need extra logic for is handling randomized seed, control net, and for img2img,
+   * the `fit` param. These are added to the graph at the end.
+   */
+
+  // copy-pasted graph from node editor, filled in with state values & friendly node ids
+  const graph: NonNullableGraph = {
+    id: TEXT_TO_IMAGE_GRAPH,
+    nodes: {
+      [POSITIVE_CONDITIONING]: {
+        type: 'compel',
+        id: POSITIVE_CONDITIONING,
+        prompt: positivePrompt,
+      },
+      [NEGATIVE_CONDITIONING]: {
+        type: 'compel',
+        id: NEGATIVE_CONDITIONING,
+        prompt: negativePrompt,
+      },
+      [RANGE_OF_SIZE]: {
+        type: 'range_of_size',
+        id: RANGE_OF_SIZE,
+        // start: 0, // seed - must be connected manually
+        size: iterations,
+        step: 1,
+      },
+      [NOISE]: {
+        type: 'noise',
+        id: NOISE,
+        width,
+        height,
+      },
+      [TEXT_TO_LATENTS]: {
+        type: 't2l',
+        id: TEXT_TO_LATENTS,
+        cfg_scale,
+        scheduler,
+        steps,
+      },
+      [MODEL_LOADER]: {
+        type: 'sd1_model_loader',
+        id: MODEL_LOADER,
+        model_name,
+      },
+      [LATENTS_TO_IMAGE]: {
+        type: 'l2i',
+        id: LATENTS_TO_IMAGE,
+      },
+      [ITERATE]: {
+        type: 'iterate',
+        id: ITERATE,
+      },
+    },
+    edges: [
+      {
+        source: {
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: TEXT_TO_LATENTS,
+          field: 'negative_conditioning',
+        },
+      },
+      {
+        source: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: TEXT_TO_LATENTS,
+          field: 'positive_conditioning',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'clip',
+        },
+        destination: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'clip',
+        },
+        destination: {
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'unet',
+        },
+        destination: {
+          node_id: TEXT_TO_LATENTS,
+          field: 'unet',
+        },
+      },
+      {
+        source: {
+          node_id: TEXT_TO_LATENTS,
+          field: 'latents',
+        },
+        destination: {
+          node_id: LATENTS_TO_IMAGE,
+          field: 'latents',
+        },
+      },
+      {
+        source: {
+          node_id: MODEL_LOADER,
+          field: 'vae',
+        },
+        destination: {
+          node_id: LATENTS_TO_IMAGE,
+          field: 'vae',
+        },
+      },
+      {
+        source: {
+          node_id: RANGE_OF_SIZE,
+          field: 'collection',
+        },
+        destination: {
+          node_id: ITERATE,
+          field: 'collection',
+        },
+      },
+      {
+        source: {
+          node_id: ITERATE,
+          field: 'item',
+        },
+        destination: {
+          node_id: NOISE,
+          field: 'seed',
+        },
+      },
+      {
+        source: {
+          node_id: NOISE,
+          field: 'noise',
+        },
+        destination: {
+          node_id: TEXT_TO_LATENTS,
+          field: 'noise',
+        },
+      },
+    ],
+  };
+
+  // handle seed
+  if (shouldRandomizeSeed) {
+    // Random int node to generate the starting seed
+    const randomIntNode: RandomIntInvocation = {
+      id: RANDOM_INT,
+      type: 'rand_int',
+    };
+
+    graph.nodes[RANDOM_INT] = randomIntNode;
+
+    // Connect random int to the start of the range of size so the range starts on the random first seed
+    graph.edges.push({
+      source: { node_id: RANDOM_INT, field: 'a' },
+      destination: { node_id: RANGE_OF_SIZE, field: 'start' },
+    });
+  } else {
+    // User specified seed, so set the start of the range of size to the seed
+    (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed;
+  }
+
+  // add controlnet
+  addControlNetToLinearGraph(graph, TEXT_TO_LATENTS, state);
+
+  return graph;
+};

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts

@@ -1,6 +1,5 @@
 import { RootState } from 'app/store/store';
 import {
-  Graph,
   ImageResizeInvocation,
   RandomIntInvocation,
   RangeOfSizeInvocation,
@@ -23,12 +22,15 @@ import {
 } from './constants';
 import { set } from 'lodash-es';
 import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph';
+
 const moduleLog = log.child({ namespace: 'nodes' });
 
 /**
  * Builds the Image to Image tab graph.
  */
-export const buildImageToImageGraph = (state: RootState): Graph => {
+export const buildLinearImageToImageGraph = (
+  state: RootState
+): NonNullableGraph => {
   const {
     positivePrompt,
     negativePrompt,
@@ -275,8 +277,8 @@ export const buildImageToImageGraph = (state: RootState): Graph => {
         image_name: initialImage.image_name,
       },
       is_intermediate: true,
-      height,
       width,
+      height,
     };
 
     graph.nodes[RESIZE] = resizeNode;

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts

@@ -1,10 +1,6 @@
 import { RootState } from 'app/store/store';
 import { NonNullableGraph } from 'features/nodes/types/types';
-import {
+import { RandomIntInvocation, RangeOfSizeInvocation } from 'services/api';
-  Graph,
-  RandomIntInvocation,
-  RangeOfSizeInvocation,
-} from 'services/api';
 import {
   ITERATE,
   LATENTS_TO_IMAGE,
@@ -19,7 +15,15 @@ import {
 } from './constants';
 import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph';
 
-export const buildTextToImageGraph = (state: RootState): Graph => {
+type TextToImageGraphOverrides = {
+  width: number;
+  height: number;
+};
+
+export const buildLinearTextToImageGraph = (
+  state: RootState,
+  overrides?: TextToImageGraphOverrides
+): NonNullableGraph => {
   const {
     positivePrompt,
     negativePrompt,
@@ -67,8 +71,8 @@ export const buildTextToImageGraph = (state: RootState): Graph => {
       [NOISE]: {
         type: 'noise',
         id: NOISE,
-        width,
+        width: overrides?.width || width,
-        height,
+        height: overrides?.height || height,
       },
       [TEXT_TO_LATENTS]: {
         type: 't2l',

invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts

@@ -1,3 +1,4 @@
+// friendly node ids
 export const POSITIVE_CONDITIONING = 'positive_conditioning';
 export const NEGATIVE_CONDITIONING = 'negative_conditioning';
 export const TEXT_TO_LATENTS = 'text_to_latents';
@@ -10,8 +11,10 @@ export const MODEL_LOADER = 'model_loader';
 export const IMAGE_TO_LATENTS = 'image_to_latents';
 export const LATENTS_TO_LATENTS = 'latents_to_latents';
 export const RESIZE = 'resize_image';
+export const INPAINT = 'inpaint';
+export const CONTROL_NET_COLLECT = 'control_net_collect';
 
+// friendly graph ids
 export const TEXT_TO_IMAGE_GRAPH = 'text_to_image_graph';
 export const IMAGE_TO_IMAGE_GRAPH = 'image_to_image_graph';
-
+export const INPAINT_GRAPH = 'inpaint_graph';
-export const CONTROL_NET_COLLECT = 'control_net_collect';

invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx

@@ -1,5 +1,4 @@
 import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons';
-import ParamSeedCollapse from 'features/parameters/components/Parameters/Seed/ParamSeedCollapse';
 import ParamVariationCollapse from 'features/parameters/components/Parameters/Variations/ParamVariationCollapse';
 import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse';
 import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse';
@@ -8,6 +7,7 @@ import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters';
 import { memo } from 'react';
 import ParamPositiveConditioning from 'features/parameters/components/Parameters/Core/ParamPositiveConditioning';
 import ParamNegativeConditioning from 'features/parameters/components/Parameters/Core/ParamNegativeConditioning';
+import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse';
 
 const UnifiedCanvasParameters = () => {
   return (
@@ -16,6 +16,7 @@ const UnifiedCanvasParameters = () => {
       <ParamNegativeConditioning />
       <ProcessButtons />
       <UnifiedCanvasCoreParameters />
+      <ParamControlNetCollapse />
       <ParamVariationCollapse />
       <ParamSymmetryCollapse />
       <ParamSeamCorrectionCollapse />