From 41442eb7f658c76bbf9971654f47011a8aa9d9b6 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Mon, 19 Jun 2023 15:55:53 +1000 Subject: [PATCH] feat(ui): convert canvas txt2img & img2img to latents - Add graph builders for canvas txt2img & img2img - they are mostly copy and paste from the linear graph builders but different in a few ways that are very tricky to work around. Just made totally new functions for them. - Canvas txt2img and img2img support ControlNet (not inpaint/outpaint). There's no way to determine in real-time which mode the canvas is in just yet, so we cannot disable the ControlNet UI when the mode will be inpaint/outpaint - it will always display. It's possible to determine this in near-real-time, will add this at some point. - Canvas inpaint/outpaint migrated to use model loader, though inpaint/outpaint are still using the non-latents nodes. --- .../listeners/userInvokedCanvas.ts | 117 +++---- .../listeners/userInvokedImageToImage.ts | 4 +- .../listeners/userInvokedTextToImage.ts | 4 +- .../util/graphBuilders/buildCanvasGraph.ts | 133 ++----- .../buildCanvasImageToImageGraph.ts | 331 ++++++++++++++++++ .../graphBuilders/buildCanvasInpaintGraph.ts | 224 ++++++++++++ .../buildCanvasTextToImageGraph.ts | 224 ++++++++++++ ...aph.ts => buildLinearImageToImageGraph.ts} | 8 +- ...raph.ts => buildLinearTextToImageGraph.ts} | 20 +- .../nodes/util/graphBuilders/constants.ts | 7 +- .../UnifiedCanvas/UnifiedCanvasParameters.tsx | 3 +- 11 files changed, 890 insertions(+), 185 deletions(-) create mode 100644 invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts create mode 100644 invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts create mode 100644 invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts rename invokeai/frontend/web/src/features/nodes/util/graphBuilders/{buildImageToImageGraph.ts => buildLinearImageToImageGraph.ts} (98%) rename invokeai/frontend/web/src/features/nodes/util/graphBuilders/{buildTextToImageGraph.ts => buildLinearTextToImageGraph.ts} (93%) diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts index 4d8177d7f3..a26d872d50 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts @@ -1,11 +1,10 @@ import { startAppListening } from '..'; import { sessionCreated } from 'services/thunks/session'; -import { buildCanvasGraphComponents } from 'features/nodes/util/graphBuilders/buildCanvasGraph'; +import { buildCanvasGraph } from 'features/nodes/util/graphBuilders/buildCanvasGraph'; import { log } from 'app/logging/useLogger'; import { canvasGraphBuilt } from 'features/nodes/store/actions'; import { imageUpdated, imageUploaded } from 'services/thunks/image'; -import { v4 as uuidv4 } from 'uuid'; -import { Graph } from 'services/api'; +import { ImageDTO } from 'services/api'; import { canvasSessionIdChanged, stagingAreaInitialized, @@ -67,112 +66,106 @@ export const addUserInvokedCanvasListener = () => { moduleLog.debug(`Generation mode: ${generationMode}`); - // Build the canvas graph - const graphComponents = await buildCanvasGraphComponents( - state, - generationMode - ); + // Temp placeholders for the init and mask images + let canvasInitImage: ImageDTO | undefined; + let canvasMaskImage: ImageDTO | undefined; - if (!graphComponents) { - moduleLog.error('Problem building graph'); - return; - } - - const { rangeNode, iterateNode, baseNode, edges } = graphComponents; - - // Assemble! Note that this graph *does not have the init or mask image set yet!* - const nodes: Graph['nodes'] = { - [rangeNode.id]: rangeNode, - [iterateNode.id]: iterateNode, - [baseNode.id]: baseNode, - }; - - const graph = { nodes, edges }; - - dispatch(canvasGraphBuilt(graph)); - - moduleLog.debug({ data: graph }, 'Canvas graph built'); - - // If we are generating img2img or inpaint, we need to upload the init images - if (baseNode.type === 'img2img' || baseNode.type === 'inpaint') { - const baseFilename = `${uuidv4()}.png`; - dispatch( + // For img2img and inpaint/outpaint, we need to upload the init images + if (['img2img', 'inpaint', 'outpaint'].includes(generationMode)) { + // upload the image, saving the request id + const { requestId: initImageUploadedRequestId } = dispatch( imageUploaded({ formData: { - file: new File([baseBlob], baseFilename, { type: 'image/png' }), + file: new File([baseBlob], 'canvasInitImage.png', { + type: 'image/png', + }), }, imageCategory: 'general', isIntermediate: true, }) ); - // Wait for the image to be uploaded - const [{ payload: baseImageDTO }] = await take( + // Wait for the image to be uploaded, matching by request id + const [{ payload }] = await take( (action): action is ReturnType => imageUploaded.fulfilled.match(action) && - action.meta.arg.formData.file.name === baseFilename + action.meta.requestId === initImageUploadedRequestId ); - // Update the base node with the image name and type - baseNode.image = { - image_name: baseImageDTO.image_name, - }; + canvasInitImage = payload; } - // For inpaint, we also need to upload the mask layer - if (baseNode.type === 'inpaint') { - const maskFilename = `${uuidv4()}.png`; - dispatch( + // For inpaint/outpaint, we also need to upload the mask layer + if (['inpaint', 'outpaint'].includes(generationMode)) { + // upload the image, saving the request id + const { requestId: maskImageUploadedRequestId } = dispatch( imageUploaded({ formData: { - file: new File([maskBlob], maskFilename, { type: 'image/png' }), + file: new File([maskBlob], 'canvasMaskImage.png', { + type: 'image/png', + }), }, imageCategory: 'mask', isIntermediate: true, }) ); - // Wait for the mask to be uploaded - const [{ payload: maskImageDTO }] = await take( + // Wait for the image to be uploaded, matching by request id + const [{ payload }] = await take( (action): action is ReturnType => imageUploaded.fulfilled.match(action) && - action.meta.arg.formData.file.name === maskFilename + action.meta.requestId === maskImageUploadedRequestId ); - // Update the base node with the image name and type - baseNode.mask = { - image_name: maskImageDTO.image_name, - }; + canvasMaskImage = payload; } - // Create the session and wait for response - dispatch(sessionCreated({ graph })); - const [sessionCreatedAction] = await take(sessionCreated.fulfilled.match); + const graph = buildCanvasGraph( + state, + generationMode, + canvasInitImage, + canvasMaskImage + ); + + moduleLog.debug({ graph }, `Canvas graph built`); + + // currently this action is just listened to for logging + dispatch(canvasGraphBuilt(graph)); + + // Create the session, store the request id + const { requestId: sessionCreatedRequestId } = dispatch( + sessionCreated({ graph }) + ); + + // Take the session created action, matching by its request id + const [sessionCreatedAction] = await take( + (action): action is ReturnType => + sessionCreated.fulfilled.match(action) && + action.meta.requestId === sessionCreatedRequestId + ); const sessionId = sessionCreatedAction.payload.id; // Associate the init image with the session, now that we have the session ID - if ( - (baseNode.type === 'img2img' || baseNode.type === 'inpaint') && - baseNode.image - ) { + if (['img2img', 'inpaint'].includes(generationMode) && canvasInitImage) { dispatch( imageUpdated({ - imageName: baseNode.image.image_name, + imageName: canvasInitImage.image_name, requestBody: { session_id: sessionId }, }) ); } // Associate the mask image with the session, now that we have the session ID - if (baseNode.type === 'inpaint' && baseNode.mask) { + if (['inpaint'].includes(generationMode) && canvasMaskImage) { dispatch( imageUpdated({ - imageName: baseNode.mask.image_name, + imageName: canvasMaskImage.image_name, requestBody: { session_id: sessionId }, }) ); } + // Prep the canvas staging area if it is not yet initialized if (!state.canvas.layerState.stagingArea.boundingBox) { dispatch( stagingAreaInitialized({ diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts index dc67ebf073..368d97a10f 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts @@ -4,7 +4,7 @@ import { log } from 'app/logging/useLogger'; import { imageToImageGraphBuilt } from 'features/nodes/store/actions'; import { userInvoked } from 'app/store/actions'; import { sessionReadyToInvoke } from 'features/system/store/actions'; -import { buildImageToImageGraph } from 'features/nodes/util/graphBuilders/buildImageToImageGraph'; +import { buildLinearImageToImageGraph } from 'features/nodes/util/graphBuilders/buildLinearImageToImageGraph'; const moduleLog = log.child({ namespace: 'invoke' }); @@ -15,7 +15,7 @@ export const addUserInvokedImageToImageListener = () => { effect: async (action, { getState, dispatch, take }) => { const state = getState(); - const graph = buildImageToImageGraph(state); + const graph = buildLinearImageToImageGraph(state); dispatch(imageToImageGraphBuilt(graph)); moduleLog.debug({ data: graph }, 'Image to Image graph built'); diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts index 0538022d39..c76e0dfd4f 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts @@ -4,7 +4,7 @@ import { log } from 'app/logging/useLogger'; import { textToImageGraphBuilt } from 'features/nodes/store/actions'; import { userInvoked } from 'app/store/actions'; import { sessionReadyToInvoke } from 'features/system/store/actions'; -import { buildTextToImageGraph } from 'features/nodes/util/graphBuilders/buildTextToImageGraph'; +import { buildLinearTextToImageGraph } from 'features/nodes/util/graphBuilders/buildLinearTextToImageGraph'; const moduleLog = log.child({ namespace: 'invoke' }); @@ -15,7 +15,7 @@ export const addUserInvokedTextToImageListener = () => { effect: async (action, { getState, dispatch, take }) => { const state = getState(); - const graph = buildTextToImageGraph(state); + const graph = buildLinearTextToImageGraph(state); dispatch(textToImageGraphBuilt(graph)); diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts index 2d23b882ea..3ea513fe7e 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts @@ -1,116 +1,39 @@ import { RootState } from 'app/store/store'; -import { - Edge, - ImageToImageInvocation, - InpaintInvocation, - IterateInvocation, - RandomRangeInvocation, - RangeInvocation, - TextToImageInvocation, -} from 'services/api'; -import { buildImg2ImgNode } from '../nodeBuilders/buildImageToImageNode'; -import { buildTxt2ImgNode } from '../nodeBuilders/buildTextToImageNode'; -import { buildRangeNode } from '../nodeBuilders/buildRangeNode'; -import { buildIterateNode } from '../nodeBuilders/buildIterateNode'; -import { buildEdges } from '../edgeBuilders/buildEdges'; +import { ImageDTO } from 'services/api'; import { log } from 'app/logging/useLogger'; -import { buildInpaintNode } from '../nodeBuilders/buildInpaintNode'; +import { forEach } from 'lodash-es'; +import { buildCanvasInpaintGraph } from './buildCanvasInpaintGraph'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { buildCanvasImageToImageGraph } from './buildCanvasImageToImageGraph'; +import { buildCanvasTextToImageGraph } from './buildCanvasTextToImageGraph'; const moduleLog = log.child({ namespace: 'nodes' }); -const buildBaseNode = ( - nodeType: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint', - state: RootState -): - | TextToImageInvocation - | ImageToImageInvocation - | InpaintInvocation - | undefined => { - const overrides = { - ...state.canvas.boundingBoxDimensions, - is_intermediate: true, - }; - - if (nodeType === 'txt2img') { - return buildTxt2ImgNode(state, overrides); - } - - if (nodeType === 'img2img') { - return buildImg2ImgNode(state, overrides); - } - - if (nodeType === 'inpaint' || nodeType === 'outpaint') { - return buildInpaintNode(state, overrides); - } -}; - -/** - * Builds the Canvas workflow graph and image blobs. - */ -export const buildCanvasGraphComponents = async ( +export const buildCanvasGraph = ( state: RootState, - generationMode: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint' -): Promise< - | { - rangeNode: RangeInvocation | RandomRangeInvocation; - iterateNode: IterateInvocation; - baseNode: - | TextToImageInvocation - | ImageToImageInvocation - | InpaintInvocation; - edges: Edge[]; - } - | undefined -> => { - // The base node is a txt2img, img2img or inpaint node - const baseNode = buildBaseNode(generationMode, state); + generationMode: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint', + canvasInitImage: ImageDTO | undefined, + canvasMaskImage: ImageDTO | undefined +) => { + let graph: NonNullableGraph; - if (!baseNode) { - moduleLog.error('Problem building base node'); - return; + if (generationMode === 'txt2img') { + graph = buildCanvasTextToImageGraph(state); + } else if (generationMode === 'img2img') { + if (!canvasInitImage) { + throw new Error('Missing canvas init image'); + } + graph = buildCanvasImageToImageGraph(state, canvasInitImage); + } else { + if (!canvasInitImage || !canvasMaskImage) { + throw new Error('Missing canvas init and mask images'); + } + graph = buildCanvasInpaintGraph(state, canvasInitImage, canvasMaskImage); } - if (baseNode.type === 'inpaint') { - const { - seamSize, - seamBlur, - seamSteps, - seamStrength, - tileSize, - infillMethod, - } = state.generation; + forEach(graph.nodes, (node) => { + graph.nodes[node.id].is_intermediate = true; + }); - const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } = - state.canvas; - - if (boundingBoxScaleMethod !== 'none') { - baseNode.inpaint_width = scaledBoundingBoxDimensions.width; - baseNode.inpaint_height = scaledBoundingBoxDimensions.height; - } - - baseNode.seam_size = seamSize; - baseNode.seam_blur = seamBlur; - baseNode.seam_strength = seamStrength; - baseNode.seam_steps = seamSteps; - baseNode.infill_method = infillMethod as InpaintInvocation['infill_method']; - - if (infillMethod === 'tile') { - baseNode.tile_size = tileSize; - } - } - - // We always range and iterate nodes, no matter the iteration count - // This is required to provide the correct seeds to the backend engine - const rangeNode = buildRangeNode(state); - const iterateNode = buildIterateNode(); - - // Build the edges for the nodes selected. - const edges = buildEdges(baseNode, rangeNode, iterateNode); - - return { - rangeNode, - iterateNode, - baseNode, - edges, - }; + return graph; }; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts new file mode 100644 index 0000000000..efaeaddff2 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts @@ -0,0 +1,331 @@ +import { RootState } from 'app/store/store'; +import { + ImageDTO, + ImageResizeInvocation, + RandomIntInvocation, + RangeOfSizeInvocation, +} from 'services/api'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { log } from 'app/logging/useLogger'; +import { + ITERATE, + LATENTS_TO_IMAGE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + NOISE, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + IMAGE_TO_IMAGE_GRAPH, + IMAGE_TO_LATENTS, + LATENTS_TO_LATENTS, + RESIZE, +} from './constants'; +import { set } from 'lodash-es'; +import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + +const moduleLog = log.child({ namespace: 'nodes' }); + +/** + * Builds the Canvas tab's Image to Image graph. + */ +export const buildCanvasImageToImageGraph = ( + state: RootState, + initialImage: ImageDTO +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + img2imgStrength: strength, + iterations, + seed, + shouldRandomizeSeed, + } = state.generation; + + // The bounding box determines width and height, not the width and height params + const { width, height } = state.canvas.boundingBoxDimensions; + + /** + * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the + * full graph here as a template. Then use the parameters from app state and set friendlier node + * ids. + * + * The only thing we need extra logic for is handling randomized seed, control net, and for img2img, + * the `fit` param. These are added to the graph at the end. + */ + + // copy-pasted graph from node editor, filled in with state values & friendly node ids + const graph: NonNullableGraph = { + id: IMAGE_TO_IMAGE_GRAPH, + nodes: { + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // seed - must be connected manually + // start: 0, + size: iterations, + step: 1, + }, + [NOISE]: { + type: 'noise', + id: NOISE, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [LATENTS_TO_IMAGE]: { + type: 'l2i', + id: LATENTS_TO_IMAGE, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + [LATENTS_TO_LATENTS]: { + type: 'l2l', + id: LATENTS_TO_LATENTS, + cfg_scale, + scheduler, + steps, + strength, + }, + [IMAGE_TO_LATENTS]: { + type: 'i2l', + id: IMAGE_TO_LATENTS, + // must be set manually later, bc `fit` parameter may require a resize node inserted + // image: { + // image_name: initialImage.image_name, + // }, + }, + }, + edges: [ + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: NOISE, + field: 'seed', + }, + }, + { + source: { + node_id: LATENTS_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'latents', + }, + }, + { + source: { + node_id: IMAGE_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'latents', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'noise', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'vae', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'positive_conditioning', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + // handle `fit` + if (initialImage.width !== width || initialImage.height !== height) { + // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS` + + // Create a resize node, explicitly setting its image + const resizeNode: ImageResizeInvocation = { + id: RESIZE, + type: 'img_resize', + image: { + image_name: initialImage.image_name, + }, + is_intermediate: true, + width, + height, + }; + + graph.nodes[RESIZE] = resizeNode; + + // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS` + graph.edges.push({ + source: { node_id: RESIZE, field: 'image' }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'image', + }, + }); + + // The `RESIZE` node also passes its width and height to `NOISE` + graph.edges.push({ + source: { node_id: RESIZE, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + + graph.edges.push({ + source: { node_id: RESIZE, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } else { + // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly + set(graph.nodes[IMAGE_TO_LATENTS], 'image', { + image_name: initialImage.image_name, + }); + + // Pass the image's dimensions to the `NOISE` node + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } + + // add controlnet + addControlNetToLinearGraph(graph, LATENTS_TO_LATENTS, state); + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts new file mode 100644 index 0000000000..785e1d2fdb --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts @@ -0,0 +1,224 @@ +import { RootState } from 'app/store/store'; +import { + ImageDTO, + InpaintInvocation, + RandomIntInvocation, + RangeOfSizeInvocation, +} from 'services/api'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { log } from 'app/logging/useLogger'; +import { + ITERATE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + INPAINT_GRAPH, + INPAINT, +} from './constants'; + +const moduleLog = log.child({ namespace: 'nodes' }); + +/** + * Builds the Canvas tab's Inpaint graph. + */ +export const buildCanvasInpaintGraph = ( + state: RootState, + canvasInitImage: ImageDTO, + canvasMaskImage: ImageDTO +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + img2imgStrength: strength, + shouldFitToWidthHeight, + iterations, + seed, + shouldRandomizeSeed, + seamSize, + seamBlur, + seamSteps, + seamStrength, + tileSize, + infillMethod, + } = state.generation; + + // The bounding box determines width and height, not the width and height params + const { width, height } = state.canvas.boundingBoxDimensions; + + // We may need to set the inpaint width and height to scale the image + const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } = state.canvas; + + const graph: NonNullableGraph = { + id: INPAINT_GRAPH, + nodes: { + [INPAINT]: { + type: 'inpaint', + id: INPAINT, + steps, + width, + height, + cfg_scale, + scheduler, + image: { + image_name: canvasInitImage.image_name, + }, + strength, + fit: shouldFitToWidthHeight, + mask: { + image_name: canvasMaskImage.image_name, + }, + seam_size: seamSize, + seam_blur: seamBlur, + seam_strength: seamStrength, + seam_steps: seamSteps, + tile_size: infillMethod === 'tile' ? tileSize : undefined, + infill_method: infillMethod as InpaintInvocation['infill_method'], + inpaint_width: + boundingBoxScaleMethod !== 'none' + ? scaledBoundingBoxDimensions.width + : undefined, + inpaint_height: + boundingBoxScaleMethod !== 'none' + ? scaledBoundingBoxDimensions.height + : undefined, + }, + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // seed - must be connected manually + // start: 0, + size: iterations, + step: 1, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + }, + edges: [ + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: INPAINT, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: INPAINT, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: INPAINT, + field: 'unet', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: INPAINT, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: INPAINT, + field: 'seed', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts new file mode 100644 index 0000000000..ca0e56e849 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts @@ -0,0 +1,224 @@ +import { RootState } from 'app/store/store'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { RandomIntInvocation, RangeOfSizeInvocation } from 'services/api'; +import { + ITERATE, + LATENTS_TO_IMAGE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + NOISE, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + TEXT_TO_IMAGE_GRAPH, + TEXT_TO_LATENTS, +} from './constants'; +import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + +/** + * Builds the Canvas tab's Text to Image graph. + */ +export const buildCanvasTextToImageGraph = ( + state: RootState +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + iterations, + seed, + shouldRandomizeSeed, + } = state.generation; + + // The bounding box determines width and height, not the width and height params + const { width, height } = state.canvas.boundingBoxDimensions; + + /** + * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the + * full graph here as a template. Then use the parameters from app state and set friendlier node + * ids. + * + * The only thing we need extra logic for is handling randomized seed, control net, and for img2img, + * the `fit` param. These are added to the graph at the end. + */ + + // copy-pasted graph from node editor, filled in with state values & friendly node ids + const graph: NonNullableGraph = { + id: TEXT_TO_IMAGE_GRAPH, + nodes: { + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // start: 0, // seed - must be connected manually + size: iterations, + step: 1, + }, + [NOISE]: { + type: 'noise', + id: NOISE, + width, + height, + }, + [TEXT_TO_LATENTS]: { + type: 't2l', + id: TEXT_TO_LATENTS, + cfg_scale, + scheduler, + steps, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [LATENTS_TO_IMAGE]: { + type: 'l2i', + id: LATENTS_TO_IMAGE, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + }, + edges: [ + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: TEXT_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'latents', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: NOISE, + field: 'seed', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'noise', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + // add controlnet + addControlNetToLinearGraph(graph, TEXT_TO_LATENTS, state); + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildImageToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts similarity index 98% rename from invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildImageToImageGraph.ts rename to invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts index a46af7199f..1f2c8327e0 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildImageToImageGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts @@ -1,6 +1,5 @@ import { RootState } from 'app/store/store'; import { - Graph, ImageResizeInvocation, RandomIntInvocation, RangeOfSizeInvocation, @@ -23,12 +22,15 @@ import { } from './constants'; import { set } from 'lodash-es'; import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + const moduleLog = log.child({ namespace: 'nodes' }); /** * Builds the Image to Image tab graph. */ -export const buildImageToImageGraph = (state: RootState): Graph => { +export const buildLinearImageToImageGraph = ( + state: RootState +): NonNullableGraph => { const { positivePrompt, negativePrompt, @@ -275,8 +277,8 @@ export const buildImageToImageGraph = (state: RootState): Graph => { image_name: initialImage.image_name, }, is_intermediate: true, - height, width, + height, }; graph.nodes[RESIZE] = resizeNode; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildTextToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts similarity index 93% rename from invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildTextToImageGraph.ts rename to invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts index 945f96a9e3..c179a89504 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildTextToImageGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts @@ -1,10 +1,6 @@ import { RootState } from 'app/store/store'; import { NonNullableGraph } from 'features/nodes/types/types'; -import { - Graph, - RandomIntInvocation, - RangeOfSizeInvocation, -} from 'services/api'; +import { RandomIntInvocation, RangeOfSizeInvocation } from 'services/api'; import { ITERATE, LATENTS_TO_IMAGE, @@ -19,7 +15,15 @@ import { } from './constants'; import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; -export const buildTextToImageGraph = (state: RootState): Graph => { +type TextToImageGraphOverrides = { + width: number; + height: number; +}; + +export const buildLinearTextToImageGraph = ( + state: RootState, + overrides?: TextToImageGraphOverrides +): NonNullableGraph => { const { positivePrompt, negativePrompt, @@ -67,8 +71,8 @@ export const buildTextToImageGraph = (state: RootState): Graph => { [NOISE]: { type: 'noise', id: NOISE, - width, - height, + width: overrides?.width || width, + height: overrides?.height || height, }, [TEXT_TO_LATENTS]: { type: 't2l', diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts index a65830e47f..39e0080d11 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts @@ -1,3 +1,4 @@ +// friendly node ids export const POSITIVE_CONDITIONING = 'positive_conditioning'; export const NEGATIVE_CONDITIONING = 'negative_conditioning'; export const TEXT_TO_LATENTS = 'text_to_latents'; @@ -10,8 +11,10 @@ export const MODEL_LOADER = 'model_loader'; export const IMAGE_TO_LATENTS = 'image_to_latents'; export const LATENTS_TO_LATENTS = 'latents_to_latents'; export const RESIZE = 'resize_image'; +export const INPAINT = 'inpaint'; +export const CONTROL_NET_COLLECT = 'control_net_collect'; +// friendly graph ids export const TEXT_TO_IMAGE_GRAPH = 'text_to_image_graph'; export const IMAGE_TO_IMAGE_GRAPH = 'image_to_image_graph'; - -export const CONTROL_NET_COLLECT = 'control_net_collect'; +export const INPAINT_GRAPH = 'inpaint_graph'; diff --git a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx index 19ef7fd6fa..8e17ff066c 100644 --- a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx +++ b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx @@ -1,5 +1,4 @@ import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons'; -import ParamSeedCollapse from 'features/parameters/components/Parameters/Seed/ParamSeedCollapse'; import ParamVariationCollapse from 'features/parameters/components/Parameters/Variations/ParamVariationCollapse'; import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse'; import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse'; @@ -8,6 +7,7 @@ import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters'; import { memo } from 'react'; import ParamPositiveConditioning from 'features/parameters/components/Parameters/Core/ParamPositiveConditioning'; import ParamNegativeConditioning from 'features/parameters/components/Parameters/Core/ParamNegativeConditioning'; +import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse'; const UnifiedCanvasParameters = () => { return ( @@ -16,6 +16,7 @@ const UnifiedCanvasParameters = () => { +