diff --git a/docs/index.md b/docs/index.md index 4587b08f18..0aa99a1747 100644 --- a/docs/index.md +++ b/docs/index.md @@ -67,7 +67,7 @@ title: Home implementation of Stable Diffusion, the open source text-to-image and image-to-image generator. It provides a streamlined process with various new features and options to aid the image generation process. It runs on Windows, -Mac and Linux machines, and runs on GPU cards with as little as 4 GB or RAM. +Mac and Linux machines, and runs on GPU cards with as little as 4 GB of RAM. **Quick links**: [Discord Server] [Code and Downloads] [ None: - stable_diffusion_step_callback( - context=context, - intermediate_state=intermediate_state, - node=self.dict(), - source_node_id=source_node_id, - ) - - def invoke(self, context: InvocationContext) -> ImageOutput: - # Handle invalid model parameter - model = context.services.model_manager.get_model(self.model,node=self,context=context) - - # loading controlnet image (currently requires pre-processed image) - control_image = ( - None if self.control_image is None - else context.services.images.get_pil_image(self.control_image.image_name) - ) - # loading controlnet model - if (self.control_model is None or self.control_model==''): - control_model = None - else: - # FIXME: change this to dropdown menu? - # FIXME: generalize so don't have to hardcode torch_dtype and device - control_model = ControlNetModel.from_pretrained(self.control_model, - torch_dtype=torch.float16).to("cuda") - - # Get the source node id (we are invoking the prepared node) - graph_execution_state = context.services.graph_execution_manager.get( - context.graph_execution_state_id - ) - source_node_id = graph_execution_state.prepared_source_mapping[self.id] - - txt2img = Txt2Img(model, control_model=control_model) - outputs = txt2img.generate( - prompt=self.prompt, - step_callback=partial(self.dispatch_progress, context, source_node_id), - control_image=control_image, - **self.dict( - exclude={"prompt", "control_image" } - ), # Shorthand for passing all of the parameters above manually - ) - # Outputs is an infinite iterator that will return a new InvokeAIGeneratorOutput object - # each time it is called. We only need the first one. - generate_output = next(outputs) - - image_dto = context.services.images.create( - image=generate_output.image, - image_origin=ResourceOrigin.INTERNAL, - image_category=ImageCategory.GENERAL, - session_id=context.graph_execution_state_id, - node_id=self.id, - is_intermediate=self.is_intermediate, - ) - - return ImageOutput( - image=ImageField(image_name=image_dto.image_name), - width=image_dto.width, - height=image_dto.height, - ) - - -class ImageToImageInvocation(TextToImageInvocation): - """Generates an image using img2img.""" - - type: Literal["img2img"] = "img2img" + unet: UNetField = Field(default=None, description="UNet model") + vae: VaeField = Field(default=None, description="Vae model") # Inputs image: Union[ImageField, None] = Field(description="The input image") @@ -144,72 +85,6 @@ class ImageToImageInvocation(TextToImageInvocation): description="Whether or not the result should be fit to the aspect ratio of the input image", ) - def dispatch_progress( - self, - context: InvocationContext, - source_node_id: str, - intermediate_state: PipelineIntermediateState, - ) -> None: - stable_diffusion_step_callback( - context=context, - intermediate_state=intermediate_state, - node=self.dict(), - source_node_id=source_node_id, - ) - - def invoke(self, context: InvocationContext) -> ImageOutput: - image = ( - None - if self.image is None - else context.services.images.get_pil_image(self.image.image_name) - ) - - if self.fit: - image = image.resize((self.width, self.height)) - - # Handle invalid model parameter - model = context.services.model_manager.get_model(self.model,node=self,context=context) - - # Get the source node id (we are invoking the prepared node) - graph_execution_state = context.services.graph_execution_manager.get( - context.graph_execution_state_id - ) - source_node_id = graph_execution_state.prepared_source_mapping[self.id] - - outputs = Img2Img(model).generate( - prompt=self.prompt, - init_image=image, - step_callback=partial(self.dispatch_progress, context, source_node_id), - **self.dict( - exclude={"prompt", "image", "mask"} - ), # Shorthand for passing all of the parameters above manually - ) - - # Outputs is an infinite iterator that will return a new InvokeAIGeneratorOutput object - # each time it is called. We only need the first one. - generator_output = next(outputs) - - image_dto = context.services.images.create( - image=generator_output.image, - image_origin=ResourceOrigin.INTERNAL, - image_category=ImageCategory.GENERAL, - session_id=context.graph_execution_state_id, - node_id=self.id, - is_intermediate=self.is_intermediate, - ) - - return ImageOutput( - image=ImageField(image_name=image_dto.image_name), - width=image_dto.width, - height=image_dto.height, - ) - - -class InpaintInvocation(ImageToImageInvocation): - """Generates an image using inpaint.""" - - type: Literal["inpaint"] = "inpaint" - # Inputs mask: Union[ImageField, None] = Field(description="The mask") seam_size: int = Field(default=96, ge=1, description="The seam inpaint size (px)") @@ -252,6 +127,14 @@ class InpaintInvocation(ImageToImageInvocation): description="The amount by which to replace masked areas with latent noise", ) + # Schema customisation + class Config(InvocationConfig): + schema_extra = { + "ui": { + "tags": ["stable-diffusion", "image"], + }, + } + def dispatch_progress( self, context: InvocationContext, @@ -265,6 +148,49 @@ class InpaintInvocation(ImageToImageInvocation): source_node_id=source_node_id, ) + def get_conditioning(self, context): + c, extra_conditioning_info = context.services.latents.get(self.positive_conditioning.conditioning_name) + uc, _ = context.services.latents.get(self.negative_conditioning.conditioning_name) + + return (uc, c, extra_conditioning_info) + + @contextmanager + def load_model_old_way(self, context, scheduler): + unet_info = context.services.model_manager.get_model(**self.unet.unet.dict()) + vae_info = context.services.model_manager.get_model(**self.vae.vae.dict()) + + #unet = unet_info.context.model + #vae = vae_info.context.model + + with ExitStack() as stack: + loras = [(stack.enter_context(context.services.model_manager.get_model(**lora.dict(exclude={"weight"}))), lora.weight) for lora in self.unet.loras] + + with vae_info as vae,\ + unet_info as unet,\ + ModelPatcher.apply_lora_unet(unet, loras): + + device = context.services.model_manager.mgr.cache.execution_device + dtype = context.services.model_manager.mgr.cache.precision + + pipeline = StableDiffusionGeneratorPipeline( + vae=vae, + text_encoder=None, + tokenizer=None, + unet=unet, + scheduler=scheduler, + safety_checker=None, + feature_extractor=None, + requires_safety_checker=False, + precision="float16" if dtype == torch.float16 else "float32", + execution_device=device, + ) + + yield OldModelInfo( + name=self.unet.unet.model_name, + hash="", + model=pipeline, + ) + def invoke(self, context: InvocationContext) -> ImageOutput: image = ( None @@ -277,25 +203,31 @@ class InpaintInvocation(ImageToImageInvocation): else context.services.images.get_pil_image(self.mask.image_name) ) - # Handle invalid model parameter - model = context.services.model_manager.get_model(self.model,node=self,context=context) - # Get the source node id (we are invoking the prepared node) graph_execution_state = context.services.graph_execution_manager.get( context.graph_execution_state_id ) source_node_id = graph_execution_state.prepared_source_mapping[self.id] - outputs = Inpaint(model).generate( - prompt=self.prompt, - init_image=image, - mask_image=mask, - step_callback=partial(self.dispatch_progress, context, source_node_id), - **self.dict( - exclude={"prompt", "image", "mask"} - ), # Shorthand for passing all of the parameters above manually + conditioning = self.get_conditioning(context) + scheduler = get_scheduler( + context=context, + scheduler_info=self.unet.scheduler, + scheduler_name=self.scheduler, ) + with self.load_model_old_way(context, scheduler) as model: + outputs = Inpaint(model).generate( + conditioning=conditioning, + scheduler=scheduler, + init_image=image, + mask_image=mask, + step_callback=partial(self.dispatch_progress, context, source_node_id), + **self.dict( + exclude={"positive_conditioning", "negative_conditioning", "scheduler", "image", "mask"} + ), # Shorthand for passing all of the parameters above manually + ) + # Outputs is an infinite iterator that will return a new InvokeAIGeneratorOutput object # each time it is called. We only need the first one. generator_output = next(outputs) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index cf216e6c54..63db3d925c 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -7,7 +7,7 @@ import einops from pydantic import BaseModel, Field, validator import torch -from diffusers import ControlNetModel +from diffusers import ControlNetModel, DPMSolverMultistepScheduler from diffusers.image_processor import VaeImageProcessor from diffusers.schedulers import SchedulerMixin as Scheduler @@ -233,7 +233,17 @@ class TextToLatentsInvocation(BaseInvocation): h_symmetry_time_pct=None,#h_symmetry_time_pct, v_symmetry_time_pct=None#v_symmetry_time_pct, ), - ).add_scheduler_args_if_applicable(scheduler, eta=0.0)#ddim_eta) + ) + + conditioning_data = conditioning_data.add_scheduler_args_if_applicable( + scheduler, + + # for ddim scheduler + eta=0.0, #ddim_eta + + # for ancestral and sde schedulers + generator=torch.Generator(device=uc.device).manual_seed(0), + ) return conditioning_data def create_pipeline(self, unet, scheduler) -> StableDiffusionGeneratorPipeline: diff --git a/invokeai/backend/__init__.py b/invokeai/backend/__init__.py index 55782bc445..ff8b4bc8c5 100644 --- a/invokeai/backend/__init__.py +++ b/invokeai/backend/__init__.py @@ -5,7 +5,6 @@ from .generator import ( InvokeAIGeneratorBasicParams, InvokeAIGenerator, InvokeAIGeneratorOutput, - Txt2Img, Img2Img, Inpaint ) diff --git a/invokeai/backend/generator/__init__.py b/invokeai/backend/generator/__init__.py index 9d6263453a..8a7f1c9167 100644 --- a/invokeai/backend/generator/__init__.py +++ b/invokeai/backend/generator/__init__.py @@ -5,7 +5,6 @@ from .base import ( InvokeAIGenerator, InvokeAIGeneratorBasicParams, InvokeAIGeneratorOutput, - Txt2Img, Img2Img, Inpaint, Generator, diff --git a/invokeai/backend/generator/base.py b/invokeai/backend/generator/base.py index fb293ab5b2..462b1a4f4b 100644 --- a/invokeai/backend/generator/base.py +++ b/invokeai/backend/generator/base.py @@ -29,7 +29,6 @@ import invokeai.backend.util.logging as logger from ..image_util import configure_model_padding from ..util.util import rand_perlin_2d from ..safety_checker import SafetyChecker -from ..prompting.conditioning import get_uc_and_c_and_ec from ..stable_diffusion.diffusers_pipeline import StableDiffusionGeneratorPipeline from ..stable_diffusion.schedulers import SCHEDULER_MAP @@ -81,13 +80,15 @@ class InvokeAIGenerator(metaclass=ABCMeta): self.params=params self.kwargs = kwargs - def generate(self, - prompt: str='', - callback: Optional[Callable]=None, - step_callback: Optional[Callable]=None, - iterations: int=1, - **keyword_args, - )->Iterator[InvokeAIGeneratorOutput]: + def generate( + self, + conditioning: tuple, + scheduler, + callback: Optional[Callable]=None, + step_callback: Optional[Callable]=None, + iterations: int=1, + **keyword_args, + )->Iterator[InvokeAIGeneratorOutput]: ''' Return an iterator across the indicated number of generations. Each time the iterator is called it will return an InvokeAIGeneratorOutput @@ -116,11 +117,6 @@ class InvokeAIGenerator(metaclass=ABCMeta): model_name = model_info.name model_hash = model_info.hash with model_info.context as model: - scheduler: Scheduler = self.get_scheduler( - model=model, - scheduler_name=generator_args.get('scheduler') - ) - uc, c, extra_conditioning_info = get_uc_and_c_and_ec(prompt,model=model) gen_class = self._generator_class() generator = gen_class(model, self.params.precision, **self.kwargs) if self.params.variation_amount > 0: @@ -143,12 +139,12 @@ class InvokeAIGenerator(metaclass=ABCMeta): iteration_count = range(iterations) if iterations else itertools.count(start=0, step=1) for i in iteration_count: - results = generator.generate(prompt, - conditioning=(uc, c, extra_conditioning_info), - step_callback=step_callback, - sampler=scheduler, - **generator_args, - ) + results = generator.generate( + conditioning=conditioning, + step_callback=step_callback, + sampler=scheduler, + **generator_args, + ) output = InvokeAIGeneratorOutput( image=results[0][0], seed=results[0][1], @@ -170,20 +166,6 @@ class InvokeAIGenerator(metaclass=ABCMeta): def load_generator(self, model: StableDiffusionGeneratorPipeline, generator_class: Type[Generator]): return generator_class(model, self.params.precision) - def get_scheduler(self, scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP['ddim']) - - scheduler_config = model.scheduler.config - if "_backup" in scheduler_config: - scheduler_config = scheduler_config["_backup"] - scheduler_config = {**scheduler_config, **scheduler_extra_config, "_backup": scheduler_config} - scheduler = scheduler_class.from_config(scheduler_config) - - # hack copied over from generate.py - if not hasattr(scheduler, 'uses_inpainting_model'): - scheduler.uses_inpainting_model = lambda: False - return scheduler - @classmethod def _generator_class(cls)->Type[Generator]: ''' @@ -193,13 +175,6 @@ class InvokeAIGenerator(metaclass=ABCMeta): ''' return Generator -# ------------------------------------ -class Txt2Img(InvokeAIGenerator): - @classmethod - def _generator_class(cls): - from .txt2img import Txt2Img - return Txt2Img - # ------------------------------------ class Img2Img(InvokeAIGenerator): def generate(self, @@ -253,24 +228,6 @@ class Inpaint(Img2Img): from .inpaint import Inpaint return Inpaint -# ------------------------------------ -class Embiggen(Txt2Img): - def generate( - self, - embiggen: list=None, - embiggen_tiles: list = None, - strength: float=0.75, - **kwargs)->Iterator[InvokeAIGeneratorOutput]: - return super().generate(embiggen=embiggen, - embiggen_tiles=embiggen_tiles, - strength=strength, - **kwargs) - - @classmethod - def _generator_class(cls): - from .embiggen import Embiggen - return Embiggen - class Generator: downsampling_factor: int latent_channels: int @@ -281,7 +238,7 @@ class Generator: self.model = model self.precision = precision self.seed = None - self.latent_channels = model.channels + self.latent_channels = model.unet.config.in_channels self.downsampling_factor = downsampling # BUG: should come from model or config self.safety_checker = None self.perlin = 0.0 @@ -292,7 +249,7 @@ class Generator: self.free_gpu_mem = None # this is going to be overridden in img2img.py, txt2img.py and inpaint.py - def get_make_image(self, prompt, **kwargs): + def get_make_image(self, **kwargs): """ Returns a function returning an image derived from the prompt and the initial image Return value depends on the seed at the time you call it @@ -308,7 +265,6 @@ class Generator: def generate( self, - prompt, width, height, sampler, @@ -333,7 +289,6 @@ class Generator: saver.get_stacked_maps_image() ) make_image = self.get_make_image( - prompt, sampler=sampler, init_image=init_image, width=width, diff --git a/invokeai/backend/generator/embiggen.py b/invokeai/backend/generator/embiggen.py deleted file mode 100644 index 6eae5732b0..0000000000 --- a/invokeai/backend/generator/embiggen.py +++ /dev/null @@ -1,559 +0,0 @@ -""" -invokeai.backend.generator.embiggen descends from .generator -and generates with .generator.img2img -""" - -import numpy as np -import torch -from PIL import Image -from tqdm import trange - -import invokeai.backend.util.logging as logger - -from .base import Generator -from .img2img import Img2Img - -class Embiggen(Generator): - def __init__(self, model, precision): - super().__init__(model, precision) - self.init_latent = None - - # Replace generate because Embiggen doesn't need/use most of what it does normallly - def generate( - self, - prompt, - iterations=1, - seed=None, - image_callback=None, - step_callback=None, - **kwargs, - ): - make_image = self.get_make_image(prompt, step_callback=step_callback, **kwargs) - results = [] - seed = seed if seed else self.new_seed() - - # Noise will be generated by the Img2Img generator when called - for _ in trange(iterations, desc="Generating"): - # make_image will call Img2Img which will do the equivalent of get_noise itself - image = make_image() - results.append([image, seed]) - if image_callback is not None: - image_callback(image, seed, prompt_in=prompt) - seed = self.new_seed() - return results - - @torch.no_grad() - def get_make_image( - self, - prompt, - sampler, - steps, - cfg_scale, - ddim_eta, - conditioning, - init_img, - strength, - width, - height, - embiggen, - embiggen_tiles, - step_callback=None, - **kwargs, - ): - """ - Returns a function returning an image derived from the prompt and multi-stage twice-baked potato layering over the img2img on the initial image - Return value depends on the seed at the time you call it - """ - assert ( - not sampler.uses_inpainting_model() - ), "--embiggen is not supported by inpainting models" - - # Construct embiggen arg array, and sanity check arguments - if embiggen == None: # embiggen can also be called with just embiggen_tiles - embiggen = [1.0] # If not specified, assume no scaling - elif embiggen[0] < 0: - embiggen[0] = 1.0 - logger.warning( - "Embiggen scaling factor cannot be negative, fell back to the default of 1.0 !" - ) - if len(embiggen) < 2: - embiggen.append(0.75) - elif embiggen[1] > 1.0 or embiggen[1] < 0: - embiggen[1] = 0.75 - logger.warning( - "Embiggen upscaling strength for ESRGAN must be between 0 and 1, fell back to the default of 0.75 !" - ) - if len(embiggen) < 3: - embiggen.append(0.25) - elif embiggen[2] < 0: - embiggen[2] = 0.25 - logger.warning( - "Overlap size for Embiggen must be a positive ratio between 0 and 1 OR a number of pixels, fell back to the default of 0.25 !" - ) - - # Convert tiles from their user-freindly count-from-one to count-from-zero, because we need to do modulo math - # and then sort them, because... people. - if embiggen_tiles: - embiggen_tiles = list(map(lambda n: n - 1, embiggen_tiles)) - embiggen_tiles.sort() - - if strength >= 0.5: - logger.warning( - f"Embiggen may produce mirror motifs if the strength (-f) is too high (currently {strength}). Try values between 0.35-0.45." - ) - - # Prep img2img generator, since we wrap over it - gen_img2img = Img2Img(self.model, self.precision) - - # Open original init image (not a tensor) to manipulate - initsuperimage = Image.open(init_img) - - with Image.open(init_img) as img: - initsuperimage = img.convert("RGB") - - # Size of the target super init image in pixels - initsuperwidth, initsuperheight = initsuperimage.size - - # Increase by scaling factor if not already resized, using ESRGAN as able - if embiggen[0] != 1.0: - initsuperwidth = round(initsuperwidth * embiggen[0]) - initsuperheight = round(initsuperheight * embiggen[0]) - if embiggen[1] > 0: # No point in ESRGAN upscaling if strength is set zero - from ..restoration.realesrgan import ESRGAN - - esrgan = ESRGAN() - logger.info( - f"ESRGAN upscaling init image prior to cutting with Embiggen with strength {embiggen[1]}" - ) - if embiggen[0] > 2: - initsuperimage = esrgan.process( - initsuperimage, - embiggen[1], # upscale strength - self.seed, - 4, # upscale scale - ) - else: - initsuperimage = esrgan.process( - initsuperimage, - embiggen[1], # upscale strength - self.seed, - 2, # upscale scale - ) - # We could keep recursively re-running ESRGAN for a requested embiggen[0] larger than 4x - # but from personal experiance it doesn't greatly improve anything after 4x - # Resize to target scaling factor resolution - initsuperimage = initsuperimage.resize( - (initsuperwidth, initsuperheight), Image.Resampling.LANCZOS - ) - - # Use width and height as tile widths and height - # Determine buffer size in pixels - if embiggen[2] < 1: - if embiggen[2] < 0: - embiggen[2] = 0 - overlap_size_x = round(embiggen[2] * width) - overlap_size_y = round(embiggen[2] * height) - else: - overlap_size_x = round(embiggen[2]) - overlap_size_y = round(embiggen[2]) - - # With overall image width and height known, determine how many tiles we need - def ceildiv(a, b): - return -1 * (-a // b) - - # X and Y needs to be determined independantly (we may have savings on one based on the buffer pixel count) - # (initsuperwidth - width) is the area remaining to the right that we need to layers tiles to fill - # (width - overlap_size_x) is how much new we can fill with a single tile - emb_tiles_x = 1 - emb_tiles_y = 1 - if (initsuperwidth - width) > 0: - emb_tiles_x = ceildiv(initsuperwidth - width, width - overlap_size_x) + 1 - if (initsuperheight - height) > 0: - emb_tiles_y = ceildiv(initsuperheight - height, height - overlap_size_y) + 1 - # Sanity - assert ( - emb_tiles_x > 1 or emb_tiles_y > 1 - ), f"ERROR: Based on the requested dimensions of {initsuperwidth}x{initsuperheight} and tiles of {width}x{height} you don't need to Embiggen! Check your arguments." - - # Prep alpha layers -------------- - # https://stackoverflow.com/questions/69321734/how-to-create-different-transparency-like-gradient-with-python-pil - # agradientL is Left-side transparent - agradientL = ( - Image.linear_gradient("L").rotate(90).resize((overlap_size_x, height)) - ) - # agradientT is Top-side transparent - agradientT = Image.linear_gradient("L").resize((width, overlap_size_y)) - # radial corner is the left-top corner, made full circle then cut to just the left-top quadrant - agradientC = Image.new("L", (256, 256)) - for y in range(256): - for x in range(256): - # Find distance to lower right corner (numpy takes arrays) - distanceToLR = np.sqrt([(255 - x) ** 2 + (255 - y) ** 2])[0] - # Clamp values to max 255 - if distanceToLR > 255: - distanceToLR = 255 - # Place the pixel as invert of distance - agradientC.putpixel((x, y), round(255 - distanceToLR)) - - # Create alternative asymmetric diagonal corner to use on "tailing" intersections to prevent hard edges - # Fits for a left-fading gradient on the bottom side and full opacity on the right side. - agradientAsymC = Image.new("L", (256, 256)) - for y in range(256): - for x in range(256): - value = round(max(0, x - (255 - y)) * (255 / max(1, y))) - # Clamp values - value = max(0, value) - value = min(255, value) - agradientAsymC.putpixel((x, y), value) - - # Create alpha layers default fully white - alphaLayerL = Image.new("L", (width, height), 255) - alphaLayerT = Image.new("L", (width, height), 255) - alphaLayerLTC = Image.new("L", (width, height), 255) - # Paste gradients into alpha layers - alphaLayerL.paste(agradientL, (0, 0)) - alphaLayerT.paste(agradientT, (0, 0)) - alphaLayerLTC.paste(agradientL, (0, 0)) - alphaLayerLTC.paste(agradientT, (0, 0)) - alphaLayerLTC.paste(agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0)) - # make masks with an asymmetric upper-right corner so when the curved transparent corner of the next tile - # to its right is placed it doesn't reveal a hard trailing semi-transparent edge in the overlapping space - alphaLayerTaC = alphaLayerT.copy() - alphaLayerTaC.paste( - agradientAsymC.rotate(270).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, 0), - ) - alphaLayerLTaC = alphaLayerLTC.copy() - alphaLayerLTaC.paste( - agradientAsymC.rotate(270).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, 0), - ) - - if embiggen_tiles: - # Individual unconnected sides - alphaLayerR = Image.new("L", (width, height), 255) - alphaLayerR.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) - alphaLayerB = Image.new("L", (width, height), 255) - alphaLayerB.paste(agradientT.rotate(180), (0, height - overlap_size_y)) - alphaLayerTB = Image.new("L", (width, height), 255) - alphaLayerTB.paste(agradientT, (0, 0)) - alphaLayerTB.paste(agradientT.rotate(180), (0, height - overlap_size_y)) - alphaLayerLR = Image.new("L", (width, height), 255) - alphaLayerLR.paste(agradientL, (0, 0)) - alphaLayerLR.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) - - # Sides and corner Layers - alphaLayerRBC = Image.new("L", (width, height), 255) - alphaLayerRBC.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) - alphaLayerRBC.paste(agradientT.rotate(180), (0, height - overlap_size_y)) - alphaLayerRBC.paste( - agradientC.rotate(180).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, height - overlap_size_y), - ) - alphaLayerLBC = Image.new("L", (width, height), 255) - alphaLayerLBC.paste(agradientL, (0, 0)) - alphaLayerLBC.paste(agradientT.rotate(180), (0, height - overlap_size_y)) - alphaLayerLBC.paste( - agradientC.rotate(90).resize((overlap_size_x, overlap_size_y)), - (0, height - overlap_size_y), - ) - alphaLayerRTC = Image.new("L", (width, height), 255) - alphaLayerRTC.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) - alphaLayerRTC.paste(agradientT, (0, 0)) - alphaLayerRTC.paste( - agradientC.rotate(270).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, 0), - ) - - # All but X layers - alphaLayerABT = Image.new("L", (width, height), 255) - alphaLayerABT.paste(alphaLayerLBC, (0, 0)) - alphaLayerABT.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) - alphaLayerABT.paste( - agradientC.rotate(180).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, height - overlap_size_y), - ) - alphaLayerABL = Image.new("L", (width, height), 255) - alphaLayerABL.paste(alphaLayerRTC, (0, 0)) - alphaLayerABL.paste(agradientT.rotate(180), (0, height - overlap_size_y)) - alphaLayerABL.paste( - agradientC.rotate(180).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, height - overlap_size_y), - ) - alphaLayerABR = Image.new("L", (width, height), 255) - alphaLayerABR.paste(alphaLayerLBC, (0, 0)) - alphaLayerABR.paste(agradientT, (0, 0)) - alphaLayerABR.paste( - agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0) - ) - alphaLayerABB = Image.new("L", (width, height), 255) - alphaLayerABB.paste(alphaLayerRTC, (0, 0)) - alphaLayerABB.paste(agradientL, (0, 0)) - alphaLayerABB.paste( - agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0) - ) - - # All-around layer - alphaLayerAA = Image.new("L", (width, height), 255) - alphaLayerAA.paste(alphaLayerABT, (0, 0)) - alphaLayerAA.paste(agradientT, (0, 0)) - alphaLayerAA.paste( - agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0) - ) - alphaLayerAA.paste( - agradientC.rotate(270).resize((overlap_size_x, overlap_size_y)), - (width - overlap_size_x, 0), - ) - - # Clean up temporary gradients - del agradientL - del agradientT - del agradientC - - def make_image(): - # Make main tiles ------------------------------------------------- - if embiggen_tiles: - logger.info(f"Making {len(embiggen_tiles)} Embiggen tiles...") - else: - logger.info( - f"Making {(emb_tiles_x * emb_tiles_y)} Embiggen tiles ({emb_tiles_x}x{emb_tiles_y})..." - ) - - emb_tile_store = [] - # Although we could use the same seed for every tile for determinism, at higher strengths this may - # produce duplicated structures for each tile and make the tiling effect more obvious - # instead track and iterate a local seed we pass to Img2Img - seed = self.seed - seedintlimit = ( - np.iinfo(np.uint32).max - 1 - ) # only retreive this one from numpy - - for tile in range(emb_tiles_x * emb_tiles_y): - # Don't iterate on first tile - if tile != 0: - if seed < seedintlimit: - seed += 1 - else: - seed = 0 - - # Determine if this is a re-run and replace - if embiggen_tiles and not tile in embiggen_tiles: - continue - # Get row and column entries - emb_row_i = tile // emb_tiles_x - emb_column_i = tile % emb_tiles_x - # Determine bounds to cut up the init image - # Determine upper-left point - if emb_column_i + 1 == emb_tiles_x: - left = initsuperwidth - width - else: - left = round(emb_column_i * (width - overlap_size_x)) - if emb_row_i + 1 == emb_tiles_y: - top = initsuperheight - height - else: - top = round(emb_row_i * (height - overlap_size_y)) - right = left + width - bottom = top + height - - # Cropped image of above dimension (does not modify the original) - newinitimage = initsuperimage.crop((left, top, right, bottom)) - # DEBUG: - # newinitimagepath = init_img[0:-4] + f'_emb_Ti{tile}.png' - # newinitimage.save(newinitimagepath) - - if embiggen_tiles: - logger.debug( - f"Making tile #{tile + 1} ({embiggen_tiles.index(tile) + 1} of {len(embiggen_tiles)} requested)" - ) - else: - logger.debug(f"Starting {tile + 1} of {(emb_tiles_x * emb_tiles_y)} tiles") - - # create a torch tensor from an Image - newinitimage = np.array(newinitimage).astype(np.float32) / 255.0 - newinitimage = newinitimage[None].transpose(0, 3, 1, 2) - newinitimage = torch.from_numpy(newinitimage) - newinitimage = 2.0 * newinitimage - 1.0 - newinitimage = newinitimage.to(self.model.device) - clear_cuda_cache = ( - kwargs["clear_cuda_cache"] if "clear_cuda_cache" in kwargs else None - ) - - tile_results = gen_img2img.generate( - prompt, - iterations=1, - seed=seed, - sampler=sampler, - steps=steps, - cfg_scale=cfg_scale, - conditioning=conditioning, - ddim_eta=ddim_eta, - image_callback=None, # called only after the final image is generated - step_callback=step_callback, # called after each intermediate image is generated - width=width, - height=height, - init_image=newinitimage, # notice that init_image is different from init_img - mask_image=None, - strength=strength, - clear_cuda_cache=clear_cuda_cache, - ) - - emb_tile_store.append(tile_results[0][0]) - # DEBUG (but, also has other uses), worth saving if you want tiles without a transparency overlap to manually composite - # emb_tile_store[-1].save(init_img[0:-4] + f'_emb_To{tile}.png') - del newinitimage - - # Sanity check we have them all - if len(emb_tile_store) == (emb_tiles_x * emb_tiles_y) or ( - embiggen_tiles != [] and len(emb_tile_store) == len(embiggen_tiles) - ): - outputsuperimage = Image.new("RGBA", (initsuperwidth, initsuperheight)) - if embiggen_tiles: - outputsuperimage.alpha_composite( - initsuperimage.convert("RGBA"), (0, 0) - ) - for tile in range(emb_tiles_x * emb_tiles_y): - if embiggen_tiles: - if tile in embiggen_tiles: - intileimage = emb_tile_store.pop(0) - else: - continue - else: - intileimage = emb_tile_store[tile] - intileimage = intileimage.convert("RGBA") - # Get row and column entries - emb_row_i = tile // emb_tiles_x - emb_column_i = tile % emb_tiles_x - if emb_row_i == 0 and emb_column_i == 0 and not embiggen_tiles: - left = 0 - top = 0 - else: - # Determine upper-left point - if emb_column_i + 1 == emb_tiles_x: - left = initsuperwidth - width - else: - left = round(emb_column_i * (width - overlap_size_x)) - if emb_row_i + 1 == emb_tiles_y: - top = initsuperheight - height - else: - top = round(emb_row_i * (height - overlap_size_y)) - # Handle gradients for various conditions - # Handle emb_rerun case - if embiggen_tiles: - # top of image - if emb_row_i == 0: - if emb_column_i == 0: - if (tile + 1) in embiggen_tiles: # Look-ahead right - if ( - tile + emb_tiles_x - ) not in embiggen_tiles: # Look-ahead down - intileimage.putalpha(alphaLayerB) - # Otherwise do nothing on this tile - elif ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down only - intileimage.putalpha(alphaLayerR) - else: - intileimage.putalpha(alphaLayerRBC) - elif emb_column_i == emb_tiles_x - 1: - if ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down - intileimage.putalpha(alphaLayerL) - else: - intileimage.putalpha(alphaLayerLBC) - else: - if (tile + 1) in embiggen_tiles: # Look-ahead right - if ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down - intileimage.putalpha(alphaLayerL) - else: - intileimage.putalpha(alphaLayerLBC) - elif ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down only - intileimage.putalpha(alphaLayerLR) - else: - intileimage.putalpha(alphaLayerABT) - # bottom of image - elif emb_row_i == emb_tiles_y - 1: - if emb_column_i == 0: - if (tile + 1) in embiggen_tiles: # Look-ahead right - intileimage.putalpha(alphaLayerTaC) - else: - intileimage.putalpha(alphaLayerRTC) - elif emb_column_i == emb_tiles_x - 1: - # No tiles to look ahead to - intileimage.putalpha(alphaLayerLTC) - else: - if (tile + 1) in embiggen_tiles: # Look-ahead right - intileimage.putalpha(alphaLayerLTaC) - else: - intileimage.putalpha(alphaLayerABB) - # vertical middle of image - else: - if emb_column_i == 0: - if (tile + 1) in embiggen_tiles: # Look-ahead right - if ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down - intileimage.putalpha(alphaLayerTaC) - else: - intileimage.putalpha(alphaLayerTB) - elif ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down only - intileimage.putalpha(alphaLayerRTC) - else: - intileimage.putalpha(alphaLayerABL) - elif emb_column_i == emb_tiles_x - 1: - if ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down - intileimage.putalpha(alphaLayerLTC) - else: - intileimage.putalpha(alphaLayerABR) - else: - if (tile + 1) in embiggen_tiles: # Look-ahead right - if ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down - intileimage.putalpha(alphaLayerLTaC) - else: - intileimage.putalpha(alphaLayerABR) - elif ( - tile + emb_tiles_x - ) in embiggen_tiles: # Look-ahead down only - intileimage.putalpha(alphaLayerABB) - else: - intileimage.putalpha(alphaLayerAA) - # Handle normal tiling case (much simpler - since we tile left to right, top to bottom) - else: - if emb_row_i == 0 and emb_column_i >= 1: - intileimage.putalpha(alphaLayerL) - elif emb_row_i >= 1 and emb_column_i == 0: - if ( - emb_column_i + 1 == emb_tiles_x - ): # If we don't have anything that can be placed to the right - intileimage.putalpha(alphaLayerT) - else: - intileimage.putalpha(alphaLayerTaC) - else: - if ( - emb_column_i + 1 == emb_tiles_x - ): # If we don't have anything that can be placed to the right - intileimage.putalpha(alphaLayerLTC) - else: - intileimage.putalpha(alphaLayerLTaC) - # Layer tile onto final image - outputsuperimage.alpha_composite(intileimage, (left, top)) - else: - logger.error( - "Could not find all Embiggen output tiles in memory? Something must have gone wrong with img2img generation." - ) - - # after internal loops and patching up return Embiggen image - return outputsuperimage - - # end of function declaration - return make_image diff --git a/invokeai/backend/generator/img2img.py b/invokeai/backend/generator/img2img.py index 2c62bec4d6..1cfbeb66c0 100644 --- a/invokeai/backend/generator/img2img.py +++ b/invokeai/backend/generator/img2img.py @@ -22,7 +22,6 @@ class Img2Img(Generator): def get_make_image( self, - prompt, sampler, steps, cfg_scale, diff --git a/invokeai/backend/generator/inpaint.py b/invokeai/backend/generator/inpaint.py index a7fec83eb7..eaf4047109 100644 --- a/invokeai/backend/generator/inpaint.py +++ b/invokeai/backend/generator/inpaint.py @@ -161,9 +161,7 @@ class Inpaint(Img2Img): im: Image.Image, seam_size: int, seam_blur: int, - prompt, seed, - sampler, steps, cfg_scale, ddim_eta, @@ -177,8 +175,6 @@ class Inpaint(Img2Img): mask = self.mask_edge(hard_mask, seam_size, seam_blur) make_image = self.get_make_image( - prompt, - sampler, steps, cfg_scale, ddim_eta, @@ -203,8 +199,6 @@ class Inpaint(Img2Img): @torch.no_grad() def get_make_image( self, - prompt, - sampler, steps, cfg_scale, ddim_eta, @@ -306,7 +300,6 @@ class Inpaint(Img2Img): # noinspection PyTypeChecker pipeline: StableDiffusionGeneratorPipeline = self.model - pipeline.scheduler = sampler # todo: support cross-attention control uc, c, _ = conditioning @@ -345,9 +338,7 @@ class Inpaint(Img2Img): result, seam_size, seam_blur, - prompt, seed, - sampler, seam_steps, cfg_scale, ddim_eta, @@ -360,8 +351,6 @@ class Inpaint(Img2Img): # Restore original settings self.get_make_image( - prompt, - sampler, steps, cfg_scale, ddim_eta, diff --git a/invokeai/backend/generator/txt2img.py b/invokeai/backend/generator/txt2img.py deleted file mode 100644 index 9ea19bd896..0000000000 --- a/invokeai/backend/generator/txt2img.py +++ /dev/null @@ -1,125 +0,0 @@ -""" -invokeai.backend.generator.txt2img inherits from invokeai.backend.generator -""" -import PIL.Image -import torch - -from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from diffusers.models.controlnet import ControlNetModel, ControlNetOutput -from diffusers.pipelines.controlnet import MultiControlNetModel - -from ..stable_diffusion import ( - ConditioningData, - PostprocessingSettings, - StableDiffusionGeneratorPipeline, -) -from .base import Generator - - -class Txt2Img(Generator): - def __init__(self, model, precision, - control_model: Optional[Union[ControlNetModel, List[ControlNetModel]]] = None, - **kwargs): - self.control_model = control_model - if isinstance(self.control_model, list): - self.control_model = MultiControlNetModel(self.control_model) - super().__init__(model, precision, **kwargs) - - @torch.no_grad() - def get_make_image( - self, - prompt, - sampler, - steps, - cfg_scale, - ddim_eta, - conditioning, - width, - height, - step_callback=None, - threshold=0.0, - warmup=0.2, - perlin=0.0, - h_symmetry_time_pct=None, - v_symmetry_time_pct=None, - attention_maps_callback=None, - **kwargs, - ): - """ - Returns a function returning an image derived from the prompt and the initial image - Return value depends on the seed at the time you call it - kwargs are 'width' and 'height' - """ - self.perlin = perlin - control_image = kwargs.get("control_image", None) - do_classifier_free_guidance = cfg_scale > 1.0 - - # noinspection PyTypeChecker - pipeline: StableDiffusionGeneratorPipeline = self.model - pipeline.control_model = self.control_model - pipeline.scheduler = sampler - - uc, c, extra_conditioning_info = conditioning - conditioning_data = ConditioningData( - uc, - c, - cfg_scale, - extra_conditioning_info, - postprocessing_settings=PostprocessingSettings( - threshold=threshold, - warmup=warmup, - h_symmetry_time_pct=h_symmetry_time_pct, - v_symmetry_time_pct=v_symmetry_time_pct, - ), - ).add_scheduler_args_if_applicable(pipeline.scheduler, eta=ddim_eta) - - # FIXME: still need to test with different widths, heights, devices, dtypes - # and add in batch_size, num_images_per_prompt? - if control_image is not None: - if isinstance(self.control_model, ControlNetModel): - control_image = pipeline.prepare_control_image( - image=control_image, - do_classifier_free_guidance=do_classifier_free_guidance, - width=width, - height=height, - # batch_size=batch_size * num_images_per_prompt, - # num_images_per_prompt=num_images_per_prompt, - device=self.control_model.device, - dtype=self.control_model.dtype, - ) - elif isinstance(self.control_model, MultiControlNetModel): - images = [] - for image_ in control_image: - image_ = pipeline.prepare_control_image( - image=image_, - do_classifier_free_guidance=do_classifier_free_guidance, - width=width, - height=height, - # batch_size=batch_size * num_images_per_prompt, - # num_images_per_prompt=num_images_per_prompt, - device=self.control_model.device, - dtype=self.control_model.dtype, - ) - images.append(image_) - control_image = images - kwargs["control_image"] = control_image - - def make_image(x_T: torch.Tensor, _: int) -> PIL.Image.Image: - pipeline_output = pipeline.image_from_embeddings( - latents=torch.zeros_like(x_T, dtype=self.torch_dtype()), - noise=x_T, - num_inference_steps=steps, - conditioning_data=conditioning_data, - callback=step_callback, - **kwargs, - ) - - if ( - pipeline_output.attention_map_saver is not None - and attention_maps_callback is not None - ): - attention_maps_callback(pipeline_output.attention_map_saver) - - return pipeline.numpy_to_pil(pipeline_output.images)[0] - - return make_image diff --git a/invokeai/backend/generator/txt2img2img.py b/invokeai/backend/generator/txt2img2img.py deleted file mode 100644 index 1257a44fb1..0000000000 --- a/invokeai/backend/generator/txt2img2img.py +++ /dev/null @@ -1,209 +0,0 @@ -""" -invokeai.backend.generator.txt2img inherits from invokeai.backend.generator -""" - -import math -from typing import Callable, Optional - -import torch -from diffusers.utils.logging import get_verbosity, set_verbosity, set_verbosity_error - -from ..stable_diffusion import PostprocessingSettings -from .base import Generator -from ..stable_diffusion.diffusers_pipeline import StableDiffusionGeneratorPipeline -from ..stable_diffusion.diffusers_pipeline import ConditioningData -from ..stable_diffusion.diffusers_pipeline import trim_to_multiple_of - -import invokeai.backend.util.logging as logger - -class Txt2Img2Img(Generator): - def __init__(self, model, precision): - super().__init__(model, precision) - self.init_latent = None # for get_noise() - - def get_make_image( - self, - prompt: str, - sampler, - steps: int, - cfg_scale: float, - ddim_eta, - conditioning, - width: int, - height: int, - strength: float, - step_callback: Optional[Callable] = None, - threshold=0.0, - warmup=0.2, - perlin=0.0, - h_symmetry_time_pct=None, - v_symmetry_time_pct=None, - attention_maps_callback=None, - **kwargs, - ): - """ - Returns a function returning an image derived from the prompt and the initial image - Return value depends on the seed at the time you call it - kwargs are 'width' and 'height' - """ - self.perlin = perlin - - # noinspection PyTypeChecker - pipeline: StableDiffusionGeneratorPipeline = self.model - pipeline.scheduler = sampler - - uc, c, extra_conditioning_info = conditioning - conditioning_data = ConditioningData( - uc, - c, - cfg_scale, - extra_conditioning_info, - postprocessing_settings=PostprocessingSettings( - threshold=threshold, - warmup=0.2, - h_symmetry_time_pct=h_symmetry_time_pct, - v_symmetry_time_pct=v_symmetry_time_pct, - ), - ).add_scheduler_args_if_applicable(pipeline.scheduler, eta=ddim_eta) - - def make_image(x_T: torch.Tensor, _: int): - first_pass_latent_output, _ = pipeline.latents_from_embeddings( - latents=torch.zeros_like(x_T), - num_inference_steps=steps, - conditioning_data=conditioning_data, - noise=x_T, - callback=step_callback, - ) - - # Get our initial generation width and height directly from the latent output so - # the message below is accurate. - init_width = first_pass_latent_output.size()[3] * self.downsampling_factor - init_height = first_pass_latent_output.size()[2] * self.downsampling_factor - logger.info( - f"Interpolating from {init_width}x{init_height} to {width}x{height} using DDIM sampling" - ) - - # resizing - resized_latents = torch.nn.functional.interpolate( - first_pass_latent_output, - size=( - height // self.downsampling_factor, - width // self.downsampling_factor, - ), - mode="bilinear", - ) - - # Free up memory from the last generation. - clear_cuda_cache = kwargs["clear_cuda_cache"] or None - if clear_cuda_cache is not None: - clear_cuda_cache() - - second_pass_noise = self.get_noise_like( - resized_latents, override_perlin=True - ) - - # Clear symmetry for the second pass - from dataclasses import replace - - new_postprocessing_settings = replace( - conditioning_data.postprocessing_settings, h_symmetry_time_pct=None - ) - new_postprocessing_settings = replace( - new_postprocessing_settings, v_symmetry_time_pct=None - ) - new_conditioning_data = replace( - conditioning_data, postprocessing_settings=new_postprocessing_settings - ) - - verbosity = get_verbosity() - set_verbosity_error() - pipeline_output = pipeline.img2img_from_latents_and_embeddings( - resized_latents, - num_inference_steps=steps, - conditioning_data=new_conditioning_data, - strength=strength, - noise=second_pass_noise, - callback=step_callback, - ) - set_verbosity(verbosity) - - if ( - pipeline_output.attention_map_saver is not None - and attention_maps_callback is not None - ): - attention_maps_callback(pipeline_output.attention_map_saver) - - return pipeline.numpy_to_pil(pipeline_output.images)[0] - - # FIXME: do we really need something entirely different for the inpainting model? - - # in the case of the inpainting model being loaded, the trick of - # providing an interpolated latent doesn't work, so we transiently - # create a 512x512 PIL image, upscale it, and run the inpainting - # over it in img2img mode. Because the inpaing model is so conservative - # it doesn't change the image (much) - - return make_image - - def get_noise_like(self, like: torch.Tensor, override_perlin: bool = False): - device = like.device - if device.type == "mps": - x = torch.randn_like(like, device="cpu", dtype=self.torch_dtype()).to( - device - ) - else: - x = torch.randn_like(like, device=device, dtype=self.torch_dtype()) - if self.perlin > 0.0 and override_perlin == False: - shape = like.shape - x = (1 - self.perlin) * x + self.perlin * self.get_perlin_noise( - shape[3], shape[2] - ) - return x - - # returns a tensor filled with random numbers from a normal distribution - def get_noise(self, width, height, scale=True): - # print(f"Get noise: {width}x{height}") - if scale: - # Scale the input width and height for the initial generation - # Make their area equivalent to the model's resolution area (e.g. 512*512 = 262144), - # while keeping the minimum dimension at least 0.5 * resolution (e.g. 512*0.5 = 256) - - aspect = width / height - dimension = self.model.unet.config.sample_size * self.model.vae_scale_factor - min_dimension = math.floor(dimension * 0.5) - model_area = ( - dimension * dimension - ) # hardcoded for now since all models are trained on square images - - if aspect > 1.0: - init_height = max(min_dimension, math.sqrt(model_area / aspect)) - init_width = init_height * aspect - else: - init_width = max(min_dimension, math.sqrt(model_area * aspect)) - init_height = init_width / aspect - - scaled_width, scaled_height = trim_to_multiple_of( - math.floor(init_width), math.floor(init_height) - ) - - else: - scaled_width = width - scaled_height = height - - device = self.model.device - channels = self.latent_channels - if channels == 9: - channels = 4 # we don't really want noise for all the mask channels - shape = ( - 1, - channels, - scaled_height // self.downsampling_factor, - scaled_width // self.downsampling_factor, - ) - if self.use_mps_noise or device.type == "mps": - tensor = torch.empty(size=shape, device="cpu") - tensor = self.get_noise_like(like=tensor).to(device) - else: - tensor = torch.empty(size=shape, device=device) - tensor = self.get_noise_like(like=tensor) - return tensor diff --git a/invokeai/backend/install/legacy_arg_parsing.py b/invokeai/backend/install/legacy_arg_parsing.py index 85ca588fe2..4a58ff8336 100644 --- a/invokeai/backend/install/legacy_arg_parsing.py +++ b/invokeai/backend/install/legacy_arg_parsing.py @@ -9,6 +9,7 @@ SAMPLER_CHOICES = [ "ddpm", "deis", "lms", + "lms_k", "pndm", "heun", "heun_k", @@ -18,8 +19,13 @@ SAMPLER_CHOICES = [ "kdpm_2", "kdpm_2_a", "dpmpp_2s", + "dpmpp_2s_k", "dpmpp_2m", "dpmpp_2m_k", + "dpmpp_2m_sde", + "dpmpp_2m_sde_k", + "dpmpp_sde", + "dpmpp_sde_k", "unipc", ] diff --git a/invokeai/backend/model_management/lora.py b/invokeai/backend/model_management/lora.py index 46638878aa..c351a76590 100644 --- a/invokeai/backend/model_management/lora.py +++ b/invokeai/backend/model_management/lora.py @@ -556,8 +556,8 @@ class ModelPatcher: new_tokens_added = None try: - ti_manager = TextualInversionManager() ti_tokenizer = copy.deepcopy(tokenizer) + ti_manager = TextualInversionManager(ti_tokenizer) init_tokens_count = text_encoder.resize_token_embeddings(None).num_embeddings def _get_trigger(ti, index): @@ -650,22 +650,24 @@ class TextualInversionModel: class TextualInversionManager(BaseTextualInversionManager): pad_tokens: Dict[int, List[int]] + tokenizer: CLIPTokenizer - def __init__(self): + def __init__(self, tokenizer: CLIPTokenizer): self.pad_tokens = dict() + self.tokenizer = tokenizer def expand_textual_inversion_token_ids_if_necessary( self, token_ids: list[int] ) -> list[int]: - #if token_ids[0] == self.tokenizer.bos_token_id: - # raise ValueError("token_ids must not start with bos_token_id") - #if token_ids[-1] == self.tokenizer.eos_token_id: - # raise ValueError("token_ids must not end with eos_token_id") - if len(self.pad_tokens) == 0: return token_ids + if token_ids[0] == self.tokenizer.bos_token_id: + raise ValueError("token_ids must not start with bos_token_id") + if token_ids[-1] == self.tokenizer.eos_token_id: + raise ValueError("token_ids must not end with eos_token_id") + new_token_ids = [] for token_id in token_ids: new_token_ids.append(token_id) diff --git a/invokeai/backend/model_management/models/textual_inversion.py b/invokeai/backend/model_management/models/textual_inversion.py index e8c96ff31e..66847f53eb 100644 --- a/invokeai/backend/model_management/models/textual_inversion.py +++ b/invokeai/backend/model_management/models/textual_inversion.py @@ -1,3 +1,4 @@ +import os import torch from typing import Optional from .base import ( diff --git a/invokeai/backend/prompting/__init__.py b/invokeai/backend/prompting/__init__.py deleted file mode 100644 index b52206dd94..0000000000 --- a/invokeai/backend/prompting/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Initialization file for invokeai.backend.prompting -""" -from .conditioning import ( - get_prompt_structure, - get_tokens_for_prompt_object, - get_uc_and_c_and_ec, - split_weighted_subprompts, -) diff --git a/invokeai/backend/prompting/conditioning.py b/invokeai/backend/prompting/conditioning.py deleted file mode 100644 index d070342794..0000000000 --- a/invokeai/backend/prompting/conditioning.py +++ /dev/null @@ -1,297 +0,0 @@ -""" -This module handles the generation of the conditioning tensors. - -Useful function exports: - -get_uc_and_c_and_ec() get the conditioned and unconditioned latent, and edited conditioning if we're doing cross-attention control - -""" -import re -import torch -from typing import Optional, Union - -from compel import Compel -from compel.prompt_parser import ( - Blend, - CrossAttentionControlSubstitute, - FlattenedPrompt, - Fragment, - PromptParser, - Conjunction, -) - -import invokeai.backend.util.logging as logger - -from invokeai.app.services.config import InvokeAIAppConfig -from ..stable_diffusion import InvokeAIDiffuserComponent -from ..util import torch_dtype - -config = InvokeAIAppConfig.get_config() - -def get_uc_and_c_and_ec(prompt_string, - model: InvokeAIDiffuserComponent, - log_tokens=False, skip_normalize_legacy_blend=False): - # lazy-load any deferred textual inversions. - # this might take a couple of seconds the first time a textual inversion is used. - model.textual_inversion_manager.create_deferred_token_ids_for_any_trigger_terms(prompt_string) - - compel = Compel(tokenizer=model.tokenizer, - text_encoder=model.text_encoder, - textual_inversion_manager=model.textual_inversion_manager, - dtype_for_device_getter=torch_dtype, - truncate_long_prompts=False, - ) - - # get rid of any newline characters - prompt_string = prompt_string.replace("\n", " ") - positive_prompt_string, negative_prompt_string = split_prompt_to_positive_and_negative(prompt_string) - - legacy_blend = try_parse_legacy_blend(positive_prompt_string, skip_normalize_legacy_blend) - positive_conjunction: Conjunction - if legacy_blend is not None: - positive_conjunction = legacy_blend - else: - positive_conjunction = Compel.parse_prompt_string(positive_prompt_string) - positive_prompt = positive_conjunction.prompts[0] - - negative_conjunction = Compel.parse_prompt_string(negative_prompt_string) - negative_prompt: FlattenedPrompt | Blend = negative_conjunction.prompts[0] - - tokens_count = get_max_token_count(model.tokenizer, positive_prompt) - if log_tokens or config.log_tokenization: - log_tokenization(positive_prompt, negative_prompt, tokenizer=model.tokenizer) - - c, options = compel.build_conditioning_tensor_for_prompt_object(positive_prompt) - uc, _ = compel.build_conditioning_tensor_for_prompt_object(negative_prompt) - [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc]) - - ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(tokens_count_including_eos_bos=tokens_count, - cross_attention_control_args=options.get( - 'cross_attention_control', None)) - return uc, c, ec - -def get_prompt_structure( - prompt_string, skip_normalize_legacy_blend: bool = False -) -> (Union[FlattenedPrompt, Blend], FlattenedPrompt): - ( - positive_prompt_string, - negative_prompt_string, - ) = split_prompt_to_positive_and_negative(prompt_string) - legacy_blend = try_parse_legacy_blend( - positive_prompt_string, skip_normalize_legacy_blend - ) - positive_prompt: Conjunction - if legacy_blend is not None: - positive_conjunction = legacy_blend - else: - positive_conjunction = Compel.parse_prompt_string(positive_prompt_string) - positive_prompt = positive_conjunction.prompts[0] - negative_conjunction = Compel.parse_prompt_string(negative_prompt_string) - negative_prompt: FlattenedPrompt|Blend = negative_conjunction.prompts[0] - - return positive_prompt, negative_prompt - -def get_max_token_count( - tokenizer, prompt: Union[FlattenedPrompt, Blend], truncate_if_too_long=False -) -> int: - if type(prompt) is Blend: - blend: Blend = prompt - return max( - [ - get_max_token_count(tokenizer, c, truncate_if_too_long) - for c in blend.prompts - ] - ) - else: - return len( - get_tokens_for_prompt_object(tokenizer, prompt, truncate_if_too_long) - ) - - -def get_tokens_for_prompt_object( - tokenizer, parsed_prompt: FlattenedPrompt, truncate_if_too_long=True -) -> [str]: - if type(parsed_prompt) is Blend: - raise ValueError( - "Blend is not supported here - you need to get tokens for each of its .children" - ) - - text_fragments = [ - x.text - if type(x) is Fragment - else ( - " ".join([f.text for f in x.original]) - if type(x) is CrossAttentionControlSubstitute - else str(x) - ) - for x in parsed_prompt.children - ] - text = " ".join(text_fragments) - tokens = tokenizer.tokenize(text) - if truncate_if_too_long: - max_tokens_length = tokenizer.model_max_length - 2 # typically 75 - tokens = tokens[0:max_tokens_length] - return tokens - - -def split_prompt_to_positive_and_negative(prompt_string_uncleaned: str): - unconditioned_words = "" - unconditional_regex = r"\[(.*?)\]" - unconditionals = re.findall(unconditional_regex, prompt_string_uncleaned) - if len(unconditionals) > 0: - unconditioned_words = " ".join(unconditionals) - - # Remove Unconditioned Words From Prompt - unconditional_regex_compile = re.compile(unconditional_regex) - clean_prompt = unconditional_regex_compile.sub(" ", prompt_string_uncleaned) - prompt_string_cleaned = re.sub(" +", " ", clean_prompt) - else: - prompt_string_cleaned = prompt_string_uncleaned - return prompt_string_cleaned, unconditioned_words - - -def log_tokenization( - positive_prompt: Union[Blend, FlattenedPrompt], - negative_prompt: Union[Blend, FlattenedPrompt], - tokenizer, -): - logger.info(f"[TOKENLOG] Parsed Prompt: {positive_prompt}") - logger.info(f"[TOKENLOG] Parsed Negative Prompt: {negative_prompt}") - - log_tokenization_for_prompt_object(positive_prompt, tokenizer) - log_tokenization_for_prompt_object( - negative_prompt, tokenizer, display_label_prefix="(negative prompt)" - ) - - -def log_tokenization_for_prompt_object( - p: Union[Blend, FlattenedPrompt], tokenizer, display_label_prefix=None -): - display_label_prefix = display_label_prefix or "" - if type(p) is Blend: - blend: Blend = p - for i, c in enumerate(blend.prompts): - log_tokenization_for_prompt_object( - c, - tokenizer, - display_label_prefix=f"{display_label_prefix}(blend part {i + 1}, weight={blend.weights[i]})", - ) - elif type(p) is FlattenedPrompt: - flattened_prompt: FlattenedPrompt = p - if flattened_prompt.wants_cross_attention_control: - original_fragments = [] - edited_fragments = [] - for f in flattened_prompt.children: - if type(f) is CrossAttentionControlSubstitute: - original_fragments += f.original - edited_fragments += f.edited - else: - original_fragments.append(f) - edited_fragments.append(f) - - original_text = " ".join([x.text for x in original_fragments]) - log_tokenization_for_text( - original_text, - tokenizer, - display_label=f"{display_label_prefix}(.swap originals)", - ) - edited_text = " ".join([x.text for x in edited_fragments]) - log_tokenization_for_text( - edited_text, - tokenizer, - display_label=f"{display_label_prefix}(.swap replacements)", - ) - else: - text = " ".join([x.text for x in flattened_prompt.children]) - log_tokenization_for_text( - text, tokenizer, display_label=display_label_prefix - ) - - -def log_tokenization_for_text(text, tokenizer, display_label=None, truncate_if_too_long=False): - """shows how the prompt is tokenized - # usually tokens have '' to indicate end-of-word, - # but for readability it has been replaced with ' ' - """ - tokens = tokenizer.tokenize(text) - tokenized = "" - discarded = "" - usedTokens = 0 - totalTokens = len(tokens) - - for i in range(0, totalTokens): - token = tokens[i].replace("", " ") - # alternate color - s = (usedTokens % 6) + 1 - if truncate_if_too_long and i >= tokenizer.model_max_length: - discarded = discarded + f"\x1b[0;3{s};40m{token}" - else: - tokenized = tokenized + f"\x1b[0;3{s};40m{token}" - usedTokens += 1 - - if usedTokens > 0: - logger.info(f'[TOKENLOG] Tokens {display_label or ""} ({usedTokens}):') - logger.debug(f"{tokenized}\x1b[0m") - - if discarded != "": - logger.info(f"[TOKENLOG] Tokens Discarded ({totalTokens - usedTokens}):") - logger.debug(f"{discarded}\x1b[0m") - -def try_parse_legacy_blend(text: str, skip_normalize: bool = False) -> Optional[Conjunction]: - weighted_subprompts = split_weighted_subprompts(text, skip_normalize=skip_normalize) - if len(weighted_subprompts) <= 1: - return None - strings = [x[0] for x in weighted_subprompts] - - pp = PromptParser() - parsed_conjunctions = [pp.parse_conjunction(x) for x in strings] - flattened_prompts = [] - weights = [] - for i, x in enumerate(parsed_conjunctions): - if len(x.prompts)>0: - flattened_prompts.append(x.prompts[0]) - weights.append(weighted_subprompts[i][1]) - return Conjunction([Blend(prompts=flattened_prompts, weights=weights, normalize_weights=not skip_normalize)]) - -def split_weighted_subprompts(text, skip_normalize=False) -> list: - """ - Legacy blend parsing. - - grabs all text up to the first occurrence of ':' - uses the grabbed text as a sub-prompt, and takes the value following ':' as weight - if ':' has no value defined, defaults to 1.0 - repeats until no text remaining - """ - prompt_parser = re.compile( - """ - (?P # capture group for 'prompt' - (?:\\\:|[^:])+ # match one or more non ':' characters or escaped colons '\:' - ) # end 'prompt' - (?: # non-capture group - :+ # match one or more ':' characters - (?P # capture group for 'weight' - -?\d+(?:\.\d+)? # match positive or negative integer or decimal number - )? # end weight capture group, make optional - \s* # strip spaces after weight - | # OR - $ # else, if no ':' then match end of line - ) # end non-capture group - """, - re.VERBOSE, - ) - parsed_prompts = [ - (match.group("prompt").replace("\\:", ":"), float(match.group("weight") or 1)) - for match in re.finditer(prompt_parser, text) - ] - if len(parsed_prompts) == 0: - return [] - if skip_normalize: - return parsed_prompts - weight_sum = sum(map(lambda x: x[1], parsed_prompts)) - if weight_sum == 0: - logger.warning( - "Subprompt weights add up to zero. Discarding and using even weights instead." - ) - equal_weight = 1 / max(len(parsed_prompts), 1) - return [(x[0], equal_weight) for x in parsed_prompts] - return [(x[0], x[1] / weight_sum) for x in parsed_prompts] diff --git a/invokeai/backend/stable_diffusion/__init__.py b/invokeai/backend/stable_diffusion/__init__.py index 55333d3589..37024ccace 100644 --- a/invokeai/backend/stable_diffusion/__init__.py +++ b/invokeai/backend/stable_diffusion/__init__.py @@ -1,7 +1,6 @@ """ Initialization file for the invokeai.backend.stable_diffusion package """ -from .concepts_lib import HuggingFaceConceptsLibrary from .diffusers_pipeline import ( ConditioningData, PipelineIntermediateState, @@ -10,4 +9,3 @@ from .diffusers_pipeline import ( from .diffusion import InvokeAIDiffuserComponent from .diffusion.cross_attention_map_saving import AttentionMapSaver from .diffusion.shared_invokeai_diffusion import PostprocessingSettings -from .textual_inversion_manager import TextualInversionManager diff --git a/invokeai/backend/stable_diffusion/concepts_lib.py b/invokeai/backend/stable_diffusion/concepts_lib.py deleted file mode 100644 index 5294150783..0000000000 --- a/invokeai/backend/stable_diffusion/concepts_lib.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -Query and install embeddings from the HuggingFace SD Concepts Library -at https://huggingface.co/sd-concepts-library. - -The interface is through the Concepts() object. -""" -import os -import re -from typing import Callable -from urllib import error as ul_error -from urllib import request - -from huggingface_hub import ( - HfApi, - HfFolder, - ModelFilter, - hf_hub_url, -) - -from invokeai.backend.util.logging import InvokeAILogger -from invokeai.app.services.config import InvokeAIAppConfig -logger = InvokeAILogger.getLogger() - -class HuggingFaceConceptsLibrary(object): - def __init__(self, root=None): - """ - Initialize the Concepts object. May optionally pass a root directory. - """ - self.config = InvokeAIAppConfig.get_config() - self.root = root or self.config.root - self.hf_api = HfApi() - self.local_concepts = dict() - self.concept_list = None - self.concepts_loaded = dict() - self.triggers = dict() # concept name to trigger phrase - self.concept_names = dict() # trigger phrase to concept name - self.match_trigger = re.compile( - "(<[\w\- >]+>)" - ) # trigger is slightly less restrictive than HF concept name - self.match_concept = re.compile( - "<([\w\-]+)>" - ) # HF concept name can only contain A-Za-z0-9_- - - def list_concepts(self) -> list: - """ - Return a list of all the concepts by name, without the 'sd-concepts-library' part. - Also adds local concepts in invokeai/embeddings folder. - """ - local_concepts_now = self.get_local_concepts( - os.path.join(self.root, "embeddings") - ) - local_concepts_to_add = set(local_concepts_now).difference( - set(self.local_concepts) - ) - self.local_concepts.update(local_concepts_now) - - if self.concept_list is not None: - if local_concepts_to_add: - self.concept_list.extend(list(local_concepts_to_add)) - return self.concept_list - return self.concept_list - elif self.config.internet_available is True: - try: - models = self.hf_api.list_models( - filter=ModelFilter(model_name="sd-concepts-library/") - ) - self.concept_list = [a.id.split("/")[1] for a in models] - # when init, add all in dir. when not init, add only concepts added between init and now - self.concept_list.extend(list(local_concepts_to_add)) - except Exception as e: - logger.warning( - f"Hugging Face textual inversion concepts libraries could not be loaded. The error was {str(e)}." - ) - logger.warning( - "You may load .bin and .pt file(s) manually using the --embedding_directory argument." - ) - return self.concept_list - else: - return self.concept_list - - def get_concept_model_path(self, concept_name: str) -> str: - """ - Returns the path to the 'learned_embeds.bin' file in - the named concept. Returns None if invalid or cannot - be downloaded. - """ - if not concept_name in self.list_concepts(): - logger.warning( - f"{concept_name} is not a local embedding trigger, nor is it a HuggingFace concept. Generation will continue without the concept." - ) - return None - return self.get_concept_file(concept_name.lower(), "learned_embeds.bin") - - def concept_to_trigger(self, concept_name: str) -> str: - """ - Given a concept name returns its trigger by looking in the - "token_identifier.txt" file. - """ - if concept_name in self.triggers: - return self.triggers[concept_name] - elif self.concept_is_local(concept_name): - trigger = f"<{concept_name}>" - self.triggers[concept_name] = trigger - self.concept_names[trigger] = concept_name - return trigger - - file = self.get_concept_file( - concept_name, "token_identifier.txt", local_only=True - ) - if not file: - return None - with open(file, "r") as f: - trigger = f.readline() - trigger = trigger.strip() - self.triggers[concept_name] = trigger - self.concept_names[trigger] = concept_name - return trigger - - def trigger_to_concept(self, trigger: str) -> str: - """ - Given a trigger phrase, maps it to the concept library name. - Only works if concept_to_trigger() has previously been called - on this library. There needs to be a persistent database for - this. - """ - concept = self.concept_names.get(trigger, None) - return f"<{concept}>" if concept else f"{trigger}" - - def replace_triggers_with_concepts(self, prompt: str) -> str: - """ - Given a prompt string that contains tags, replace these - tags with the concept name. The reason for this is so that the - concept names get stored in the prompt metadata. There is no - controlling of colliding triggers in the SD library, so it is - better to store the concept name (unique) than the concept trigger - (not necessarily unique!) - """ - if not prompt: - return prompt - triggers = self.match_trigger.findall(prompt) - if not triggers: - return prompt - - def do_replace(match) -> str: - return self.trigger_to_concept(match.group(1)) or f"<{match.group(1)}>" - - return self.match_trigger.sub(do_replace, prompt) - - def replace_concepts_with_triggers( - self, - prompt: str, - load_concepts_callback: Callable[[list], any], - excluded_tokens: list[str], - ) -> str: - """ - Given a prompt string that contains `` tags, replace - these tags with the appropriate trigger. - - If any `` tags are found, `load_concepts_callback()` is called with a list - of `concepts_name` strings. - - `excluded_tokens` are any tokens that should not be replaced, typically because they - are trigger tokens from a locally-loaded embedding. - """ - concepts = self.match_concept.findall(prompt) - if not concepts: - return prompt - load_concepts_callback(concepts) - - def do_replace(match) -> str: - if excluded_tokens and f"<{match.group(1)}>" in excluded_tokens: - return f"<{match.group(1)}>" - return self.concept_to_trigger(match.group(1)) or f"<{match.group(1)}>" - - return self.match_concept.sub(do_replace, prompt) - - def get_concept_file( - self, - concept_name: str, - file_name: str = "learned_embeds.bin", - local_only: bool = False, - ) -> str: - if not ( - self.concept_is_downloaded(concept_name) - or self.concept_is_local(concept_name) - or local_only - ): - self.download_concept(concept_name) - - # get local path in invokeai/embeddings if local concept - if self.concept_is_local(concept_name): - concept_path = self._concept_local_path(concept_name) - path = concept_path - else: - concept_path = self._concept_path(concept_name) - path = os.path.join(concept_path, file_name) - return path if os.path.exists(path) else None - - def concept_is_local(self, concept_name) -> bool: - return concept_name in self.local_concepts - - def concept_is_downloaded(self, concept_name) -> bool: - concept_directory = self._concept_path(concept_name) - return os.path.exists(concept_directory) - - def download_concept(self, concept_name) -> bool: - repo_id = self._concept_id(concept_name) - dest = self._concept_path(concept_name) - - access_token = HfFolder.get_token() - header = [("Authorization", f"Bearer {access_token}")] if access_token else [] - opener = request.build_opener() - opener.addheaders = header - request.install_opener(opener) - - os.makedirs(dest, exist_ok=True) - succeeded = True - - bytes = 0 - - def tally_download_size(chunk, size, total): - nonlocal bytes - if chunk == 0: - bytes += total - - logger.info(f"Downloading {repo_id}...", end="") - try: - for file in ( - "README.md", - "learned_embeds.bin", - "token_identifier.txt", - "type_of_concept.txt", - ): - url = hf_hub_url(repo_id, file) - request.urlretrieve( - url, os.path.join(dest, file), reporthook=tally_download_size - ) - except ul_error.HTTPError as e: - if e.code == 404: - logger.warning( - f"Concept {concept_name} is not known to the Hugging Face library. Generation will continue without the concept." - ) - else: - logger.warning( - f"Failed to download {concept_name}/{file} ({str(e)}. Generation will continue without the concept.)" - ) - os.rmdir(dest) - return False - except ul_error.URLError as e: - logger.error( - f"an error occurred while downloading {concept_name}: {str(e)}. This may reflect a network issue. Generation will continue without the concept." - ) - os.rmdir(dest) - return False - logger.info("...{:.2f}Kb".format(bytes / 1024)) - return succeeded - - def _concept_id(self, concept_name: str) -> str: - return f"sd-concepts-library/{concept_name}" - - def _concept_path(self, concept_name: str) -> str: - return os.path.join(self.root, "models", "sd-concepts-library", concept_name) - - def _concept_local_path(self, concept_name: str) -> str: - filename = self.local_concepts[concept_name] - return os.path.join(self.root, "embeddings", filename) - - def get_local_concepts(self, loc_dir: str): - locs_dic = dict() - if os.path.isdir(loc_dir): - for file in os.listdir(loc_dir): - f = os.path.splitext(file) - if f[1] == ".bin" or f[1] == ".pt": - locs_dic[f[0]] = file - return locs_dic diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index f4afd880d3..0010f33a0d 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -16,7 +16,6 @@ from accelerate.utils import set_seed import psutil import torch import torchvision.transforms as T -from compel import EmbeddingsProvider from diffusers.models import AutoencoderKL, UNet2DConditionModel from diffusers.models.controlnet import ControlNetModel, ControlNetOutput from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput @@ -48,7 +47,6 @@ from .diffusion import ( PostprocessingSettings, ) from .offloading import FullyLoadedModelGroup, LazilyLoadedModelGroup, ModelGroup -from .textual_inversion_manager import TextualInversionManager @dataclass class PipelineIntermediateState: @@ -317,6 +315,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): requires_safety_checker: bool = False, precision: str = "float32", control_model: ControlNetModel = None, + execution_device: Optional[torch.device] = None, ): super().__init__( vae, @@ -341,22 +340,10 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): # control_model=control_model, ) self.invokeai_diffuser = InvokeAIDiffuserComponent( - self.unet, self._unet_forward, is_running_diffusers=True - ) - use_full_precision = precision == "float32" or precision == "autocast" - self.textual_inversion_manager = TextualInversionManager( - tokenizer=self.tokenizer, - text_encoder=self.text_encoder, - full_precision=use_full_precision, - ) - # InvokeAI's interface for text embeddings and whatnot - self.embeddings_provider = EmbeddingsProvider( - tokenizer=self.tokenizer, - text_encoder=self.text_encoder, - textual_inversion_manager=self.textual_inversion_manager, + self.unet, self._unet_forward ) - self._model_group = FullyLoadedModelGroup(self.unet.device) + self._model_group = FullyLoadedModelGroup(execution_device or self.unet.device) self._model_group.install(*self._submodels) self.control_model = control_model @@ -404,50 +391,6 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): else: self.disable_attention_slicing() - def enable_offload_submodels(self, device: torch.device): - """ - Offload each submodel when it's not in use. - - Useful for low-vRAM situations where the size of the model in memory is a big chunk of - the total available resource, and you want to free up as much for inference as possible. - - This requires more moving parts and may add some delay as the U-Net is swapped out for the - VAE and vice-versa. - """ - models = self._submodels - if self._model_group is not None: - self._model_group.uninstall(*models) - group = LazilyLoadedModelGroup(device) - group.install(*models) - self._model_group = group - - def disable_offload_submodels(self): - """ - Leave all submodels loaded. - - Appropriate for cases where the size of the model in memory is small compared to the memory - required for inference. Avoids the delay and complexity of shuffling the submodels to and - from the GPU. - """ - models = self._submodels - if self._model_group is not None: - self._model_group.uninstall(*models) - group = FullyLoadedModelGroup(self._model_group.execution_device) - group.install(*models) - self._model_group = group - - def offload_all(self): - """Offload all this pipeline's models to CPU.""" - self._model_group.offload_current() - - def ready(self): - """ - Ready this pipeline's models. - - i.e. preload them to the GPU if appropriate. - """ - self._model_group.ready() - def to(self, torch_device: Optional[Union[str, torch.device]] = None, silence_dtype_warnings=False): # overridden method; types match the superclass. if torch_device is None: @@ -991,25 +934,6 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): device = self._model_group.device_for(self.safety_checker) return super().run_safety_checker(image, device, dtype) - @torch.inference_mode() - def get_learned_conditioning( - self, c: List[List[str]], *, return_tokens=True, fragment_weights=None - ): - """ - Compatibility function for invokeai.models.diffusion.ddpm.LatentDiffusion. - """ - return self.embeddings_provider.get_embeddings_for_weighted_prompt_fragments( - text_batch=c, - fragment_weights_batch=fragment_weights, - should_return_tokens=return_tokens, - device=self._model_group.device_for(self.unet), - ) - - @property - def channels(self) -> int: - """Compatible with DiffusionWrapper""" - return self.unet.config.in_channels - def decode_latents(self, latents): # Explicit call to get the vae loaded, since `decode` isn't the forward method. self._model_group.load(self.vae) diff --git a/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py b/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py index eec8097857..f3b09f6a9f 100644 --- a/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py +++ b/invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py @@ -18,7 +18,6 @@ from .cross_attention_control import ( CrossAttentionType, SwapCrossAttnContext, get_cross_attention_modules, - restore_default_cross_attention, setup_cross_attention_control_attention_processors, ) from .cross_attention_map_saving import AttentionMapSaver @@ -66,7 +65,6 @@ class InvokeAIDiffuserComponent: self, model, model_forward_callback: ModelForwardCallback, - is_running_diffusers: bool = False, ): """ :param model: the unet model to pass through to cross attention control @@ -75,7 +73,6 @@ class InvokeAIDiffuserComponent: config = InvokeAIAppConfig.get_config() self.conditioning = None self.model = model - self.is_running_diffusers = is_running_diffusers self.model_forward_callback = model_forward_callback self.cross_attention_control_context = None self.sequential_guidance = config.sequential_guidance @@ -112,37 +109,6 @@ class InvokeAIDiffuserComponent: # TODO resuscitate attention map saving # self.remove_attention_map_saving() - # apparently unused code - # TODO: delete - # def override_cross_attention( - # self, conditioning: ExtraConditioningInfo, step_count: int - # ) -> Dict[str, AttentionProcessor]: - # """ - # setup cross attention .swap control. for diffusers this replaces the attention processor, so - # the previous attention processor is returned so that the caller can restore it later. - # """ - # self.conditioning = conditioning - # self.cross_attention_control_context = Context( - # arguments=self.conditioning.cross_attention_control_args, - # step_count=step_count, - # ) - # return override_cross_attention( - # self.model, - # self.cross_attention_control_context, - # is_running_diffusers=self.is_running_diffusers, - # ) - - def restore_default_cross_attention( - self, restore_attention_processor: Optional["AttentionProcessor"] = None - ): - self.conditioning = None - self.cross_attention_control_context = None - restore_default_cross_attention( - self.model, - is_running_diffusers=self.is_running_diffusers, - restore_attention_processor=restore_attention_processor, - ) - def setup_attention_map_saving(self, saver: AttentionMapSaver): def callback(slice, dim, offset, slice_size, key): if dim is not None: @@ -204,9 +170,7 @@ class InvokeAIDiffuserComponent: cross_attention_control_types_to_do = [] context: Context = self.cross_attention_control_context if self.cross_attention_control_context is not None: - percent_through = self.calculate_percent_through( - sigma, step_index, total_step_count - ) + percent_through = step_index / total_step_count cross_attention_control_types_to_do = ( context.get_active_cross_attention_control_types_for_step( percent_through @@ -264,9 +228,7 @@ class InvokeAIDiffuserComponent: total_step_count, ) -> torch.Tensor: if postprocessing_settings is not None: - percent_through = self.calculate_percent_through( - sigma, step_index, total_step_count - ) + percent_through = step_index / total_step_count latents = self.apply_threshold( postprocessing_settings, latents, percent_through ) @@ -275,22 +237,6 @@ class InvokeAIDiffuserComponent: ) return latents - def calculate_percent_through(self, sigma, step_index, total_step_count): - if step_index is not None and total_step_count is not None: - # 🧨diffusers codepath - percent_through = ( - step_index / total_step_count - ) # will never reach 1.0 - this is deliberate - else: - # legacy compvis codepath - # TODO remove when compvis codepath support is dropped - if step_index is None and sigma is None: - raise ValueError( - "Either step_index or sigma is required when doing cross attention control, but both are None." - ) - percent_through = self.estimate_percent_through(step_index, sigma) - return percent_through - # methods below are called from do_diffusion_step and should be considered private to this class. def _apply_standard_conditioning(self, x, sigma, unconditioning, conditioning, **kwargs): @@ -323,6 +269,7 @@ class InvokeAIDiffuserComponent: conditioned_next_x = conditioned_next_x.clone() return unconditioned_next_x, conditioned_next_x + # TODO: looks unused def _apply_hybrid_conditioning(self, x, sigma, unconditioning, conditioning, **kwargs): assert isinstance(conditioning, dict) assert isinstance(unconditioning, dict) @@ -350,34 +297,6 @@ class InvokeAIDiffuserComponent: conditioning, cross_attention_control_types_to_do, **kwargs, - ): - if self.is_running_diffusers: - return self._apply_cross_attention_controlled_conditioning__diffusers( - x, - sigma, - unconditioning, - conditioning, - cross_attention_control_types_to_do, - **kwargs, - ) - else: - return self._apply_cross_attention_controlled_conditioning__compvis( - x, - sigma, - unconditioning, - conditioning, - cross_attention_control_types_to_do, - **kwargs, - ) - - def _apply_cross_attention_controlled_conditioning__diffusers( - self, - x: torch.Tensor, - sigma, - unconditioning, - conditioning, - cross_attention_control_types_to_do, - **kwargs, ): context: Context = self.cross_attention_control_context @@ -409,54 +328,6 @@ class InvokeAIDiffuserComponent: ) return unconditioned_next_x, conditioned_next_x - def _apply_cross_attention_controlled_conditioning__compvis( - self, - x: torch.Tensor, - sigma, - unconditioning, - conditioning, - cross_attention_control_types_to_do, - **kwargs, - ): - # print('pct', percent_through, ': doing cross attention control on', cross_attention_control_types_to_do) - # slower non-batched path (20% slower on mac MPS) - # We are only interested in using attention maps for conditioned_next_x, but batching them with generation of - # unconditioned_next_x causes attention maps to *also* be saved for the unconditioned_next_x. - # This messes app their application later, due to mismatched shape of dim 0 (seems to be 16 for batched vs. 8) - # (For the batched invocation the `wrangler` function gets attention tensor with shape[0]=16, - # representing batched uncond + cond, but then when it comes to applying the saved attention, the - # wrangler gets an attention tensor which only has shape[0]=8, representing just self.edited_conditionings.) - # todo: give CrossAttentionControl's `wrangler` function more info so it can work with a batched call as well. - context: Context = self.cross_attention_control_context - - try: - unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning, **kwargs) - - # process x using the original prompt, saving the attention maps - # print("saving attention maps for", cross_attention_control_types_to_do) - for ca_type in cross_attention_control_types_to_do: - context.request_save_attention_maps(ca_type) - _ = self.model_forward_callback(x, sigma, conditioning, **kwargs,) - context.clear_requests(cleanup=False) - - # process x again, using the saved attention maps to control where self.edited_conditioning will be applied - # print("applying saved attention maps for", cross_attention_control_types_to_do) - for ca_type in cross_attention_control_types_to_do: - context.request_apply_saved_attention_maps(ca_type) - edited_conditioning = ( - self.conditioning.cross_attention_control_args.edited_conditioning - ) - conditioned_next_x = self.model_forward_callback( - x, sigma, edited_conditioning, **kwargs, - ) - context.clear_requests(cleanup=True) - - except: - context.clear_requests(cleanup=True) - raise - - return unconditioned_next_x, conditioned_next_x - def _combine(self, unconditioned_next_x, conditioned_next_x, guidance_scale): # to scale how much effect conditioning has, calculate the changes it does and then scale that scaled_delta = (conditioned_next_x - unconditioned_next_x) * guidance_scale diff --git a/invokeai/backend/stable_diffusion/schedulers/schedulers.py b/invokeai/backend/stable_diffusion/schedulers/schedulers.py index 08f85cf559..77c45d5eb8 100644 --- a/invokeai/backend/stable_diffusion/schedulers/schedulers.py +++ b/invokeai/backend/stable_diffusion/schedulers/schedulers.py @@ -1,13 +1,14 @@ from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, KDPM2DiscreteScheduler, \ KDPM2AncestralDiscreteScheduler, EulerDiscreteScheduler, EulerAncestralDiscreteScheduler, \ HeunDiscreteScheduler, LMSDiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler, \ - DPMSolverSinglestepScheduler, DEISMultistepScheduler, DDPMScheduler + DPMSolverSinglestepScheduler, DEISMultistepScheduler, DDPMScheduler, DPMSolverSDEScheduler SCHEDULER_MAP = dict( ddim=(DDIMScheduler, dict()), ddpm=(DDPMScheduler, dict()), deis=(DEISMultistepScheduler, dict()), - lms=(LMSDiscreteScheduler, dict()), + lms=(LMSDiscreteScheduler, dict(use_karras_sigmas=False)), + lms_k=(LMSDiscreteScheduler, dict(use_karras_sigmas=True)), pndm=(PNDMScheduler, dict()), heun=(HeunDiscreteScheduler, dict(use_karras_sigmas=False)), heun_k=(HeunDiscreteScheduler, dict(use_karras_sigmas=True)), @@ -16,8 +17,13 @@ SCHEDULER_MAP = dict( euler_a=(EulerAncestralDiscreteScheduler, dict()), kdpm_2=(KDPM2DiscreteScheduler, dict()), kdpm_2_a=(KDPM2AncestralDiscreteScheduler, dict()), - dpmpp_2s=(DPMSolverSinglestepScheduler, dict()), + dpmpp_2s=(DPMSolverSinglestepScheduler, dict(use_karras_sigmas=False)), + dpmpp_2s_k=(DPMSolverSinglestepScheduler, dict(use_karras_sigmas=True)), dpmpp_2m=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=False)), dpmpp_2m_k=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=True)), + dpmpp_2m_sde=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=False, algorithm_type='sde-dpmsolver++')), + dpmpp_2m_sde_k=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=True, algorithm_type='sde-dpmsolver++')), + dpmpp_sde=(DPMSolverSDEScheduler, dict(use_karras_sigmas=False, noise_sampler_seed=0)), + dpmpp_sde_k=(DPMSolverSDEScheduler, dict(use_karras_sigmas=True, noise_sampler_seed=0)), unipc=(UniPCMultistepScheduler, dict(cpu_only=True)) ) diff --git a/invokeai/backend/stable_diffusion/textual_inversion_manager.py b/invokeai/backend/stable_diffusion/textual_inversion_manager.py deleted file mode 100644 index 9476c12dc5..0000000000 --- a/invokeai/backend/stable_diffusion/textual_inversion_manager.py +++ /dev/null @@ -1,429 +0,0 @@ -import traceback -from dataclasses import dataclass -from pathlib import Path -from typing import Optional, Union, List - -import safetensors.torch -import torch - -from compel.embeddings_provider import BaseTextualInversionManager -from picklescan.scanner import scan_file_path -from transformers import CLIPTextModel, CLIPTokenizer - -import invokeai.backend.util.logging as logger -from .concepts_lib import HuggingFaceConceptsLibrary - -@dataclass -class EmbeddingInfo: - name: str - embedding: torch.Tensor - num_vectors_per_token: int - token_dim: int - trained_steps: int = None - trained_model_name: str = None - trained_model_checksum: str = None - -@dataclass -class TextualInversion: - trigger_string: str - embedding: torch.Tensor - trigger_token_id: Optional[int] = None - pad_token_ids: Optional[list[int]] = None - - @property - def embedding_vector_length(self) -> int: - return self.embedding.shape[0] - - -class TextualInversionManager(BaseTextualInversionManager): - def __init__( - self, - tokenizer: CLIPTokenizer, - text_encoder: CLIPTextModel, - full_precision: bool = True, - ): - self.tokenizer = tokenizer - self.text_encoder = text_encoder - self.full_precision = full_precision - self.hf_concepts_library = HuggingFaceConceptsLibrary() - self.trigger_to_sourcefile = dict() - default_textual_inversions: list[TextualInversion] = [] - self.textual_inversions = default_textual_inversions - - def load_huggingface_concepts(self, concepts: list[str]): - for concept_name in concepts: - if concept_name in self.hf_concepts_library.concepts_loaded: - continue - trigger = self.hf_concepts_library.concept_to_trigger(concept_name) - if ( - self.has_textual_inversion_for_trigger_string(trigger) - or self.has_textual_inversion_for_trigger_string(concept_name) - or self.has_textual_inversion_for_trigger_string(f"<{concept_name}>") - ): # in case a token with literal angle brackets encountered - logger.info(f"Loaded local embedding for trigger {concept_name}") - continue - bin_file = self.hf_concepts_library.get_concept_model_path(concept_name) - if not bin_file: - continue - logger.info(f"Loaded remote embedding for trigger {concept_name}") - self.load_textual_inversion(bin_file) - self.hf_concepts_library.concepts_loaded[concept_name] = True - - def get_all_trigger_strings(self) -> list[str]: - return [ti.trigger_string for ti in self.textual_inversions] - - def load_textual_inversion( - self, ckpt_path: Union[str, Path], defer_injecting_tokens: bool = False - ): - ckpt_path = Path(ckpt_path) - - if not ckpt_path.is_file(): - return - - if str(ckpt_path).endswith(".DS_Store"): - return - - embedding_list = self._parse_embedding(str(ckpt_path)) - for embedding_info in embedding_list: - if (self.text_encoder.get_input_embeddings().weight.data[0].shape[0] != embedding_info.token_dim): - logger.warning( - f"Notice: {ckpt_path.parents[0].name}/{ckpt_path.name} was trained on a model with an incompatible token dimension: {self.text_encoder.get_input_embeddings().weight.data[0].shape[0]} vs {embedding_info.token_dim}." - ) - continue - - # Resolve the situation in which an earlier embedding has claimed the same - # trigger string. We replace the trigger with '', as we used to. - trigger_str = embedding_info.name - sourcefile = ( - f"{ckpt_path.parent.name}/{ckpt_path.name}" - if ckpt_path.name == "learned_embeds.bin" - else ckpt_path.name - ) - - if trigger_str in self.trigger_to_sourcefile: - replacement_trigger_str = ( - f"<{ckpt_path.parent.name}>" - if ckpt_path.name == "learned_embeds.bin" - else f"<{ckpt_path.stem}>" - ) - logger.info( - f"{sourcefile}: Trigger token '{trigger_str}' is already claimed by '{self.trigger_to_sourcefile[trigger_str]}'. Trigger this concept with {replacement_trigger_str}" - ) - trigger_str = replacement_trigger_str - - try: - self._add_textual_inversion( - trigger_str, - embedding_info.embedding, - defer_injecting_tokens=defer_injecting_tokens, - ) - # remember which source file claims this trigger - self.trigger_to_sourcefile[trigger_str] = sourcefile - - except ValueError as e: - logger.debug(f'Ignoring incompatible embedding {embedding_info["name"]}') - logger.debug(f"The error was {str(e)}") - - def _add_textual_inversion( - self, trigger_str, embedding, defer_injecting_tokens=False - ) -> Optional[TextualInversion]: - """ - Add a textual inversion to be recognised. - :param trigger_str: The trigger text in the prompt that activates this textual inversion. If unknown to the embedder's tokenizer, will be added. - :param embedding: The actual embedding data that will be inserted into the conditioning at the point where the token_str appears. - :return: The token id for the added embedding, either existing or newly-added. - """ - if trigger_str in [ti.trigger_string for ti in self.textual_inversions]: - logger.warning( - f"TextualInversionManager refusing to overwrite already-loaded token '{trigger_str}'" - ) - return - if not self.full_precision: - embedding = embedding.half() - if len(embedding.shape) == 1: - embedding = embedding.unsqueeze(0) - elif len(embedding.shape) > 2: - raise ValueError( - f"** TextualInversionManager cannot add {trigger_str} because the embedding shape {embedding.shape} is incorrect. The embedding must have shape [token_dim] or [V, token_dim] where V is vector length and token_dim is 768 for SD1 or 1280 for SD2." - ) - - try: - ti = TextualInversion(trigger_string=trigger_str, embedding=embedding) - if not defer_injecting_tokens: - self._inject_tokens_and_assign_embeddings(ti) - self.textual_inversions.append(ti) - return ti - - except ValueError as e: - if str(e).startswith("Warning"): - logger.warning(f"{str(e)}") - else: - traceback.print_exc() - logger.error( - f"TextualInversionManager was unable to add a textual inversion with trigger string {trigger_str}." - ) - raise - - def _inject_tokens_and_assign_embeddings(self, ti: TextualInversion) -> int: - if ti.trigger_token_id is not None: - raise ValueError( - f"Tokens already injected for textual inversion with trigger '{ti.trigger_string}'" - ) - - trigger_token_id = self._get_or_create_token_id_and_assign_embedding( - ti.trigger_string, ti.embedding[0] - ) - - if ti.embedding_vector_length > 1: - # for embeddings with vector length > 1 - pad_token_strings = [ - ti.trigger_string + "-!pad-" + str(pad_index) - for pad_index in range(1, ti.embedding_vector_length) - ] - # todo: batched UI for faster loading when vector length >2 - pad_token_ids = [ - self._get_or_create_token_id_and_assign_embedding( - pad_token_str, ti.embedding[1 + i] - ) - for (i, pad_token_str) in enumerate(pad_token_strings) - ] - else: - pad_token_ids = [] - - ti.trigger_token_id = trigger_token_id - ti.pad_token_ids = pad_token_ids - return ti.trigger_token_id - - def has_textual_inversion_for_trigger_string(self, trigger_string: str) -> bool: - try: - ti = self.get_textual_inversion_for_trigger_string(trigger_string) - return ti is not None - except StopIteration: - return False - - def get_textual_inversion_for_trigger_string( - self, trigger_string: str - ) -> TextualInversion: - return next( - ti for ti in self.textual_inversions if ti.trigger_string == trigger_string - ) - - def get_textual_inversion_for_token_id(self, token_id: int) -> TextualInversion: - return next( - ti for ti in self.textual_inversions if ti.trigger_token_id == token_id - ) - - def create_deferred_token_ids_for_any_trigger_terms( - self, prompt_string: str - ) -> list[int]: - injected_token_ids = [] - for ti in self.textual_inversions: - if ti.trigger_token_id is None and ti.trigger_string in prompt_string: - if ti.embedding_vector_length > 1: - logger.info( - f"Preparing tokens for textual inversion {ti.trigger_string}..." - ) - try: - self._inject_tokens_and_assign_embeddings(ti) - except ValueError as e: - logger.debug( - f"Ignoring incompatible embedding trigger {ti.trigger_string}" - ) - logger.debug(f"The error was {str(e)}") - continue - injected_token_ids.append(ti.trigger_token_id) - injected_token_ids.extend(ti.pad_token_ids) - return injected_token_ids - - def expand_textual_inversion_token_ids_if_necessary( - self, prompt_token_ids: list[int] - ) -> list[int]: - """ - Insert padding tokens as necessary into the passed-in list of token ids to match any textual inversions it includes. - - :param prompt_token_ids: The prompt as a list of token ids (`int`s). Should not include bos and eos markers. - :return: The prompt token ids with any necessary padding to account for textual inversions inserted. May be too - long - caller is responsible for prepending/appending eos and bos token ids, and truncating if necessary. - """ - if len(prompt_token_ids) == 0: - return prompt_token_ids - - if prompt_token_ids[0] == self.tokenizer.bos_token_id: - raise ValueError("prompt_token_ids must not start with bos_token_id") - if prompt_token_ids[-1] == self.tokenizer.eos_token_id: - raise ValueError("prompt_token_ids must not end with eos_token_id") - textual_inversion_trigger_token_ids = [ - ti.trigger_token_id for ti in self.textual_inversions - ] - prompt_token_ids = prompt_token_ids.copy() - for i, token_id in reversed(list(enumerate(prompt_token_ids))): - if token_id in textual_inversion_trigger_token_ids: - textual_inversion = next( - ti - for ti in self.textual_inversions - if ti.trigger_token_id == token_id - ) - for pad_idx in range(0, textual_inversion.embedding_vector_length - 1): - prompt_token_ids.insert( - i + pad_idx + 1, textual_inversion.pad_token_ids[pad_idx] - ) - - return prompt_token_ids - - def _get_or_create_token_id_and_assign_embedding( - self, token_str: str, embedding: torch.Tensor - ) -> int: - if len(embedding.shape) != 1: - raise ValueError( - "Embedding has incorrect shape - must be [token_dim] where token_dim is 768 for SD1 or 1280 for SD2" - ) - existing_token_id = self.tokenizer.convert_tokens_to_ids(token_str) - if existing_token_id == self.tokenizer.unk_token_id: - num_tokens_added = self.tokenizer.add_tokens(token_str) - current_embeddings = self.text_encoder.resize_token_embeddings(None) - current_token_count = current_embeddings.num_embeddings - new_token_count = current_token_count + num_tokens_added - # the following call is slow - todo make batched for better performance with vector length >1 - self.text_encoder.resize_token_embeddings(new_token_count) - - token_id = self.tokenizer.convert_tokens_to_ids(token_str) - if token_id == self.tokenizer.unk_token_id: - raise RuntimeError(f"Unable to find token id for token '{token_str}'") - if ( - self.text_encoder.get_input_embeddings().weight.data[token_id].shape - != embedding.shape - ): - raise ValueError( - f"Warning. Cannot load embedding for {token_str}. It was trained on a model with token dimension {embedding.shape[0]}, but the current model has token dimension {self.text_encoder.get_input_embeddings().weight.data[token_id].shape[0]}." - ) - self.text_encoder.get_input_embeddings().weight.data[token_id] = embedding - - return token_id - - - def _parse_embedding(self, embedding_file: str)->List[EmbeddingInfo]: - suffix = Path(embedding_file).suffix - try: - if suffix in [".pt",".ckpt",".bin"]: - scan_result = scan_file_path(embedding_file) - if scan_result.infected_files > 0: - logger.critical( - f"Security Issues Found in Model: {scan_result.issues_count}" - ) - logger.critical("For your safety, InvokeAI will not load this embed.") - return list() - ckpt = torch.load(embedding_file,map_location="cpu") - else: - ckpt = safetensors.torch.load_file(embedding_file) - except Exception as e: - logger.warning(f"Notice: unrecognized embedding file format: {embedding_file}: {e}") - return list() - - # try to figure out what kind of embedding file it is and parse accordingly - keys = list(ckpt.keys()) - if all(x in keys for x in ['string_to_token','string_to_param','name','step']): - return self._parse_embedding_v1(ckpt, embedding_file) # example rem_rezero.pt - - elif all(x in keys for x in ['string_to_token','string_to_param']): - return self._parse_embedding_v2(ckpt, embedding_file) # example midj-strong.pt - - elif 'emb_params' in keys: - return self._parse_embedding_v3(ckpt, embedding_file) # example easynegative.safetensors - - else: - return self._parse_embedding_v4(ckpt, embedding_file) # usually a '.bin' file - - def _parse_embedding_v1(self, embedding_ckpt: dict, file_path: str)->List[EmbeddingInfo]: - basename = Path(file_path).stem - logger.debug(f'Loading v1 embedding file: {basename}') - - embeddings = list() - token_counter = -1 - for token,embedding in embedding_ckpt["string_to_param"].items(): - if token_counter < 0: - trigger = embedding_ckpt["name"] - elif token_counter == 0: - trigger = '' - else: - trigger = f'<{basename}-{int(token_counter:=token_counter)}>' - token_counter += 1 - embedding_info = EmbeddingInfo( - name = trigger, - embedding = embedding, - num_vectors_per_token = embedding.size()[0], - token_dim = embedding.size()[1], - trained_steps = embedding_ckpt["step"], - trained_model_name = embedding_ckpt["sd_checkpoint_name"], - trained_model_checksum = embedding_ckpt["sd_checkpoint"] - ) - embeddings.append(embedding_info) - return embeddings - - def _parse_embedding_v2 ( - self, embedding_ckpt: dict, file_path: str - ) -> List[EmbeddingInfo]: - """ - This handles embedding .pt file variant #2. - """ - basename = Path(file_path).stem - logger.debug(f'Loading v2 embedding file: {basename}') - embeddings = list() - - if isinstance( - list(embedding_ckpt["string_to_token"].values())[0], torch.Tensor - ): - token_counter = 0 - for token,embedding in embedding_ckpt["string_to_param"].items(): - trigger = token if token != '*' \ - else f'<{basename}>' if token_counter == 0 \ - else f'<{basename}-{int(token_counter:=token_counter+1)}>' - embedding_info = EmbeddingInfo( - name = trigger, - embedding = embedding, - num_vectors_per_token = embedding.size()[0], - token_dim = embedding.size()[1], - ) - embeddings.append(embedding_info) - else: - logger.warning(f"{basename}: Unrecognized embedding format") - - return embeddings - - def _parse_embedding_v3(self, embedding_ckpt: dict, file_path: str)->List[EmbeddingInfo]: - """ - Parse 'version 3' of the .pt textual inversion embedding files. - """ - basename = Path(file_path).stem - logger.debug(f'Loading v3 embedding file: {basename}') - embedding = embedding_ckpt['emb_params'] - embedding_info = EmbeddingInfo( - name = f'<{basename}>', - embedding = embedding, - num_vectors_per_token = embedding.size()[0], - token_dim = embedding.size()[1], - ) - return [embedding_info] - - def _parse_embedding_v4(self, embedding_ckpt: dict, filepath: str)->List[EmbeddingInfo]: - """ - Parse 'version 4' of the textual inversion embedding files. This one - is usually associated with .bin files trained by HuggingFace diffusers. - """ - basename = Path(filepath).stem - short_path = Path(filepath).parents[0].name+'/'+Path(filepath).name - - logger.debug(f'Loading v4 embedding file: {short_path}') - - embeddings = list() - if list(embedding_ckpt.keys()) == 0: - logger.warning(f"Invalid embeddings file: {short_path}") - else: - for token,embedding in embedding_ckpt.items(): - embedding_info = EmbeddingInfo( - name = token or f"<{basename}>", - embedding = embedding, - num_vectors_per_token = 1, # All Concepts seem to default to 1 - token_dim = embedding.size()[0], - ) - embeddings.append(embedding_info) - return embeddings diff --git a/invokeai/backend/web/modules/parameters.py b/invokeai/backend/web/modules/parameters.py index 9a4bc0aec3..440f21a947 100644 --- a/invokeai/backend/web/modules/parameters.py +++ b/invokeai/backend/web/modules/parameters.py @@ -7,6 +7,7 @@ SAMPLER_CHOICES = [ "ddpm", "deis", "lms", + "lms_k", "pndm", "heun", 'heun_k', @@ -16,8 +17,13 @@ SAMPLER_CHOICES = [ "kdpm_2", "kdpm_2_a", "dpmpp_2s", + "dpmpp_2s_k", "dpmpp_2m", "dpmpp_2m_k", + "dpmpp_2m_sde", + "dpmpp_2m_sde_k", + "dpmpp_sde", + "dpmpp_sde_k", "unipc", ] diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index 7a73bae411..eae0c07eff 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -547,7 +547,8 @@ "general": "General", "generation": "Generation", "ui": "User Interface", - "availableSchedulers": "Available Schedulers" + "favoriteSchedulers": "Favorite Schedulers", + "favoriteSchedulersPlaceholder": "No schedulers favorited" }, "toast": { "serverError": "Server Error", diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index c2e525ad7d..5fd413d915 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -1,25 +1,62 @@ -// TODO: use Enums? +import { SchedulerParam } from 'features/parameters/store/parameterZodSchemas'; -export const SCHEDULERS = [ - 'ddim', - 'lms', +// zod needs the array to be `as const` to infer the type correctly +// this is the source of the `SchedulerParam` type, which is generated by zod +export const SCHEDULER_NAMES_AS_CONST = [ 'euler', - 'euler_k', - 'euler_a', + 'deis', + 'ddim', + 'ddpm', 'dpmpp_2s', 'dpmpp_2m', - 'dpmpp_2m_k', - 'kdpm_2', - 'kdpm_2_a', - 'deis', - 'ddpm', - 'pndm', + 'dpmpp_2m_sde', + 'dpmpp_sde', 'heun', - 'heun_k', + 'kdpm_2', + 'lms', + 'pndm', 'unipc', + 'euler_k', + 'dpmpp_2s_k', + 'dpmpp_2m_k', + 'dpmpp_2m_sde_k', + 'dpmpp_sde_k', + 'heun_k', + 'lms_k', + 'euler_a', + 'kdpm_2_a', ] as const; -export type Scheduler = (typeof SCHEDULERS)[number]; +export const DEFAULT_SCHEDULER_NAME = 'euler'; + +export const SCHEDULER_NAMES: SchedulerParam[] = [...SCHEDULER_NAMES_AS_CONST]; + +export const SCHEDULER_LABEL_MAP: Record = { + euler: 'Euler', + deis: 'DEIS', + ddim: 'DDIM', + ddpm: 'DDPM', + dpmpp_sde: 'DPM++ SDE', + dpmpp_2s: 'DPM++ 2S', + dpmpp_2m: 'DPM++ 2M', + dpmpp_2m_sde: 'DPM++ 2M SDE', + heun: 'Heun', + kdpm_2: 'KDPM 2', + lms: 'LMS', + pndm: 'PNDM', + unipc: 'UniPC', + euler_k: 'Euler Karras', + dpmpp_sde_k: 'DPM++ SDE Karras', + dpmpp_2s_k: 'DPM++ 2S Karras', + dpmpp_2m_k: 'DPM++ 2M Karras', + dpmpp_2m_sde_k: 'DPM++ 2M SDE Karras', + heun_k: 'Heun Karras', + lms_k: 'LMS Karras', + euler_a: 'Euler Ancestral', + kdpm_2_a: 'KDPM 2 Ancestral', +}; + +export type Scheduler = (typeof SCHEDULER_NAMES)[number]; // Valid upscaling levels export const UPSCALING_LEVELS: Array<{ label: string; value: string }> = [ diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts index 4d8177d7f3..a26d872d50 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedCanvas.ts @@ -1,11 +1,10 @@ import { startAppListening } from '..'; import { sessionCreated } from 'services/thunks/session'; -import { buildCanvasGraphComponents } from 'features/nodes/util/graphBuilders/buildCanvasGraph'; +import { buildCanvasGraph } from 'features/nodes/util/graphBuilders/buildCanvasGraph'; import { log } from 'app/logging/useLogger'; import { canvasGraphBuilt } from 'features/nodes/store/actions'; import { imageUpdated, imageUploaded } from 'services/thunks/image'; -import { v4 as uuidv4 } from 'uuid'; -import { Graph } from 'services/api'; +import { ImageDTO } from 'services/api'; import { canvasSessionIdChanged, stagingAreaInitialized, @@ -67,112 +66,106 @@ export const addUserInvokedCanvasListener = () => { moduleLog.debug(`Generation mode: ${generationMode}`); - // Build the canvas graph - const graphComponents = await buildCanvasGraphComponents( - state, - generationMode - ); + // Temp placeholders for the init and mask images + let canvasInitImage: ImageDTO | undefined; + let canvasMaskImage: ImageDTO | undefined; - if (!graphComponents) { - moduleLog.error('Problem building graph'); - return; - } - - const { rangeNode, iterateNode, baseNode, edges } = graphComponents; - - // Assemble! Note that this graph *does not have the init or mask image set yet!* - const nodes: Graph['nodes'] = { - [rangeNode.id]: rangeNode, - [iterateNode.id]: iterateNode, - [baseNode.id]: baseNode, - }; - - const graph = { nodes, edges }; - - dispatch(canvasGraphBuilt(graph)); - - moduleLog.debug({ data: graph }, 'Canvas graph built'); - - // If we are generating img2img or inpaint, we need to upload the init images - if (baseNode.type === 'img2img' || baseNode.type === 'inpaint') { - const baseFilename = `${uuidv4()}.png`; - dispatch( + // For img2img and inpaint/outpaint, we need to upload the init images + if (['img2img', 'inpaint', 'outpaint'].includes(generationMode)) { + // upload the image, saving the request id + const { requestId: initImageUploadedRequestId } = dispatch( imageUploaded({ formData: { - file: new File([baseBlob], baseFilename, { type: 'image/png' }), + file: new File([baseBlob], 'canvasInitImage.png', { + type: 'image/png', + }), }, imageCategory: 'general', isIntermediate: true, }) ); - // Wait for the image to be uploaded - const [{ payload: baseImageDTO }] = await take( + // Wait for the image to be uploaded, matching by request id + const [{ payload }] = await take( (action): action is ReturnType => imageUploaded.fulfilled.match(action) && - action.meta.arg.formData.file.name === baseFilename + action.meta.requestId === initImageUploadedRequestId ); - // Update the base node with the image name and type - baseNode.image = { - image_name: baseImageDTO.image_name, - }; + canvasInitImage = payload; } - // For inpaint, we also need to upload the mask layer - if (baseNode.type === 'inpaint') { - const maskFilename = `${uuidv4()}.png`; - dispatch( + // For inpaint/outpaint, we also need to upload the mask layer + if (['inpaint', 'outpaint'].includes(generationMode)) { + // upload the image, saving the request id + const { requestId: maskImageUploadedRequestId } = dispatch( imageUploaded({ formData: { - file: new File([maskBlob], maskFilename, { type: 'image/png' }), + file: new File([maskBlob], 'canvasMaskImage.png', { + type: 'image/png', + }), }, imageCategory: 'mask', isIntermediate: true, }) ); - // Wait for the mask to be uploaded - const [{ payload: maskImageDTO }] = await take( + // Wait for the image to be uploaded, matching by request id + const [{ payload }] = await take( (action): action is ReturnType => imageUploaded.fulfilled.match(action) && - action.meta.arg.formData.file.name === maskFilename + action.meta.requestId === maskImageUploadedRequestId ); - // Update the base node with the image name and type - baseNode.mask = { - image_name: maskImageDTO.image_name, - }; + canvasMaskImage = payload; } - // Create the session and wait for response - dispatch(sessionCreated({ graph })); - const [sessionCreatedAction] = await take(sessionCreated.fulfilled.match); + const graph = buildCanvasGraph( + state, + generationMode, + canvasInitImage, + canvasMaskImage + ); + + moduleLog.debug({ graph }, `Canvas graph built`); + + // currently this action is just listened to for logging + dispatch(canvasGraphBuilt(graph)); + + // Create the session, store the request id + const { requestId: sessionCreatedRequestId } = dispatch( + sessionCreated({ graph }) + ); + + // Take the session created action, matching by its request id + const [sessionCreatedAction] = await take( + (action): action is ReturnType => + sessionCreated.fulfilled.match(action) && + action.meta.requestId === sessionCreatedRequestId + ); const sessionId = sessionCreatedAction.payload.id; // Associate the init image with the session, now that we have the session ID - if ( - (baseNode.type === 'img2img' || baseNode.type === 'inpaint') && - baseNode.image - ) { + if (['img2img', 'inpaint'].includes(generationMode) && canvasInitImage) { dispatch( imageUpdated({ - imageName: baseNode.image.image_name, + imageName: canvasInitImage.image_name, requestBody: { session_id: sessionId }, }) ); } // Associate the mask image with the session, now that we have the session ID - if (baseNode.type === 'inpaint' && baseNode.mask) { + if (['inpaint'].includes(generationMode) && canvasMaskImage) { dispatch( imageUpdated({ - imageName: baseNode.mask.image_name, + imageName: canvasMaskImage.image_name, requestBody: { session_id: sessionId }, }) ); } + // Prep the canvas staging area if it is not yet initialized if (!state.canvas.layerState.stagingArea.boundingBox) { dispatch( stagingAreaInitialized({ diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts index 7dcbe8a41d..368d97a10f 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedImageToImage.ts @@ -1,10 +1,10 @@ import { startAppListening } from '..'; -import { buildImageToImageGraph } from 'features/nodes/util/graphBuilders/buildImageToImageGraph'; import { sessionCreated } from 'services/thunks/session'; import { log } from 'app/logging/useLogger'; import { imageToImageGraphBuilt } from 'features/nodes/store/actions'; import { userInvoked } from 'app/store/actions'; import { sessionReadyToInvoke } from 'features/system/store/actions'; +import { buildLinearImageToImageGraph } from 'features/nodes/util/graphBuilders/buildLinearImageToImageGraph'; const moduleLog = log.child({ namespace: 'invoke' }); @@ -15,7 +15,7 @@ export const addUserInvokedImageToImageListener = () => { effect: async (action, { getState, dispatch, take }) => { const state = getState(); - const graph = buildImageToImageGraph(state); + const graph = buildLinearImageToImageGraph(state); dispatch(imageToImageGraphBuilt(graph)); moduleLog.debug({ data: graph }, 'Image to Image graph built'); diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts index 6042d86cb7..c76e0dfd4f 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/userInvokedTextToImage.ts @@ -1,10 +1,10 @@ import { startAppListening } from '..'; -import { buildTextToImageGraph } from 'features/nodes/util/graphBuilders/buildTextToImageGraph'; import { sessionCreated } from 'services/thunks/session'; import { log } from 'app/logging/useLogger'; import { textToImageGraphBuilt } from 'features/nodes/store/actions'; import { userInvoked } from 'app/store/actions'; import { sessionReadyToInvoke } from 'features/system/store/actions'; +import { buildLinearTextToImageGraph } from 'features/nodes/util/graphBuilders/buildLinearTextToImageGraph'; const moduleLog = log.child({ namespace: 'invoke' }); @@ -15,7 +15,7 @@ export const addUserInvokedTextToImageListener = () => { effect: async (action, { getState, dispatch, take }) => { const state = getState(); - const graph = buildTextToImageGraph(state); + const graph = buildLinearTextToImageGraph(state); dispatch(textToImageGraphBuilt(graph)); diff --git a/invokeai/frontend/web/src/common/components/IAIMantineMultiSelect.tsx b/invokeai/frontend/web/src/common/components/IAIMantineMultiSelect.tsx new file mode 100644 index 0000000000..c7ce1de4c1 --- /dev/null +++ b/invokeai/frontend/web/src/common/components/IAIMantineMultiSelect.tsx @@ -0,0 +1,94 @@ +import { Tooltip } from '@chakra-ui/react'; +import { MultiSelect, MultiSelectProps } from '@mantine/core'; +import { memo } from 'react'; + +type IAIMultiSelectProps = MultiSelectProps & { + tooltip?: string; +}; + +const IAIMantineMultiSelect = (props: IAIMultiSelectProps) => { + const { searchable = true, tooltip, ...rest } = props; + return ( + + ({ + label: { + color: 'var(--invokeai-colors-base-300)', + fontWeight: 'normal', + }, + searchInput: { + '::placeholder': { + color: 'var(--invokeai-colors-base-700)', + }, + }, + input: { + backgroundColor: 'var(--invokeai-colors-base-900)', + borderWidth: '2px', + borderColor: 'var(--invokeai-colors-base-800)', + color: 'var(--invokeai-colors-base-100)', + padding: 10, + paddingRight: 24, + fontWeight: 600, + '&:hover': { borderColor: 'var(--invokeai-colors-base-700)' }, + '&:focus': { + borderColor: 'var(--invokeai-colors-accent-600)', + }, + '&:focus-within': { + borderColor: 'var(--invokeai-colors-accent-600)', + }, + }, + value: { + backgroundColor: 'var(--invokeai-colors-base-800)', + color: 'var(--invokeai-colors-base-100)', + button: { + color: 'var(--invokeai-colors-base-100)', + }, + '&:hover': { + backgroundColor: 'var(--invokeai-colors-base-700)', + cursor: 'pointer', + }, + }, + dropdown: { + backgroundColor: 'var(--invokeai-colors-base-800)', + borderColor: 'var(--invokeai-colors-base-700)', + }, + item: { + backgroundColor: 'var(--invokeai-colors-base-800)', + color: 'var(--invokeai-colors-base-200)', + padding: 6, + '&[data-hovered]': { + color: 'var(--invokeai-colors-base-100)', + backgroundColor: 'var(--invokeai-colors-base-750)', + }, + '&[data-active]': { + backgroundColor: 'var(--invokeai-colors-base-750)', + '&:hover': { + color: 'var(--invokeai-colors-base-100)', + backgroundColor: 'var(--invokeai-colors-base-750)', + }, + }, + '&[data-selected]': { + color: 'var(--invokeai-colors-base-50)', + backgroundColor: 'var(--invokeai-colors-accent-650)', + fontWeight: 600, + '&:hover': { + backgroundColor: 'var(--invokeai-colors-accent-600)', + }, + }, + }, + rightSection: { + width: 24, + padding: 20, + button: { + color: 'var(--invokeai-colors-base-100)', + }, + }, + })} + {...rest} + /> + + ); +}; + +export default memo(IAIMantineMultiSelect); diff --git a/invokeai/frontend/web/src/features/nodes/util/addControlNetToLinearGraph.ts b/invokeai/frontend/web/src/features/nodes/util/addControlNetToLinearGraph.ts index 1fd7eb2dba..dd5a97e2f1 100644 --- a/invokeai/frontend/web/src/features/nodes/util/addControlNetToLinearGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/addControlNetToLinearGraph.ts @@ -2,8 +2,7 @@ import { RootState } from 'app/store/store'; import { filter, forEach, size } from 'lodash-es'; import { CollectInvocation, ControlNetInvocation } from 'services/api'; import { NonNullableGraph } from '../types/types'; - -const CONTROL_NET_COLLECT = 'control_net_collect'; +import { CONTROL_NET_COLLECT } from './graphBuilders/constants'; export const addControlNetToLinearGraph = ( graph: NonNullableGraph, @@ -37,7 +36,7 @@ export const addControlNetToLinearGraph = ( }); } - forEach(controlNets, (controlNet, index) => { + forEach(controlNets, (controlNet) => { const { controlNetId, isEnabled, diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts index 2d23b882ea..3ea513fe7e 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasGraph.ts @@ -1,116 +1,39 @@ import { RootState } from 'app/store/store'; -import { - Edge, - ImageToImageInvocation, - InpaintInvocation, - IterateInvocation, - RandomRangeInvocation, - RangeInvocation, - TextToImageInvocation, -} from 'services/api'; -import { buildImg2ImgNode } from '../nodeBuilders/buildImageToImageNode'; -import { buildTxt2ImgNode } from '../nodeBuilders/buildTextToImageNode'; -import { buildRangeNode } from '../nodeBuilders/buildRangeNode'; -import { buildIterateNode } from '../nodeBuilders/buildIterateNode'; -import { buildEdges } from '../edgeBuilders/buildEdges'; +import { ImageDTO } from 'services/api'; import { log } from 'app/logging/useLogger'; -import { buildInpaintNode } from '../nodeBuilders/buildInpaintNode'; +import { forEach } from 'lodash-es'; +import { buildCanvasInpaintGraph } from './buildCanvasInpaintGraph'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { buildCanvasImageToImageGraph } from './buildCanvasImageToImageGraph'; +import { buildCanvasTextToImageGraph } from './buildCanvasTextToImageGraph'; const moduleLog = log.child({ namespace: 'nodes' }); -const buildBaseNode = ( - nodeType: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint', - state: RootState -): - | TextToImageInvocation - | ImageToImageInvocation - | InpaintInvocation - | undefined => { - const overrides = { - ...state.canvas.boundingBoxDimensions, - is_intermediate: true, - }; - - if (nodeType === 'txt2img') { - return buildTxt2ImgNode(state, overrides); - } - - if (nodeType === 'img2img') { - return buildImg2ImgNode(state, overrides); - } - - if (nodeType === 'inpaint' || nodeType === 'outpaint') { - return buildInpaintNode(state, overrides); - } -}; - -/** - * Builds the Canvas workflow graph and image blobs. - */ -export const buildCanvasGraphComponents = async ( +export const buildCanvasGraph = ( state: RootState, - generationMode: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint' -): Promise< - | { - rangeNode: RangeInvocation | RandomRangeInvocation; - iterateNode: IterateInvocation; - baseNode: - | TextToImageInvocation - | ImageToImageInvocation - | InpaintInvocation; - edges: Edge[]; - } - | undefined -> => { - // The base node is a txt2img, img2img or inpaint node - const baseNode = buildBaseNode(generationMode, state); + generationMode: 'txt2img' | 'img2img' | 'inpaint' | 'outpaint', + canvasInitImage: ImageDTO | undefined, + canvasMaskImage: ImageDTO | undefined +) => { + let graph: NonNullableGraph; - if (!baseNode) { - moduleLog.error('Problem building base node'); - return; + if (generationMode === 'txt2img') { + graph = buildCanvasTextToImageGraph(state); + } else if (generationMode === 'img2img') { + if (!canvasInitImage) { + throw new Error('Missing canvas init image'); + } + graph = buildCanvasImageToImageGraph(state, canvasInitImage); + } else { + if (!canvasInitImage || !canvasMaskImage) { + throw new Error('Missing canvas init and mask images'); + } + graph = buildCanvasInpaintGraph(state, canvasInitImage, canvasMaskImage); } - if (baseNode.type === 'inpaint') { - const { - seamSize, - seamBlur, - seamSteps, - seamStrength, - tileSize, - infillMethod, - } = state.generation; + forEach(graph.nodes, (node) => { + graph.nodes[node.id].is_intermediate = true; + }); - const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } = - state.canvas; - - if (boundingBoxScaleMethod !== 'none') { - baseNode.inpaint_width = scaledBoundingBoxDimensions.width; - baseNode.inpaint_height = scaledBoundingBoxDimensions.height; - } - - baseNode.seam_size = seamSize; - baseNode.seam_blur = seamBlur; - baseNode.seam_strength = seamStrength; - baseNode.seam_steps = seamSteps; - baseNode.infill_method = infillMethod as InpaintInvocation['infill_method']; - - if (infillMethod === 'tile') { - baseNode.tile_size = tileSize; - } - } - - // We always range and iterate nodes, no matter the iteration count - // This is required to provide the correct seeds to the backend engine - const rangeNode = buildRangeNode(state); - const iterateNode = buildIterateNode(); - - // Build the edges for the nodes selected. - const edges = buildEdges(baseNode, rangeNode, iterateNode); - - return { - rangeNode, - iterateNode, - baseNode, - edges, - }; + return graph; }; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts new file mode 100644 index 0000000000..efaeaddff2 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts @@ -0,0 +1,331 @@ +import { RootState } from 'app/store/store'; +import { + ImageDTO, + ImageResizeInvocation, + RandomIntInvocation, + RangeOfSizeInvocation, +} from 'services/api'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { log } from 'app/logging/useLogger'; +import { + ITERATE, + LATENTS_TO_IMAGE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + NOISE, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + IMAGE_TO_IMAGE_GRAPH, + IMAGE_TO_LATENTS, + LATENTS_TO_LATENTS, + RESIZE, +} from './constants'; +import { set } from 'lodash-es'; +import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + +const moduleLog = log.child({ namespace: 'nodes' }); + +/** + * Builds the Canvas tab's Image to Image graph. + */ +export const buildCanvasImageToImageGraph = ( + state: RootState, + initialImage: ImageDTO +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + img2imgStrength: strength, + iterations, + seed, + shouldRandomizeSeed, + } = state.generation; + + // The bounding box determines width and height, not the width and height params + const { width, height } = state.canvas.boundingBoxDimensions; + + /** + * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the + * full graph here as a template. Then use the parameters from app state and set friendlier node + * ids. + * + * The only thing we need extra logic for is handling randomized seed, control net, and for img2img, + * the `fit` param. These are added to the graph at the end. + */ + + // copy-pasted graph from node editor, filled in with state values & friendly node ids + const graph: NonNullableGraph = { + id: IMAGE_TO_IMAGE_GRAPH, + nodes: { + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // seed - must be connected manually + // start: 0, + size: iterations, + step: 1, + }, + [NOISE]: { + type: 'noise', + id: NOISE, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [LATENTS_TO_IMAGE]: { + type: 'l2i', + id: LATENTS_TO_IMAGE, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + [LATENTS_TO_LATENTS]: { + type: 'l2l', + id: LATENTS_TO_LATENTS, + cfg_scale, + scheduler, + steps, + strength, + }, + [IMAGE_TO_LATENTS]: { + type: 'i2l', + id: IMAGE_TO_LATENTS, + // must be set manually later, bc `fit` parameter may require a resize node inserted + // image: { + // image_name: initialImage.image_name, + // }, + }, + }, + edges: [ + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: NOISE, + field: 'seed', + }, + }, + { + source: { + node_id: LATENTS_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'latents', + }, + }, + { + source: { + node_id: IMAGE_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'latents', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'noise', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'vae', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'positive_conditioning', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + // handle `fit` + if (initialImage.width !== width || initialImage.height !== height) { + // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS` + + // Create a resize node, explicitly setting its image + const resizeNode: ImageResizeInvocation = { + id: RESIZE, + type: 'img_resize', + image: { + image_name: initialImage.image_name, + }, + is_intermediate: true, + width, + height, + }; + + graph.nodes[RESIZE] = resizeNode; + + // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS` + graph.edges.push({ + source: { node_id: RESIZE, field: 'image' }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'image', + }, + }); + + // The `RESIZE` node also passes its width and height to `NOISE` + graph.edges.push({ + source: { node_id: RESIZE, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + + graph.edges.push({ + source: { node_id: RESIZE, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } else { + // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly + set(graph.nodes[IMAGE_TO_LATENTS], 'image', { + image_name: initialImage.image_name, + }); + + // Pass the image's dimensions to the `NOISE` node + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } + + // add controlnet + addControlNetToLinearGraph(graph, LATENTS_TO_LATENTS, state); + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts new file mode 100644 index 0000000000..785e1d2fdb --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts @@ -0,0 +1,224 @@ +import { RootState } from 'app/store/store'; +import { + ImageDTO, + InpaintInvocation, + RandomIntInvocation, + RangeOfSizeInvocation, +} from 'services/api'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { log } from 'app/logging/useLogger'; +import { + ITERATE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + INPAINT_GRAPH, + INPAINT, +} from './constants'; + +const moduleLog = log.child({ namespace: 'nodes' }); + +/** + * Builds the Canvas tab's Inpaint graph. + */ +export const buildCanvasInpaintGraph = ( + state: RootState, + canvasInitImage: ImageDTO, + canvasMaskImage: ImageDTO +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + img2imgStrength: strength, + shouldFitToWidthHeight, + iterations, + seed, + shouldRandomizeSeed, + seamSize, + seamBlur, + seamSteps, + seamStrength, + tileSize, + infillMethod, + } = state.generation; + + // The bounding box determines width and height, not the width and height params + const { width, height } = state.canvas.boundingBoxDimensions; + + // We may need to set the inpaint width and height to scale the image + const { scaledBoundingBoxDimensions, boundingBoxScaleMethod } = state.canvas; + + const graph: NonNullableGraph = { + id: INPAINT_GRAPH, + nodes: { + [INPAINT]: { + type: 'inpaint', + id: INPAINT, + steps, + width, + height, + cfg_scale, + scheduler, + image: { + image_name: canvasInitImage.image_name, + }, + strength, + fit: shouldFitToWidthHeight, + mask: { + image_name: canvasMaskImage.image_name, + }, + seam_size: seamSize, + seam_blur: seamBlur, + seam_strength: seamStrength, + seam_steps: seamSteps, + tile_size: infillMethod === 'tile' ? tileSize : undefined, + infill_method: infillMethod as InpaintInvocation['infill_method'], + inpaint_width: + boundingBoxScaleMethod !== 'none' + ? scaledBoundingBoxDimensions.width + : undefined, + inpaint_height: + boundingBoxScaleMethod !== 'none' + ? scaledBoundingBoxDimensions.height + : undefined, + }, + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // seed - must be connected manually + // start: 0, + size: iterations, + step: 1, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + }, + edges: [ + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: INPAINT, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: INPAINT, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: INPAINT, + field: 'unet', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: INPAINT, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: INPAINT, + field: 'seed', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts new file mode 100644 index 0000000000..ca0e56e849 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts @@ -0,0 +1,224 @@ +import { RootState } from 'app/store/store'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { RandomIntInvocation, RangeOfSizeInvocation } from 'services/api'; +import { + ITERATE, + LATENTS_TO_IMAGE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + NOISE, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + TEXT_TO_IMAGE_GRAPH, + TEXT_TO_LATENTS, +} from './constants'; +import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + +/** + * Builds the Canvas tab's Text to Image graph. + */ +export const buildCanvasTextToImageGraph = ( + state: RootState +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + iterations, + seed, + shouldRandomizeSeed, + } = state.generation; + + // The bounding box determines width and height, not the width and height params + const { width, height } = state.canvas.boundingBoxDimensions; + + /** + * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the + * full graph here as a template. Then use the parameters from app state and set friendlier node + * ids. + * + * The only thing we need extra logic for is handling randomized seed, control net, and for img2img, + * the `fit` param. These are added to the graph at the end. + */ + + // copy-pasted graph from node editor, filled in with state values & friendly node ids + const graph: NonNullableGraph = { + id: TEXT_TO_IMAGE_GRAPH, + nodes: { + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // start: 0, // seed - must be connected manually + size: iterations, + step: 1, + }, + [NOISE]: { + type: 'noise', + id: NOISE, + width, + height, + }, + [TEXT_TO_LATENTS]: { + type: 't2l', + id: TEXT_TO_LATENTS, + cfg_scale, + scheduler, + steps, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [LATENTS_TO_IMAGE]: { + type: 'l2i', + id: LATENTS_TO_IMAGE, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + }, + edges: [ + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: TEXT_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'latents', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: NOISE, + field: 'seed', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'noise', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + // add controlnet + addControlNetToLinearGraph(graph, TEXT_TO_LATENTS, state); + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildImageToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildImageToImageGraph.ts deleted file mode 100644 index 4986d86713..0000000000 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildImageToImageGraph.ts +++ /dev/null @@ -1,416 +0,0 @@ -import { RootState } from 'app/store/store'; -import { - CompelInvocation, - Graph, - ImageResizeInvocation, - ImageToLatentsInvocation, - IterateInvocation, - LatentsToImageInvocation, - LatentsToLatentsInvocation, - NoiseInvocation, - RandomIntInvocation, - RangeOfSizeInvocation, -} from 'services/api'; -import { NonNullableGraph } from 'features/nodes/types/types'; -import { log } from 'app/logging/useLogger'; -import { set } from 'lodash-es'; -import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; - -const moduleLog = log.child({ namespace: 'nodes' }); - -const POSITIVE_CONDITIONING = 'positive_conditioning'; -const NEGATIVE_CONDITIONING = 'negative_conditioning'; -const IMAGE_TO_LATENTS = 'image_to_latents'; -const LATENTS_TO_LATENTS = 'latents_to_latents'; -const LATENTS_TO_IMAGE = 'latents_to_image'; -const RESIZE = 'resize_image'; -const NOISE = 'noise'; -const RANDOM_INT = 'rand_int'; -const RANGE_OF_SIZE = 'range_of_size'; -const ITERATE = 'iterate'; - -/** - * Builds the Image to Image tab graph. - */ -export const buildImageToImageGraph = (state: RootState): Graph => { - const { - positivePrompt, - negativePrompt, - model, - cfgScale: cfg_scale, - scheduler, - steps, - initialImage, - img2imgStrength: strength, - shouldFitToWidthHeight, - width, - height, - iterations, - seed, - shouldRandomizeSeed, - } = state.generation; - - if (!initialImage) { - moduleLog.error('No initial image found in state'); - throw new Error('No initial image found in state'); - } - - const graph: NonNullableGraph = { - nodes: {}, - edges: [], - }; - - // Create the positive conditioning (prompt) node - const positiveConditioningNode: CompelInvocation = { - id: POSITIVE_CONDITIONING, - type: 'compel', - prompt: positivePrompt, - model, - }; - - // Negative conditioning - const negativeConditioningNode: CompelInvocation = { - id: NEGATIVE_CONDITIONING, - type: 'compel', - prompt: negativePrompt, - model, - }; - - // This will encode the raster image to latents - but it may get its `image` from a resize node, - // so we do not set its `image` property yet - const imageToLatentsNode: ImageToLatentsInvocation = { - id: IMAGE_TO_LATENTS, - type: 'i2l', - model, - }; - - // This does the actual img2img inference - const latentsToLatentsNode: LatentsToLatentsInvocation = { - id: LATENTS_TO_LATENTS, - type: 'l2l', - cfg_scale, - model, - scheduler, - steps, - strength, - }; - - // Finally we decode the latents back to an image - const latentsToImageNode: LatentsToImageInvocation = { - id: LATENTS_TO_IMAGE, - type: 'l2i', - model, - }; - - // Add all those nodes to the graph - graph.nodes[POSITIVE_CONDITIONING] = positiveConditioningNode; - graph.nodes[NEGATIVE_CONDITIONING] = negativeConditioningNode; - graph.nodes[IMAGE_TO_LATENTS] = imageToLatentsNode; - graph.nodes[LATENTS_TO_LATENTS] = latentsToLatentsNode; - graph.nodes[LATENTS_TO_IMAGE] = latentsToImageNode; - - // Connect the prompt nodes to the imageToLatents node - graph.edges.push({ - source: { node_id: POSITIVE_CONDITIONING, field: 'conditioning' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'positive_conditioning', - }, - }); - graph.edges.push({ - source: { node_id: NEGATIVE_CONDITIONING, field: 'conditioning' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'negative_conditioning', - }, - }); - - // Connect the image-encoding node - graph.edges.push({ - source: { node_id: IMAGE_TO_LATENTS, field: 'latents' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'latents', - }, - }); - - // Connect the image-decoding node - graph.edges.push({ - source: { node_id: LATENTS_TO_LATENTS, field: 'latents' }, - destination: { - node_id: LATENTS_TO_IMAGE, - field: 'latents', - }, - }); - - /** - * Now we need to handle iterations and random seeds. There are four possible scenarios: - * - Single iteration, explicit seed - * - Single iteration, random seed - * - Multiple iterations, explicit seed - * - Multiple iterations, random seed - * - * They all have different graphs and connections. - */ - - // Single iteration, explicit seed - if (!shouldRandomizeSeed && iterations === 1) { - // Noise node using the explicit seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - seed: seed, - }; - - graph.nodes[NOISE] = noiseNode; - - // Connect noise to l2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'noise', - }, - }); - } - - // Single iteration, random seed - if (shouldRandomizeSeed && iterations === 1) { - // Random int node to generate the seed - const randomIntNode: RandomIntInvocation = { - id: RANDOM_INT, - type: 'rand_int', - }; - - // Noise node without any seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - }; - - graph.nodes[RANDOM_INT] = randomIntNode; - graph.nodes[NOISE] = noiseNode; - - // Connect random int to the seed of the noise node - graph.edges.push({ - source: { node_id: RANDOM_INT, field: 'a' }, - destination: { - node_id: NOISE, - field: 'seed', - }, - }); - - // Connect noise to l2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'noise', - }, - }); - } - - // Multiple iterations, explicit seed - if (!shouldRandomizeSeed && iterations > 1) { - // Range of size node to generate `iterations` count of seeds - range of size generates a collection - // of ints from `start` to `start + size`. The `start` is the seed, and the `size` is the number of - // iterations. - const rangeOfSizeNode: RangeOfSizeInvocation = { - id: RANGE_OF_SIZE, - type: 'range_of_size', - start: seed, - size: iterations, - }; - - // Iterate node to iterate over the seeds generated by the range of size node - const iterateNode: IterateInvocation = { - id: ITERATE, - type: 'iterate', - }; - - // Noise node without any seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - }; - - // Adding to the graph - graph.nodes[RANGE_OF_SIZE] = rangeOfSizeNode; - graph.nodes[ITERATE] = iterateNode; - graph.nodes[NOISE] = noiseNode; - - // Connect range of size to iterate - graph.edges.push({ - source: { node_id: RANGE_OF_SIZE, field: 'collection' }, - destination: { - node_id: ITERATE, - field: 'collection', - }, - }); - - // Connect iterate to noise - graph.edges.push({ - source: { - node_id: ITERATE, - field: 'item', - }, - destination: { - node_id: NOISE, - field: 'seed', - }, - }); - - // Connect noise to l2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'noise', - }, - }); - } - - // Multiple iterations, random seed - if (shouldRandomizeSeed && iterations > 1) { - // Random int node to generate the seed - const randomIntNode: RandomIntInvocation = { - id: RANDOM_INT, - type: 'rand_int', - }; - - // Range of size node to generate `iterations` count of seeds - range of size generates a collection - const rangeOfSizeNode: RangeOfSizeInvocation = { - id: RANGE_OF_SIZE, - type: 'range_of_size', - size: iterations, - }; - - // Iterate node to iterate over the seeds generated by the range of size node - const iterateNode: IterateInvocation = { - id: ITERATE, - type: 'iterate', - }; - - // Noise node without any seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - width, - height, - }; - - // Adding to the graph - graph.nodes[RANDOM_INT] = randomIntNode; - graph.nodes[RANGE_OF_SIZE] = rangeOfSizeNode; - graph.nodes[ITERATE] = iterateNode; - graph.nodes[NOISE] = noiseNode; - - // Connect random int to the start of the range of size so the range starts on the random first seed - graph.edges.push({ - source: { node_id: RANDOM_INT, field: 'a' }, - destination: { node_id: RANGE_OF_SIZE, field: 'start' }, - }); - - // Connect range of size to iterate - graph.edges.push({ - source: { node_id: RANGE_OF_SIZE, field: 'collection' }, - destination: { - node_id: ITERATE, - field: 'collection', - }, - }); - - // Connect iterate to noise - graph.edges.push({ - source: { - node_id: ITERATE, - field: 'item', - }, - destination: { - node_id: NOISE, - field: 'seed', - }, - }); - - // Connect noise to l2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: LATENTS_TO_LATENTS, - field: 'noise', - }, - }); - } - - if ( - shouldFitToWidthHeight && - (initialImage.width !== width || initialImage.height !== height) - ) { - // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS` - - // Create a resize node, explicitly setting its image - const resizeNode: ImageResizeInvocation = { - id: RESIZE, - type: 'img_resize', - image: { - image_name: initialImage.image_name, - }, - is_intermediate: true, - height, - width, - }; - - graph.nodes[RESIZE] = resizeNode; - - // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS` - graph.edges.push({ - source: { node_id: RESIZE, field: 'image' }, - destination: { - node_id: IMAGE_TO_LATENTS, - field: 'image', - }, - }); - - // The `RESIZE` node also passes its width and height to `NOISE` - graph.edges.push({ - source: { node_id: RESIZE, field: 'width' }, - destination: { - node_id: NOISE, - field: 'width', - }, - }); - - graph.edges.push({ - source: { node_id: RESIZE, field: 'height' }, - destination: { - node_id: NOISE, - field: 'height', - }, - }); - } else { - // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly - set(graph.nodes[IMAGE_TO_LATENTS], 'image', { - image_name: initialImage.image_name, - }); - - // Pass the image's dimensions to the `NOISE` node - graph.edges.push({ - source: { node_id: IMAGE_TO_LATENTS, field: 'width' }, - destination: { - node_id: NOISE, - field: 'width', - }, - }); - graph.edges.push({ - source: { node_id: IMAGE_TO_LATENTS, field: 'height' }, - destination: { - node_id: NOISE, - field: 'height', - }, - }); - } - - addControlNetToLinearGraph(graph, LATENTS_TO_LATENTS, state); - - return graph; -}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts new file mode 100644 index 0000000000..1f2c8327e0 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearImageToImageGraph.ts @@ -0,0 +1,338 @@ +import { RootState } from 'app/store/store'; +import { + ImageResizeInvocation, + RandomIntInvocation, + RangeOfSizeInvocation, +} from 'services/api'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { log } from 'app/logging/useLogger'; +import { + ITERATE, + LATENTS_TO_IMAGE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + NOISE, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + IMAGE_TO_IMAGE_GRAPH, + IMAGE_TO_LATENTS, + LATENTS_TO_LATENTS, + RESIZE, +} from './constants'; +import { set } from 'lodash-es'; +import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + +const moduleLog = log.child({ namespace: 'nodes' }); + +/** + * Builds the Image to Image tab graph. + */ +export const buildLinearImageToImageGraph = ( + state: RootState +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + initialImage, + img2imgStrength: strength, + shouldFitToWidthHeight, + width, + height, + iterations, + seed, + shouldRandomizeSeed, + } = state.generation; + + /** + * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the + * full graph here as a template. Then use the parameters from app state and set friendlier node + * ids. + * + * The only thing we need extra logic for is handling randomized seed, control net, and for img2img, + * the `fit` param. These are added to the graph at the end. + */ + + if (!initialImage) { + moduleLog.error('No initial image found in state'); + throw new Error('No initial image found in state'); + } + + // copy-pasted graph from node editor, filled in with state values & friendly node ids + const graph: NonNullableGraph = { + id: IMAGE_TO_IMAGE_GRAPH, + nodes: { + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // seed - must be connected manually + // start: 0, + size: iterations, + step: 1, + }, + [NOISE]: { + type: 'noise', + id: NOISE, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [LATENTS_TO_IMAGE]: { + type: 'l2i', + id: LATENTS_TO_IMAGE, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + [LATENTS_TO_LATENTS]: { + type: 'l2l', + id: LATENTS_TO_LATENTS, + cfg_scale, + scheduler, + steps, + strength, + }, + [IMAGE_TO_LATENTS]: { + type: 'i2l', + id: IMAGE_TO_LATENTS, + // must be set manually later, bc `fit` parameter may require a resize node inserted + // image: { + // image_name: initialImage.image_name, + // }, + }, + }, + edges: [ + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: NOISE, + field: 'seed', + }, + }, + { + source: { + node_id: LATENTS_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'latents', + }, + }, + { + source: { + node_id: IMAGE_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'latents', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'noise', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'vae', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: LATENTS_TO_LATENTS, + field: 'positive_conditioning', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + // handle `fit` + if ( + shouldFitToWidthHeight && + (initialImage.width !== width || initialImage.height !== height) + ) { + // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS` + + // Create a resize node, explicitly setting its image + const resizeNode: ImageResizeInvocation = { + id: RESIZE, + type: 'img_resize', + image: { + image_name: initialImage.image_name, + }, + is_intermediate: true, + width, + height, + }; + + graph.nodes[RESIZE] = resizeNode; + + // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS` + graph.edges.push({ + source: { node_id: RESIZE, field: 'image' }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'image', + }, + }); + + // The `RESIZE` node also passes its width and height to `NOISE` + graph.edges.push({ + source: { node_id: RESIZE, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + + graph.edges.push({ + source: { node_id: RESIZE, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } else { + // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly + set(graph.nodes[IMAGE_TO_LATENTS], 'image', { + image_name: initialImage.image_name, + }); + + // Pass the image's dimensions to the `NOISE` node + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } + + // add controlnet + addControlNetToLinearGraph(graph, LATENTS_TO_LATENTS, state); + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts new file mode 100644 index 0000000000..c179a89504 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildLinearTextToImageGraph.ts @@ -0,0 +1,226 @@ +import { RootState } from 'app/store/store'; +import { NonNullableGraph } from 'features/nodes/types/types'; +import { RandomIntInvocation, RangeOfSizeInvocation } from 'services/api'; +import { + ITERATE, + LATENTS_TO_IMAGE, + MODEL_LOADER, + NEGATIVE_CONDITIONING, + NOISE, + POSITIVE_CONDITIONING, + RANDOM_INT, + RANGE_OF_SIZE, + TEXT_TO_IMAGE_GRAPH, + TEXT_TO_LATENTS, +} from './constants'; +import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; + +type TextToImageGraphOverrides = { + width: number; + height: number; +}; + +export const buildLinearTextToImageGraph = ( + state: RootState, + overrides?: TextToImageGraphOverrides +): NonNullableGraph => { + const { + positivePrompt, + negativePrompt, + model: model_name, + cfgScale: cfg_scale, + scheduler, + steps, + width, + height, + iterations, + seed, + shouldRandomizeSeed, + } = state.generation; + + /** + * The easiest way to build linear graphs is to do it in the node editor, then copy and paste the + * full graph here as a template. Then use the parameters from app state and set friendlier node + * ids. + * + * The only thing we need extra logic for is handling randomized seed, control net, and for img2img, + * the `fit` param. These are added to the graph at the end. + */ + + // copy-pasted graph from node editor, filled in with state values & friendly node ids + const graph: NonNullableGraph = { + id: TEXT_TO_IMAGE_GRAPH, + nodes: { + [POSITIVE_CONDITIONING]: { + type: 'compel', + id: POSITIVE_CONDITIONING, + prompt: positivePrompt, + }, + [NEGATIVE_CONDITIONING]: { + type: 'compel', + id: NEGATIVE_CONDITIONING, + prompt: negativePrompt, + }, + [RANGE_OF_SIZE]: { + type: 'range_of_size', + id: RANGE_OF_SIZE, + // start: 0, // seed - must be connected manually + size: iterations, + step: 1, + }, + [NOISE]: { + type: 'noise', + id: NOISE, + width: overrides?.width || width, + height: overrides?.height || height, + }, + [TEXT_TO_LATENTS]: { + type: 't2l', + id: TEXT_TO_LATENTS, + cfg_scale, + scheduler, + steps, + }, + [MODEL_LOADER]: { + type: 'sd1_model_loader', + id: MODEL_LOADER, + model_name, + }, + [LATENTS_TO_IMAGE]: { + type: 'l2i', + id: LATENTS_TO_IMAGE, + }, + [ITERATE]: { + type: 'iterate', + id: ITERATE, + }, + }, + edges: [ + { + source: { + node_id: NEGATIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'negative_conditioning', + }, + }, + { + source: { + node_id: POSITIVE_CONDITIONING, + field: 'conditioning', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'positive_conditioning', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: POSITIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'clip', + }, + destination: { + node_id: NEGATIVE_CONDITIONING, + field: 'clip', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'unet', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'unet', + }, + }, + { + source: { + node_id: TEXT_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'latents', + }, + }, + { + source: { + node_id: MODEL_LOADER, + field: 'vae', + }, + destination: { + node_id: LATENTS_TO_IMAGE, + field: 'vae', + }, + }, + { + source: { + node_id: RANGE_OF_SIZE, + field: 'collection', + }, + destination: { + node_id: ITERATE, + field: 'collection', + }, + }, + { + source: { + node_id: ITERATE, + field: 'item', + }, + destination: { + node_id: NOISE, + field: 'seed', + }, + }, + { + source: { + node_id: NOISE, + field: 'noise', + }, + destination: { + node_id: TEXT_TO_LATENTS, + field: 'noise', + }, + }, + ], + }; + + // handle seed + if (shouldRandomizeSeed) { + // Random int node to generate the starting seed + const randomIntNode: RandomIntInvocation = { + id: RANDOM_INT, + type: 'rand_int', + }; + + graph.nodes[RANDOM_INT] = randomIntNode; + + // Connect random int to the start of the range of size so the range starts on the random first seed + graph.edges.push({ + source: { node_id: RANDOM_INT, field: 'a' }, + destination: { node_id: RANGE_OF_SIZE, field: 'start' }, + }); + } else { + // User specified seed, so set the start of the range of size to the seed + (graph.nodes[RANGE_OF_SIZE] as RangeOfSizeInvocation).start = seed; + } + + // add controlnet + addControlNetToLinearGraph(graph, TEXT_TO_LATENTS, state); + + return graph; +}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildTextToImageGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildTextToImageGraph.ts deleted file mode 100644 index ae71f569b6..0000000000 --- a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildTextToImageGraph.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { RootState } from 'app/store/store'; -import { - CompelInvocation, - Graph, - IterateInvocation, - LatentsToImageInvocation, - NoiseInvocation, - RandomIntInvocation, - RangeOfSizeInvocation, - TextToLatentsInvocation, -} from 'services/api'; -import { NonNullableGraph } from 'features/nodes/types/types'; -import { addControlNetToLinearGraph } from '../addControlNetToLinearGraph'; - -const POSITIVE_CONDITIONING = 'positive_conditioning'; -const NEGATIVE_CONDITIONING = 'negative_conditioning'; -const TEXT_TO_LATENTS = 'text_to_latents'; -const LATENTS_TO_IMAGE = 'latents_to_image'; -const NOISE = 'noise'; -const RANDOM_INT = 'rand_int'; -const RANGE_OF_SIZE = 'range_of_size'; -const ITERATE = 'iterate'; - -/** - * Builds the Text to Image tab graph. - */ -export const buildTextToImageGraph = (state: RootState): Graph => { - const { - positivePrompt, - negativePrompt, - model, - cfgScale: cfg_scale, - scheduler, - steps, - width, - height, - iterations, - seed, - shouldRandomizeSeed, - } = state.generation; - - const graph: NonNullableGraph = { - nodes: {}, - edges: [], - }; - - // Create the conditioning, t2l and l2i nodes - const positiveConditioningNode: CompelInvocation = { - id: POSITIVE_CONDITIONING, - type: 'compel', - prompt: positivePrompt, - model, - }; - - const negativeConditioningNode: CompelInvocation = { - id: NEGATIVE_CONDITIONING, - type: 'compel', - prompt: negativePrompt, - model, - }; - - const textToLatentsNode: TextToLatentsInvocation = { - id: TEXT_TO_LATENTS, - type: 't2l', - cfg_scale, - model, - scheduler, - steps, - }; - - const latentsToImageNode: LatentsToImageInvocation = { - id: LATENTS_TO_IMAGE, - type: 'l2i', - model, - }; - - // Add to the graph - graph.nodes[POSITIVE_CONDITIONING] = positiveConditioningNode; - graph.nodes[NEGATIVE_CONDITIONING] = negativeConditioningNode; - graph.nodes[TEXT_TO_LATENTS] = textToLatentsNode; - graph.nodes[LATENTS_TO_IMAGE] = latentsToImageNode; - - // Connect them - graph.edges.push({ - source: { node_id: POSITIVE_CONDITIONING, field: 'conditioning' }, - destination: { - node_id: TEXT_TO_LATENTS, - field: 'positive_conditioning', - }, - }); - - graph.edges.push({ - source: { node_id: NEGATIVE_CONDITIONING, field: 'conditioning' }, - destination: { - node_id: TEXT_TO_LATENTS, - field: 'negative_conditioning', - }, - }); - - graph.edges.push({ - source: { node_id: TEXT_TO_LATENTS, field: 'latents' }, - destination: { - node_id: LATENTS_TO_IMAGE, - field: 'latents', - }, - }); - - /** - * Now we need to handle iterations and random seeds. There are four possible scenarios: - * - Single iteration, explicit seed - * - Single iteration, random seed - * - Multiple iterations, explicit seed - * - Multiple iterations, random seed - * - * They all have different graphs and connections. - */ - - // Single iteration, explicit seed - if (!shouldRandomizeSeed && iterations === 1) { - // Noise node using the explicit seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - seed: seed, - width, - height, - }; - - graph.nodes[NOISE] = noiseNode; - - // Connect noise to l2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: TEXT_TO_LATENTS, - field: 'noise', - }, - }); - } - - // Single iteration, random seed - if (shouldRandomizeSeed && iterations === 1) { - // Random int node to generate the seed - const randomIntNode: RandomIntInvocation = { - id: RANDOM_INT, - type: 'rand_int', - }; - - // Noise node without any seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - width, - height, - }; - - graph.nodes[RANDOM_INT] = randomIntNode; - graph.nodes[NOISE] = noiseNode; - - // Connect random int to the seed of the noise node - graph.edges.push({ - source: { node_id: RANDOM_INT, field: 'a' }, - destination: { - node_id: NOISE, - field: 'seed', - }, - }); - - // Connect noise to t2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: TEXT_TO_LATENTS, - field: 'noise', - }, - }); - } - - // Multiple iterations, explicit seed - if (!shouldRandomizeSeed && iterations > 1) { - // Range of size node to generate `iterations` count of seeds - range of size generates a collection - // of ints from `start` to `start + size`. The `start` is the seed, and the `size` is the number of - // iterations. - const rangeOfSizeNode: RangeOfSizeInvocation = { - id: RANGE_OF_SIZE, - type: 'range_of_size', - start: seed, - size: iterations, - }; - - // Iterate node to iterate over the seeds generated by the range of size node - const iterateNode: IterateInvocation = { - id: ITERATE, - type: 'iterate', - }; - - // Noise node without any seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - width, - height, - }; - - // Adding to the graph - graph.nodes[RANGE_OF_SIZE] = rangeOfSizeNode; - graph.nodes[ITERATE] = iterateNode; - graph.nodes[NOISE] = noiseNode; - - // Connect range of size to iterate - graph.edges.push({ - source: { node_id: RANGE_OF_SIZE, field: 'collection' }, - destination: { - node_id: ITERATE, - field: 'collection', - }, - }); - - // Connect iterate to noise - graph.edges.push({ - source: { - node_id: ITERATE, - field: 'item', - }, - destination: { - node_id: NOISE, - field: 'seed', - }, - }); - - // Connect noise to t2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: TEXT_TO_LATENTS, - field: 'noise', - }, - }); - } - - // Multiple iterations, random seed - if (shouldRandomizeSeed && iterations > 1) { - // Random int node to generate the seed - const randomIntNode: RandomIntInvocation = { - id: RANDOM_INT, - type: 'rand_int', - }; - - // Range of size node to generate `iterations` count of seeds - range of size generates a collection - const rangeOfSizeNode: RangeOfSizeInvocation = { - id: RANGE_OF_SIZE, - type: 'range_of_size', - size: iterations, - }; - - // Iterate node to iterate over the seeds generated by the range of size node - const iterateNode: IterateInvocation = { - id: ITERATE, - type: 'iterate', - }; - - // Noise node without any seed - const noiseNode: NoiseInvocation = { - id: NOISE, - type: 'noise', - width, - height, - }; - - // Adding to the graph - graph.nodes[RANDOM_INT] = randomIntNode; - graph.nodes[RANGE_OF_SIZE] = rangeOfSizeNode; - graph.nodes[ITERATE] = iterateNode; - graph.nodes[NOISE] = noiseNode; - - // Connect random int to the start of the range of size so the range starts on the random first seed - graph.edges.push({ - source: { node_id: RANDOM_INT, field: 'a' }, - destination: { node_id: RANGE_OF_SIZE, field: 'start' }, - }); - - // Connect range of size to iterate - graph.edges.push({ - source: { node_id: RANGE_OF_SIZE, field: 'collection' }, - destination: { - node_id: ITERATE, - field: 'collection', - }, - }); - - // Connect iterate to noise - graph.edges.push({ - source: { - node_id: ITERATE, - field: 'item', - }, - destination: { - node_id: NOISE, - field: 'seed', - }, - }); - - // Connect noise to t2l - graph.edges.push({ - source: { node_id: NOISE, field: 'noise' }, - destination: { - node_id: TEXT_TO_LATENTS, - field: 'noise', - }, - }); - } - - addControlNetToLinearGraph(graph, TEXT_TO_LATENTS, state); - - return graph; -}; diff --git a/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts new file mode 100644 index 0000000000..39e0080d11 --- /dev/null +++ b/invokeai/frontend/web/src/features/nodes/util/graphBuilders/constants.ts @@ -0,0 +1,20 @@ +// friendly node ids +export const POSITIVE_CONDITIONING = 'positive_conditioning'; +export const NEGATIVE_CONDITIONING = 'negative_conditioning'; +export const TEXT_TO_LATENTS = 'text_to_latents'; +export const LATENTS_TO_IMAGE = 'latents_to_image'; +export const NOISE = 'noise'; +export const RANDOM_INT = 'rand_int'; +export const RANGE_OF_SIZE = 'range_of_size'; +export const ITERATE = 'iterate'; +export const MODEL_LOADER = 'model_loader'; +export const IMAGE_TO_LATENTS = 'image_to_latents'; +export const LATENTS_TO_LATENTS = 'latents_to_latents'; +export const RESIZE = 'resize_image'; +export const INPAINT = 'inpaint'; +export const CONTROL_NET_COLLECT = 'control_net_collect'; + +// friendly graph ids +export const TEXT_TO_IMAGE_GRAPH = 'text_to_image_graph'; +export const IMAGE_TO_IMAGE_GRAPH = 'image_to_image_graph'; +export const INPAINT_GRAPH = 'inpaint_graph'; diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamScheduler.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamScheduler.tsx index cf29636ea3..8818dcba9b 100644 --- a/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamScheduler.tsx +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamScheduler.tsx @@ -1,12 +1,11 @@ import { createSelector } from '@reduxjs/toolkit'; -import { Scheduler } from 'app/constants'; +import { SCHEDULER_LABEL_MAP, SCHEDULER_NAMES } from 'app/constants'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import { defaultSelectorOptions } from 'app/store/util/defaultMemoizeOptions'; -import IAIMantineSelect, { - IAISelectDataType, -} from 'common/components/IAIMantineSelect'; +import IAIMantineSelect from 'common/components/IAIMantineSelect'; import { generationSelector } from 'features/parameters/store/generationSelectors'; import { setScheduler } from 'features/parameters/store/generationSlice'; +import { SchedulerParam } from 'features/parameters/store/parameterZodSchemas'; import { uiSelector } from 'features/ui/store/uiSelectors'; import { memo, useCallback } from 'react'; import { useTranslation } from 'react-i18next'; @@ -14,30 +13,36 @@ import { useTranslation } from 'react-i18next'; const selector = createSelector( [uiSelector, generationSelector], (ui, generation) => { - const allSchedulers: string[] = ui.schedulers - .slice() - .sort((a, b) => a.localeCompare(b)); + const { scheduler } = generation; + const { favoriteSchedulers: enabledSchedulers } = ui; + + const data = SCHEDULER_NAMES.map((schedulerName) => ({ + value: schedulerName, + label: SCHEDULER_LABEL_MAP[schedulerName as SchedulerParam], + group: enabledSchedulers.includes(schedulerName) + ? 'Favorites' + : undefined, + })).sort((a, b) => a.label.localeCompare(b.label)); return { - scheduler: generation.scheduler, - allSchedulers, + scheduler, + data, }; }, defaultSelectorOptions ); const ParamScheduler = () => { - const { allSchedulers, scheduler } = useAppSelector(selector); - const dispatch = useAppDispatch(); const { t } = useTranslation(); + const { scheduler, data } = useAppSelector(selector); const handleChange = useCallback( (v: string | null) => { if (!v) { return; } - dispatch(setScheduler(v as Scheduler)); + dispatch(setScheduler(v as SchedulerParam)); }, [dispatch] ); @@ -46,7 +51,7 @@ const ParamScheduler = () => { ); diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSchedulerAndModel.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSchedulerAndModel.tsx index 3b53f5005c..65da89b94d 100644 --- a/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSchedulerAndModel.tsx +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSchedulerAndModel.tsx @@ -1,12 +1,12 @@ import { Box, Flex } from '@chakra-ui/react'; -import { memo } from 'react'; import ModelSelect from 'features/system/components/ModelSelect'; +import { memo } from 'react'; import ParamScheduler from './ParamScheduler'; const ParamSchedulerAndModel = () => { return ( - + diff --git a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts index f516229efe..961ea1b8af 100644 --- a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts +++ b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts @@ -1,10 +1,10 @@ import type { PayloadAction } from '@reduxjs/toolkit'; import { createSlice } from '@reduxjs/toolkit'; -import { clamp, sortBy } from 'lodash-es'; -import { receivedModels } from 'services/thunks/model'; -import { Scheduler } from 'app/constants'; -import { ImageDTO } from 'services/api'; import { configChanged } from 'features/system/store/configSlice'; +import { clamp, sortBy } from 'lodash-es'; +import { ImageDTO } from 'services/api'; +import { imageUrlsReceived } from 'services/thunks/image'; +import { receivedModels } from 'services/thunks/model'; import { CfgScaleParam, HeightParam, @@ -17,7 +17,7 @@ import { StrengthParam, WidthParam, } from './parameterZodSchemas'; -import { imageUrlsReceived } from 'services/thunks/image'; +import { DEFAULT_SCHEDULER_NAME } from 'app/constants'; export interface GenerationState { cfgScale: CfgScaleParam; @@ -63,7 +63,7 @@ export const initialGenerationState: GenerationState = { perlin: 0, positivePrompt: '', negativePrompt: '', - scheduler: 'euler', + scheduler: DEFAULT_SCHEDULER_NAME, seamBlur: 16, seamSize: 96, seamSteps: 30, @@ -133,7 +133,7 @@ export const generationSlice = createSlice({ setWidth: (state, action: PayloadAction) => { state.width = action.payload; }, - setScheduler: (state, action: PayloadAction) => { + setScheduler: (state, action: PayloadAction) => { state.scheduler = action.payload; }, setSeed: (state, action: PayloadAction) => { diff --git a/invokeai/frontend/web/src/features/parameters/store/parameterZodSchemas.ts b/invokeai/frontend/web/src/features/parameters/store/parameterZodSchemas.ts index b99e57bfbb..61567d3fb8 100644 --- a/invokeai/frontend/web/src/features/parameters/store/parameterZodSchemas.ts +++ b/invokeai/frontend/web/src/features/parameters/store/parameterZodSchemas.ts @@ -1,4 +1,4 @@ -import { NUMPY_RAND_MAX, SCHEDULERS } from 'app/constants'; +import { NUMPY_RAND_MAX, SCHEDULER_NAMES_AS_CONST } from 'app/constants'; import { z } from 'zod'; /** @@ -73,7 +73,7 @@ export const isValidCfgScale = (val: unknown): val is CfgScaleParam => /** * Zod schema for scheduler parameter */ -export const zScheduler = z.enum(SCHEDULERS); +export const zScheduler = z.enum(SCHEDULER_NAMES_AS_CONST); /** * Type alias for scheduler parameter, inferred from its zod schema */ diff --git a/invokeai/frontend/web/src/features/system/components/SettingsModal/SettingsSchedulers.tsx b/invokeai/frontend/web/src/features/system/components/SettingsModal/SettingsSchedulers.tsx index e5f4a4cbf7..2e0b3234c7 100644 --- a/invokeai/frontend/web/src/features/system/components/SettingsModal/SettingsSchedulers.tsx +++ b/invokeai/frontend/web/src/features/system/components/SettingsModal/SettingsSchedulers.tsx @@ -1,47 +1,44 @@ -import { - Menu, - MenuButton, - MenuItemOption, - MenuList, - MenuOptionGroup, -} from '@chakra-ui/react'; -import { SCHEDULERS } from 'app/constants'; - +import { SCHEDULER_LABEL_MAP, SCHEDULER_NAMES } from 'app/constants'; import { RootState } from 'app/store/store'; + import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; -import IAIButton from 'common/components/IAIButton'; -import { setSchedulers } from 'features/ui/store/uiSlice'; -import { isArray } from 'lodash-es'; +import IAIMantineMultiSelect from 'common/components/IAIMantineMultiSelect'; +import { SchedulerParam } from 'features/parameters/store/parameterZodSchemas'; +import { favoriteSchedulersChanged } from 'features/ui/store/uiSlice'; +import { map } from 'lodash-es'; +import { useCallback } from 'react'; import { useTranslation } from 'react-i18next'; -export default function SettingsSchedulers() { - const schedulers = useAppSelector((state: RootState) => state.ui.schedulers); +const data = map(SCHEDULER_NAMES, (s) => ({ + value: s, + label: SCHEDULER_LABEL_MAP[s], +})).sort((a, b) => a.label.localeCompare(b.label)); +export default function SettingsSchedulers() { const dispatch = useAppDispatch(); const { t } = useTranslation(); - const schedulerSettingsHandler = (v: string | string[]) => { - if (isArray(v)) dispatch(setSchedulers(v.sort())); - }; + const enabledSchedulers = useAppSelector( + (state: RootState) => state.ui.favoriteSchedulers + ); + + const handleChange = useCallback( + (v: string[]) => { + dispatch(favoriteSchedulersChanged(v as SchedulerParam[])); + }, + [dispatch] + ); return ( - - - {t('settings.availableSchedulers')} - - - - {SCHEDULERS.map((scheduler) => ( - - {scheduler} - - ))} - - - + ); } diff --git a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx index 19ef7fd6fa..8e17ff066c 100644 --- a/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx +++ b/invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx @@ -1,5 +1,4 @@ import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons'; -import ParamSeedCollapse from 'features/parameters/components/Parameters/Seed/ParamSeedCollapse'; import ParamVariationCollapse from 'features/parameters/components/Parameters/Variations/ParamVariationCollapse'; import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse'; import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse'; @@ -8,6 +7,7 @@ import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters'; import { memo } from 'react'; import ParamPositiveConditioning from 'features/parameters/components/Parameters/Core/ParamPositiveConditioning'; import ParamNegativeConditioning from 'features/parameters/components/Parameters/Core/ParamNegativeConditioning'; +import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse'; const UnifiedCanvasParameters = () => { return ( @@ -16,6 +16,7 @@ const UnifiedCanvasParameters = () => { + diff --git a/invokeai/frontend/web/src/features/ui/store/uiSlice.ts b/invokeai/frontend/web/src/features/ui/store/uiSlice.ts index 65a48bc92c..36c514e995 100644 --- a/invokeai/frontend/web/src/features/ui/store/uiSlice.ts +++ b/invokeai/frontend/web/src/features/ui/store/uiSlice.ts @@ -1,10 +1,10 @@ import type { PayloadAction } from '@reduxjs/toolkit'; import { createSlice } from '@reduxjs/toolkit'; +import { initialImageChanged } from 'features/parameters/store/generationSlice'; import { setActiveTabReducer } from './extraReducers'; import { InvokeTabName } from './tabMap'; import { AddNewModelType, UIState } from './uiTypes'; -import { initialImageChanged } from 'features/parameters/store/generationSlice'; -import { SCHEDULERS } from 'app/constants'; +import { SchedulerParam } from 'features/parameters/store/parameterZodSchemas'; export const initialUIState: UIState = { activeTab: 0, @@ -20,7 +20,7 @@ export const initialUIState: UIState = { shouldShowGallery: true, shouldHidePreview: false, shouldShowProgressInViewer: true, - schedulers: SCHEDULERS, + favoriteSchedulers: [], }; export const uiSlice = createSlice({ @@ -94,9 +94,11 @@ export const uiSlice = createSlice({ setShouldShowProgressInViewer: (state, action: PayloadAction) => { state.shouldShowProgressInViewer = action.payload; }, - setSchedulers: (state, action: PayloadAction) => { - state.schedulers = []; - state.schedulers = action.payload; + favoriteSchedulersChanged: ( + state, + action: PayloadAction + ) => { + state.favoriteSchedulers = action.payload; }, }, extraReducers(builder) { @@ -124,7 +126,7 @@ export const { toggleParametersPanel, toggleGalleryPanel, setShouldShowProgressInViewer, - setSchedulers, + favoriteSchedulersChanged, } = uiSlice.actions; export default uiSlice.reducer; diff --git a/invokeai/frontend/web/src/features/ui/store/uiTypes.ts b/invokeai/frontend/web/src/features/ui/store/uiTypes.ts index 18a758cdd6..2a9a82fbe8 100644 --- a/invokeai/frontend/web/src/features/ui/store/uiTypes.ts +++ b/invokeai/frontend/web/src/features/ui/store/uiTypes.ts @@ -1,3 +1,5 @@ +import { SchedulerParam } from 'features/parameters/store/parameterZodSchemas'; + export type AddNewModelType = 'ckpt' | 'diffusers' | null; export type Coordinates = { @@ -26,5 +28,5 @@ export interface UIState { shouldPinGallery: boolean; shouldShowGallery: boolean; shouldShowProgressInViewer: boolean; - schedulers: string[]; + favoriteSchedulers: SchedulerParam[]; } diff --git a/invokeai/frontend/web/src/services/api/index.ts b/invokeai/frontend/web/src/services/api/index.ts index cd83555f15..7481a5daad 100644 --- a/invokeai/frontend/web/src/services/api/index.ts +++ b/invokeai/frontend/web/src/services/api/index.ts @@ -7,9 +7,11 @@ export { OpenAPI } from './core/OpenAPI'; export type { OpenAPIConfig } from './core/OpenAPI'; export type { AddInvocation } from './models/AddInvocation'; +export type { BaseModelType } from './models/BaseModelType'; export type { Body_upload_image } from './models/Body_upload_image'; export type { CannyImageProcessorInvocation } from './models/CannyImageProcessorInvocation'; export type { CkptModelInfo } from './models/CkptModelInfo'; +export type { ClipField } from './models/ClipField'; export type { CollectInvocation } from './models/CollectInvocation'; export type { CollectInvocationOutput } from './models/CollectInvocationOutput'; export type { ColorField } from './models/ColorField'; @@ -53,7 +55,6 @@ export type { ImageProcessorInvocation } from './models/ImageProcessorInvocation export type { ImageRecordChanges } from './models/ImageRecordChanges'; export type { ImageResizeInvocation } from './models/ImageResizeInvocation'; export type { ImageScaleInvocation } from './models/ImageScaleInvocation'; -export type { ImageToImageInvocation } from './models/ImageToImageInvocation'; export type { ImageToLatentsInvocation } from './models/ImageToLatentsInvocation'; export type { ImageUrlsDTO } from './models/ImageUrlsDTO'; export type { InfillColorInvocation } from './models/InfillColorInvocation'; @@ -62,6 +63,14 @@ export type { InfillTileInvocation } from './models/InfillTileInvocation'; export type { InpaintInvocation } from './models/InpaintInvocation'; export type { IntCollectionOutput } from './models/IntCollectionOutput'; export type { IntOutput } from './models/IntOutput'; +export type { invokeai__backend__model_management__models__controlnet__ControlNetModel__Config } from './models/invokeai__backend__model_management__models__controlnet__ControlNetModel__Config'; +export type { invokeai__backend__model_management__models__lora__LoRAModel__Config } from './models/invokeai__backend__model_management__models__lora__LoRAModel__Config'; +export type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig } from './models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig'; +export type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig } from './models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig'; +export type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig } from './models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig'; +export type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig } from './models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig'; +export type { invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config } from './models/invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config'; +export type { invokeai__backend__model_management__models__vae__VaeModel__Config } from './models/invokeai__backend__model_management__models__vae__VaeModel__Config'; export type { IterateInvocation } from './models/IterateInvocation'; export type { IterateInvocationOutput } from './models/IterateInvocationOutput'; export type { LatentsField } from './models/LatentsField'; @@ -71,12 +80,20 @@ export type { LatentsToLatentsInvocation } from './models/LatentsToLatentsInvoca export type { LineartAnimeImageProcessorInvocation } from './models/LineartAnimeImageProcessorInvocation'; export type { LineartImageProcessorInvocation } from './models/LineartImageProcessorInvocation'; export type { LoadImageInvocation } from './models/LoadImageInvocation'; +export type { LoraInfo } from './models/LoraInfo'; +export type { LoraLoaderInvocation } from './models/LoraLoaderInvocation'; +export type { LoraLoaderOutput } from './models/LoraLoaderOutput'; export type { MaskFromAlphaInvocation } from './models/MaskFromAlphaInvocation'; export type { MaskOutput } from './models/MaskOutput'; export type { MediapipeFaceProcessorInvocation } from './models/MediapipeFaceProcessorInvocation'; export type { MidasDepthImageProcessorInvocation } from './models/MidasDepthImageProcessorInvocation'; export type { MlsdImageProcessorInvocation } from './models/MlsdImageProcessorInvocation'; +export type { ModelError } from './models/ModelError'; +export type { ModelInfo } from './models/ModelInfo'; +export type { ModelLoaderOutput } from './models/ModelLoaderOutput'; export type { ModelsList } from './models/ModelsList'; +export type { ModelType } from './models/ModelType'; +export type { ModelVariantType } from './models/ModelVariantType'; export type { MultiplyInvocation } from './models/MultiplyInvocation'; export type { NoiseInvocation } from './models/NoiseInvocation'; export type { NoiseOutput } from './models/NoiseOutput'; @@ -97,12 +114,17 @@ export type { ResizeLatentsInvocation } from './models/ResizeLatentsInvocation'; export type { ResourceOrigin } from './models/ResourceOrigin'; export type { RestoreFaceInvocation } from './models/RestoreFaceInvocation'; export type { ScaleLatentsInvocation } from './models/ScaleLatentsInvocation'; +export type { SchedulerPredictionType } from './models/SchedulerPredictionType'; +export type { SD1ModelLoaderInvocation } from './models/SD1ModelLoaderInvocation'; +export type { SD2ModelLoaderInvocation } from './models/SD2ModelLoaderInvocation'; export type { ShowImageInvocation } from './models/ShowImageInvocation'; export type { StepParamEasingInvocation } from './models/StepParamEasingInvocation'; +export type { SubModelType } from './models/SubModelType'; export type { SubtractInvocation } from './models/SubtractInvocation'; -export type { TextToImageInvocation } from './models/TextToImageInvocation'; export type { TextToLatentsInvocation } from './models/TextToLatentsInvocation'; +export type { UNetField } from './models/UNetField'; export type { UpscaleInvocation } from './models/UpscaleInvocation'; +export type { VaeField } from './models/VaeField'; export type { VaeRepo } from './models/VaeRepo'; export type { ValidationError } from './models/ValidationError'; export type { ZoeDepthImageProcessorInvocation } from './models/ZoeDepthImageProcessorInvocation'; diff --git a/invokeai/frontend/web/src/services/api/models/BaseModelType.ts b/invokeai/frontend/web/src/services/api/models/BaseModelType.ts new file mode 100644 index 0000000000..3f72e68fa4 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/BaseModelType.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type BaseModelType = 'sd-1' | 'sd-2'; diff --git a/invokeai/frontend/web/src/services/api/models/CkptModelInfo.ts b/invokeai/frontend/web/src/services/api/models/CkptModelInfo.ts index 2ae7c09674..cfa4357725 100644 --- a/invokeai/frontend/web/src/services/api/models/CkptModelInfo.ts +++ b/invokeai/frontend/web/src/services/api/models/CkptModelInfo.ts @@ -7,6 +7,14 @@ export type CkptModelInfo = { * A description of the model */ description?: string; + /** + * The name of the model + */ + model_name: string; + /** + * The type of the model + */ + model_type: string; format?: 'ckpt'; /** * The path to the model config diff --git a/invokeai/frontend/web/src/services/api/models/ClipField.ts b/invokeai/frontend/web/src/services/api/models/ClipField.ts new file mode 100644 index 0000000000..f9ef2cc683 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/ClipField.ts @@ -0,0 +1,22 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LoraInfo } from './LoraInfo'; +import type { ModelInfo } from './ModelInfo'; + +export type ClipField = { + /** + * Info to load tokenizer submodel + */ + tokenizer: ModelInfo; + /** + * Info to load text_encoder submodel + */ + text_encoder: ModelInfo; + /** + * Loras to apply on model loading + */ + loras: Array; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/CompelInvocation.ts b/invokeai/frontend/web/src/services/api/models/CompelInvocation.ts index 1dc390c1be..dd381ef22c 100644 --- a/invokeai/frontend/web/src/services/api/models/CompelInvocation.ts +++ b/invokeai/frontend/web/src/services/api/models/CompelInvocation.ts @@ -2,6 +2,8 @@ /* tslint:disable */ /* eslint-disable */ +import type { ClipField } from './ClipField'; + /** * Parse prompt using compel package to conditioning. */ @@ -20,8 +22,8 @@ export type CompelInvocation = { */ prompt?: string; /** - * Model to use + * Clip to use */ - model?: string; + clip?: ClipField; }; diff --git a/invokeai/frontend/web/src/services/api/models/DiffusersModelInfo.ts b/invokeai/frontend/web/src/services/api/models/DiffusersModelInfo.ts index 5be4801cdd..4e722ddb80 100644 --- a/invokeai/frontend/web/src/services/api/models/DiffusersModelInfo.ts +++ b/invokeai/frontend/web/src/services/api/models/DiffusersModelInfo.ts @@ -9,7 +9,15 @@ export type DiffusersModelInfo = { * A description of the model */ description?: string; - format?: 'diffusers'; + /** + * The name of the model + */ + model_name: string; + /** + * The type of the model + */ + model_type: string; + format?: 'folder'; /** * The VAE repo to use for this model */ diff --git a/invokeai/frontend/web/src/services/api/models/Graph.ts b/invokeai/frontend/web/src/services/api/models/Graph.ts index efac5dabcc..e148954f16 100644 --- a/invokeai/frontend/web/src/services/api/models/Graph.ts +++ b/invokeai/frontend/web/src/services/api/models/Graph.ts @@ -26,7 +26,6 @@ import type { ImagePasteInvocation } from './ImagePasteInvocation'; import type { ImageProcessorInvocation } from './ImageProcessorInvocation'; import type { ImageResizeInvocation } from './ImageResizeInvocation'; import type { ImageScaleInvocation } from './ImageScaleInvocation'; -import type { ImageToImageInvocation } from './ImageToImageInvocation'; import type { ImageToLatentsInvocation } from './ImageToLatentsInvocation'; import type { InfillColorInvocation } from './InfillColorInvocation'; import type { InfillPatchMatchInvocation } from './InfillPatchMatchInvocation'; @@ -38,6 +37,7 @@ import type { LatentsToLatentsInvocation } from './LatentsToLatentsInvocation'; import type { LineartAnimeImageProcessorInvocation } from './LineartAnimeImageProcessorInvocation'; import type { LineartImageProcessorInvocation } from './LineartImageProcessorInvocation'; import type { LoadImageInvocation } from './LoadImageInvocation'; +import type { LoraLoaderInvocation } from './LoraLoaderInvocation'; import type { MaskFromAlphaInvocation } from './MaskFromAlphaInvocation'; import type { MediapipeFaceProcessorInvocation } from './MediapipeFaceProcessorInvocation'; import type { MidasDepthImageProcessorInvocation } from './MidasDepthImageProcessorInvocation'; @@ -56,10 +56,11 @@ import type { RangeOfSizeInvocation } from './RangeOfSizeInvocation'; import type { ResizeLatentsInvocation } from './ResizeLatentsInvocation'; import type { RestoreFaceInvocation } from './RestoreFaceInvocation'; import type { ScaleLatentsInvocation } from './ScaleLatentsInvocation'; +import type { SD1ModelLoaderInvocation } from './SD1ModelLoaderInvocation'; +import type { SD2ModelLoaderInvocation } from './SD2ModelLoaderInvocation'; import type { ShowImageInvocation } from './ShowImageInvocation'; import type { StepParamEasingInvocation } from './StepParamEasingInvocation'; import type { SubtractInvocation } from './SubtractInvocation'; -import type { TextToImageInvocation } from './TextToImageInvocation'; import type { TextToLatentsInvocation } from './TextToLatentsInvocation'; import type { UpscaleInvocation } from './UpscaleInvocation'; import type { ZoeDepthImageProcessorInvocation } from './ZoeDepthImageProcessorInvocation'; @@ -72,7 +73,7 @@ export type Graph = { /** * The nodes in this graph */ - nodes?: Record; + nodes?: Record; /** * The connections between nodes and their fields in this graph */ diff --git a/invokeai/frontend/web/src/services/api/models/GraphExecutionState.ts b/invokeai/frontend/web/src/services/api/models/GraphExecutionState.ts index ccd5d6f499..602e7a2ebc 100644 --- a/invokeai/frontend/web/src/services/api/models/GraphExecutionState.ts +++ b/invokeai/frontend/web/src/services/api/models/GraphExecutionState.ts @@ -14,7 +14,9 @@ import type { IntCollectionOutput } from './IntCollectionOutput'; import type { IntOutput } from './IntOutput'; import type { IterateInvocationOutput } from './IterateInvocationOutput'; import type { LatentsOutput } from './LatentsOutput'; +import type { LoraLoaderOutput } from './LoraLoaderOutput'; import type { MaskOutput } from './MaskOutput'; +import type { ModelLoaderOutput } from './ModelLoaderOutput'; import type { NoiseOutput } from './NoiseOutput'; import type { PromptCollectionOutput } from './PromptCollectionOutput'; import type { PromptOutput } from './PromptOutput'; @@ -46,7 +48,7 @@ export type GraphExecutionState = { /** * The results of node executions */ - results: Record; + results: Record; /** * Errors raised when executing nodes */ diff --git a/invokeai/frontend/web/src/services/api/models/ImageToImageInvocation.ts b/invokeai/frontend/web/src/services/api/models/ImageToImageInvocation.ts deleted file mode 100644 index e63ec93ada..0000000000 --- a/invokeai/frontend/web/src/services/api/models/ImageToImageInvocation.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ - -import type { ImageField } from './ImageField'; - -/** - * Generates an image using img2img. - */ -export type ImageToImageInvocation = { - /** - * The id of this node. Must be unique among all nodes. - */ - id: string; - /** - * Whether or not this node is an intermediate node. - */ - is_intermediate?: boolean; - type?: 'img2img'; - /** - * The prompt to generate an image from - */ - prompt?: string; - /** - * The seed to use (omit for random) - */ - seed?: number; - /** - * The number of steps to use to generate the image - */ - steps?: number; - /** - * The width of the resulting image - */ - width?: number; - /** - * The height of the resulting image - */ - height?: number; - /** - * The Classifier-Free Guidance, higher values may result in a result closer to the prompt - */ - cfg_scale?: number; - /** - * The scheduler to use - */ - scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'unipc'; - /** - * The model to use (currently ignored) - */ - model?: string; - /** - * Whether or not to produce progress images during generation - */ - progress_images?: boolean; - /** - * The control model to use - */ - control_model?: string; - /** - * The processed control image - */ - control_image?: ImageField; - /** - * The input image - */ - image?: ImageField; - /** - * The strength of the original image - */ - strength?: number; - /** - * Whether or not the result should be fit to the aspect ratio of the input image - */ - fit?: boolean; -}; - diff --git a/invokeai/frontend/web/src/services/api/models/ImageToLatentsInvocation.ts b/invokeai/frontend/web/src/services/api/models/ImageToLatentsInvocation.ts index 5569c2fa86..ace0ed8e3c 100644 --- a/invokeai/frontend/web/src/services/api/models/ImageToLatentsInvocation.ts +++ b/invokeai/frontend/web/src/services/api/models/ImageToLatentsInvocation.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ImageField } from './ImageField'; +import type { VaeField } from './VaeField'; /** * Encodes an image into latents. @@ -22,8 +23,12 @@ export type ImageToLatentsInvocation = { */ image?: ImageField; /** - * The model to use + * Vae submodel */ - model?: string; + vae?: VaeField; + /** + * Encode latents by overlaping tiles(less memory consumption) + */ + tiled?: boolean; }; diff --git a/invokeai/frontend/web/src/services/api/models/InpaintInvocation.ts b/invokeai/frontend/web/src/services/api/models/InpaintInvocation.ts index b8ed268ef9..8fb9ad3d54 100644 --- a/invokeai/frontend/web/src/services/api/models/InpaintInvocation.ts +++ b/invokeai/frontend/web/src/services/api/models/InpaintInvocation.ts @@ -3,7 +3,10 @@ /* eslint-disable */ import type { ColorField } from './ColorField'; +import type { ConditioningField } from './ConditioningField'; import type { ImageField } from './ImageField'; +import type { UNetField } from './UNetField'; +import type { VaeField } from './VaeField'; /** * Generates an image using inpaint. @@ -19,9 +22,13 @@ export type InpaintInvocation = { is_intermediate?: boolean; type?: 'inpaint'; /** - * The prompt to generate an image from + * Positive conditioning for generation */ - prompt?: string; + positive_conditioning?: ConditioningField; + /** + * Negative conditioning for generation + */ + negative_conditioning?: ConditioningField; /** * The seed to use (omit for random) */ @@ -45,23 +52,15 @@ export type InpaintInvocation = { /** * The scheduler to use */ - scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'unipc'; + scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'lms_k' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2s_k' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'dpmpp_2m_sde' | 'dpmpp_2m_sde_k' | 'dpmpp_sde' | 'dpmpp_sde_k' | 'unipc'; /** - * The model to use (currently ignored) + * UNet model */ - model?: string; + unet?: UNetField; /** - * Whether or not to produce progress images during generation + * Vae model */ - progress_images?: boolean; - /** - * The control model to use - */ - control_model?: string; - /** - * The processed control image - */ - control_image?: ImageField; + vae?: VaeField; /** * The input image */ diff --git a/invokeai/frontend/web/src/services/api/models/LatentsToImageInvocation.ts b/invokeai/frontend/web/src/services/api/models/LatentsToImageInvocation.ts index fcaa37d7e8..865eeff554 100644 --- a/invokeai/frontend/web/src/services/api/models/LatentsToImageInvocation.ts +++ b/invokeai/frontend/web/src/services/api/models/LatentsToImageInvocation.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { LatentsField } from './LatentsField'; +import type { VaeField } from './VaeField'; /** * Generates an image from latents. @@ -22,8 +23,12 @@ export type LatentsToImageInvocation = { */ latents?: LatentsField; /** - * The model to use + * Vae submodel */ - model?: string; + vae?: VaeField; + /** + * Decode latents by overlaping tiles(less memory consumption) + */ + tiled?: boolean; }; diff --git a/invokeai/frontend/web/src/services/api/models/LatentsToLatentsInvocation.ts b/invokeai/frontend/web/src/services/api/models/LatentsToLatentsInvocation.ts index 60504459e7..4273115963 100644 --- a/invokeai/frontend/web/src/services/api/models/LatentsToLatentsInvocation.ts +++ b/invokeai/frontend/web/src/services/api/models/LatentsToLatentsInvocation.ts @@ -5,6 +5,7 @@ import type { ConditioningField } from './ConditioningField'; import type { ControlField } from './ControlField'; import type { LatentsField } from './LatentsField'; +import type { UNetField } from './UNetField'; /** * Generates latents using latents as base image. @@ -42,11 +43,11 @@ export type LatentsToLatentsInvocation = { /** * The scheduler to use */ - scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'unipc'; + scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'lms_k' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2s_k' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'dpmpp_2m_sde' | 'dpmpp_2m_sde_k' | 'dpmpp_sde' | 'dpmpp_sde_k' | 'unipc'; /** - * The model to use (currently ignored) + * UNet submodel */ - model?: string; + unet?: UNetField; /** * The control to use */ diff --git a/invokeai/frontend/web/src/services/api/models/LoraInfo.ts b/invokeai/frontend/web/src/services/api/models/LoraInfo.ts new file mode 100644 index 0000000000..1a575d4147 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/LoraInfo.ts @@ -0,0 +1,31 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { BaseModelType } from './BaseModelType'; +import type { ModelType } from './ModelType'; +import type { SubModelType } from './SubModelType'; + +export type LoraInfo = { + /** + * Info to load submodel + */ + model_name: string; + /** + * Base model + */ + base_model: BaseModelType; + /** + * Info to load submodel + */ + model_type: ModelType; + /** + * Info to load submodel + */ + submodel?: SubModelType; + /** + * Lora's weight which to use when apply to model + */ + weight: number; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/LoraLoaderInvocation.ts b/invokeai/frontend/web/src/services/api/models/LoraLoaderInvocation.ts new file mode 100644 index 0000000000..b93281c5a7 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/LoraLoaderInvocation.ts @@ -0,0 +1,38 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ClipField } from './ClipField'; +import type { UNetField } from './UNetField'; + +/** + * Apply selected lora to unet and text_encoder. + */ +export type LoraLoaderInvocation = { + /** + * The id of this node. Must be unique among all nodes. + */ + id: string; + /** + * Whether or not this node is an intermediate node. + */ + is_intermediate?: boolean; + type?: 'lora_loader'; + /** + * Lora model name + */ + lora_name: string; + /** + * With what weight to apply lora + */ + weight?: number; + /** + * UNet model for applying lora + */ + unet?: UNetField; + /** + * Clip model for applying lora + */ + clip?: ClipField; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/LoraLoaderOutput.ts b/invokeai/frontend/web/src/services/api/models/LoraLoaderOutput.ts new file mode 100644 index 0000000000..1fed1ebc58 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/LoraLoaderOutput.ts @@ -0,0 +1,22 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ClipField } from './ClipField'; +import type { UNetField } from './UNetField'; + +/** + * Model loader output + */ +export type LoraLoaderOutput = { + type?: 'lora_loader_output'; + /** + * UNet submodel + */ + unet?: UNetField; + /** + * Tokenizer and text_encoder submodels + */ + clip?: ClipField; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/ModelError.ts b/invokeai/frontend/web/src/services/api/models/ModelError.ts new file mode 100644 index 0000000000..3151a764d6 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/ModelError.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type ModelError = 'not_found'; diff --git a/invokeai/frontend/web/src/services/api/models/ModelInfo.ts b/invokeai/frontend/web/src/services/api/models/ModelInfo.ts new file mode 100644 index 0000000000..e87799d142 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/ModelInfo.ts @@ -0,0 +1,27 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { BaseModelType } from './BaseModelType'; +import type { ModelType } from './ModelType'; +import type { SubModelType } from './SubModelType'; + +export type ModelInfo = { + /** + * Info to load submodel + */ + model_name: string; + /** + * Base model + */ + base_model: BaseModelType; + /** + * Info to load submodel + */ + model_type: ModelType; + /** + * Info to load submodel + */ + submodel?: SubModelType; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/ModelLoaderOutput.ts b/invokeai/frontend/web/src/services/api/models/ModelLoaderOutput.ts new file mode 100644 index 0000000000..5b5b51e71f --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/ModelLoaderOutput.ts @@ -0,0 +1,27 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ClipField } from './ClipField'; +import type { UNetField } from './UNetField'; +import type { VaeField } from './VaeField'; + +/** + * Model loader output + */ +export type ModelLoaderOutput = { + type?: 'model_loader_output'; + /** + * UNet submodel + */ + unet?: UNetField; + /** + * Tokenizer and text_encoder submodels + */ + clip?: ClipField; + /** + * Vae submodel + */ + vae?: VaeField; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/ModelType.ts b/invokeai/frontend/web/src/services/api/models/ModelType.ts new file mode 100644 index 0000000000..7d7abcafae --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/ModelType.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type ModelType = 'pipeline' | 'vae' | 'lora' | 'controlnet' | 'embedding'; diff --git a/invokeai/frontend/web/src/services/api/models/ModelVariantType.ts b/invokeai/frontend/web/src/services/api/models/ModelVariantType.ts new file mode 100644 index 0000000000..0527c40bcf --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/ModelVariantType.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type ModelVariantType = 'normal' | 'inpaint' | 'depth'; diff --git a/invokeai/frontend/web/src/services/api/models/ModelsList.ts b/invokeai/frontend/web/src/services/api/models/ModelsList.ts index 7a7449542d..a2d88d1967 100644 --- a/invokeai/frontend/web/src/services/api/models/ModelsList.ts +++ b/invokeai/frontend/web/src/services/api/models/ModelsList.ts @@ -2,10 +2,16 @@ /* tslint:disable */ /* eslint-disable */ -import type { CkptModelInfo } from './CkptModelInfo'; -import type { DiffusersModelInfo } from './DiffusersModelInfo'; +import type { invokeai__backend__model_management__models__controlnet__ControlNetModel__Config } from './invokeai__backend__model_management__models__controlnet__ControlNetModel__Config'; +import type { invokeai__backend__model_management__models__lora__LoRAModel__Config } from './invokeai__backend__model_management__models__lora__LoRAModel__Config'; +import type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig } from './invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig'; +import type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig } from './invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig'; +import type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig } from './invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig'; +import type { invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig } from './invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig'; +import type { invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config } from './invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config'; +import type { invokeai__backend__model_management__models__vae__VaeModel__Config } from './invokeai__backend__model_management__models__vae__VaeModel__Config'; export type ModelsList = { - models: Record; + models: Record>>; }; diff --git a/invokeai/frontend/web/src/services/api/models/SD1ModelLoaderInvocation.ts b/invokeai/frontend/web/src/services/api/models/SD1ModelLoaderInvocation.ts new file mode 100644 index 0000000000..9a8a23077a --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/SD1ModelLoaderInvocation.ts @@ -0,0 +1,23 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Loading submodels of selected model. + */ +export type SD1ModelLoaderInvocation = { + /** + * The id of this node. Must be unique among all nodes. + */ + id: string; + /** + * Whether or not this node is an intermediate node. + */ + is_intermediate?: boolean; + type?: 'sd1_model_loader'; + /** + * Model to load + */ + model_name?: string; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/SD2ModelLoaderInvocation.ts b/invokeai/frontend/web/src/services/api/models/SD2ModelLoaderInvocation.ts new file mode 100644 index 0000000000..f477c11a8d --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/SD2ModelLoaderInvocation.ts @@ -0,0 +1,23 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Loading submodels of selected model. + */ +export type SD2ModelLoaderInvocation = { + /** + * The id of this node. Must be unique among all nodes. + */ + id: string; + /** + * Whether or not this node is an intermediate node. + */ + is_intermediate?: boolean; + type?: 'sd2_model_loader'; + /** + * Model to load + */ + model_name?: string; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/SchedulerPredictionType.ts b/invokeai/frontend/web/src/services/api/models/SchedulerPredictionType.ts new file mode 100644 index 0000000000..fa24aab5a1 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/SchedulerPredictionType.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type SchedulerPredictionType = 'epsilon' | 'v_prediction' | 'sample'; diff --git a/invokeai/frontend/web/src/services/api/models/SubModelType.ts b/invokeai/frontend/web/src/services/api/models/SubModelType.ts new file mode 100644 index 0000000000..12b055994c --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/SubModelType.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type SubModelType = 'unet' | 'text_encoder' | 'tokenizer' | 'vae' | 'scheduler' | 'safety_checker'; diff --git a/invokeai/frontend/web/src/services/api/models/TextToImageInvocation.ts b/invokeai/frontend/web/src/services/api/models/TextToImageInvocation.ts deleted file mode 100644 index 7128ea8440..0000000000 --- a/invokeai/frontend/web/src/services/api/models/TextToImageInvocation.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ - -import type { ImageField } from './ImageField'; - -/** - * Generates an image using text2img. - */ -export type TextToImageInvocation = { - /** - * The id of this node. Must be unique among all nodes. - */ - id: string; - /** - * Whether or not this node is an intermediate node. - */ - is_intermediate?: boolean; - type?: 'txt2img'; - /** - * The prompt to generate an image from - */ - prompt?: string; - /** - * The seed to use (omit for random) - */ - seed?: number; - /** - * The number of steps to use to generate the image - */ - steps?: number; - /** - * The width of the resulting image - */ - width?: number; - /** - * The height of the resulting image - */ - height?: number; - /** - * The Classifier-Free Guidance, higher values may result in a result closer to the prompt - */ - cfg_scale?: number; - /** - * The scheduler to use - */ - scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'unipc'; - /** - * The model to use (currently ignored) - */ - model?: string; - /** - * Whether or not to produce progress images during generation - */ - progress_images?: boolean; - /** - * The control model to use - */ - control_model?: string; - /** - * The processed control image - */ - control_image?: ImageField; -}; - diff --git a/invokeai/frontend/web/src/services/api/models/TextToLatentsInvocation.ts b/invokeai/frontend/web/src/services/api/models/TextToLatentsInvocation.ts index 2db0657e25..cf8229b1f7 100644 --- a/invokeai/frontend/web/src/services/api/models/TextToLatentsInvocation.ts +++ b/invokeai/frontend/web/src/services/api/models/TextToLatentsInvocation.ts @@ -5,6 +5,7 @@ import type { ConditioningField } from './ConditioningField'; import type { ControlField } from './ControlField'; import type { LatentsField } from './LatentsField'; +import type { UNetField } from './UNetField'; /** * Generates latents from conditionings. @@ -42,11 +43,11 @@ export type TextToLatentsInvocation = { /** * The scheduler to use */ - scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'unipc'; + scheduler?: 'ddim' | 'ddpm' | 'deis' | 'lms' | 'lms_k' | 'pndm' | 'heun' | 'heun_k' | 'euler' | 'euler_k' | 'euler_a' | 'kdpm_2' | 'kdpm_2_a' | 'dpmpp_2s' | 'dpmpp_2s_k' | 'dpmpp_2m' | 'dpmpp_2m_k' | 'dpmpp_2m_sde' | 'dpmpp_2m_sde_k' | 'dpmpp_sde' | 'dpmpp_sde_k' | 'unipc'; /** - * The model to use (currently ignored) + * UNet submodel */ - model?: string; + unet?: UNetField; /** * The control to use */ diff --git a/invokeai/frontend/web/src/services/api/models/UNetField.ts b/invokeai/frontend/web/src/services/api/models/UNetField.ts new file mode 100644 index 0000000000..ad3b1ddb5b --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/UNetField.ts @@ -0,0 +1,22 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LoraInfo } from './LoraInfo'; +import type { ModelInfo } from './ModelInfo'; + +export type UNetField = { + /** + * Info to load unet submodel + */ + unet: ModelInfo; + /** + * Info to load scheduler submodel + */ + scheduler: ModelInfo; + /** + * Loras to apply on model loading + */ + loras: Array; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/VaeField.ts b/invokeai/frontend/web/src/services/api/models/VaeField.ts new file mode 100644 index 0000000000..bfe2793887 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/VaeField.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelInfo } from './ModelInfo'; + +export type VaeField = { + /** + * Info to load vae submodel + */ + vae: ModelInfo; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__controlnet__ControlNetModel__Config.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__controlnet__ControlNetModel__Config.ts new file mode 100644 index 0000000000..f8decdb341 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__controlnet__ControlNetModel__Config.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; + +export type invokeai__backend__model_management__models__controlnet__ControlNetModel__Config = { + path: string; + description?: string; + format: ('checkpoint' | 'diffusers'); + default?: boolean; + error?: ModelError; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__lora__LoRAModel__Config.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__lora__LoRAModel__Config.ts new file mode 100644 index 0000000000..614749a2c5 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__lora__LoRAModel__Config.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; + +export type invokeai__backend__model_management__models__lora__LoRAModel__Config = { + path: string; + description?: string; + format: ('lycoris' | 'diffusers'); + default?: boolean; + error?: ModelError; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig.ts new file mode 100644 index 0000000000..6bdcb87dd4 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig.ts @@ -0,0 +1,18 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; +import type { ModelVariantType } from './ModelVariantType'; + +export type invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__CheckpointConfig = { + path: string; + description?: string; + format: 'checkpoint'; + default?: boolean; + error?: ModelError; + vae?: string; + config?: string; + variant: ModelVariantType; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig.ts new file mode 100644 index 0000000000..c88e042178 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig.ts @@ -0,0 +1,17 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; +import type { ModelVariantType } from './ModelVariantType'; + +export type invokeai__backend__model_management__models__stable_diffusion__StableDiffusion1Model__DiffusersConfig = { + path: string; + description?: string; + format: 'diffusers'; + default?: boolean; + error?: ModelError; + vae?: string; + variant: ModelVariantType; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig.ts new file mode 100644 index 0000000000..ec2ae4a845 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig.ts @@ -0,0 +1,21 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; +import type { ModelVariantType } from './ModelVariantType'; +import type { SchedulerPredictionType } from './SchedulerPredictionType'; + +export type invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__CheckpointConfig = { + path: string; + description?: string; + format: 'checkpoint'; + default?: boolean; + error?: ModelError; + vae?: string; + config?: string; + variant: ModelVariantType; + prediction_type: SchedulerPredictionType; + upcast_attention: boolean; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig.ts new file mode 100644 index 0000000000..67b897d9d9 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig.ts @@ -0,0 +1,20 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; +import type { ModelVariantType } from './ModelVariantType'; +import type { SchedulerPredictionType } from './SchedulerPredictionType'; + +export type invokeai__backend__model_management__models__stable_diffusion__StableDiffusion2Model__DiffusersConfig = { + path: string; + description?: string; + format: 'diffusers'; + default?: boolean; + error?: ModelError; + vae?: string; + variant: ModelVariantType; + prediction_type: SchedulerPredictionType; + upcast_attention: boolean; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config.ts new file mode 100644 index 0000000000..f23d5002e3 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; + +export type invokeai__backend__model_management__models__textual_inversion__TextualInversionModel__Config = { + path: string; + description?: string; + format: null; + default?: boolean; + error?: ModelError; +}; + diff --git a/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__vae__VaeModel__Config.ts b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__vae__VaeModel__Config.ts new file mode 100644 index 0000000000..d9314a6063 --- /dev/null +++ b/invokeai/frontend/web/src/services/api/models/invokeai__backend__model_management__models__vae__VaeModel__Config.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ModelError } from './ModelError'; + +export type invokeai__backend__model_management__models__vae__VaeModel__Config = { + path: string; + description?: string; + format: ('checkpoint' | 'diffusers'); + default?: boolean; + error?: ModelError; +}; + diff --git a/invokeai/frontend/web/src/services/api/services/ModelsService.ts b/invokeai/frontend/web/src/services/api/services/ModelsService.ts index 3f8ae6bf7b..54580ce204 100644 --- a/invokeai/frontend/web/src/services/api/services/ModelsService.ts +++ b/invokeai/frontend/web/src/services/api/services/ModelsService.ts @@ -1,8 +1,10 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ +import type { BaseModelType } from '../models/BaseModelType'; import type { CreateModelRequest } from '../models/CreateModelRequest'; import type { ModelsList } from '../models/ModelsList'; +import type { ModelType } from '../models/ModelType'; import type { CancelablePromise } from '../core/CancelablePromise'; import { OpenAPI } from '../core/OpenAPI'; @@ -16,10 +18,29 @@ export class ModelsService { * @returns ModelsList Successful Response * @throws ApiError */ - public static listModels(): CancelablePromise { + public static listModels({ + baseModel, + modelType, + }: { + /** + * Base model + */ + baseModel?: BaseModelType, + /** + * The type of model to get + */ + modelType?: ModelType, + }): CancelablePromise { return __request(OpenAPI, { method: 'GET', url: '/api/v1/models/', + query: { + 'base_model': baseModel, + 'model_type': modelType, + }, + errors: { + 422: `Validation Error`, + }, }); } diff --git a/invokeai/frontend/web/src/services/api/services/SessionsService.ts b/invokeai/frontend/web/src/services/api/services/SessionsService.ts index d850a1ed38..2e4a83b25f 100644 --- a/invokeai/frontend/web/src/services/api/services/SessionsService.ts +++ b/invokeai/frontend/web/src/services/api/services/SessionsService.ts @@ -27,7 +27,6 @@ import type { ImagePasteInvocation } from '../models/ImagePasteInvocation'; import type { ImageProcessorInvocation } from '../models/ImageProcessorInvocation'; import type { ImageResizeInvocation } from '../models/ImageResizeInvocation'; import type { ImageScaleInvocation } from '../models/ImageScaleInvocation'; -import type { ImageToImageInvocation } from '../models/ImageToImageInvocation'; import type { ImageToLatentsInvocation } from '../models/ImageToLatentsInvocation'; import type { InfillColorInvocation } from '../models/InfillColorInvocation'; import type { InfillPatchMatchInvocation } from '../models/InfillPatchMatchInvocation'; @@ -39,6 +38,7 @@ import type { LatentsToLatentsInvocation } from '../models/LatentsToLatentsInvoc import type { LineartAnimeImageProcessorInvocation } from '../models/LineartAnimeImageProcessorInvocation'; import type { LineartImageProcessorInvocation } from '../models/LineartImageProcessorInvocation'; import type { LoadImageInvocation } from '../models/LoadImageInvocation'; +import type { LoraLoaderInvocation } from '../models/LoraLoaderInvocation'; import type { MaskFromAlphaInvocation } from '../models/MaskFromAlphaInvocation'; import type { MediapipeFaceProcessorInvocation } from '../models/MediapipeFaceProcessorInvocation'; import type { MidasDepthImageProcessorInvocation } from '../models/MidasDepthImageProcessorInvocation'; @@ -58,10 +58,11 @@ import type { RangeOfSizeInvocation } from '../models/RangeOfSizeInvocation'; import type { ResizeLatentsInvocation } from '../models/ResizeLatentsInvocation'; import type { RestoreFaceInvocation } from '../models/RestoreFaceInvocation'; import type { ScaleLatentsInvocation } from '../models/ScaleLatentsInvocation'; +import type { SD1ModelLoaderInvocation } from '../models/SD1ModelLoaderInvocation'; +import type { SD2ModelLoaderInvocation } from '../models/SD2ModelLoaderInvocation'; import type { ShowImageInvocation } from '../models/ShowImageInvocation'; import type { StepParamEasingInvocation } from '../models/StepParamEasingInvocation'; import type { SubtractInvocation } from '../models/SubtractInvocation'; -import type { TextToImageInvocation } from '../models/TextToImageInvocation'; import type { TextToLatentsInvocation } from '../models/TextToLatentsInvocation'; import type { UpscaleInvocation } from '../models/UpscaleInvocation'; import type { ZoeDepthImageProcessorInvocation } from '../models/ZoeDepthImageProcessorInvocation'; @@ -174,7 +175,7 @@ export class SessionsService { * The id of the session */ sessionId: string, - requestBody: (LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | DynamicPromptInvocation | CompelInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | CvInpaintInvocation | RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | UpscaleInvocation | RestoreFaceInvocation | TextToImageInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | LatentsToLatentsInvocation | ImageToImageInvocation | InpaintInvocation), + requestBody: (LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | SD1ModelLoaderInvocation | SD2ModelLoaderInvocation | LoraLoaderInvocation | DynamicPromptInvocation | CompelInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | CvInpaintInvocation | RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | UpscaleInvocation | RestoreFaceInvocation | InpaintInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | LatentsToLatentsInvocation), }): CancelablePromise { return __request(OpenAPI, { method: 'POST', @@ -211,7 +212,7 @@ export class SessionsService { * The path to the node in the graph */ nodePath: string, - requestBody: (LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | DynamicPromptInvocation | CompelInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | CvInpaintInvocation | RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | UpscaleInvocation | RestoreFaceInvocation | TextToImageInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | LatentsToLatentsInvocation | ImageToImageInvocation | InpaintInvocation), + requestBody: (LoadImageInvocation | ShowImageInvocation | ImageCropInvocation | ImagePasteInvocation | MaskFromAlphaInvocation | ImageMultiplyInvocation | ImageChannelInvocation | ImageConvertInvocation | ImageBlurInvocation | ImageResizeInvocation | ImageScaleInvocation | ImageLerpInvocation | ImageInverseLerpInvocation | ControlNetInvocation | ImageProcessorInvocation | SD1ModelLoaderInvocation | SD2ModelLoaderInvocation | LoraLoaderInvocation | DynamicPromptInvocation | CompelInvocation | AddInvocation | SubtractInvocation | MultiplyInvocation | DivideInvocation | RandomIntInvocation | ParamIntInvocation | ParamFloatInvocation | NoiseInvocation | TextToLatentsInvocation | LatentsToImageInvocation | ResizeLatentsInvocation | ScaleLatentsInvocation | ImageToLatentsInvocation | CvInpaintInvocation | RangeInvocation | RangeOfSizeInvocation | RandomRangeInvocation | FloatLinearRangeInvocation | StepParamEasingInvocation | UpscaleInvocation | RestoreFaceInvocation | InpaintInvocation | InfillColorInvocation | InfillTileInvocation | InfillPatchMatchInvocation | GraphInvocation | IterateInvocation | CollectInvocation | CannyImageProcessorInvocation | HedImageProcessorInvocation | LineartImageProcessorInvocation | LineartAnimeImageProcessorInvocation | OpenposeImageProcessorInvocation | MidasDepthImageProcessorInvocation | NormalbaeImageProcessorInvocation | MlsdImageProcessorInvocation | PidiImageProcessorInvocation | ContentShuffleImageProcessorInvocation | ZoeDepthImageProcessorInvocation | MediapipeFaceProcessorInvocation | LatentsToLatentsInvocation), }): CancelablePromise { return __request(OpenAPI, { method: 'PUT',