From 06b5800d28667ff085c90beb9088bfbfcdd3e3fa Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Thu, 11 May 2023 20:52:37 +1200 Subject: [PATCH 01/14] Add UniPC Scheduler --- invokeai/app/invocations/latent.py | 32 +++++++++--------- invokeai/backend/args.py | 1 + invokeai/backend/generate.py | 25 +++++++------- invokeai/backend/generator/base.py | 28 ++++++++-------- .../convert_ckpt_to_diffusers.py | 3 ++ .../stable_diffusion/diffusers_pipeline.py | 33 ++++++++++--------- invokeai/backend/web/modules/parameters.py | 1 + invokeai/frontend/web/src/app/constants.ts | 1 + 8 files changed, 67 insertions(+), 57 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index c6ddcb0396..82396e358a 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -54,16 +54,17 @@ class NoiseOutput(BaseInvocationOutput): # TODO: this seems like a hack scheduler_map = dict( - ddim=diffusers.DDIMScheduler, - dpmpp_2=diffusers.DPMSolverMultistepScheduler, - k_dpm_2=diffusers.KDPM2DiscreteScheduler, - k_dpm_2_a=diffusers.KDPM2AncestralDiscreteScheduler, - k_dpmpp_2=diffusers.DPMSolverMultistepScheduler, - k_euler=diffusers.EulerDiscreteScheduler, - k_euler_a=diffusers.EulerAncestralDiscreteScheduler, - k_heun=diffusers.HeunDiscreteScheduler, - k_lms=diffusers.LMSDiscreteScheduler, - plms=diffusers.PNDMScheduler, + ddim=(diffusers.DDIMScheduler, dict()), + dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), + k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict()), + k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict()), + k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), + k_euler=(diffusers.EulerDiscreteScheduler, dict()), + k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict()), + k_heun=(diffusers.HeunDiscreteScheduler, dict()), + k_lms=(diffusers.LMSDiscreteScheduler, dict()), + plms=(diffusers.PNDMScheduler, dict()), + unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) ) @@ -73,8 +74,9 @@ SAMPLER_NAME_VALUES = Literal[ def get_scheduler(scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class = scheduler_map.get(scheduler_name,'ddim') - scheduler = scheduler_class.from_config(model.scheduler.config) + scheduler_class, scheduler_extra_config = scheduler_map.get(scheduler_name,'ddim') + scheduler_config = {**model.scheduler.config, **scheduler_extra_config} + scheduler = scheduler_class.from_config(scheduler_config) # hack copied over from generate.py if not hasattr(scheduler, 'uses_inpainting_model'): scheduler.uses_inpainting_model = lambda: False @@ -293,11 +295,7 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation): latent, device=model.device, dtype=latent.dtype ) - timesteps, _ = model.get_img2img_timesteps( - self.steps, - self.strength, - device=model.device, - ) + timesteps, _ = model.get_img2img_timesteps(self.steps, self.strength) result_latents, result_attention_map_saver = model.latents_from_embeddings( latents=initial_latents, diff --git a/invokeai/backend/args.py b/invokeai/backend/args.py index eb8b396ee0..8cbeea10ad 100644 --- a/invokeai/backend/args.py +++ b/invokeai/backend/args.py @@ -119,6 +119,7 @@ SAMPLER_CHOICES = [ "plms", # diffusers: "pndm", + "unipc" ] PRECISION_CHOICES = [ diff --git a/invokeai/backend/generate.py b/invokeai/backend/generate.py index 4f3df60f1c..237f609b2f 100644 --- a/invokeai/backend/generate.py +++ b/invokeai/backend/generate.py @@ -1049,27 +1049,28 @@ class Generate: # See https://github.com/huggingface/diffusers/issues/277#issuecomment-1371428672 scheduler_map = dict( - ddim=diffusers.DDIMScheduler, - dpmpp_2=diffusers.DPMSolverMultistepScheduler, - k_dpm_2=diffusers.KDPM2DiscreteScheduler, - k_dpm_2_a=diffusers.KDPM2AncestralDiscreteScheduler, + ddim=(diffusers.DDIMScheduler, dict()), + dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), + k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict()), + k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict()), # DPMSolverMultistepScheduler is technically not `k_` anything, as it is neither # the k-diffusers implementation nor included in EDM (Karras 2022), but we can # provide an alias for compatibility. - k_dpmpp_2=diffusers.DPMSolverMultistepScheduler, - k_euler=diffusers.EulerDiscreteScheduler, - k_euler_a=diffusers.EulerAncestralDiscreteScheduler, - k_heun=diffusers.HeunDiscreteScheduler, - k_lms=diffusers.LMSDiscreteScheduler, - plms=diffusers.PNDMScheduler, + k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), + k_euler=(diffusers.EulerDiscreteScheduler, dict()), + k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict()), + k_heun=(diffusers.HeunDiscreteScheduler, dict()), + k_lms=(diffusers.LMSDiscreteScheduler, dict()), + plms=(diffusers.PNDMScheduler, dict()), + unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) ) if self.sampler_name in scheduler_map: - sampler_class = scheduler_map[self.sampler_name] + sampler_class, sampler_extra_config = scheduler_map[self.sampler_name] msg = ( f"Setting Sampler to {self.sampler_name} ({sampler_class.__name__})" ) - self.sampler = sampler_class.from_config(self.model.scheduler.config) + self.sampler = sampler_class.from_config({**self.model.scheduler.config, **sampler_extra_config}) else: msg = ( f" Unsupported Sampler: {self.sampler_name} "+ diff --git a/invokeai/backend/generator/base.py b/invokeai/backend/generator/base.py index 9887434e90..f898d18345 100644 --- a/invokeai/backend/generator/base.py +++ b/invokeai/backend/generator/base.py @@ -72,17 +72,18 @@ class InvokeAIGeneratorOutput: # old code that calls Generate will continue to work. class InvokeAIGenerator(metaclass=ABCMeta): scheduler_map = dict( - ddim=diffusers.DDIMScheduler, - dpmpp_2=diffusers.DPMSolverMultistepScheduler, - k_dpm_2=diffusers.KDPM2DiscreteScheduler, - k_dpm_2_a=diffusers.KDPM2AncestralDiscreteScheduler, - k_dpmpp_2=diffusers.DPMSolverMultistepScheduler, - k_euler=diffusers.EulerDiscreteScheduler, - k_euler_a=diffusers.EulerAncestralDiscreteScheduler, - k_heun=diffusers.HeunDiscreteScheduler, - k_lms=diffusers.LMSDiscreteScheduler, - plms=diffusers.PNDMScheduler, - ) + ddim=(diffusers.DDIMScheduler, dict()), + dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), + k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict()), + k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict()), + k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), + k_euler=(diffusers.EulerDiscreteScheduler, dict()), + k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict()), + k_heun=(diffusers.HeunDiscreteScheduler, dict()), + k_lms=(diffusers.LMSDiscreteScheduler, dict()), + plms=(diffusers.PNDMScheduler, dict()), + unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) + ) def __init__(self, model_info: dict, @@ -181,8 +182,9 @@ class InvokeAIGenerator(metaclass=ABCMeta): return generator_class(model, self.params.precision) def get_scheduler(self, scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class = self.scheduler_map.get(scheduler_name,'ddim') - scheduler = scheduler_class.from_config(model.scheduler.config) + scheduler_class, scheduler_extra_config = self.scheduler_map.get(scheduler_name,'ddim') + scheduler_config = {**model.scheduler.config, **scheduler_extra_config} + scheduler = scheduler_class.from_config(scheduler_config) # hack copied over from generate.py if not hasattr(scheduler, 'uses_inpainting_model'): scheduler.uses_inpainting_model = lambda: False diff --git a/invokeai/backend/model_management/convert_ckpt_to_diffusers.py b/invokeai/backend/model_management/convert_ckpt_to_diffusers.py index 8aec5a01d9..1ebd044d54 100644 --- a/invokeai/backend/model_management/convert_ckpt_to_diffusers.py +++ b/invokeai/backend/model_management/convert_ckpt_to_diffusers.py @@ -47,6 +47,7 @@ from diffusers import ( LDMTextToImagePipeline, LMSDiscreteScheduler, PNDMScheduler, + UniPCMultistepScheduler, StableDiffusionPipeline, UNet2DConditionModel, ) @@ -1209,6 +1210,8 @@ def load_pipeline_from_original_stable_diffusion_ckpt( scheduler = EulerAncestralDiscreteScheduler.from_config(scheduler.config) elif scheduler_type == "dpm": scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + elif scheduler_type == 'unipc': + scheduler = UniPCMultistepScheduler.from_config(scheduler.config) elif scheduler_type == "ddim": scheduler = scheduler else: diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index 94ec9da7e8..c8a932b9e9 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -509,10 +509,13 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): run_id=None, callback: Callable[[PipelineIntermediateState], None] = None, ) -> tuple[torch.Tensor, Optional[AttentionMapSaver]]: + if self.scheduler.config.get("cpu_only", False): + scheduler_device = torch.device('cpu') + else: + scheduler_device = self._model_group.device_for(self.unet) + if timesteps is None: - self.scheduler.set_timesteps( - num_inference_steps, device=self._model_group.device_for(self.unet) - ) + self.scheduler.set_timesteps(num_inference_steps, device=scheduler_device) timesteps = self.scheduler.timesteps infer_latents_from_embeddings = GeneratorToCallbackinator( self.generate_latents_from_embeddings, PipelineIntermediateState @@ -725,12 +728,8 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): noise: torch.Tensor, run_id=None, callback=None, - ) -> InvokeAIStableDiffusionPipelineOutput: - timesteps, _ = self.get_img2img_timesteps( - num_inference_steps, - strength, - device=self._model_group.device_for(self.unet), - ) + ) -> InvokeAIStableDiffusionPipelineOutput: + timesteps, _ = self.get_img2img_timesteps(num_inference_steps, strength) result_latents, result_attention_maps = self.latents_from_embeddings( latents=initial_latents if strength < 1.0 else torch.zeros_like( initial_latents, device=initial_latents.device, dtype=initial_latents.dtype @@ -756,13 +755,19 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): return self.check_for_safety(output, dtype=conditioning_data.dtype) def get_img2img_timesteps( - self, num_inference_steps: int, strength: float, device + self, num_inference_steps: int, strength: float, device=None ) -> (torch.Tensor, int): img2img_pipeline = StableDiffusionImg2ImgPipeline(**self.components) assert img2img_pipeline.scheduler is self.scheduler - img2img_pipeline.scheduler.set_timesteps(num_inference_steps, device=device) + + if self.scheduler.config.get("cpu_only", False): + scheduler_device = torch.device('cpu') + else: + scheduler_device = self._model_group.device_for(self.unet) + + img2img_pipeline.scheduler.set_timesteps(num_inference_steps, device=scheduler_device) timesteps, adjusted_steps = img2img_pipeline.get_timesteps( - num_inference_steps, strength, device=device + num_inference_steps, strength, device=scheduler_device ) # Workaround for low strength resulting in zero timesteps. # TODO: submit upstream fix for zero-step img2img @@ -796,9 +801,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): if init_image.dim() == 3: init_image = init_image.unsqueeze(0) - timesteps, _ = self.get_img2img_timesteps( - num_inference_steps, strength, device=device - ) + timesteps, _ = self.get_img2img_timesteps(num_inference_steps, strength) # 6. Prepare latent variables # can't quite use upstream StableDiffusionImg2ImgPipeline.prepare_latents diff --git a/invokeai/backend/web/modules/parameters.py b/invokeai/backend/web/modules/parameters.py index 3c9c530dd2..9c06f68097 100644 --- a/invokeai/backend/web/modules/parameters.py +++ b/invokeai/backend/web/modules/parameters.py @@ -15,6 +15,7 @@ SAMPLER_CHOICES = [ "plms", # diffusers: "pndm", + "unipc" ] diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index 534ca9e29a..2d89ffe31e 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -11,6 +11,7 @@ export const DIFFUSERS_SCHEDULERS: Array = [ 'k_euler', 'k_euler_a', 'k_heun', + 'unipc', ]; // Valid image widths From d1029138d289a2713bb5b47ce2bdf30617342b24 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Thu, 11 May 2023 22:49:28 +1200 Subject: [PATCH 02/14] Default to DDIM if scheduler is missing --- invokeai/app/invocations/latent.py | 2 +- invokeai/backend/generator/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 82396e358a..cbd39fdb5f 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -74,7 +74,7 @@ SAMPLER_NAME_VALUES = Literal[ def get_scheduler(scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class, scheduler_extra_config = scheduler_map.get(scheduler_name,'ddim') + scheduler_class, scheduler_extra_config = scheduler_map.get(scheduler_name, scheduler_map['ddim']) scheduler_config = {**model.scheduler.config, **scheduler_extra_config} scheduler = scheduler_class.from_config(scheduler_config) # hack copied over from generate.py diff --git a/invokeai/backend/generator/base.py b/invokeai/backend/generator/base.py index f898d18345..f286ad0ad7 100644 --- a/invokeai/backend/generator/base.py +++ b/invokeai/backend/generator/base.py @@ -182,7 +182,7 @@ class InvokeAIGenerator(metaclass=ABCMeta): return generator_class(model, self.params.precision) def get_scheduler(self, scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class, scheduler_extra_config = self.scheduler_map.get(scheduler_name,'ddim') + scheduler_class, scheduler_extra_config = self.scheduler_map.get(scheduler_name, self.scheduler_map['ddim']) scheduler_config = {**model.scheduler.config, **scheduler_extra_config} scheduler = scheduler_class.from_config(scheduler_config) # hack copied over from generate.py From 3ffff023b2d63377397b4f6114cd0e209c48f179 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 00:07:36 +1200 Subject: [PATCH 03/14] Add missing key to scheduler_map It was breaking coz the sampler was not being reset. So needs a key on each. Will simplify this later. --- invokeai/app/invocations/latent.py | 20 ++++++++++---------- invokeai/backend/generate.py | 20 ++++++++++---------- invokeai/backend/generator/base.py | 20 ++++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index cbd39fdb5f..da0e6c0354 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -54,16 +54,16 @@ class NoiseOutput(BaseInvocationOutput): # TODO: this seems like a hack scheduler_map = dict( - ddim=(diffusers.DDIMScheduler, dict()), - dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), - k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict()), - k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict()), - k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), - k_euler=(diffusers.EulerDiscreteScheduler, dict()), - k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict()), - k_heun=(diffusers.HeunDiscreteScheduler, dict()), - k_lms=(diffusers.LMSDiscreteScheduler, dict()), - plms=(diffusers.PNDMScheduler, dict()), + ddim=(diffusers.DDIMScheduler, dict(cpu_only=False)), + dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict(cpu_only=False)), + k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), + k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_euler=(diffusers.EulerDiscreteScheduler, dict(cpu_only=False)), + k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict(cpu_only=False)), + k_heun=(diffusers.HeunDiscreteScheduler, dict(cpu_only=False)), + k_lms=(diffusers.LMSDiscreteScheduler, dict(cpu_only=False)), + plms=(diffusers.PNDMScheduler, dict(cpu_only=False)), unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) ) diff --git a/invokeai/backend/generate.py b/invokeai/backend/generate.py index 237f609b2f..c64c0e3320 100644 --- a/invokeai/backend/generate.py +++ b/invokeai/backend/generate.py @@ -1049,19 +1049,19 @@ class Generate: # See https://github.com/huggingface/diffusers/issues/277#issuecomment-1371428672 scheduler_map = dict( - ddim=(diffusers.DDIMScheduler, dict()), - dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), - k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict()), - k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict()), + ddim=(diffusers.DDIMScheduler, dict(cpu_only=False)), + dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict(cpu_only=False)), + k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), # DPMSolverMultistepScheduler is technically not `k_` anything, as it is neither # the k-diffusers implementation nor included in EDM (Karras 2022), but we can # provide an alias for compatibility. - k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), - k_euler=(diffusers.EulerDiscreteScheduler, dict()), - k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict()), - k_heun=(diffusers.HeunDiscreteScheduler, dict()), - k_lms=(diffusers.LMSDiscreteScheduler, dict()), - plms=(diffusers.PNDMScheduler, dict()), + k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_euler=(diffusers.EulerDiscreteScheduler, dict(cpu_only=False)), + k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict(cpu_only=False)), + k_heun=(diffusers.HeunDiscreteScheduler, dict(cpu_only=False)), + k_lms=(diffusers.LMSDiscreteScheduler, dict(cpu_only=False)), + plms=(diffusers.PNDMScheduler, dict(cpu_only=False)), unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) ) diff --git a/invokeai/backend/generator/base.py b/invokeai/backend/generator/base.py index f286ad0ad7..33a625de30 100644 --- a/invokeai/backend/generator/base.py +++ b/invokeai/backend/generator/base.py @@ -72,16 +72,16 @@ class InvokeAIGeneratorOutput: # old code that calls Generate will continue to work. class InvokeAIGenerator(metaclass=ABCMeta): scheduler_map = dict( - ddim=(diffusers.DDIMScheduler, dict()), - dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), - k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict()), - k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict()), - k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict()), - k_euler=(diffusers.EulerDiscreteScheduler, dict()), - k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict()), - k_heun=(diffusers.HeunDiscreteScheduler, dict()), - k_lms=(diffusers.LMSDiscreteScheduler, dict()), - plms=(diffusers.PNDMScheduler, dict()), + ddim=(diffusers.DDIMScheduler, dict(cpu_only=False)), + dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict(cpu_only=False)), + k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), + k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_euler=(diffusers.EulerDiscreteScheduler, dict(cpu_only=False)), + k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict(cpu_only=False)), + k_heun=(diffusers.HeunDiscreteScheduler, dict(cpu_only=False)), + k_lms=(diffusers.LMSDiscreteScheduler, dict(cpu_only=False)), + plms=(diffusers.PNDMScheduler, dict(cpu_only=False)), unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) ) From 9a383e456dbcdacc6eb6e79c043b2483d597c776 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 00:40:03 +1200 Subject: [PATCH 04/14] Codesplit SCHEDULER_MAP for reusage --- invokeai/app/invocations/latent.py | 21 +++-------------- invokeai/backend/generate.py | 23 +++---------------- invokeai/backend/generator/base.py | 19 +++------------ .../stable_diffusion/schedulers/__init__.py | 1 + .../stable_diffusion/schedulers/schedulers.py | 17 ++++++++++++++ 5 files changed, 27 insertions(+), 54 deletions(-) create mode 100644 invokeai/backend/stable_diffusion/schedulers/__init__.py create mode 100644 invokeai/backend/stable_diffusion/schedulers/schedulers.py diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 4ce98f97f4..349a13bf3d 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -17,6 +17,7 @@ from ...backend.stable_diffusion.diffusion.shared_invokeai_diffusion import Post from ...backend.image_util.seamless import configure_model_padding from ...backend.prompting.conditioning import get_uc_and_c_and_ec from ...backend.stable_diffusion.diffusers_pipeline import ConditioningData, StableDiffusionGeneratorPipeline, image_resized_to_grid_as_tensor +from ...backend.stable_diffusion.schedulers import SCHEDULER_MAP from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationContext, InvocationConfig import numpy as np from ..services.image_storage import ImageType @@ -52,29 +53,13 @@ class NoiseOutput(BaseInvocationOutput): #fmt: on -# TODO: this seems like a hack -scheduler_map = dict( - ddim=(diffusers.DDIMScheduler, dict(cpu_only=False)), - dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict(cpu_only=False)), - k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), - k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_euler=(diffusers.EulerDiscreteScheduler, dict(cpu_only=False)), - k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict(cpu_only=False)), - k_heun=(diffusers.HeunDiscreteScheduler, dict(cpu_only=False)), - k_lms=(diffusers.LMSDiscreteScheduler, dict(cpu_only=False)), - plms=(diffusers.PNDMScheduler, dict(cpu_only=False)), - unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) -) - - SAMPLER_NAME_VALUES = Literal[ - tuple(list(scheduler_map.keys())) + tuple(list(SCHEDULER_MAP.keys())) ] def get_scheduler(scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class, scheduler_extra_config = scheduler_map.get(scheduler_name, scheduler_map['ddim']) + scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP['ddim']) scheduler_config = {**model.scheduler.config, **scheduler_extra_config} scheduler = scheduler_class.from_config(scheduler_config) # hack copied over from generate.py diff --git a/invokeai/backend/generate.py b/invokeai/backend/generate.py index c64c0e3320..dc5f9d9ed0 100644 --- a/invokeai/backend/generate.py +++ b/invokeai/backend/generate.py @@ -37,6 +37,7 @@ from .safety_checker import SafetyChecker from .prompting import get_uc_and_c_and_ec from .prompting.conditioning import log_tokenization from .stable_diffusion import HuggingFaceConceptsLibrary +from .stable_diffusion.schedulers import SCHEDULER_MAP from .util import choose_precision, choose_torch_device def fix_func(orig): @@ -1047,26 +1048,8 @@ class Generate: def _set_scheduler(self): default = self.model.scheduler - # See https://github.com/huggingface/diffusers/issues/277#issuecomment-1371428672 - scheduler_map = dict( - ddim=(diffusers.DDIMScheduler, dict(cpu_only=False)), - dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict(cpu_only=False)), - k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), - # DPMSolverMultistepScheduler is technically not `k_` anything, as it is neither - # the k-diffusers implementation nor included in EDM (Karras 2022), but we can - # provide an alias for compatibility. - k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_euler=(diffusers.EulerDiscreteScheduler, dict(cpu_only=False)), - k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict(cpu_only=False)), - k_heun=(diffusers.HeunDiscreteScheduler, dict(cpu_only=False)), - k_lms=(diffusers.LMSDiscreteScheduler, dict(cpu_only=False)), - plms=(diffusers.PNDMScheduler, dict(cpu_only=False)), - unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) - ) - - if self.sampler_name in scheduler_map: - sampler_class, sampler_extra_config = scheduler_map[self.sampler_name] + if self.sampler_name in SCHEDULER_MAP: + sampler_class, sampler_extra_config = SCHEDULER_MAP[self.sampler_name] msg = ( f"Setting Sampler to {self.sampler_name} ({sampler_class.__name__})" ) diff --git a/invokeai/backend/generator/base.py b/invokeai/backend/generator/base.py index 33a625de30..317110b35b 100644 --- a/invokeai/backend/generator/base.py +++ b/invokeai/backend/generator/base.py @@ -31,6 +31,7 @@ from ..util.util import rand_perlin_2d from ..safety_checker import SafetyChecker from ..prompting.conditioning import get_uc_and_c_and_ec from ..stable_diffusion.diffusers_pipeline import StableDiffusionGeneratorPipeline +from ..stable_diffusion.schedulers import SCHEDULER_MAP downsampling = 8 @@ -71,20 +72,6 @@ class InvokeAIGeneratorOutput: # we are interposing a wrapper around the original Generator classes so that # old code that calls Generate will continue to work. class InvokeAIGenerator(metaclass=ABCMeta): - scheduler_map = dict( - ddim=(diffusers.DDIMScheduler, dict(cpu_only=False)), - dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_dpm_2=(diffusers.KDPM2DiscreteScheduler, dict(cpu_only=False)), - k_dpm_2_a=(diffusers.KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), - k_dpmpp_2=(diffusers.DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_euler=(diffusers.EulerDiscreteScheduler, dict(cpu_only=False)), - k_euler_a=(diffusers.EulerAncestralDiscreteScheduler, dict(cpu_only=False)), - k_heun=(diffusers.HeunDiscreteScheduler, dict(cpu_only=False)), - k_lms=(diffusers.LMSDiscreteScheduler, dict(cpu_only=False)), - plms=(diffusers.PNDMScheduler, dict(cpu_only=False)), - unipc=(diffusers.UniPCMultistepScheduler, dict(cpu_only=True)) - ) - def __init__(self, model_info: dict, params: InvokeAIGeneratorBasicParams=InvokeAIGeneratorBasicParams(), @@ -176,13 +163,13 @@ class InvokeAIGenerator(metaclass=ABCMeta): ''' Return list of all the schedulers that we currently handle. ''' - return list(self.scheduler_map.keys()) + return list(SCHEDULER_MAP.keys()) def load_generator(self, model: StableDiffusionGeneratorPipeline, generator_class: Type[Generator]): return generator_class(model, self.params.precision) def get_scheduler(self, scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: - scheduler_class, scheduler_extra_config = self.scheduler_map.get(scheduler_name, self.scheduler_map['ddim']) + scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP['ddim']) scheduler_config = {**model.scheduler.config, **scheduler_extra_config} scheduler = scheduler_class.from_config(scheduler_config) # hack copied over from generate.py diff --git a/invokeai/backend/stable_diffusion/schedulers/__init__.py b/invokeai/backend/stable_diffusion/schedulers/__init__.py new file mode 100644 index 0000000000..b2df0df231 --- /dev/null +++ b/invokeai/backend/stable_diffusion/schedulers/__init__.py @@ -0,0 +1 @@ +from .schedulers import SCHEDULER_MAP \ No newline at end of file diff --git a/invokeai/backend/stable_diffusion/schedulers/schedulers.py b/invokeai/backend/stable_diffusion/schedulers/schedulers.py new file mode 100644 index 0000000000..00d3675fe5 --- /dev/null +++ b/invokeai/backend/stable_diffusion/schedulers/schedulers.py @@ -0,0 +1,17 @@ +from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, KDPM2DiscreteScheduler, \ + KDPM2AncestralDiscreteScheduler, EulerDiscreteScheduler, EulerAncestralDiscreteScheduler, \ + HeunDiscreteScheduler, LMSDiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler + +SCHEDULER_MAP = dict( + ddim=(DDIMScheduler, dict(cpu_only=False)), + dpmpp_2=(DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_dpm_2=(KDPM2DiscreteScheduler, dict(cpu_only=False)), + k_dpm_2_a=(KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), + k_dpmpp_2=(DPMSolverMultistepScheduler, dict(cpu_only=False)), + k_euler=(EulerDiscreteScheduler, dict(cpu_only=False)), + k_euler_a=(EulerAncestralDiscreteScheduler, dict(cpu_only=False)), + k_heun=(HeunDiscreteScheduler, dict(cpu_only=False)), + k_lms=(LMSDiscreteScheduler, dict(cpu_only=False)), + plms=(PNDMScheduler, dict(cpu_only=False)), + unipc=(UniPCMultistepScheduler, dict(cpu_only=True)) +) From 95c36445647f5953331fd74eb5620398eb99ffb7 Mon Sep 17 00:00:00 2001 From: Mary Hipp Date: Thu, 11 May 2023 10:09:01 -0400 Subject: [PATCH 05/14] fix it again --- invokeai/frontend/web/src/i18n.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/frontend/web/src/i18n.ts b/invokeai/frontend/web/src/i18n.ts index f28365db20..71d4dfb35f 100644 --- a/invokeai/frontend/web/src/i18n.ts +++ b/invokeai/frontend/web/src/i18n.ts @@ -3,7 +3,7 @@ import LanguageDetector from 'i18next-browser-languagedetector'; import Backend from 'i18next-http-backend'; import { initReactI18next } from 'react-i18next'; -import translationEN from '../dist/locales/en.json'; +import translationEN from '../public/locales/en.json'; import { LOCALSTORAGE_PREFIX } from 'app/store/constants'; if (import.meta.env.MODE === 'package') { From 8a836247c82650a2edae75c8d2248d31a31cf7c5 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 02:23:33 +1200 Subject: [PATCH 06/14] Add DPMPP Single, Euler Karras and DPMPP2 Multi Karras Schedulers --- invokeai/app/invocations/latent.py | 7 +++++- invokeai/backend/args.py | 20 +++++++-------- invokeai/backend/generator/base.py | 7 +++++- .../stable_diffusion/schedulers/schedulers.py | 25 +++++++++++-------- invokeai/backend/web/modules/parameters.py | 20 +++++++-------- invokeai/frontend/web/src/app/constants.ts | 12 +++++---- .../frontend/web/src/app/types/invokeai.ts | 19 ++++++++------ 7 files changed, 64 insertions(+), 46 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 349a13bf3d..75fa719c07 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -60,8 +60,13 @@ SAMPLER_NAME_VALUES = Literal[ def get_scheduler(scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP['ddim']) - scheduler_config = {**model.scheduler.config, **scheduler_extra_config} + + scheduler_config = model.scheduler.config + if "_backup" in scheduler_config: + scheduler_config = scheduler_config["_backup"] + scheduler_config = {**scheduler_config, **scheduler_extra_config, "_backup": scheduler_config} scheduler = scheduler_class.from_config(scheduler_config) + # hack copied over from generate.py if not hasattr(scheduler, 'uses_inpainting_model'): scheduler.uses_inpainting_model = lambda: False diff --git a/invokeai/backend/args.py b/invokeai/backend/args.py index 8cbeea10ad..0efaf6fd26 100644 --- a/invokeai/backend/args.py +++ b/invokeai/backend/args.py @@ -108,18 +108,18 @@ APP_VERSION = invokeai.version.__version__ SAMPLER_CHOICES = [ "ddim", - "k_dpm_2_a", - "k_dpm_2", - "k_dpmpp_2_a", - "k_dpmpp_2", - "k_euler_a", - "k_euler", - "k_heun", "k_lms", "plms", - # diffusers: - "pndm", - "unipc" + "k_heun", + "k_euler", + "euler_karras", + "k_euler_a", + "k_dpm_2", + "k_dpm_2_a", + "dpmpp_2s", + "k_dpmpp_2", + "k_dpmpp_2_karras", + "unipc", ] PRECISION_CHOICES = [ diff --git a/invokeai/backend/generator/base.py b/invokeai/backend/generator/base.py index 317110b35b..8f5b1a8395 100644 --- a/invokeai/backend/generator/base.py +++ b/invokeai/backend/generator/base.py @@ -170,8 +170,13 @@ class InvokeAIGenerator(metaclass=ABCMeta): def get_scheduler(self, scheduler_name:str, model: StableDiffusionGeneratorPipeline)->Scheduler: scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP['ddim']) - scheduler_config = {**model.scheduler.config, **scheduler_extra_config} + + scheduler_config = model.scheduler.config + if "_backup" in scheduler_config: + scheduler_config = scheduler_config["_backup"] + scheduler_config = {**scheduler_config, **scheduler_extra_config, "_backup": scheduler_config} scheduler = scheduler_class.from_config(scheduler_config) + # hack copied over from generate.py if not hasattr(scheduler, 'uses_inpainting_model'): scheduler.uses_inpainting_model = lambda: False diff --git a/invokeai/backend/stable_diffusion/schedulers/schedulers.py b/invokeai/backend/stable_diffusion/schedulers/schedulers.py index 00d3675fe5..98d73ae10b 100644 --- a/invokeai/backend/stable_diffusion/schedulers/schedulers.py +++ b/invokeai/backend/stable_diffusion/schedulers/schedulers.py @@ -1,17 +1,20 @@ from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, KDPM2DiscreteScheduler, \ KDPM2AncestralDiscreteScheduler, EulerDiscreteScheduler, EulerAncestralDiscreteScheduler, \ - HeunDiscreteScheduler, LMSDiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler + HeunDiscreteScheduler, LMSDiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler, \ + DPMSolverSinglestepScheduler SCHEDULER_MAP = dict( - ddim=(DDIMScheduler, dict(cpu_only=False)), - dpmpp_2=(DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_dpm_2=(KDPM2DiscreteScheduler, dict(cpu_only=False)), - k_dpm_2_a=(KDPM2AncestralDiscreteScheduler, dict(cpu_only=False)), - k_dpmpp_2=(DPMSolverMultistepScheduler, dict(cpu_only=False)), - k_euler=(EulerDiscreteScheduler, dict(cpu_only=False)), - k_euler_a=(EulerAncestralDiscreteScheduler, dict(cpu_only=False)), - k_heun=(HeunDiscreteScheduler, dict(cpu_only=False)), - k_lms=(LMSDiscreteScheduler, dict(cpu_only=False)), - plms=(PNDMScheduler, dict(cpu_only=False)), + ddim=(DDIMScheduler, dict()), + k_lms=(LMSDiscreteScheduler, dict()), + plms=(PNDMScheduler, dict()), + k_euler=(EulerDiscreteScheduler, dict(use_karras_sigmas=False)), + euler_karras=(EulerDiscreteScheduler, dict(use_karras_sigmas=True)), + k_euler_a=(EulerAncestralDiscreteScheduler, dict()), + k_dpm_2=(KDPM2DiscreteScheduler, dict()), + k_dpm_2_a=(KDPM2AncestralDiscreteScheduler, dict()), + dpmpp_2s=(DPMSolverSinglestepScheduler, dict()), + k_dpmpp_2=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=False)), + k_dpmpp_2_karras=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=True)), + k_heun=(HeunDiscreteScheduler, dict()), unipc=(UniPCMultistepScheduler, dict(cpu_only=True)) ) diff --git a/invokeai/backend/web/modules/parameters.py b/invokeai/backend/web/modules/parameters.py index 9c06f68097..f25864765b 100644 --- a/invokeai/backend/web/modules/parameters.py +++ b/invokeai/backend/web/modules/parameters.py @@ -4,18 +4,18 @@ from .parse_seed_weights import parse_seed_weights SAMPLER_CHOICES = [ "ddim", - "k_dpm_2_a", - "k_dpm_2", - "k_dpmpp_2_a", - "k_dpmpp_2", - "k_euler_a", - "k_euler", - "k_heun", "k_lms", "plms", - # diffusers: - "pndm", - "unipc" + "k_heun", + "k_euler", + "euler_karras", + "k_euler_a", + "k_dpm_2", + "k_dpm_2_a", + "dpmpp_2s", + "k_dpmpp_2", + "k_dpmpp_2_karras", + "unipc", ] diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index 2d89ffe31e..d9a606435c 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -2,15 +2,17 @@ export const DIFFUSERS_SCHEDULERS: Array = [ 'ddim', - 'plms', 'k_lms', - 'dpmpp_2', + 'plms', + 'k_heun', + 'k_euler', + 'euler_karras', + 'k_euler_a', 'k_dpm_2', 'k_dpm_2_a', + 'dpmpp_2s', 'k_dpmpp_2', - 'k_euler', - 'k_euler_a', - 'k_heun', + 'k_dpmpp_2_karras', 'unipc', ]; diff --git a/invokeai/frontend/web/src/app/types/invokeai.ts b/invokeai/frontend/web/src/app/types/invokeai.ts index 4023f7665d..0d8fe4bbdc 100644 --- a/invokeai/frontend/web/src/app/types/invokeai.ts +++ b/invokeai/frontend/web/src/app/types/invokeai.ts @@ -47,15 +47,18 @@ export type CommonGeneratedImageMetadata = { postprocessing: null | Array; sampler: | 'ddim' - | 'k_dpm_2_a' - | 'k_dpm_2' - | 'k_dpmpp_2_a' - | 'k_dpmpp_2' - | 'k_euler_a' - | 'k_euler' - | 'k_heun' | 'k_lms' - | 'plms'; + | 'plms' + | 'k_heun' + | 'k_euler' + | 'euler_karras' + | 'k_euler_a' + | 'k_dpm_2' + | 'k_dpm_2_a' + | 'dpmpp_2s' + | 'k_dpmpp_2' + | 'k_dpmpp_2_karras' + | 'unipc'; prompt: Prompt; seed: number; variations: SeedWeights; From b928d7a6e68810b91221e7ee8cb731042cade695 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 02:59:43 +1200 Subject: [PATCH 07/14] Change scheduler names to be accurate _a = Ancestral _k = Karras --- invokeai/backend/args.py | 20 +++++++++---------- .../stable_diffusion/schedulers/schedulers.py | 20 +++++++++---------- invokeai/backend/web/modules/parameters.py | 20 +++++++++---------- invokeai/frontend/web/src/app/constants.ts | 20 +++++++++---------- .../frontend/web/src/app/types/invokeai.ts | 20 +++++++++---------- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/invokeai/backend/args.py b/invokeai/backend/args.py index 0efaf6fd26..0925e19edf 100644 --- a/invokeai/backend/args.py +++ b/invokeai/backend/args.py @@ -108,17 +108,17 @@ APP_VERSION = invokeai.version.__version__ SAMPLER_CHOICES = [ "ddim", - "k_lms", - "plms", - "k_heun", - "k_euler", - "euler_karras", - "k_euler_a", - "k_dpm_2", - "k_dpm_2_a", + "lms", + "pndm", + "heun", + "euler", + "euler_k", + "euler_a", + "kdpm_2", + "kdpm_2_a", "dpmpp_2s", - "k_dpmpp_2", - "k_dpmpp_2_karras", + "dpmpp_2m", + "dpmpp_2m_k", "unipc", ] diff --git a/invokeai/backend/stable_diffusion/schedulers/schedulers.py b/invokeai/backend/stable_diffusion/schedulers/schedulers.py index 98d73ae10b..5df6d045c1 100644 --- a/invokeai/backend/stable_diffusion/schedulers/schedulers.py +++ b/invokeai/backend/stable_diffusion/schedulers/schedulers.py @@ -5,16 +5,16 @@ from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, KDPM2DiscreteS SCHEDULER_MAP = dict( ddim=(DDIMScheduler, dict()), - k_lms=(LMSDiscreteScheduler, dict()), - plms=(PNDMScheduler, dict()), - k_euler=(EulerDiscreteScheduler, dict(use_karras_sigmas=False)), - euler_karras=(EulerDiscreteScheduler, dict(use_karras_sigmas=True)), - k_euler_a=(EulerAncestralDiscreteScheduler, dict()), - k_dpm_2=(KDPM2DiscreteScheduler, dict()), - k_dpm_2_a=(KDPM2AncestralDiscreteScheduler, dict()), + lms=(LMSDiscreteScheduler, dict()), + pndm=(PNDMScheduler, dict()), + heun=(HeunDiscreteScheduler, dict()), + euler=(EulerDiscreteScheduler, dict(use_karras_sigmas=False)), + euler_k=(EulerDiscreteScheduler, dict(use_karras_sigmas=True)), + euler_a=(EulerAncestralDiscreteScheduler, dict()), + kdpm_2=(KDPM2DiscreteScheduler, dict()), + kdpm_2_a=(KDPM2AncestralDiscreteScheduler, dict()), dpmpp_2s=(DPMSolverSinglestepScheduler, dict()), - k_dpmpp_2=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=False)), - k_dpmpp_2_karras=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=True)), - k_heun=(HeunDiscreteScheduler, dict()), + dpmpp_2m=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=False)), + dpmpp_2m_k=(DPMSolverMultistepScheduler, dict(use_karras_sigmas=True)), unipc=(UniPCMultistepScheduler, dict(cpu_only=True)) ) diff --git a/invokeai/backend/web/modules/parameters.py b/invokeai/backend/web/modules/parameters.py index f25864765b..22bc2f353e 100644 --- a/invokeai/backend/web/modules/parameters.py +++ b/invokeai/backend/web/modules/parameters.py @@ -4,17 +4,17 @@ from .parse_seed_weights import parse_seed_weights SAMPLER_CHOICES = [ "ddim", - "k_lms", - "plms", - "k_heun", - "k_euler", - "euler_karras", - "k_euler_a", - "k_dpm_2", - "k_dpm_2_a", + "lms", + "pndm", + "heun", + "euler", + "euler_k", + "euler_a", + "kdpm_2", + "kdpm_2_a", "dpmpp_2s", - "k_dpmpp_2", - "k_dpmpp_2_karras", + "dpmpp_2m", + "dpmpp_2m_k", "unipc", ] diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index d9a606435c..ab1c313abe 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -2,17 +2,17 @@ export const DIFFUSERS_SCHEDULERS: Array = [ 'ddim', - 'k_lms', - 'plms', - 'k_heun', - 'k_euler', - 'euler_karras', - 'k_euler_a', - 'k_dpm_2', - 'k_dpm_2_a', + 'lms', + 'pndm', + 'heun', + 'euler', + 'euler_k', + 'euler_a', + 'kdpm_2', + 'kdpm_2_a', 'dpmpp_2s', - 'k_dpmpp_2', - 'k_dpmpp_2_karras', + 'dpmpp_2m', + 'dpmpp_2m_k', 'unipc', ]; diff --git a/invokeai/frontend/web/src/app/types/invokeai.ts b/invokeai/frontend/web/src/app/types/invokeai.ts index 0d8fe4bbdc..998bb00d47 100644 --- a/invokeai/frontend/web/src/app/types/invokeai.ts +++ b/invokeai/frontend/web/src/app/types/invokeai.ts @@ -47,17 +47,17 @@ export type CommonGeneratedImageMetadata = { postprocessing: null | Array; sampler: | 'ddim' - | 'k_lms' - | 'plms' - | 'k_heun' - | 'k_euler' - | 'euler_karras' - | 'k_euler_a' - | 'k_dpm_2' - | 'k_dpm_2_a' + | 'lms' + | 'pndm' + | 'heun' + | 'euler' + | 'euler_k' + | 'euler_a' + | 'kdpm_2' + | 'kdpm_2_a' | 'dpmpp_2s' - | 'k_dpmpp_2' - | 'k_dpmpp_2_karras' + | 'dpmpp_2m' + | 'dpmpp_2m_k' | 'unipc'; prompt: Prompt; seed: number; From 46ca7718d9e69f559bdec6cd0f679db5d2fc7bd9 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 03:10:30 +1200 Subject: [PATCH 08/14] Add DEIS Scheduler --- invokeai/backend/args.py | 1 + invokeai/backend/stable_diffusion/schedulers/schedulers.py | 3 ++- invokeai/backend/web/modules/parameters.py | 1 + invokeai/frontend/web/src/app/constants.ts | 1 + invokeai/frontend/web/src/app/types/invokeai.ts | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/invokeai/backend/args.py b/invokeai/backend/args.py index 0925e19edf..4af9e43f42 100644 --- a/invokeai/backend/args.py +++ b/invokeai/backend/args.py @@ -108,6 +108,7 @@ APP_VERSION = invokeai.version.__version__ SAMPLER_CHOICES = [ "ddim", + "deis", "lms", "pndm", "heun", diff --git a/invokeai/backend/stable_diffusion/schedulers/schedulers.py b/invokeai/backend/stable_diffusion/schedulers/schedulers.py index 5df6d045c1..752e7cdb0c 100644 --- a/invokeai/backend/stable_diffusion/schedulers/schedulers.py +++ b/invokeai/backend/stable_diffusion/schedulers/schedulers.py @@ -1,10 +1,11 @@ from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, KDPM2DiscreteScheduler, \ KDPM2AncestralDiscreteScheduler, EulerDiscreteScheduler, EulerAncestralDiscreteScheduler, \ HeunDiscreteScheduler, LMSDiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler, \ - DPMSolverSinglestepScheduler + DPMSolverSinglestepScheduler, DEISMultistepScheduler SCHEDULER_MAP = dict( ddim=(DDIMScheduler, dict()), + deis=(DEISMultistepScheduler, dict()), lms=(LMSDiscreteScheduler, dict()), pndm=(PNDMScheduler, dict()), heun=(HeunDiscreteScheduler, dict()), diff --git a/invokeai/backend/web/modules/parameters.py b/invokeai/backend/web/modules/parameters.py index 22bc2f353e..940732898d 100644 --- a/invokeai/backend/web/modules/parameters.py +++ b/invokeai/backend/web/modules/parameters.py @@ -4,6 +4,7 @@ from .parse_seed_weights import parse_seed_weights SAMPLER_CHOICES = [ "ddim", + "deis", "lms", "pndm", "heun", diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index ab1c313abe..f0dc48f6d5 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -2,6 +2,7 @@ export const DIFFUSERS_SCHEDULERS: Array = [ 'ddim', + 'deis', 'lms', 'pndm', 'heun', diff --git a/invokeai/frontend/web/src/app/types/invokeai.ts b/invokeai/frontend/web/src/app/types/invokeai.ts index 998bb00d47..2ae221b0a8 100644 --- a/invokeai/frontend/web/src/app/types/invokeai.ts +++ b/invokeai/frontend/web/src/app/types/invokeai.ts @@ -47,6 +47,7 @@ export type CommonGeneratedImageMetadata = { postprocessing: null | Array; sampler: | 'ddim' + | 'deis' | 'lms' | 'pndm' | 'heun' From 4b957edfecbcfa1545c99d6252ecb5aeaa159d80 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 03:18:34 +1200 Subject: [PATCH 09/14] Add DDPM Scheduler --- invokeai/backend/args.py | 1 + invokeai/backend/stable_diffusion/schedulers/schedulers.py | 3 ++- invokeai/backend/web/modules/parameters.py | 1 + invokeai/frontend/web/src/app/constants.ts | 1 + invokeai/frontend/web/src/app/types/invokeai.ts | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/invokeai/backend/args.py b/invokeai/backend/args.py index 4af9e43f42..856272c45b 100644 --- a/invokeai/backend/args.py +++ b/invokeai/backend/args.py @@ -108,6 +108,7 @@ APP_VERSION = invokeai.version.__version__ SAMPLER_CHOICES = [ "ddim", + "ddpm", "deis", "lms", "pndm", diff --git a/invokeai/backend/stable_diffusion/schedulers/schedulers.py b/invokeai/backend/stable_diffusion/schedulers/schedulers.py index 752e7cdb0c..fab28aca8c 100644 --- a/invokeai/backend/stable_diffusion/schedulers/schedulers.py +++ b/invokeai/backend/stable_diffusion/schedulers/schedulers.py @@ -1,10 +1,11 @@ from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, KDPM2DiscreteScheduler, \ KDPM2AncestralDiscreteScheduler, EulerDiscreteScheduler, EulerAncestralDiscreteScheduler, \ HeunDiscreteScheduler, LMSDiscreteScheduler, PNDMScheduler, UniPCMultistepScheduler, \ - DPMSolverSinglestepScheduler, DEISMultistepScheduler + DPMSolverSinglestepScheduler, DEISMultistepScheduler, DDPMScheduler SCHEDULER_MAP = dict( ddim=(DDIMScheduler, dict()), + ddpm=(DDPMScheduler, dict()), deis=(DEISMultistepScheduler, dict()), lms=(LMSDiscreteScheduler, dict()), pndm=(PNDMScheduler, dict()), diff --git a/invokeai/backend/web/modules/parameters.py b/invokeai/backend/web/modules/parameters.py index 940732898d..72211857a3 100644 --- a/invokeai/backend/web/modules/parameters.py +++ b/invokeai/backend/web/modules/parameters.py @@ -4,6 +4,7 @@ from .parse_seed_weights import parse_seed_weights SAMPLER_CHOICES = [ "ddim", + "ddpm", "deis", "lms", "pndm", diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index f0dc48f6d5..a2dc04fd22 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -2,6 +2,7 @@ export const DIFFUSERS_SCHEDULERS: Array = [ 'ddim', + 'ddpm', 'deis', 'lms', 'pndm', diff --git a/invokeai/frontend/web/src/app/types/invokeai.ts b/invokeai/frontend/web/src/app/types/invokeai.ts index 2ae221b0a8..b79504d8f2 100644 --- a/invokeai/frontend/web/src/app/types/invokeai.ts +++ b/invokeai/frontend/web/src/app/types/invokeai.ts @@ -47,6 +47,7 @@ export type CommonGeneratedImageMetadata = { postprocessing: null | Array; sampler: | 'ddim' + | 'ddpm' | 'deis' | 'lms' | 'pndm' From f7dc171c4fb33525919993cc2cb415ba19051170 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 03:44:20 +1200 Subject: [PATCH 10/14] Rename default schedulers across the app --- invokeai/app/invocations/generate.py | 2 +- invokeai/app/invocations/latent.py | 2 +- invokeai/backend/args.py | 2 +- invokeai/backend/generate.py | 2 +- .../frontend/web/src/features/nodes/examples/iterationGraph.ts | 2 +- .../web/src/features/parameters/store/generationSlice.ts | 2 +- tests/inpainting/original.json | 2 +- tests/nodes/test_png_metadata_service.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/invokeai/app/invocations/generate.py b/invokeai/app/invocations/generate.py index 9a29502048..bc72bbe2b3 100644 --- a/invokeai/app/invocations/generate.py +++ b/invokeai/app/invocations/generate.py @@ -52,7 +52,7 @@ class TextToImageInvocation(BaseInvocation, SDImageInvocation): width: int = Field(default=512, multiple_of=8, gt=0, description="The width of the resulting image", ) height: int = Field(default=512, multiple_of=8, gt=0, description="The height of the resulting image", ) cfg_scale: float = Field(default=7.5, ge=1, description="The Classifier-Free Guidance, higher values may result in a result closer to the prompt", ) - scheduler: SAMPLER_NAME_VALUES = Field(default="k_lms", description="The scheduler to use" ) + scheduler: SAMPLER_NAME_VALUES = Field(default="lms", description="The scheduler to use" ) model: str = Field(default="", description="The model to use (currently ignored)") # fmt: on diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 75fa719c07..60c77b5286 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -140,7 +140,7 @@ class TextToLatentsInvocation(BaseInvocation): noise: Optional[LatentsField] = Field(description="The noise to use") steps: int = Field(default=10, gt=0, description="The number of steps to use to generate the image") cfg_scale: float = Field(default=7.5, gt=0, description="The Classifier-Free Guidance, higher values may result in a result closer to the prompt", ) - scheduler: SAMPLER_NAME_VALUES = Field(default="k_lms", description="The scheduler to use" ) + scheduler: SAMPLER_NAME_VALUES = Field(default="lms", description="The scheduler to use" ) model: str = Field(default="", description="The model to use (currently ignored)") seamless: bool = Field(default=False, description="Whether or not to generate an image that can tile without seams", ) seamless_axes: str = Field(default="", description="The axes to tile the image on, 'x' and/or 'y'") diff --git a/invokeai/backend/args.py b/invokeai/backend/args.py index 856272c45b..db6fbe08df 100644 --- a/invokeai/backend/args.py +++ b/invokeai/backend/args.py @@ -634,7 +634,7 @@ class Args(object): choices=SAMPLER_CHOICES, metavar="SAMPLER_NAME", help=f'Set the default sampler. Supported samplers: {", ".join(SAMPLER_CHOICES)}', - default="k_lms", + default="lms", ) render_group.add_argument( "--log_tokenization", diff --git a/invokeai/backend/generate.py b/invokeai/backend/generate.py index dc5f9d9ed0..3db987bca8 100644 --- a/invokeai/backend/generate.py +++ b/invokeai/backend/generate.py @@ -142,7 +142,7 @@ class Generate: model=None, conf="configs/models.yaml", embedding_path=None, - sampler_name="k_lms", + sampler_name="lms", ddim_eta=0.0, # deterministic full_precision=False, precision="auto", diff --git a/invokeai/frontend/web/src/features/nodes/examples/iterationGraph.ts b/invokeai/frontend/web/src/features/nodes/examples/iterationGraph.ts index 46ee5289b2..3a5b6fb888 100644 --- a/invokeai/frontend/web/src/features/nodes/examples/iterationGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/examples/iterationGraph.ts @@ -20,7 +20,7 @@ export const iterationGraph = { model: '', progress_images: false, prompt: 'dog', - sampler_name: 'k_lms', + sampler_name: 'lms', seamless: false, steps: 11, type: 'txt2img', diff --git a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts index b0adc578a0..5ad0e4973c 100644 --- a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts +++ b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts @@ -51,7 +51,7 @@ export const initialGenerationState: GenerationState = { perlin: 0, prompt: '', negativePrompt: '', - sampler: 'k_lms', + sampler: 'lms', seamBlur: 16, seamSize: 96, seamSteps: 30, diff --git a/tests/inpainting/original.json b/tests/inpainting/original.json index f057f6d0dc..1a9320da03 100644 --- a/tests/inpainting/original.json +++ b/tests/inpainting/original.json @@ -23,7 +23,7 @@ ], "threshold": 0, "postprocessing": null, - "sampler": "k_lms", + "sampler": "lms", "variations": [], "type": "txt2img" } diff --git a/tests/nodes/test_png_metadata_service.py b/tests/nodes/test_png_metadata_service.py index c724074518..975e716fa9 100644 --- a/tests/nodes/test_png_metadata_service.py +++ b/tests/nodes/test_png_metadata_service.py @@ -17,7 +17,7 @@ valid_metadata = { "width": 512, "height": 512, "cfg_scale": 7.5, - "scheduler": "k_lms", + "scheduler": "lms", "model": "stable-diffusion-1.5", }, } From 27dc07d95ac9d090eb02d7fc847e3fd687f5c877 Mon Sep 17 00:00:00 2001 From: Sergey Borisov Date: Thu, 11 May 2023 18:49:27 +0300 Subject: [PATCH 11/14] Set zero eta by default(fix ddim scheduler error) --- invokeai/app/invocations/latent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 60c77b5286..825847cf79 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -208,7 +208,7 @@ class TextToLatentsInvocation(BaseInvocation): h_symmetry_time_pct=None,#h_symmetry_time_pct, v_symmetry_time_pct=None#v_symmetry_time_pct, ), - ).add_scheduler_args_if_applicable(model.scheduler, eta=None)#ddim_eta) + ).add_scheduler_args_if_applicable(model.scheduler, eta=0.0)#ddim_eta) return conditioning_data From 97127e560e8732ae25347e6b7dc1e6e1cb0c7d3b Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Fri, 12 May 2023 04:51:58 +1200 Subject: [PATCH 12/14] Disable dpmpp_2s in img2img & unifiedCanvas ... until upstream bug is fixed. --- invokeai/frontend/web/src/app/constants.ts | 6 ++++++ .../components/Parameters/Core/ParamSampler.tsx | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/invokeai/frontend/web/src/app/constants.ts b/invokeai/frontend/web/src/app/constants.ts index a2dc04fd22..189fbc9dd4 100644 --- a/invokeai/frontend/web/src/app/constants.ts +++ b/invokeai/frontend/web/src/app/constants.ts @@ -18,6 +18,12 @@ export const DIFFUSERS_SCHEDULERS: Array = [ 'unipc', ]; +export const IMG2IMG_DIFFUSERS_SCHEDULERS = DIFFUSERS_SCHEDULERS.filter( + (scheduler) => { + return scheduler !== 'dpmpp_2s'; + } +); + // Valid image widths export const WIDTHS: Array = Array.from(Array(64)).map( (_x, i) => (i + 1) * 64 diff --git a/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSampler.tsx b/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSampler.tsx index 5a20f54438..9bd22d9abe 100644 --- a/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSampler.tsx +++ b/invokeai/frontend/web/src/features/parameters/components/Parameters/Core/ParamSampler.tsx @@ -1,8 +1,12 @@ -import { DIFFUSERS_SCHEDULERS } from 'app/constants'; +import { + DIFFUSERS_SCHEDULERS, + IMG2IMG_DIFFUSERS_SCHEDULERS, +} from 'app/constants'; import { RootState } from 'app/store/store'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import IAISelect from 'common/components/IAISelect'; import { setSampler } from 'features/parameters/store/generationSlice'; +import { activeTabNameSelector } from 'features/ui/store/uiSelectors'; import { ChangeEvent, memo, useCallback } from 'react'; import { useTranslation } from 'react-i18next'; @@ -10,6 +14,9 @@ const ParamSampler = () => { const sampler = useAppSelector( (state: RootState) => state.generation.sampler ); + + const activeTabName = useAppSelector(activeTabNameSelector); + const dispatch = useAppDispatch(); const { t } = useTranslation(); @@ -23,7 +30,11 @@ const ParamSampler = () => { label={t('parameters.sampler')} value={sampler} onChange={handleChange} - validValues={DIFFUSERS_SCHEDULERS} + validValues={ + activeTabName === 'img2img' || activeTabName == 'unifiedCanvas' + ? IMG2IMG_DIFFUSERS_SCHEDULERS + : DIFFUSERS_SCHEDULERS + } minWidth={36} /> ); From 6de2f66b50b1be7dd38f93da84de909bf75788ad Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 12 May 2023 11:11:59 +1000 Subject: [PATCH 13/14] docs(ui): update ui readme --- invokeai/frontend/web/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/frontend/web/docs/README.md b/invokeai/frontend/web/docs/README.md index 787725cdda..323dcc5bc7 100644 --- a/invokeai/frontend/web/docs/README.md +++ b/invokeai/frontend/web/docs/README.md @@ -37,7 +37,7 @@ From `invokeai/frontend/web/` run `yarn install` to get everything set up. Start everything in dev mode: 1. Start the dev server: `yarn dev` -2. Start the InvokeAI UI per usual: `invokeai --web` +2. Start the InvokeAI Nodes backend: `python scripts/invokeai-new.py --web # run from the repo root` 3. Point your browser to the dev server address e.g. ### Production builds From 4caa1f19b2e81a3c54e0aa04abf0df73b17cbaa2 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Thu, 11 May 2023 19:06:02 -0700 Subject: [PATCH 14/14] fix(model manager): fix string formatting error on model checksum timer --- invokeai/backend/model_management/model_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/invokeai/backend/model_management/model_manager.py b/invokeai/backend/model_management/model_manager.py index a0a899a319..4f94395a86 100644 --- a/invokeai/backend/model_management/model_manager.py +++ b/invokeai/backend/model_management/model_manager.py @@ -30,7 +30,7 @@ from diffusers import ( UNet2DConditionModel, SchedulerMixin, logging as dlogging, -) +) from huggingface_hub import scan_cache_dir from omegaconf import OmegaConf from omegaconf.dictconfig import DictConfig @@ -68,7 +68,7 @@ class SDModelComponent(Enum): scheduler="scheduler" safety_checker="safety_checker" feature_extractor="feature_extractor" - + DEFAULT_MAX_MODELS = 2 class ModelManager(object): @@ -182,7 +182,7 @@ class ModelManager(object): vae from the model currently in the GPU. """ return self._get_sub_model(model_name, SDModelComponent.vae) - + def get_model_tokenizer(self, model_name: str=None)->CLIPTokenizer: """Given a model name identified in models.yaml, load the model into GPU if necessary and return its assigned CLIPTokenizer. If no @@ -190,12 +190,12 @@ class ModelManager(object): currently in the GPU. """ return self._get_sub_model(model_name, SDModelComponent.tokenizer) - + def get_model_unet(self, model_name: str=None)->UNet2DConditionModel: """Given a model name identified in models.yaml, load the model into GPU if necessary and return its assigned UNet2DConditionModel. If no model name is provided, return the UNet from the model - currently in the GPU. + currently in the GPU. """ return self._get_sub_model(model_name, SDModelComponent.unet) @@ -222,7 +222,7 @@ class ModelManager(object): currently in the GPU. """ return self._get_sub_model(model_name, SDModelComponent.scheduler) - + def _get_sub_model( self, model_name: str=None, @@ -1228,7 +1228,7 @@ class ModelManager(object): sha.update(chunk) hash = sha.hexdigest() toc = time.time() - self.logger.debug(f"sha256 = {hash} ({count} files hashed in", "%4.2fs)" % (toc - tic)) + self.logger.debug(f"sha256 = {hash} ({count} files hashed in {toc - tic:4.2f}s)") with open(hashpath, "w") as f: f.write(hash) return hash