From ca1f76b7ba04e1ebe0e473d7c178403aee082383 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Mon, 28 Nov 2022 18:26:52 -0800 Subject: [PATCH] create an embedding_manager for diffusers --- ldm/invoke/generator/diffusers_pipeline.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ldm/invoke/generator/diffusers_pipeline.py b/ldm/invoke/generator/diffusers_pipeline.py index 2d3f694687..714d07faac 100644 --- a/ldm/invoke/generator/diffusers_pipeline.py +++ b/ldm/invoke/generator/diffusers_pipeline.py @@ -16,6 +16,7 @@ from diffusers.schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMSchedu from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer from ldm.models.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent +from ldm.modules.embedding_manager import EmbeddingManager from ldm.modules.encoders.modules import WeightedFrozenCLIPEmbedder @@ -28,6 +29,16 @@ class PipelineIntermediateState: predicted_original: Optional[torch.Tensor] = None +# copied from configs/stable-diffusion/v1-inference.yaml +_default_personalization_config_params = dict( + placeholder_strings=["*"], + initializer_wods=["sculpture"], + per_image_tokens=False, + num_vectors_per_token=8, + progressive_words=False +) + + class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): r""" Pipeline for text-to-image generation using Stable Diffusion. @@ -89,6 +100,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): transformer=self.text_encoder ) self.invokeai_diffuser = InvokeAIDiffuserComponent(self.unet, self._unet_forward) + self.embedding_manager = EmbeddingManager(self.clip_embedder, **_default_personalization_config_params) def image_from_embeddings(self, latents: torch.Tensor, num_inference_steps: int, text_embeddings: torch.Tensor, unconditioned_embeddings: torch.Tensor,