From 4fa91724d96dcfe3ab52b7167cf480753163aeab Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 18 Feb 2023 11:07:38 -0500 Subject: [PATCH 1/2] fix conversion of checkpoints into incompatible diffusers models - The checkpoint conversion script was generating diffusers models with the safety checker set to null. This resulted in models that could not be merged with ones that have the safety checker activated. - This PR fixes the issue by incorporating the safety checker into all 1.x-derived checkpoints, regardless of user's nsfw_checker setting. --- ldm/invoke/CLI.py | 1 + ldm/invoke/ckpt_to_diffuser.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ldm/invoke/CLI.py b/ldm/invoke/CLI.py index f72f6058aa..ddcd1a64c7 100644 --- a/ldm/invoke/CLI.py +++ b/ldm/invoke/CLI.py @@ -786,6 +786,7 @@ def optimize_model(model_name_or_path: Union[Path,str], gen, opt, completer): model_name_or_path = model_name_or_path.replace('\\','/') # windows manager = gen.model_manager ckpt_path = None + original_config_file=None if model_name_or_path == gen.model_name: print("** Can't convert the active model. !switch to another model first. **") diff --git a/ldm/invoke/ckpt_to_diffuser.py b/ldm/invoke/ckpt_to_diffuser.py index 895484828f..a1c99bc5b1 100644 --- a/ldm/invoke/ckpt_to_diffuser.py +++ b/ldm/invoke/ckpt_to_diffuser.py @@ -53,6 +53,7 @@ from diffusers import ( ) from diffusers.pipelines.latent_diffusion.pipeline_latent_diffusion import LDMBertConfig, LDMBertModel from diffusers.pipelines.paint_by_example import PaintByExampleImageEncoder, PaintByExamplePipeline +from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker from diffusers.utils import is_safetensors_available from transformers import AutoFeatureExtractor, BertTokenizerFast, CLIPTextModel, CLIPTokenizer, CLIPVisionConfig @@ -984,6 +985,7 @@ def load_pipeline_from_original_stable_diffusion_ckpt( elif model_type in ['FrozenCLIPEmbedder','WeightedFrozenCLIPEmbedder']: text_model = convert_ldm_clip_checkpoint(checkpoint) tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14",cache_dir=cache_dir) + safety_checker = StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker',cache_dir=global_cache_dir("hub")) feature_extractor = AutoFeatureExtractor.from_pretrained("CompVis/stable-diffusion-safety-checker",cache_dir=cache_dir) pipe = pipeline_class( vae=vae, @@ -991,7 +993,7 @@ def load_pipeline_from_original_stable_diffusion_ckpt( tokenizer=tokenizer, unet=unet, scheduler=scheduler, - safety_checker=None, + safety_checker=safety_checker, feature_extractor=feature_extractor, ) else: From d01b7ea2d29a0f45bfe11aaa518815bf780c6a54 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 18 Feb 2023 11:19:06 -0500 Subject: [PATCH 2/2] remove debug statement & actually do merge --- ldm/invoke/merge_diffusers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ldm/invoke/merge_diffusers.py b/ldm/invoke/merge_diffusers.py index 1c8a2c63bb..edd0e778d6 100644 --- a/ldm/invoke/merge_diffusers.py +++ b/ldm/invoke/merge_diffusers.py @@ -419,8 +419,7 @@ def run_gui(args: Namespace): mergeapp.run() args = mergeapp.merge_arguments - print(f'DEBUG: {args}') - #merge_diffusion_models_and_commit(**args) + merge_diffusion_models_and_commit(**args) print(f'>> Models merged into new model: "{args["merged_model_name"]}".')