merge with main, fix conflicts

invokeai/backend/install/model_install_backend.py

@@ -173,15 +173,19 @@ class ModelInstall(object):
         # add requested models
         for path in selections.install_models:
             logger.info(f'Installing {path} [{job}/{jobs}]')
-            self.heuristic_import(path)
+            try:
+                self.heuristic_import(path)
+            except (ValueError, KeyError) as e:
+                logger.error(str(e))
             job += 1
             
         dlogging.set_verbosity(verbosity)
         self.mgr.commit()
 
     def heuristic_import(self,
-                          model_path_id_or_url: Union[str,Path],
-                          models_installed: Set[Path]=None)->Dict[str, AddModelResult]:
+                         model_path_id_or_url: Union[str,Path],
+                         models_installed: Set[Path]=None,
+                         )->Dict[str, AddModelResult]:
         '''
         :param model_path_id_or_url: A Path to a local model to import, or a string representing its repo_id or URL
         :param models_installed: Set of installed models, used for recursive invocation
@@ -194,62 +198,53 @@ class ModelInstall(object):
         # A little hack to allow nested routines to retrieve info on the requested ID
         self.current_id = model_path_id_or_url
         path = Path(model_path_id_or_url)
-        
-        try:
-            # checkpoint file, or similar
-            if path.is_file():
-                models_installed.update(self._install_path(path))
+        # checkpoint file, or similar
+        if path.is_file():
+            models_installed.update({str(path):self._install_path(path)})
 
-            # folders style or similar
-            elif path.is_dir() and any([(path/x).exists() for x in \
-                                        {'config.json','model_index.json','learned_embeds.bin','pytorch_lora_weights.bin'}
-                                        ]
-                                       ):
-                models_installed.update(self._install_path(path))
+        # folders style or similar
+        elif path.is_dir() and any([(path/x).exists() for x in \
+                                    {'config.json','model_index.json','learned_embeds.bin','pytorch_lora_weights.bin'}
+                                    ]
+                                   ):
+            models_installed.update(self._install_path(path))
 
-            # recursive scan
-            elif path.is_dir():
-                for child in path.iterdir():
-                    self.heuristic_import(child, models_installed=models_installed)
+        # recursive scan
+        elif path.is_dir():
+            for child in path.iterdir():
+                self.heuristic_import(child, models_installed=models_installed)
 
-            # huggingface repo
-            elif len(str(model_path_id_or_url).split('/')) == 2:
-                models_installed.update(self._install_repo(str(model_path_id_or_url)))
+        # huggingface repo
+        elif len(str(model_path_id_or_url).split('/')) == 2:
+            models_installed.update({str(model_path_id_or_url): self._install_repo(str(model_path_id_or_url))})
 
-            # a URL
-            elif model_path_id_or_url.startswith(("http:", "https:", "ftp:")):
-                models_installed.update(self._install_url(model_path_id_or_url))
+        # a URL
+        elif str(model_path_id_or_url).startswith(("http:", "https:", "ftp:")):
+            models_installed.update({str(model_path_id_or_url): self._install_url(model_path_id_or_url)})
 
-            else:
-                logger.warning(f'{str(model_path_id_or_url)} is not recognized as a local path, repo ID or URL. Skipping')
-            
-        except ValueError as e:
-            logger.error(str(e))
+        else:
+            raise KeyError(f'{str(model_path_id_or_url)} is not recognized as a local path, repo ID or URL. Skipping')
 
         return models_installed
 
     # install a model from a local path. The optional info parameter is there to prevent
     # the model from being probed twice in the event that it has already been probed.
-    def _install_path(self, path: Path, info: ModelProbeInfo=None)->Dict[str, AddModelResult]:
-        try:
-            model_result = None
-            info = info or ModelProbe().heuristic_probe(path,self.prediction_helper)
-            model_name = path.stem if path.is_file() else path.name
-            if self.mgr.model_exists(model_name, info.base_type, info.model_type):
-                raise ValueError(f'A model named "{model_name}" is already installed.')
-            attributes = self._make_attributes(path,info)
-            model_result = self.mgr.add_model(model_name = model_name,
-                                              base_model = info.base_type,
-                                              model_type = info.model_type,
-                                              model_attributes = attributes,
-                                              )
-        except Exception as e:
-            logger.warning(f'{str(e)} Skipping registration.')
-            return {}
-        return {str(path): model_result}
+    def _install_path(self, path: Path, info: ModelProbeInfo=None)->AddModelResult:
+        info = info or ModelProbe().heuristic_probe(path,self.prediction_helper)
+        if not info:
+            logger.warning(f'Unable to parse format of {path}')
+            return None
+        model_name = path.stem if path.is_file() else path.name
+        if self.mgr.model_exists(model_name, info.base_type, info.model_type):
+            raise ValueError(f'A model named "{model_name}" is already installed.')
+        attributes = self._make_attributes(path,info)
+        return self.mgr.add_model(model_name = model_name,
+                                  base_model = info.base_type,
+                                  model_type = info.model_type,
+                                  model_attributes = attributes,
+                                  )
 
-    def _install_url(self, url: str)->dict:
-        # copy to a staging area, probe, import and delete
+    def _install_url(self, url: str)->AddModelResult:
         with TemporaryDirectory(dir=self.config.models_path) as staging:
             location = download_with_resume(url,Path(staging))
             if not location:
@@ -261,7 +256,7 @@ class ModelInstall(object):
         # staged version will be garbage-collected at this time
         return self._install_path(Path(models_path), info)
 
-    def _install_repo(self, repo_id: str)->dict:
+    def _install_repo(self, repo_id: str)->AddModelResult:
         hinfo = HfApi().model_info(repo_id)
         
         # we try to figure out how to download this most economically

invokeai/backend/model_management/__init__.py

@@ -1,7 +1,8 @@
 """
 Initialization file for invokeai.backend.model_management
 """
-from .model_manager import ModelManager, ModelInfo, AddModelResult
+from .model_manager import ModelManager, ModelInfo, AddModelResult, SchedulerPredictionType
 from .model_cache import ModelCache
 from .models import BaseModelType, ModelType, SubModelType, ModelVariantType
+from .model_merge import ModelMerger, MergeInterpolationMethod
 

invokeai/backend/model_management/lora.py

@@ -2,16 +2,14 @@ from __future__ import annotations
 
 import copy
 from contextlib import contextmanager
+from typing import Optional, Dict, Tuple, Any, Union, List
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
 
 import torch
 from compel.embeddings_provider import BaseTextualInversionManager
 from diffusers.models import UNet2DConditionModel
 from safetensors.torch import load_file
-from torch.utils.hooks import RemovableHandle
-from transformers import CLIPTextModel
-
+from transformers import CLIPTextModel, CLIPTokenizer
 
 class LoRALayerBase:
     #rank: Optional[int]
@@ -123,8 +121,8 @@ class LoRALayer(LoRALayerBase):
 
     def get_weight(self):
         if self.mid is not None:
-            up = self.up.reshape(up.shape[0], up.shape[1])
-            down = self.down.reshape(up.shape[0], up.shape[1])
+            up = self.up.reshape(self.up.shape[0], self.up.shape[1])
+            down = self.down.reshape(self.down.shape[0], self.down.shape[1])
             weight = torch.einsum("m n w h, i m, n j -> i j w h", self.mid, up, down)
         else:
             weight = self.up.reshape(self.up.shape[0], -1) @ self.down.reshape(self.down.shape[0], -1)
@@ -410,7 +408,7 @@ class LoRAModel: #(torch.nn.Module):
             else:
                 # TODO: diff/ia3/... format
                 print(
-                    f">> Encountered unknown lora layer module in {self.name}: {layer_key}"
+                    f">> Encountered unknown lora layer module in {model.name}: {layer_key}"
                 )
                 return
 

invokeai/backend/model_management/model_manager.py

@@ -234,7 +234,7 @@ import textwrap
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional, List, Tuple, Union, Dict, Set, Callable, types
-from shutil import rmtree
+from shutil import rmtree, move
 
 import torch
 from omegaconf import OmegaConf
@@ -279,7 +279,7 @@ class InvalidModelError(Exception):
     pass
 
 class AddModelResult(BaseModel):
-    name: str = Field(description="The name of the model after import")
+    name: str = Field(description="The name of the model after installation")
     model_type: ModelType = Field(description="The type of model")
     base_model: BaseModelType = Field(description="The base model")
     config: ModelConfigBase = Field(description="The configuration of the model")
@@ -490,17 +490,32 @@ class ModelManager(object):
         """
         return [(self.parse_key(x)) for x in self.models.keys()]
 
+    def list_model(
+            self,
+            model_name: str,
+            base_model: BaseModelType,
+            model_type: ModelType,
+    ) -> dict:
+        """
+        Returns a dict describing one installed model, using
+        the combined format of the list_models() method.
+        """
+        models = self.list_models(base_model,model_type,model_name)
+        return models[0] if models else None
+
     def list_models(
         self,
         base_model: Optional[BaseModelType] = None,
         model_type: Optional[ModelType] = None,
+        model_name: Optional[str] = None,
     ) -> list[dict]:
         """
         Return a list of models.
         """
 
+        model_keys = [self.create_key(model_name, base_model, model_type)] if model_name else sorted(self.models, key=str.casefold)
         models = []
-        for model_key in sorted(self.models, key=str.casefold):
+        for model_key in model_keys:
             model_config = self.models[model_key]
 
             cur_model_name, cur_base_model, cur_model_type = self.parse_key(model_key)
@@ -545,10 +560,7 @@ class ModelManager(object):
         model_cfg = self.models.pop(model_key, None)
 
         if model_cfg is None:
-            self.logger.error(
-                f"Unknown model {model_key}"
-            )
-            return
+            raise KeyError(f"Unknown model {model_key}")
 
         # note: it not garantie to release memory(model can has other references)
         cache_ids = self.cache_keys.pop(model_key, [])
@@ -614,6 +626,7 @@ class ModelManager(object):
                 self.cache.uncache_model(cache_id)
 
         self.models[model_key] = model_config
+        self.commit()
         return AddModelResult(
             name = model_name,
             model_type = model_type,
@@ -621,6 +634,60 @@ class ModelManager(object):
             config = model_config,
         )
 
+    def convert_model (
+            self,
+            model_name: str,
+            base_model: BaseModelType,
+            model_type: Union[ModelType.Main,ModelType.Vae],
+    ) -> AddModelResult:
+        '''
+        Convert a checkpoint file into a diffusers folder, deleting the cached
+        version and deleting the original checkpoint file if it is in the models
+        directory.
+        :param model_name: Name of the model to convert
+        :param base_model: Base model type
+        :param model_type: Type of model ['vae' or 'main']
+
+        This will raise a ValueError unless the model is a checkpoint.
+        '''
+        info = self.model_info(model_name, base_model, model_type)
+        if info["model_format"] != "checkpoint":
+            raise ValueError(f"not a checkpoint format model: {model_name}")
+
+        # We are taking advantage of a side effect of get_model() that converts check points
+        # into cached diffusers directories stored at `location`. It doesn't matter
+        # what submodeltype we request here, so we get the smallest.
+        submodel = {"submodel_type": SubModelType.Tokenizer} if model_type==ModelType.Main else {}
+        model = self.get_model(model_name,
+                               base_model,
+                               model_type,
+                               **submodel,
+                               )
+        checkpoint_path = self.app_config.root_path / info["path"]
+        old_diffusers_path = self.app_config.models_path / model.location
+        new_diffusers_path = self.app_config.models_path / base_model.value / model_type.value / model_name
+        if new_diffusers_path.exists():
+            raise ValueError(f"A diffusers model already exists at {new_diffusers_path}")
+
+        try:
+            move(old_diffusers_path,new_diffusers_path)
+            info["model_format"] = "diffusers"
+            info["path"] = str(new_diffusers_path.relative_to(self.app_config.root_path))
+            info.pop('config')
+
+            result = self.add_model(model_name, base_model, model_type,
+                                    model_attributes = info,
+                                    clobber=True)
+        except:
+            # something went wrong, so don't leave dangling diffusers model in directory or it will cause a duplicate model error!
+            rmtree(new_diffusers_path)
+            raise
+        
+        if checkpoint_path.exists() and checkpoint_path.is_relative_to(self.app_config.models_path):
+            checkpoint_path.unlink()
+        
+        return result
+    
     def search_models(self, search_folder):
         self.logger.info(f"Finding Models In: {search_folder}")
         models_folder_ckpt = Path(search_folder).glob("**/*.ckpt")
@@ -821,6 +888,10 @@ class ModelManager(object):
         The result is a set of successfully installed models. Each element
         of the set is a dict corresponding to the newly-created OmegaConf stanza for
         that model.
+
+        May return the following exceptions:
+        - KeyError   - one or more of the items to import is not a valid path, repo_id or URL
+        - ValueError - a corresponding model already exists
         '''
         # avoid circular import here
         from invokeai.backend.install.model_install_backend import ModelInstall
@@ -830,11 +901,7 @@ class ModelManager(object):
                                  prediction_type_helper = prediction_type_helper,
                                  model_manager = self)
         for thing in items_to_import:
-            try:
-                installed = installer.heuristic_import(thing)
-                successfully_installed.update(installed)
-            except Exception as e:
-                self.logger.warning(f'{thing} could not be imported: {str(e)}')
-                
+            installed = installer.heuristic_import(thing)
+            successfully_installed.update(installed)
         self.commit()                
         return successfully_installed

invokeai/backend/model_management/model_merge.py

@@ -0,0 +1,131 @@
+"""
+invokeai.backend.model_management.model_merge exports:
+merge_diffusion_models() -- combine multiple models by location and return a pipeline object
+merge_diffusion_models_and_commit() -- combine multiple models by ModelManager ID and write to models.yaml
+
+Copyright (c) 2023 Lincoln Stein and the InvokeAI Development Team
+"""
+
+import warnings
+from enum import Enum
+from pathlib import Path
+from diffusers import DiffusionPipeline
+from diffusers import logging as dlogging
+from typing import List, Union
+
+import invokeai.backend.util.logging as logger
+
+from ...backend.model_management import ModelManager, ModelType, BaseModelType, ModelVariantType, AddModelResult
+
+class MergeInterpolationMethod(str, Enum):
+    WeightedSum = "weighted_sum"
+    Sigmoid = "sigmoid"
+    InvSigmoid = "inv_sigmoid"
+    AddDifference = "add_difference"
+
+class ModelMerger(object):
+    def __init__(self, manager: ModelManager):
+        self.manager = manager
+
+    def merge_diffusion_models(
+            self,
+            model_paths: List[Path],
+            alpha: float = 0.5,
+            interp: MergeInterpolationMethod = None,
+            force: bool = False,
+            **kwargs,
+    ) -> DiffusionPipeline:
+        """
+        :param model_paths:  up to three models, designated by their local paths or HuggingFace repo_ids
+        :param alpha: The interpolation parameter. Ranges from 0 to 1.  It affects the ratio in which the checkpoints are merged. A 0.8 alpha
+                   would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2
+        :param interp: The interpolation method to use for the merging. Supports "sigmoid", "inv_sigmoid", "add_difference" and None.
+                   Passing None uses the default interpolation which is weighted sum interpolation. For merging three checkpoints, only "add_difference" is supported.
+        :param force:  Whether to ignore mismatch in model_config.json for the current models. Defaults to False.
+
+        **kwargs - the default DiffusionPipeline.get_config_dict kwargs:
+             cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map
+        """
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            verbosity = dlogging.get_verbosity()
+            dlogging.set_verbosity_error()
+
+            pipe = DiffusionPipeline.from_pretrained(
+                model_paths[0],
+                custom_pipeline="checkpoint_merger",
+            )
+            merged_pipe = pipe.merge(
+                pretrained_model_name_or_path_list=model_paths,
+                alpha=alpha,
+                interp=interp.value if interp else None, #diffusers API treats None as "weighted sum"
+                force=force,
+                **kwargs,
+            )
+            dlogging.set_verbosity(verbosity)
+        return merged_pipe
+
+
+    def merge_diffusion_models_and_save (
+            self,
+            model_names: List[str],
+            base_model: Union[BaseModelType,str],
+            merged_model_name: str,
+            alpha: float = 0.5,
+            interp: MergeInterpolationMethod = None,
+            force: bool = False,
+            **kwargs,
+    ) -> AddModelResult:
+        """
+        :param models: up to three models, designated by their InvokeAI models.yaml model name
+        :param base_model: base model (must be the same for all merged models!)
+        :param merged_model_name: name for new model
+        :param alpha: The interpolation parameter. Ranges from 0 to 1.  It affects the ratio in which the checkpoints are merged. A 0.8 alpha
+                   would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2
+        :param interp: The interpolation method to use for the merging. Supports "weighted_average", "sigmoid", "inv_sigmoid", "add_difference" and None.
+                   Passing None uses the default interpolation which is weighted sum interpolation. For merging three checkpoints, only "add_difference" is supported. Add_difference is A+(B-C).
+        :param force:  Whether to ignore mismatch in model_config.json for the current models. Defaults to False.
+
+        **kwargs - the default DiffusionPipeline.get_config_dict kwargs:
+             cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map
+        """
+        model_paths = list()
+        config = self.manager.app_config
+        base_model = BaseModelType(base_model)
+        vae = None
+        
+        for mod in model_names:
+            info = self.manager.list_model(mod, base_model=base_model, model_type=ModelType.Main)
+            assert info,                                f"model {mod}, base_model {base_model}, is unknown"
+            assert info["model_format"] == "diffusers", f"{mod} is not a diffusers model. It must be optimized before merging"
+            assert info["variant"] == "normal",         f"{mod} is a {info['variant']} model, which cannot currently be merged"
+            assert len(model_names) <= 2 or \
+                interp==MergeInterpolationMethod.AddDifference, "When merging three models, only the 'add_difference' merge method is supported"
+            # pick up the first model's vae
+            if mod == model_names[0]:
+                vae = info.get("vae")
+            model_paths.extend([config.root_path / info["path"]])
+
+        merge_method = None if interp == 'weighted_sum' else MergeInterpolationMethod(interp)
+        logger.debug(f'interp = {interp}, merge_method={merge_method}')
+        merged_pipe = self.merge_diffusion_models(
+           model_paths, alpha, merge_method, force, **kwargs
+        )
+        dump_path = config.models_path / base_model.value / ModelType.Main.value
+        dump_path.mkdir(parents=True, exist_ok=True)
+        dump_path = dump_path / merged_model_name
+
+        merged_pipe.save_pretrained(dump_path, safe_serialization=1)
+        attributes = dict(
+            path = str(dump_path),
+            description = f"Merge of models {', '.join(model_names)}",
+            model_format = "diffusers",
+            variant = ModelVariantType.Normal.value,
+            vae = vae,
+        )
+        return self.manager.add_model(merged_model_name,
+                                      base_model = base_model,
+                                      model_type = ModelType.Main,
+                                      model_attributes = attributes,
+                                      clobber = True
+                                      )

invokeai/backend/model_management/models/stable_diffusion.py

@@ -116,7 +116,7 @@ class StableDiffusion1Model(DiffusersModel):
                 version=BaseModelType.StableDiffusion1,
                 model_config=config,
                 output_path=output_path,
-            )
+           )
         else:
             return model_path