From 44216381cbf2f47f737d29012c36ec109931cfa9 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 7 Oct 2023 15:29:28 -0400 Subject: [PATCH] fix conversion call --- invokeai/app/api/routers/models.py | 4 +-- .../app/services/model_install_service.py | 4 +-- invokeai/backend/model_manager/config.py | 7 ++++- invokeai/backend/model_manager/install.py | 27 ++++++++++++------- .../backend/model_manager/storage/__init__.py | 2 +- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/invokeai/app/api/routers/models.py b/invokeai/app/api/routers/models.py index 72263d2640..9e1b617bfa 100644 --- a/invokeai/app/api/routers/models.py +++ b/invokeai/app/api/routers/models.py @@ -272,7 +272,7 @@ async def delete_model( response_model=InvokeAIModelConfig, ) async def convert_model( - key: str = Path(description="Unique key of model to remove from model registry."), + key: str = Path(description="Unique key of model to convert from checkpoint/safetensors to diffusers format."), convert_dest_directory: Optional[str] = Query( default=None, description="Save the converted model to the designated directory" ), @@ -281,7 +281,7 @@ async def convert_model( try: dest = pathlib.Path(convert_dest_directory) if convert_dest_directory else None installer = ApiDependencies.invoker.services.model_installer - model_config = installer.convert_model(key, convert_dest_directory=dest) + model_config = installer.convert_model(key, dest_directory=dest) response = parse_obj_as(InvokeAIModelConfig, model_config.dict()) except UnknownModelException as e: raise HTTPException(status_code=404, detail=f"Model '{key}' not found: {str(e)}") diff --git a/invokeai/app/services/model_install_service.py b/invokeai/app/services/model_install_service.py index b6f4b543c7..4f859cbe59 100644 --- a/invokeai/app/services/model_install_service.py +++ b/invokeai/app/services/model_install_service.py @@ -296,13 +296,13 @@ class ModelInstallService(ModelInstall, ModelInstallServiceBase): :param key: Key of the model to convert :param convert_dest_directory: Save the converted model to the designated directory (`models/etc/etc` by default) - This will raise a ValueError unless the model is not a checkpoint. It will + This will raise a ValueError unless the model is a checkpoint. It will also raise a ValueError in the event that there is a similarly-named diffusers directory already in place. """ model_info = self.store.get_model(key) self.logger.info(f"Converting model {model_info.name} into a diffusers") - return self.convert_model(key, dest_directory) + return super().convert_model(key, dest_directory) @property def logger(self): diff --git a/invokeai/backend/model_manager/config.py b/invokeai/backend/model_manager/config.py index 9a2c4e5d35..7c33716305 100644 --- a/invokeai/backend/model_manager/config.py +++ b/invokeai/backend/model_manager/config.py @@ -118,7 +118,12 @@ class ModelConfigBase(BaseModel): base_model: BaseModelType model_type: ModelType model_format: ModelFormat - key: str = Field(description="hash key for model", default="") # this will get added by the store + key: str = Field( + description="key for model derived from original hash", default="" + ) # assigned on the first install + hash: Optional[str] = Field( + description="current hash key for model", default=None + ) # if model is converted or otherwise modified, this will hold updated hash description: Optional[str] = Field(None) author: Optional[str] = Field(description="Model author") license: Optional[str] = Field(description="License string") diff --git a/invokeai/backend/model_manager/install.py b/invokeai/backend/model_manager/install.py index a8569b8544..19f9e58783 100644 --- a/invokeai/backend/model_manager/install.py +++ b/invokeai/backend/model_manager/install.py @@ -417,6 +417,7 @@ class ModelInstall(ModelInstallBase): base_model=info.base_type, model_type=info.model_type, model_format=info.format, + hash=key, ) # add 'main' specific fields if info.model_type == ModelType.Main: @@ -577,7 +578,7 @@ class ModelInstall(ModelInstallBase): elif job.status == "cancelled": self._logger.warning(f"{job.source}: Model installation cancelled at caller's request.") - def sync_model_path(self, key) -> ModelConfigBase: + def sync_model_path(self, key: str, ignore_hash_change: bool = False) -> ModelConfigBase: """ Move model into the location indicated by its basetype, type and name. @@ -596,12 +597,15 @@ class ModelInstall(ModelInstallBase): return model new_path = models_dir / model.base_model.value / model.model_type.value / model.name - self._logger.info( - f"{old_path.name} is not in the right directory for a model of its type. Moving to {new_path}." - ) - model.path = self._move_model(old_path, new_path).as_posix() - new_hash = self.hash(model.path) - assert new_hash == key, f"{model.name}: Model hash changed during installation, possibly corrupted." + self._logger.info(f"Moving {model.name} to {new_path}.") + new_path = self._move_model(old_path, new_path) + model.hash = self.hash(new_path) + model.path = new_path.relative_to(models_dir).as_posix() + if model.hash != key: + assert ( + ignore_hash_change + ), f"{model.name}: Model hash changed during installation, model is possibly corrupted" + self._logger.info(f"Model has new hash {model.hash}, but will continue to be identified by {key}") self._store.update_model(key, model) return model @@ -692,7 +696,7 @@ class ModelInstall(ModelInstallBase): # We are taking advantage of a side effect of get_model() that converts check points # into cached diffusers directories stored at `path`. It doesn't matter # what submodel type we request here, so we get the smallest. - loader = ModelLoad(self._app_config) + loader = ModelLoad(self._app_config, self.store) submodel = {"submodel_type": SubModelType.Scheduler} if info.model_type == ModelType.Main else {} converted_model: ModelInfo = loader.get_model(key, **submodel) @@ -703,7 +707,7 @@ class ModelInstall(ModelInstallBase): update = info.dict() update.pop("config") update["model_format"] = "diffusers" - update["path"] = converted_model.location + update["path"] = str(converted_model.location) if dest_directory: new_diffusers_path = Path(dest_directory) / info.name @@ -713,7 +717,7 @@ class ModelInstall(ModelInstallBase): update["path"] = new_diffusers_path.as_posix() self._store.update_model(key, update) - result = self.sync_model_path(key) + result = self.sync_model_path(key, ignore_hash_change=True) except Exception as excp: # something went wrong, so don't leave dangling diffusers model in directory or it will cause a duplicate model error! if new_diffusers_path: @@ -752,6 +756,9 @@ class ModelInstall(ModelInstallBase): def sync_to_config(self): """Synchronize models on disk to those in memory.""" self.scan_models_directory() + if autoimport := self._app_config.autoimport_dir: + self._logger.info("Scanning autoimport directory for new models") + self.scan_directory(self._app_config.root_path / autoimport) def scan_models_directory(self): """ diff --git a/invokeai/backend/model_manager/storage/__init__.py b/invokeai/backend/model_manager/storage/__init__.py index 5046d9ca24..cdb7eea958 100644 --- a/invokeai/backend/model_manager/storage/__init__.py +++ b/invokeai/backend/model_manager/storage/__init__.py @@ -22,5 +22,5 @@ def get_config_store(location: pathlib.Path) -> ModelConfigStore: return ModelConfigStoreSQL(location) else: raise Exception( - "Unable to determine type of configuration file '{location}'. Type 'auto' is not supported outside the app." + f"Unable to determine type of configuration file '{location}'. Type 'auto' is not supported outside the app." )