From 1c7d9dbf40d194405a6c03195739c60918ce5414 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 14 Aug 2023 21:10:45 -0400 Subject: [PATCH 1/3] start installer module --- invokeai/backend/model_manager/__init__.py | 16 ++++++------ .../backend/model_manager/model_install.py | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 invokeai/backend/model_manager/model_install.py diff --git a/invokeai/backend/model_manager/__init__.py b/invokeai/backend/model_manager/__init__.py index 7700a112d9..494a141367 100644 --- a/invokeai/backend/model_manager/__init__.py +++ b/invokeai/backend/model_manager/__init__.py @@ -1,15 +1,15 @@ """ Initialization file for invokeai.backend.model_manager.config """ -from invokeai.backend.model_manager.config import ( - ModelConfigFactory, - ModelConfigBase, - InvalidModelConfigException, +from .config import ( BaseModelType, - ModelType, - SubModelType, - ModelVariantType, + InvalidModelConfigException, + ModelConfigBase, + ModelConfigFactory, ModelFormat, + ModelType, + ModelVariantType, SchedulerPredictionType, + SubModelType, ) - +from .model_install import ModelInstall diff --git a/invokeai/backend/model_manager/model_install.py b/invokeai/backend/model_manager/model_install.py new file mode 100644 index 0000000000..21af78ca3f --- /dev/null +++ b/invokeai/backend/model_manager/model_install.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023 Lincoln D. Stein and the InvokeAI Development Team +""" +Install/delete models. + +Typical usage: + + from invokeai.app.services.config import InvokeAIAppConfig + from invokeai.backend.model_manager import ModelInstall + from invokeai.backend.model_manager.storage import ModelConfigStoreSQL + + config = InvokeAIAppConfig.get_config() + store = ModelConfigStoreSQL(config.db_path) + installer = ModelInstall(store=store, config=config) + + id: str = installer.install_model('/path/to/model') + installer.delete_model(id) + + hash: str = installer.hash('/path/to/model') # should be same as id above + +The following exceptions may be raised: + DuplicateModelException + UnknownModelTypeException +""" + + From e83d00595dfb0336a767787b9280e195f7bc5678 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 14 Aug 2023 21:49:32 -0400 Subject: [PATCH 2/3] module skeleton written --- .../backend/model_manager/model_install.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/invokeai/backend/model_manager/model_install.py b/invokeai/backend/model_manager/model_install.py index 21af78ca3f..ae9c70f16a 100644 --- a/invokeai/backend/model_manager/model_install.py +++ b/invokeai/backend/model_manager/model_install.py @@ -12,14 +12,95 @@ Typical usage: store = ModelConfigStoreSQL(config.db_path) installer = ModelInstall(store=store, config=config) + # register config, don't move path + id: str = installer.register_model('/path/to/model') + + # register config, and install model in `models` id: str = installer.install_model('/path/to/model') + + # unregister, don't delete + installer.forget(id) + + # unregister and delete model from disk installer.delete_model(id) + # scan directory recursively and install all new models found + ids: List[str] = installer.scan_directory('/path/to/directory') + + # unregister any model whose path is no longer valid + ids: List[str] = installer.garbage_collect() + hash: str = installer.hash('/path/to/model') # should be same as id above The following exceptions may be raised: DuplicateModelException UnknownModelTypeException """ +from pathlib import Path +from typing import Optional +from invokeai.app.services.config import InvokeAIAppConfig +from invokeai.backend.util.logging import InvokeAILogger +from .storage import ModelConfigStore +class ModelInstall(object): + """Model installer class handles installation from a local path.""" + + _config: InvokeAIAppConfig + _logger: InvokeAILogger + _store: ModelConfigStore + + def __init__(self, + store: Optional[ModelConfigStore] = None, + config: Optional[InvokeAIAppConfig] = None, + logger: Optional[InvokeAILogger] = None + ): + """ + Create ModelInstall object. + + :param store: Optional ModelConfigStore. If None passed, + defaults to `configs/models.yaml`. + :param config: Optional InvokeAIAppConfig. If None passed, + uses the system-wide default app config. + :param logger: Optional InvokeAILogger. If None passed, + uses the system-wide default logger. + """ + self._config = config or InvokeAIAppConfig.get_config() + self._logger = logger or InvokeAILogger.getLogger() + if store is None: + from .storage import ModelConfigStoreYAML + store = ModelConfigStoreYAML(config.model_conf_path) + self._store = store + + + def register(self, model_path: Path) -> str: + """Probe and register the model at model_path.""" + pass + + def install(self, model_path: Path) -> str: + """Probe, register and Install the model in the models directory.""" + pass + + def forget(self, id: str) -> str: + """Unregister the model identified by id.""" + pass + + def delete(self, id: str) -> str: + """ + Unregister and delete the model identified by id. + Note that this deletes the model unconditionally. + """ + pass + + def scan_directory(self, scan_dir: Path, install: bool=False) -> List[str]: + """Scan directory for new models and register or install them.""" + pass + + def garbage_collect(self): + """Unregister any models whose paths are no longer valid.""" + pass + + def hash(self, model_path: Path) -> str: + """Compute the fast hash of the model.""" + pass + From 916cc261932256c35ffbbbd6ea11a3c80e47b141 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Wed, 16 Aug 2023 21:21:42 -0400 Subject: [PATCH 3/3] partial rewrite of checkpoint template creator --- scripts/create_checkpoint_template.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/create_checkpoint_template.py b/scripts/create_checkpoint_template.py index 7ff201c841..4226c46d8a 100755 --- a/scripts/create_checkpoint_template.py +++ b/scripts/create_checkpoint_template.py @@ -10,11 +10,21 @@ import json from pathlib import Path -from invokeai.backend.model_management.models.base import read_checkpoint_meta +from invokeai.backend.model_management.models.base import read_checkpoint_meta, ModelType parser = argparse.ArgumentParser(description="Create a .json template from checkpoint/safetensors model") -parser.add_argument("--checkpoint", "--in", type=Path, help="Path to the input checkpoint/safetensors file") +parser.add_argument(type=Path, help="Path to the input checkpoint/safetensors file") parser.add_argument("--template", "--out", type=Path, help="Path to the output .json file") +parser.add_argument("--base-type", + type=str, + choices=['sd-1','sd-2','sdxl'], + help="Base type of the model", + ) +parser.add_argument("--model-type", + type=str, + choices=[x.value for x in ModelType], + help="Base type of the model", + ) opt = parser.parse_args() ckpt = read_checkpoint_meta(opt.checkpoint) @@ -26,9 +36,15 @@ tmpl = {} for key, tensor in ckpt.items(): tmpl[key] = list(tensor.shape) +meta = { + 'base_type': opt.base_type, + 'model_type': opt.model_type, + 'template': tmpl +} + try: with open(opt.template, "w") as f: - json.dump(tmpl, f) + json.dump(meta, f) print(f"Template written out as {opt.template}") except Exception as e: print(f"An exception occurred while writing template: {str(e)}")