Model Manager Refactor Phase 1 - SQL-based config storage (#5039)

## What type of PR is this? (check all applicable)

- [X] Refactor


## Have you discussed this change with the InvokeAI team?
- [X] Extensively
- [ ] No, because:

      
## Have you updated all relevant documentation?
- [X] Yes
- [ ] No


## Description

As discussed with @psychedelicious and @RyanJDick, this is the first
phase of the model manager refactor. In this phase, I've added support
for storing model configuration information the `invokeai.db` SQL3
database. All the code is separate from the original model manager, so
for the time being the frontend is still using the original YAML-based
configuration, so the web app still works.

To keep things clean, I've added a new FastAPI route called
`model_records` which can add, update, retrieve and delete model
records.

The architecture is described in the first section of
`docs/contributing/MODEL_MANAGER.md`.

## QA Instructions, Screenshots, Recordings

There is a pytest for the model sql storage backend in
`tests/backend/model_manager_2/test_model_storage_sql.py`.

To populate `invokeai.db` with models from your current `models.yaml`,
do the following:

1. Stop the running server
2. Back up `invokeai.db`
3. Run `pip install -e .` to install the command used in the next step.
4. Run `invokeai-migrate-models-to-db`

This will iterate through `models.yaml` and create equivalent database
entries in the `model_config` table of `invokeai.db`. Only the models
named in the yaml file will be migrated, so anything that is autoloaded
will be ignored.

Note that in order to get the `model_records` router to be recognized by
the swagger API, I had to rebuild the frontend. Not sure why this was
necessary and would appreciate a pointer on a less radical way to do
this.

## Added/updated tests?

- [X] Yes
- [ ] No

invokeai/app/api/dependencies.py

@@ -24,6 +24,7 @@ from ..services.item_storage.item_storage_sqlite import SqliteItemStorage
 from ..services.latents_storage.latents_storage_disk import DiskLatentsStorage
 from ..services.latents_storage.latents_storage_forward_cache import ForwardCacheLatentsStorage
 from ..services.model_manager.model_manager_default import ModelManagerService
+from ..services.model_records import ModelRecordServiceSQL
 from ..services.names.names_default import SimpleNameService
 from ..services.session_processor.session_processor_default import DefaultSessionProcessor
 from ..services.session_queue.session_queue_sqlite import SqliteSessionQueue
@@ -85,6 +86,7 @@ class ApiDependencies:
         invocation_cache = MemoryInvocationCache(max_cache_size=config.node_cache_size)
         latents = ForwardCacheLatentsStorage(DiskLatentsStorage(f"{output_folder}/latents"))
         model_manager = ModelManagerService(config, logger)
+        model_record_service = ModelRecordServiceSQL(db=db)
         names = SimpleNameService()
         performance_statistics = InvocationStatsService()
         processor = DefaultInvocationProcessor()
@@ -111,6 +113,7 @@ class ApiDependencies:
             latents=latents,
             logger=logger,
             model_manager=model_manager,
+            model_records=model_record_service,
             names=names,
             performance_statistics=performance_statistics,
             processor=processor,

invokeai/app/api/routers/model_records.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2023 Lincoln D. Stein
+"""FastAPI route for model configuration records."""
+
+
+from hashlib import sha1
+from random import randbytes
+from typing import List, Optional
+
+from fastapi import Body, Path, Query, Response
+from fastapi.routing import APIRouter
+from pydantic import BaseModel, ConfigDict
+from starlette.exceptions import HTTPException
+from typing_extensions import Annotated
+
+from invokeai.app.services.model_records import (
+    DuplicateModelException,
+    InvalidModelException,
+    UnknownModelException,
+)
+from invokeai.backend.model_manager.config import (
+    AnyModelConfig,
+    BaseModelType,
+    ModelType,
+)
+
+from ..dependencies import ApiDependencies
+
+model_records_router = APIRouter(prefix="/v1/model/record", tags=["models"])
+
+
+class ModelsList(BaseModel):
+    """Return list of configs."""
+
+    models: list[AnyModelConfig]
+
+    model_config = ConfigDict(use_enum_values=True)
+
+
+@model_records_router.get(
+    "/",
+    operation_id="list_model_records",
+)
+async def list_model_records(
+    base_models: Optional[List[BaseModelType]] = Query(default=None, description="Base models to include"),
+    model_type: Optional[ModelType] = Query(default=None, description="The type of model to get"),
+) -> ModelsList:
+    """Get a list of models."""
+    record_store = ApiDependencies.invoker.services.model_records
+    found_models: list[AnyModelConfig] = []
+    if base_models:
+        for base_model in base_models:
+            found_models.extend(record_store.search_by_attr(base_model=base_model, model_type=model_type))
+    else:
+        found_models.extend(record_store.search_by_attr(model_type=model_type))
+    return ModelsList(models=found_models)
+
+
+@model_records_router.get(
+    "/i/{key}",
+    operation_id="get_model_record",
+    responses={
+        200: {"description": "Success"},
+        400: {"description": "Bad request"},
+        404: {"description": "The model could not be found"},
+    },
+)
+async def get_model_record(
+    key: str = Path(description="Key of the model record to fetch."),
+) -> AnyModelConfig:
+    """Get a model record"""
+    record_store = ApiDependencies.invoker.services.model_records
+    try:
+        return record_store.get_model(key)
+    except UnknownModelException as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+
+@model_records_router.patch(
+    "/i/{key}",
+    operation_id="update_model_record",
+    responses={
+        200: {"description": "The model was updated successfully"},
+        400: {"description": "Bad request"},
+        404: {"description": "The model could not be found"},
+        409: {"description": "There is already a model corresponding to the new name"},
+    },
+    status_code=200,
+    response_model=AnyModelConfig,
+)
+async def update_model_record(
+    key: Annotated[str, Path(description="Unique key of model")],
+    info: Annotated[AnyModelConfig, Body(description="Model config", discriminator="type")],
+) -> AnyModelConfig:
+    """Update model contents with a new config. If the model name or base fields are changed, then the model is renamed."""
+    logger = ApiDependencies.invoker.services.logger
+    record_store = ApiDependencies.invoker.services.model_records
+    try:
+        model_response = record_store.update_model(key, config=info)
+        logger.info(f"Updated model: {key}")
+    except UnknownModelException as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except ValueError as e:
+        logger.error(str(e))
+        raise HTTPException(status_code=409, detail=str(e))
+    return model_response
+
+
+@model_records_router.delete(
+    "/i/{key}",
+    operation_id="del_model_record",
+    responses={
+        204: {"description": "Model deleted successfully"},
+        404: {"description": "Model not found"},
+    },
+    status_code=204,
+)
+async def del_model_record(
+    key: str = Path(description="Unique key of model to remove from model registry."),
+) -> Response:
+    """Delete Model"""
+    logger = ApiDependencies.invoker.services.logger
+
+    try:
+        record_store = ApiDependencies.invoker.services.model_records
+        record_store.del_model(key)
+        logger.info(f"Deleted model: {key}")
+        return Response(status_code=204)
+    except UnknownModelException as e:
+        logger.error(str(e))
+        raise HTTPException(status_code=404, detail=str(e))
+
+
+@model_records_router.post(
+    "/i/",
+    operation_id="add_model_record",
+    responses={
+        201: {"description": "The model added successfully"},
+        409: {"description": "There is already a model corresponding to this path or repo_id"},
+        415: {"description": "Unrecognized file/folder format"},
+    },
+    status_code=201,
+)
+async def add_model_record(
+    config: Annotated[AnyModelConfig, Body(description="Model config", discriminator="type")]
+) -> AnyModelConfig:
+    """
+    Add a model using the configuration information appropriate for its type.
+    """
+    logger = ApiDependencies.invoker.services.logger
+    record_store = ApiDependencies.invoker.services.model_records
+    if config.key == "<NOKEY>":
+        config.key = sha1(randbytes(100)).hexdigest()
+        logger.info(f"Created model {config.key} for {config.name}")
+    try:
+        record_store.add_model(config.key, config)
+    except DuplicateModelException as e:
+        logger.error(str(e))
+        raise HTTPException(status_code=409, detail=str(e))
+    except InvalidModelException as e:
+        logger.error(str(e))
+        raise HTTPException(status_code=415)
+
+    # now fetch it out
+    return record_store.get_model(config.key)

invokeai/app/api/routers/models.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654), 2023 Kent Keirsey (https://github.com/hipsterusername), 2023 Lincoln D. Stein
 
-
 import pathlib
 from typing import Annotated, List, Literal, Optional, Union
 

invokeai/app/api_app.py

@@ -43,6 +43,7 @@ if True:  # hack to make flake8 happy with imports coming after setting up the c
         board_images,
         boards,
         images,
+        model_records,
         models,
         session_queue,
         sessions,
@@ -106,6 +107,7 @@ app.include_router(sessions.session_router, prefix="/api")
 
 app.include_router(utilities.utilities_router, prefix="/api")
 app.include_router(models.models_router, prefix="/api")
+app.include_router(model_records.model_records_router, prefix="/api")
 app.include_router(images.images_router, prefix="/api")
 app.include_router(boards.boards_router, prefix="/api")
 app.include_router(board_images.board_images_router, prefix="/api")

invokeai/app/services/invocation_services.py

@@ -22,6 +22,7 @@ if TYPE_CHECKING:
     from .item_storage.item_storage_base import ItemStorageABC
     from .latents_storage.latents_storage_base import LatentsStorageBase
     from .model_manager.model_manager_base import ModelManagerServiceBase
+    from .model_records import ModelRecordServiceBase
     from .names.names_base import NameServiceBase
     from .session_processor.session_processor_base import SessionProcessorBase
     from .session_queue.session_queue_base import SessionQueueBase
@@ -49,6 +50,7 @@ class InvocationServices:
     latents: "LatentsStorageBase"
     logger: "Logger"
     model_manager: "ModelManagerServiceBase"
+    model_records: "ModelRecordServiceBase"
     processor: "InvocationProcessorABC"
     performance_statistics: "InvocationStatsServiceBase"
     queue: "InvocationQueueABC"
@@ -76,6 +78,7 @@ class InvocationServices:
         latents: "LatentsStorageBase",
         logger: "Logger",
         model_manager: "ModelManagerServiceBase",
+        model_records: "ModelRecordServiceBase",
         processor: "InvocationProcessorABC",
         performance_statistics: "InvocationStatsServiceBase",
         queue: "InvocationQueueABC",
@@ -101,6 +104,7 @@ class InvocationServices:
         self.latents = latents
         self.logger = logger
         self.model_manager = model_manager
+        self.model_records = model_records
         self.processor = processor
         self.performance_statistics = performance_statistics
         self.queue = queue

invokeai/app/services/model_records/__init__.py

@@ -0,0 +1,8 @@
+"""Init file for model record services."""
+from .model_records_base import (  # noqa F401
+    DuplicateModelException,
+    InvalidModelException,
+    ModelRecordServiceBase,
+    UnknownModelException,
+)
+from .model_records_sql import ModelRecordServiceSQL  # noqa F401

invokeai/app/services/model_records/model_records_base.py

@@ -0,0 +1,169 @@
+# Copyright (c) 2023 Lincoln D. Stein and the InvokeAI Development Team
+"""
+Abstract base class for storing and retrieving model configuration records.
+"""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Optional, Union
+
+from invokeai.backend.model_manager.config import AnyModelConfig, BaseModelType, ModelType
+
+# should match the InvokeAI version when this is first released.
+CONFIG_FILE_VERSION = "3.2.0"
+
+
+class DuplicateModelException(Exception):
+    """Raised on an attempt to add a model with the same key twice."""
+
+
+class InvalidModelException(Exception):
+    """Raised when an invalid model is detected."""
+
+
+class UnknownModelException(Exception):
+    """Raised on an attempt to fetch or delete a model with a nonexistent key."""
+
+
+class ConfigFileVersionMismatchException(Exception):
+    """Raised on an attempt to open a config with an incompatible version."""
+
+
+class ModelRecordServiceBase(ABC):
+    """Abstract base class for storage and retrieval of model configs."""
+
+    @property
+    @abstractmethod
+    def version(self) -> str:
+        """Return the config file/database schema version."""
+        pass
+
+    @abstractmethod
+    def add_model(self, key: str, config: Union[dict, AnyModelConfig]) -> AnyModelConfig:
+        """
+        Add a model to the database.
+
+        :param key: Unique key for the model
+        :param config: Model configuration record, either a dict with the
+         required fields or a ModelConfigBase instance.
+
+        Can raise DuplicateModelException and InvalidModelConfigException exceptions.
+        """
+        pass
+
+    @abstractmethod
+    def del_model(self, key: str) -> None:
+        """
+        Delete a model.
+
+        :param key: Unique key for the model to be deleted
+
+        Can raise an UnknownModelException
+        """
+        pass
+
+    @abstractmethod
+    def update_model(self, key: str, config: Union[dict, AnyModelConfig]) -> AnyModelConfig:
+        """
+        Update the model, returning the updated version.
+
+        :param key: Unique key for the model to be updated
+        :param config: Model configuration record. Either a dict with the
+         required fields, or a ModelConfigBase instance.
+        """
+        pass
+
+    @abstractmethod
+    def get_model(self, key: str) -> AnyModelConfig:
+        """
+        Retrieve the configuration for the indicated model.
+
+        :param key: Key of model config to be fetched.
+
+        Exceptions: UnknownModelException
+        """
+        pass
+
+    @abstractmethod
+    def exists(self, key: str) -> bool:
+        """
+        Return True if a model with the indicated key exists in the databse.
+
+        :param key: Unique key for the model to be deleted
+        """
+        pass
+
+    @abstractmethod
+    def search_by_path(
+        self,
+        path: Union[str, Path],
+    ) -> List[AnyModelConfig]:
+        """Return the model(s) having the indicated path."""
+        pass
+
+    @abstractmethod
+    def search_by_hash(
+        self,
+        hash: str,
+    ) -> List[AnyModelConfig]:
+        """Return the model(s) having the indicated original hash."""
+        pass
+
+    @abstractmethod
+    def search_by_attr(
+        self,
+        model_name: Optional[str] = None,
+        base_model: Optional[BaseModelType] = None,
+        model_type: Optional[ModelType] = None,
+    ) -> List[AnyModelConfig]:
+        """
+        Return models matching name, base and/or type.
+
+        :param model_name: Filter by name of model (optional)
+        :param base_model: Filter by base model (optional)
+        :param model_type: Filter by type of model (optional)
+
+        If none of the optional filters are passed, will return all
+        models in the database.
+        """
+        pass
+
+    def all_models(self) -> List[AnyModelConfig]:
+        """Return all the model configs in the database."""
+        return self.search_by_attr()
+
+    def model_info_by_name(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> AnyModelConfig:
+        """
+        Return information about a single model using its name, base type and model type.
+
+        If there are more than one model that match, raises a DuplicateModelException.
+        If no model matches, raises an UnknownModelException
+        """
+        model_configs = self.search_by_attr(model_name=model_name, base_model=base_model, model_type=model_type)
+        if len(model_configs) > 1:
+            raise DuplicateModelException(
+                f"More than one model matched the search criteria: base_model='{base_model}', model_type='{model_type}', model_name='{model_name}'."
+            )
+        if len(model_configs) == 0:
+            raise UnknownModelException(
+                f"More than one model matched the search criteria: base_model='{base_model}', model_type='{model_type}', model_name='{model_name}'."
+            )
+        return model_configs[0]
+
+    def rename_model(
+        self,
+        key: str,
+        new_name: str,
+    ) -> AnyModelConfig:
+        """
+        Rename the indicated model. Just a special case of update_model().
+
+        In some implementations, renaming the model may involve changing where
+        it is stored on the filesystem. So this is broken out.
+
+        :param key: Model key
+        :param new_name: New name for model
+        """
+        config = self.get_model(key)
+        config.name = new_name
+        return self.update_model(key, config)

invokeai/app/services/model_records/model_records_sql.py

@@ -0,0 +1,397 @@
+# Copyright (c) 2023 Lincoln D. Stein and the InvokeAI Development Team
+"""
+SQL Implementation of the ModelRecordServiceBase API
+
+Typical usage:
+
+  from invokeai.backend.model_manager import ModelConfigStoreSQL
+  store = ModelConfigStoreSQL(sqlite_db)
+  config = dict(
+        path='/tmp/pokemon.bin',
+        name='old name',
+        base_model='sd-1',
+        type='embedding',
+        format='embedding_file',
+     )
+
+   # adding - the key becomes the model's "key" field
+   store.add_model('key1', config)
+
+   # updating
+   config.name='new name'
+   store.update_model('key1', config)
+
+   # checking for existence
+   if store.exists('key1'):
+      print("yes")
+
+   # fetching config
+   new_config = store.get_model('key1')
+   print(new_config.name, new_config.base)
+   assert new_config.key == 'key1'
+
+  # deleting
+  store.del_model('key1')
+
+  # searching
+  configs = store.search_by_path(path='/tmp/pokemon.bin')
+  configs = store.search_by_hash('750a499f35e43b7e1b4d15c207aa2f01')
+  configs = store.search_by_attr(base_model='sd-2', model_type='main')
+"""
+
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import List, Optional, Union
+
+from invokeai.backend.model_manager.config import (
+    AnyModelConfig,
+    BaseModelType,
+    ModelConfigBase,
+    ModelConfigFactory,
+    ModelType,
+)
+
+from ..shared.sqlite import SqliteDatabase
+from .model_records_base import (
+    CONFIG_FILE_VERSION,
+    DuplicateModelException,
+    ModelRecordServiceBase,
+    UnknownModelException,
+)
+
+
+class ModelRecordServiceSQL(ModelRecordServiceBase):
+    """Implementation of the ModelConfigStore ABC using a SQL database."""
+
+    _db: SqliteDatabase
+    _cursor: sqlite3.Cursor
+
+    def __init__(self, db: SqliteDatabase):
+        """
+        Initialize a new object from preexisting sqlite3 connection and threading lock objects.
+
+        :param conn: sqlite3 connection object
+        :param lock: threading Lock object
+        """
+        super().__init__()
+        self._db = db
+        self._cursor = self._db.conn.cursor()
+
+        with self._db.lock:
+            # Enable foreign keys
+            self._db.conn.execute("PRAGMA foreign_keys = ON;")
+            self._create_tables()
+            self._db.conn.commit()
+        assert (
+            str(self.version) == CONFIG_FILE_VERSION
+        ), f"Model config version {self.version} does not match expected version {CONFIG_FILE_VERSION}"
+
+    def _create_tables(self) -> None:
+        """Create sqlite3 tables."""
+        #  model_config table breaks out the fields that are common to all config objects
+        # and puts class-specific ones in a serialized json object
+        self._cursor.execute(
+            """--sql
+            CREATE TABLE IF NOT EXISTS model_config (
+                id TEXT NOT NULL PRIMARY KEY,
+                -- The next 3 fields are enums in python, unrestricted string here
+                base TEXT NOT NULL,
+                type TEXT NOT NULL,
+                name TEXT NOT NULL,
+                path TEXT NOT NULL,
+                original_hash TEXT, -- could be null
+                -- Serialized JSON representation of the whole config object,
+                -- which will contain additional fields from subclasses
+                config TEXT NOT NULL,
+                created_at DATETIME NOT NULL DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')),
+                -- Updated via trigger
+                updated_at DATETIME NOT NULL DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')),
+                -- unique constraint on combo of name, base and type
+                UNIQUE(name, base, type)
+            );
+            """
+        )
+
+        #  metadata table
+        self._cursor.execute(
+            """--sql
+            CREATE TABLE IF NOT EXISTS model_manager_metadata (
+                metadata_key TEXT NOT NULL PRIMARY KEY,
+                metadata_value TEXT NOT NULL
+            );
+            """
+        )
+
+        # Add trigger for `updated_at`.
+        self._cursor.execute(
+            """--sql
+            CREATE TRIGGER IF NOT EXISTS model_config_updated_at
+            AFTER UPDATE
+            ON model_config FOR EACH ROW
+            BEGIN
+                UPDATE model_config SET updated_at = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')
+                    WHERE id = old.id;
+            END;
+            """
+        )
+
+        # Add indexes for searchable fields
+        for stmt in [
+            "CREATE INDEX IF NOT EXISTS base_index ON model_config(base);",
+            "CREATE INDEX IF NOT EXISTS type_index ON model_config(type);",
+            "CREATE INDEX IF NOT EXISTS name_index ON model_config(name);",
+            "CREATE UNIQUE INDEX IF NOT EXISTS path_index ON model_config(path);",
+        ]:
+            self._cursor.execute(stmt)
+
+        # Add our version to the metadata table
+        self._cursor.execute(
+            """--sql
+            INSERT OR IGNORE into model_manager_metadata (
+               metadata_key,
+               metadata_value
+            )
+            VALUES (?,?);
+            """,
+            ("version", CONFIG_FILE_VERSION),
+        )
+
+    def add_model(self, key: str, config: Union[dict, ModelConfigBase]) -> AnyModelConfig:
+        """
+        Add a model to the database.
+
+        :param key: Unique key for the model
+        :param config: Model configuration record, either a dict with the
+         required fields or a ModelConfigBase instance.
+
+        Can raise DuplicateModelException and InvalidModelConfigException exceptions.
+        """
+        record = ModelConfigFactory.make_config(config, key=key)  # ensure it is a valid config obect.
+        json_serialized = record.model_dump_json()  # and turn it into a json string.
+        with self._db.lock:
+            try:
+                self._cursor.execute(
+                    """--sql
+                    INSERT INTO model_config (
+                       id,
+                       base,
+                       type,
+                       name,
+                       path,
+                       original_hash,
+                       config
+                      )
+                    VALUES (?,?,?,?,?,?,?);
+                    """,
+                    (
+                        key,
+                        record.base,
+                        record.type,
+                        record.name,
+                        record.path,
+                        record.original_hash,
+                        json_serialized,
+                    ),
+                )
+                self._db.conn.commit()
+
+            except sqlite3.IntegrityError as e:
+                self._db.conn.rollback()
+                if "UNIQUE constraint failed" in str(e):
+                    if "model_config.path" in str(e):
+                        msg = f"A model with path '{record.path}' is already installed"
+                    elif "model_config.name" in str(e):
+                        msg = f"A model with name='{record.name}', type='{record.type}', base='{record.base}' is already installed"
+                    else:
+                        msg = f"A model with key '{key}' is already installed"
+                    raise DuplicateModelException(msg) from e
+                else:
+                    raise e
+            except sqlite3.Error as e:
+                self._db.conn.rollback()
+                raise e
+
+        return self.get_model(key)
+
+    @property
+    def version(self) -> str:
+        """Return the version of the database schema."""
+        with self._db.lock:
+            self._cursor.execute(
+                """--sql
+                SELECT metadata_value FROM model_manager_metadata
+                WHERE metadata_key=?;
+                """,
+                ("version",),
+            )
+            rows = self._cursor.fetchone()
+            if not rows:
+                raise KeyError("Models database does not have metadata key 'version'")
+            return rows[0]
+
+    def del_model(self, key: str) -> None:
+        """
+        Delete a model.
+
+        :param key: Unique key for the model to be deleted
+
+        Can raise an UnknownModelException
+        """
+        with self._db.lock:
+            try:
+                self._cursor.execute(
+                    """--sql
+                    DELETE FROM model_config
+                    WHERE id=?;
+                    """,
+                    (key,),
+                )
+                if self._cursor.rowcount == 0:
+                    raise UnknownModelException("model not found")
+                self._db.conn.commit()
+            except sqlite3.Error as e:
+                self._db.conn.rollback()
+                raise e
+
+    def update_model(self, key: str, config: ModelConfigBase) -> AnyModelConfig:
+        """
+        Update the model, returning the updated version.
+
+        :param key: Unique key for the model to be updated
+        :param config: Model configuration record. Either a dict with the
+         required fields, or a ModelConfigBase instance.
+        """
+        record = ModelConfigFactory.make_config(config, key=key)  # ensure it is a valid config obect
+        json_serialized = record.model_dump_json()  # and turn it into a json string.
+        with self._db.lock:
+            try:
+                self._cursor.execute(
+                    """--sql
+                    UPDATE model_config
+                    SET base=?,
+                        type=?,
+                        name=?,
+                        path=?,
+                        config=?
+                    WHERE id=?;
+                    """,
+                    (record.base, record.type, record.name, record.path, json_serialized, key),
+                )
+                if self._cursor.rowcount == 0:
+                    raise UnknownModelException("model not found")
+                self._db.conn.commit()
+            except sqlite3.Error as e:
+                self._db.conn.rollback()
+                raise e
+
+        return self.get_model(key)
+
+    def get_model(self, key: str) -> AnyModelConfig:
+        """
+        Retrieve the ModelConfigBase instance for the indicated model.
+
+        :param key: Key of model config to be fetched.
+
+        Exceptions: UnknownModelException
+        """
+        with self._db.lock:
+            self._cursor.execute(
+                """--sql
+                SELECT config FROM model_config
+                WHERE id=?;
+                """,
+                (key,),
+            )
+            rows = self._cursor.fetchone()
+            if not rows:
+                raise UnknownModelException("model not found")
+            model = ModelConfigFactory.make_config(json.loads(rows[0]))
+        return model
+
+    def exists(self, key: str) -> bool:
+        """
+        Return True if a model with the indicated key exists in the databse.
+
+        :param key: Unique key for the model to be deleted
+        """
+        count = 0
+        with self._db.lock:
+            self._cursor.execute(
+                """--sql
+                select count(*) FROM model_config
+                WHERE id=?;
+                """,
+                (key,),
+            )
+            count = self._cursor.fetchone()[0]
+        return count > 0
+
+    def search_by_attr(
+        self,
+        model_name: Optional[str] = None,
+        base_model: Optional[BaseModelType] = None,
+        model_type: Optional[ModelType] = None,
+    ) -> List[AnyModelConfig]:
+        """
+        Return models matching name, base and/or type.
+
+        :param model_name: Filter by name of model (optional)
+        :param base_model: Filter by base model (optional)
+        :param model_type: Filter by type of model (optional)
+
+        If none of the optional filters are passed, will return all
+        models in the database.
+        """
+        results = []
+        where_clause = []
+        bindings = []
+        if model_name:
+            where_clause.append("name=?")
+            bindings.append(model_name)
+        if base_model:
+            where_clause.append("base=?")
+            bindings.append(base_model)
+        if model_type:
+            where_clause.append("type=?")
+            bindings.append(model_type)
+        where = f"WHERE {' AND '.join(where_clause)}" if where_clause else ""
+        with self._db.lock:
+            self._cursor.execute(
+                f"""--sql
+                select config FROM model_config
+                {where};
+                """,
+                tuple(bindings),
+            )
+            results = [ModelConfigFactory.make_config(json.loads(x[0])) for x in self._cursor.fetchall()]
+        return results
+
+    def search_by_path(self, path: Union[str, Path]) -> List[ModelConfigBase]:
+        """Return models with the indicated path."""
+        results = []
+        with self._db.lock:
+            self._cursor.execute(
+                """--sql
+                SELECT config FROM model_config
+                WHERE model_path=?;
+                """,
+                (str(path),),
+            )
+            results = [ModelConfigFactory.make_config(json.loads(x[0])) for x in self._cursor.fetchall()]
+        return results
+
+    def search_by_hash(self, hash: str) -> List[ModelConfigBase]:
+        """Return models with the indicated original_hash."""
+        results = []
+        with self._db.lock:
+            self._cursor.execute(
+                """--sql
+                SELECT config FROM model_config
+                WHERE original_hash=?;
+                """,
+                (hash,),
+            )
+            results = [ModelConfigFactory.make_config(json.loads(x[0])) for x in self._cursor.fetchall()]
+        return results

invokeai/backend/model_manager/config.py

@@ -0,0 +1,323 @@
+# Copyright (c) 2023 Lincoln D. Stein and the InvokeAI Development Team
+"""
+Configuration definitions for image generation models.
+
+Typical usage:
+
+  from invokeai.backend.model_manager import ModelConfigFactory
+  raw = dict(path='models/sd-1/main/foo.ckpt',
+             name='foo',
+             base='sd-1',
+             type='main',
+             config='configs/stable-diffusion/v1-inference.yaml',
+             variant='normal',
+             format='checkpoint'
+            )
+  config = ModelConfigFactory.make_config(raw)
+  print(config.name)
+
+Validation errors will raise an InvalidModelConfigException error.
+
+"""
+from enum import Enum
+from typing import Literal, Optional, Type, Union
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
+from typing_extensions import Annotated
+
+
+class InvalidModelConfigException(Exception):
+    """Exception for when config parser doesn't recognized this combination of model type and format."""
+
+
+class BaseModelType(str, Enum):
+    """Base model type."""
+
+    Any = "any"
+    StableDiffusion1 = "sd-1"
+    StableDiffusion2 = "sd-2"
+    StableDiffusionXL = "sdxl"
+    StableDiffusionXLRefiner = "sdxl-refiner"
+    # Kandinsky2_1 = "kandinsky-2.1"
+
+
+class ModelType(str, Enum):
+    """Model type."""
+
+    ONNX = "onnx"
+    Main = "main"
+    Vae = "vae"
+    Lora = "lora"
+    ControlNet = "controlnet"  # used by model_probe
+    TextualInversion = "embedding"
+    IPAdapter = "ip_adapter"
+    CLIPVision = "clip_vision"
+    T2IAdapter = "t2i_adapter"
+
+
+class SubModelType(str, Enum):
+    """Submodel type."""
+
+    UNet = "unet"
+    TextEncoder = "text_encoder"
+    TextEncoder2 = "text_encoder_2"
+    Tokenizer = "tokenizer"
+    Tokenizer2 = "tokenizer_2"
+    Vae = "vae"
+    VaeDecoder = "vae_decoder"
+    VaeEncoder = "vae_encoder"
+    Scheduler = "scheduler"
+    SafetyChecker = "safety_checker"
+
+
+class ModelVariantType(str, Enum):
+    """Variant type."""
+
+    Normal = "normal"
+    Inpaint = "inpaint"
+    Depth = "depth"
+
+
+class ModelFormat(str, Enum):
+    """Storage format of model."""
+
+    Diffusers = "diffusers"
+    Checkpoint = "checkpoint"
+    Lycoris = "lycoris"
+    Onnx = "onnx"
+    Olive = "olive"
+    EmbeddingFile = "embedding_file"
+    EmbeddingFolder = "embedding_folder"
+    InvokeAI = "invokeai"
+
+
+class SchedulerPredictionType(str, Enum):
+    """Scheduler prediction type."""
+
+    Epsilon = "epsilon"
+    VPrediction = "v_prediction"
+    Sample = "sample"
+
+
+class ModelConfigBase(BaseModel):
+    """Base class for model configuration information."""
+
+    path: str
+    name: str
+    base: BaseModelType
+    type: ModelType
+    format: ModelFormat
+    key: str = Field(description="unique key for model", default="<NOKEY>")
+    original_hash: Optional[str] = Field(
+        description="original fasthash of model contents", default=None
+    )  # this is assigned at install time and will not change
+    current_hash: Optional[str] = Field(
+        description="current fasthash of model contents", default=None
+    )  # if model is converted or otherwise modified, this will hold updated hash
+    description: Optional[str] = Field(default=None)
+    source: Optional[str] = Field(description="Model download source (URL or repo_id)", default=None)
+
+    model_config = ConfigDict(
+        use_enum_values=False,
+        validate_assignment=True,
+    )
+
+    def update(self, attributes: dict):
+        """Update the object with fields in dict."""
+        for key, value in attributes.items():
+            setattr(self, key, value)  # may raise a validation error
+
+
+class _CheckpointConfig(ModelConfigBase):
+    """Model config for checkpoint-style models."""
+
+    format: Literal[ModelFormat.Checkpoint] = ModelFormat.Checkpoint
+    config: str = Field(description="path to the checkpoint model config file")
+
+
+class _DiffusersConfig(ModelConfigBase):
+    """Model config for diffusers-style models."""
+
+    format: Literal[ModelFormat.Diffusers] = ModelFormat.Diffusers
+
+
+class LoRAConfig(ModelConfigBase):
+    """Model config for LoRA/Lycoris models."""
+
+    type: Literal[ModelType.Lora] = ModelType.Lora
+    format: Literal[ModelFormat.Lycoris, ModelFormat.Diffusers]
+
+
+class VaeCheckpointConfig(ModelConfigBase):
+    """Model config for standalone VAE models."""
+
+    type: Literal[ModelType.Vae] = ModelType.Vae
+    format: Literal[ModelFormat.Checkpoint] = ModelFormat.Checkpoint
+
+
+class VaeDiffusersConfig(ModelConfigBase):
+    """Model config for standalone VAE models (diffusers version)."""
+
+    type: Literal[ModelType.Vae] = ModelType.Vae
+    format: Literal[ModelFormat.Diffusers] = ModelFormat.Diffusers
+
+
+class ControlNetDiffusersConfig(_DiffusersConfig):
+    """Model config for ControlNet models (diffusers version)."""
+
+    type: Literal[ModelType.ControlNet] = ModelType.ControlNet
+    format: Literal[ModelFormat.Diffusers] = ModelFormat.Diffusers
+
+
+class ControlNetCheckpointConfig(_CheckpointConfig):
+    """Model config for ControlNet models (diffusers version)."""
+
+    type: Literal[ModelType.ControlNet] = ModelType.ControlNet
+    format: Literal[ModelFormat.Checkpoint] = ModelFormat.Checkpoint
+
+
+class TextualInversionConfig(ModelConfigBase):
+    """Model config for textual inversion embeddings."""
+
+    type: Literal[ModelType.TextualInversion] = ModelType.TextualInversion
+    format: Literal[ModelFormat.EmbeddingFile, ModelFormat.EmbeddingFolder]
+
+
+class _MainConfig(ModelConfigBase):
+    """Model config for main models."""
+
+    vae: Optional[str] = Field(default=None)
+    variant: ModelVariantType = ModelVariantType.Normal
+    ztsnr_training: bool = False
+
+
+class MainCheckpointConfig(_CheckpointConfig, _MainConfig):
+    """Model config for main checkpoint models."""
+
+    type: Literal[ModelType.Main] = ModelType.Main
+    # Note that we do not need prediction_type or upcast_attention here
+    # because they are provided in the checkpoint's own config file.
+
+
+class MainDiffusersConfig(_DiffusersConfig, _MainConfig):
+    """Model config for main diffusers models."""
+
+    type: Literal[ModelType.Main] = ModelType.Main
+    prediction_type: SchedulerPredictionType = SchedulerPredictionType.Epsilon
+    upcast_attention: bool = False
+
+
+class ONNXSD1Config(_MainConfig):
+    """Model config for ONNX format models based on sd-1."""
+
+    type: Literal[ModelType.ONNX] = ModelType.ONNX
+    format: Literal[ModelFormat.Onnx, ModelFormat.Olive]
+    base: Literal[BaseModelType.StableDiffusion1] = BaseModelType.StableDiffusion1
+    prediction_type: SchedulerPredictionType = SchedulerPredictionType.Epsilon
+    upcast_attention: bool = False
+
+
+class ONNXSD2Config(_MainConfig):
+    """Model config for ONNX format models based on sd-2."""
+
+    type: Literal[ModelType.ONNX] = ModelType.ONNX
+    format: Literal[ModelFormat.Onnx, ModelFormat.Olive]
+    # No yaml config file for ONNX, so these are part of config
+    base: Literal[BaseModelType.StableDiffusion2] = BaseModelType.StableDiffusion2
+    prediction_type: SchedulerPredictionType = SchedulerPredictionType.VPrediction
+    upcast_attention: bool = True
+
+
+class IPAdapterConfig(ModelConfigBase):
+    """Model config for IP Adaptor format models."""
+
+    type: Literal[ModelType.IPAdapter] = ModelType.IPAdapter
+    format: Literal[ModelFormat.InvokeAI]
+
+
+class CLIPVisionDiffusersConfig(ModelConfigBase):
+    """Model config for ClipVision."""
+
+    type: Literal[ModelType.CLIPVision] = ModelType.CLIPVision
+    format: Literal[ModelFormat.Diffusers]
+
+
+class T2IConfig(ModelConfigBase):
+    """Model config for T2I."""
+
+    type: Literal[ModelType.T2IAdapter] = ModelType.T2IAdapter
+    format: Literal[ModelFormat.Diffusers]
+
+
+_ONNXConfig = Annotated[Union[ONNXSD1Config, ONNXSD2Config], Field(discriminator="base")]
+_ControlNetConfig = Annotated[
+    Union[ControlNetDiffusersConfig, ControlNetCheckpointConfig],
+    Field(discriminator="format"),
+]
+_VaeConfig = Annotated[Union[VaeDiffusersConfig, VaeCheckpointConfig], Field(discriminator="format")]
+_MainModelConfig = Annotated[Union[MainDiffusersConfig, MainCheckpointConfig], Field(discriminator="format")]
+
+AnyModelConfig = Union[
+    _MainModelConfig,
+    _ONNXConfig,
+    _VaeConfig,
+    _ControlNetConfig,
+    LoRAConfig,
+    TextualInversionConfig,
+    IPAdapterConfig,
+    CLIPVisionDiffusersConfig,
+    T2IConfig,
+]
+
+AnyModelConfigValidator = TypeAdapter(AnyModelConfig)
+
+# IMPLEMENTATION NOTE:
+# The preferred alternative to the above is a discriminated Union as shown
+# below. However, it breaks FastAPI when used as the input Body parameter in a route.
+# This is a known issue. Please see:
+#   https://github.com/tiangolo/fastapi/discussions/9761 and
+#   https://github.com/tiangolo/fastapi/discussions/9287
+# AnyModelConfig = Annotated[
+#     Union[
+#         _MainModelConfig,
+#         _ONNXConfig,
+#         _VaeConfig,
+#         _ControlNetConfig,
+#         LoRAConfig,
+#         TextualInversionConfig,
+#         IPAdapterConfig,
+#         CLIPVisionDiffusersConfig,
+#         T2IConfig,
+#     ],
+#     Field(discriminator="type"),
+# ]
+
+
+class ModelConfigFactory(object):
+    """Class for parsing config dicts into StableDiffusion Config obects."""
+
+    @classmethod
+    def make_config(
+        cls,
+        model_data: Union[dict, AnyModelConfig],
+        key: Optional[str] = None,
+        dest_class: Optional[Type] = None,
+    ) -> AnyModelConfig:
+        """
+        Return the appropriate config object from raw dict values.
+
+        :param model_data: A raw dict corresponding the obect fields to be
+        parsed into a ModelConfigBase obect (or descendent), or a ModelConfigBase
+        object, which will be passed through unchanged.
+        :param dest_class: The config class to be returned. If not provided, will
+        be selected automatically.
+        """
+        if isinstance(model_data, ModelConfigBase):
+            model = model_data
+        elif dest_class:
+            model = dest_class.validate_python(model_data)
+        else:
+            model = AnyModelConfigValidator.validate_python(model_data)
+        if key:
+            model.key = key
+        return model

invokeai/backend/model_manager/hash.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2023 Lincoln D. Stein and the InvokeAI Development Team
+"""
+Fast hashing of diffusers and checkpoint-style models.
+
+Usage:
+from invokeai.backend.model_managre.model_hash import FastModelHash
+>>> FastModelHash.hash('/home/models/stable-diffusion-v1.5')
+'a8e693a126ea5b831c96064dc569956f'
+"""
+
+import hashlib
+import os
+from pathlib import Path
+from typing import Dict, Union
+
+from imohash import hashfile
+
+
+class FastModelHash(object):
+    """FastModelHash obect provides one public class method, hash()."""
+
+    @classmethod
+    def hash(cls, model_location: Union[str, Path]) -> str:
+        """
+        Return hexdigest string for model located at model_location.
+
+        :param model_location: Path to the model
+        """
+        model_location = Path(model_location)
+        if model_location.is_file():
+            return cls._hash_file(model_location)
+        elif model_location.is_dir():
+            return cls._hash_dir(model_location)
+        else:
+            raise OSError(f"Not a valid file or directory: {model_location}")
+
+    @classmethod
+    def _hash_file(cls, model_location: Union[str, Path]) -> str:
+        """
+        Fasthash a single file and return its hexdigest.
+
+        :param model_location: Path to the model file
+        """
+        # we return md5 hash of the filehash to make it shorter
+        # cryptographic security not needed here
+        return hashlib.md5(hashfile(model_location)).hexdigest()
+
+    @classmethod
+    def _hash_dir(cls, model_location: Union[str, Path]) -> str:
+        components: Dict[str, str] = {}
+
+        for root, _dirs, files in os.walk(model_location):
+            for file in files:
+                # only tally tensor files because diffusers config files change slightly
+                # depending on how the model was downloaded/converted.
+                if not file.endswith((".ckpt", ".safetensors", ".bin", ".pt", ".pth")):
+                    continue
+                path = (Path(root) / file).as_posix()
+                fast_hash = cls._hash_file(path)
+                components.update({path: fast_hash})
+
+        # hash all the model hashes together, using alphabetic file order
+        md5 = hashlib.md5()
+        for _path, fast_hash in sorted(components.items()):
+            md5.update(fast_hash.encode("utf-8"))
+        return md5.hexdigest()

invokeai/backend/model_manager/migrate_to_db.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2023 Lincoln D. Stein
+"""Migrate from the InvokeAI v2 models.yaml format to the v3 sqlite format."""
+
+from hashlib import sha1
+
+from omegaconf import DictConfig, OmegaConf
+from pydantic import TypeAdapter
+
+from invokeai.app.services.config import InvokeAIAppConfig
+from invokeai.app.services.model_records import (
+    DuplicateModelException,
+    ModelRecordServiceSQL,
+)
+from invokeai.app.services.shared.sqlite import SqliteDatabase
+from invokeai.backend.model_manager.config import (
+    AnyModelConfig,
+    BaseModelType,
+    ModelType,
+)
+from invokeai.backend.model_manager.hash import FastModelHash
+from invokeai.backend.util.logging import InvokeAILogger
+
+ModelsValidator = TypeAdapter(AnyModelConfig)
+
+
+class MigrateModelYamlToDb:
+    """
+    Migrate the InvokeAI models.yaml format (VERSION 3.0.0) to SQL3 database format (VERSION 3.2.0)
+
+    The class has one externally useful method, migrate(), which scans the
+    currently models.yaml file and imports all its entries into invokeai.db.
+
+    Use this way:
+
+      from invokeai.backend.model_manager/migrate_to_db import MigrateModelYamlToDb
+      MigrateModelYamlToDb().migrate()
+
+    """
+
+    config: InvokeAIAppConfig
+    logger: InvokeAILogger
+
+    def __init__(self):
+        self.config = InvokeAIAppConfig.get_config()
+        self.config.parse_args()
+        self.logger = InvokeAILogger.get_logger()
+
+    def get_db(self) -> ModelRecordServiceSQL:
+        """Fetch the sqlite3 database for this installation."""
+        db = SqliteDatabase(self.config, self.logger)
+        return ModelRecordServiceSQL(db)
+
+    def get_yaml(self) -> DictConfig:
+        """Fetch the models.yaml DictConfig for this installation."""
+        yaml_path = self.config.model_conf_path
+        return OmegaConf.load(yaml_path)
+
+    def migrate(self):
+        """Do the migration from models.yaml to invokeai.db."""
+        db = self.get_db()
+        yaml = self.get_yaml()
+
+        for model_key, stanza in yaml.items():
+            if model_key == "__metadata__":
+                assert (
+                    stanza["version"] == "3.0.0"
+                ), f"This script works on version 3.0.0 yaml files, but your configuration points to a {stanza['version']} version"
+                continue
+
+            base_type, model_type, model_name = str(model_key).split("/")
+            hash = FastModelHash.hash(self.config.models_path / stanza.path)
+            new_key = sha1(model_key.encode("utf-8")).hexdigest()
+
+            stanza["base"] = BaseModelType(base_type)
+            stanza["type"] = ModelType(model_type)
+            stanza["name"] = model_name
+            stanza["original_hash"] = hash
+            stanza["current_hash"] = hash
+
+            new_config = ModelsValidator.validate_python(stanza)
+            self.logger.info(f"Adding model {model_name} with key {model_key}")
+            try:
+                db.add_model(new_key, new_config)
+            except DuplicateModelException:
+                self.logger.warning(f"Model {model_name} is already in the database")
+
+
+def main():
+    MigrateModelYamlToDb().migrate()
+
+
+if __name__ == "__main__":
+    main()

invokeai/backend/util/util.py

@@ -5,6 +5,7 @@ import math
 import multiprocessing as mp
 import os
 import re
+import warnings
 from collections import abc
 from inspect import isfunction
 from pathlib import Path
@@ -14,8 +15,10 @@ from threading import Thread
 import numpy as np
 import requests
 import torch
+from diffusers import logging as diffusers_logging
 from PIL import Image, ImageDraw, ImageFont
 from tqdm import tqdm
+from transformers import logging as transformers_logging
 
 import invokeai.backend.util.logging as logger
 
@@ -379,3 +382,21 @@ class Chdir(object):
 
     def __exit__(self, *args):
         os.chdir(self.original)
+
+
+class SilenceWarnings(object):
+    """Context manager to temporarily lower verbosity of diffusers & transformers warning messages."""
+
+    def __enter__(self):
+        """Set verbosity to error."""
+        self.transformers_verbosity = transformers_logging.get_verbosity()
+        self.diffusers_verbosity = diffusers_logging.get_verbosity()
+        transformers_logging.set_verbosity_error()
+        diffusers_logging.set_verbosity_error()
+        warnings.simplefilter("ignore")
+
+    def __exit__(self, type, value, traceback):
+        """Restore logger verbosity to state before context was entered."""
+        transformers_logging.set_verbosity(self.transformers_verbosity)
+        diffusers_logging.set_verbosity(self.diffusers_verbosity)
+        warnings.simplefilter("default")

invokeai/frontend/web/dist/assets/ThemeLocaleProvider-58d6b3b6.js

@@ -0,0 +1,280 @@
+import{w as s,ia as T,v as l,a2 as I,ib as R,ae as V,ic as z,id as j,ie as D,ig as F,ih as G,ii as W,ij as K,aG as H,ik as U,il as Y}from"./index-54a1ea80.js";import{M as Z}from"./MantineProvider-17a58e64.js";var P=String.raw,E=P`
+  :root,
+  :host {
+    --chakra-vh: 100vh;
+  }
+
+  @supports (height: -webkit-fill-available) {
+    :root,
+    :host {
+      --chakra-vh: -webkit-fill-available;
+    }
+  }
+
+  @supports (height: -moz-fill-available) {
+    :root,
+    :host {
+      --chakra-vh: -moz-fill-available;
+    }
+  }
+
+  @supports (height: 100dvh) {
+    :root,
+    :host {
+      --chakra-vh: 100dvh;
+    }
+  }
+`,B=()=>s.jsx(T,{styles:E}),J=({scope:e=""})=>s.jsx(T,{styles:P`
+      html {
+        line-height: 1.5;
+        -webkit-text-size-adjust: 100%;
+        font-family: system-ui, sans-serif;
+        -webkit-font-smoothing: antialiased;
+        text-rendering: optimizeLegibility;
+        -moz-osx-font-smoothing: grayscale;
+        touch-action: manipulation;
+      }
+
+      body {
+        position: relative;
+        min-height: 100%;
+        margin: 0;
+        font-feature-settings: "kern";
+      }
+
+      ${e} :where(*, *::before, *::after) {
+        border-width: 0;
+        border-style: solid;
+        box-sizing: border-box;
+        word-wrap: break-word;
+      }
+
+      main {
+        display: block;
+      }
+
+      ${e} hr {
+        border-top-width: 1px;
+        box-sizing: content-box;
+        height: 0;
+        overflow: visible;
+      }
+
+      ${e} :where(pre, code, kbd,samp) {
+        font-family: SFMono-Regular, Menlo, Monaco, Consolas, monospace;
+        font-size: 1em;
+      }
+
+      ${e} a {
+        background-color: transparent;
+        color: inherit;
+        text-decoration: inherit;
+      }
+
+      ${e} abbr[title] {
+        border-bottom: none;
+        text-decoration: underline;
+        -webkit-text-decoration: underline dotted;
+        text-decoration: underline dotted;
+      }
+
+      ${e} :where(b, strong) {
+        font-weight: bold;
+      }
+
+      ${e} small {
+        font-size: 80%;
+      }
+
+      ${e} :where(sub,sup) {
+        font-size: 75%;
+        line-height: 0;
+        position: relative;
+        vertical-align: baseline;
+      }
+
+      ${e} sub {
+        bottom: -0.25em;
+      }
+
+      ${e} sup {
+        top: -0.5em;
+      }
+
+      ${e} img {
+        border-style: none;
+      }
+
+      ${e} :where(button, input, optgroup, select, textarea) {
+        font-family: inherit;
+        font-size: 100%;
+        line-height: 1.15;
+        margin: 0;
+      }
+
+      ${e} :where(button, input) {
+        overflow: visible;
+      }
+
+      ${e} :where(button, select) {
+        text-transform: none;
+      }
+
+      ${e} :where(
+          button::-moz-focus-inner,
+          [type="button"]::-moz-focus-inner,
+          [type="reset"]::-moz-focus-inner,
+          [type="submit"]::-moz-focus-inner
+        ) {
+        border-style: none;
+        padding: 0;
+      }
+
+      ${e} fieldset {
+        padding: 0.35em 0.75em 0.625em;
+      }
+
+      ${e} legend {
+        box-sizing: border-box;
+        color: inherit;
+        display: table;
+        max-width: 100%;
+        padding: 0;
+        white-space: normal;
+      }
+
+      ${e} progress {
+        vertical-align: baseline;
+      }
+
+      ${e} textarea {
+        overflow: auto;
+      }
+
+      ${e} :where([type="checkbox"], [type="radio"]) {
+        box-sizing: border-box;
+        padding: 0;
+      }
+
+      ${e} input[type="number"]::-webkit-inner-spin-button,
+      ${e} input[type="number"]::-webkit-outer-spin-button {
+        -webkit-appearance: none !important;
+      }
+
+      ${e} input[type="number"] {
+        -moz-appearance: textfield;
+      }
+
+      ${e} input[type="search"] {
+        -webkit-appearance: textfield;
+        outline-offset: -2px;
+      }
+
+      ${e} input[type="search"]::-webkit-search-decoration {
+        -webkit-appearance: none !important;
+      }
+
+      ${e} ::-webkit-file-upload-button {
+        -webkit-appearance: button;
+        font: inherit;
+      }
+
+      ${e} details {
+        display: block;
+      }
+
+      ${e} summary {
+        display: list-item;
+      }
+
+      template {
+        display: none;
+      }
+
+      [hidden] {
+        display: none !important;
+      }
+
+      ${e} :where(
+          blockquote,
+          dl,
+          dd,
+          h1,
+          h2,
+          h3,
+          h4,
+          h5,
+          h6,
+          hr,
+          figure,
+          p,
+          pre
+        ) {
+        margin: 0;
+      }
+
+      ${e} button {
+        background: transparent;
+        padding: 0;
+      }
+
+      ${e} fieldset {
+        margin: 0;
+        padding: 0;
+      }
+
+      ${e} :where(ol, ul) {
+        margin: 0;
+        padding: 0;
+      }
+
+      ${e} textarea {
+        resize: vertical;
+      }
+
+      ${e} :where(button, [role="button"]) {
+        cursor: pointer;
+      }
+
+      ${e} button::-moz-focus-inner {
+        border: 0 !important;
+      }
+
+      ${e} table {
+        border-collapse: collapse;
+      }
+
+      ${e} :where(h1, h2, h3, h4, h5, h6) {
+        font-size: inherit;
+        font-weight: inherit;
+      }
+
+      ${e} :where(button, input, optgroup, select, textarea) {
+        padding: 0;
+        line-height: inherit;
+        color: inherit;
+      }
+
+      ${e} :where(img, svg, video, canvas, audio, iframe, embed, object) {
+        display: block;
+      }
+
+      ${e} :where(img, video) {
+        max-width: 100%;
+        height: auto;
+      }
+
+      [data-js-focus-visible]
+        :focus:not([data-focus-visible-added]):not(
+          [data-focus-visible-disabled]
+        ) {
+        outline: none;
+        box-shadow: none;
+      }
+
+      ${e} select::-ms-expand {
+        display: none;
+      }
+
+      ${E}
+    `}),g={light:"chakra-ui-light",dark:"chakra-ui-dark"};function Q(e={}){const{preventTransition:o=!0}=e,n={setDataset:r=>{const t=o?n.preventTransition():void 0;document.documentElement.dataset.theme=r,document.documentElement.style.colorScheme=r,t==null||t()},setClassName(r){document.body.classList.add(r?g.dark:g.light),document.body.classList.remove(r?g.light:g.dark)},query(){return window.matchMedia("(prefers-color-scheme: dark)")},getSystemTheme(r){var t;return((t=n.query().matches)!=null?t:r==="dark")?"dark":"light"},addListener(r){const t=n.query(),i=a=>{r(a.matches?"dark":"light")};return typeof t.addListener=="function"?t.addListener(i):t.addEventListener("change",i),()=>{typeof t.removeListener=="function"?t.removeListener(i):t.removeEventListener("change",i)}},preventTransition(){const r=document.createElement("style");return r.appendChild(document.createTextNode("*{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;-ms-transition:none!important;transition:none!important}")),document.head.appendChild(r),()=>{window.getComputedStyle(document.body),requestAnimationFrame(()=>{requestAnimationFrame(()=>{document.head.removeChild(r)})})}}};return n}var X="chakra-ui-color-mode";function L(e){return{ssr:!1,type:"localStorage",get(o){if(!(globalThis!=null&&globalThis.document))return o;let n;try{n=localStorage.getItem(e)||o}catch{}return n||o},set(o){try{localStorage.setItem(e,o)}catch{}}}}var ee=L(X),M=()=>{};function S(e,o){return e.type==="cookie"&&e.ssr?e.get(o):o}function O(e){const{value:o,children:n,options:{useSystemColorMode:r,initialColorMode:t,disableTransitionOnChange:i}={},colorModeManager:a=ee}=e,d=t==="dark"?"dark":"light",[u,p]=l.useState(()=>S(a,d)),[y,b]=l.useState(()=>S(a)),{getSystemTheme:w,setClassName:k,setDataset:x,addListener:$}=l.useMemo(()=>Q({preventTransition:i}),[i]),v=t==="system"&&!u?y:u,c=l.useCallback(h=>{const f=h==="system"?w():h;p(f),k(f==="dark"),x(f),a.set(f)},[a,w,k,x]);I(()=>{t==="system"&&b(w())},[]),l.useEffect(()=>{const h=a.get();if(h){c(h);return}if(t==="system"){c("system");return}c(d)},[a,d,t,c]);const C=l.useCallback(()=>{c(v==="dark"?"light":"dark")},[v,c]);l.useEffect(()=>{if(r)return $(c)},[r,$,c]);const A=l.useMemo(()=>({colorMode:o??v,toggleColorMode:o?M:C,setColorMode:o?M:c,forced:o!==void 0}),[v,C,c,o]);return s.jsx(R.Provider,{value:A,children:n})}O.displayName="ColorModeProvider";var te=["borders","breakpoints","colors","components","config","direction","fonts","fontSizes","fontWeights","letterSpacings","lineHeights","radii","shadows","sizes","space","styles","transition","zIndices"];function re(e){return V(e)?te.every(o=>Object.prototype.hasOwnProperty.call(e,o)):!1}function m(e){return typeof e=="function"}function oe(...e){return o=>e.reduce((n,r)=>r(n),o)}var ne=e=>function(...n){let r=[...n],t=n[n.length-1];return re(t)&&r.length>1?r=r.slice(0,r.length-1):t=e,oe(...r.map(i=>a=>m(i)?i(a):ae(a,i)))(t)},ie=ne(j);function ae(...e){return z({},...e,_)}function _(e,o,n,r){if((m(e)||m(o))&&Object.prototype.hasOwnProperty.call(r,n))return(...t)=>{const i=m(e)?e(...t):e,a=m(o)?o(...t):o;return z({},i,a,_)}}var q=l.createContext({getDocument(){return document},getWindow(){return window}});q.displayName="EnvironmentContext";function N(e){const{children:o,environment:n,disabled:r}=e,t=l.useRef(null),i=l.useMemo(()=>n||{getDocument:()=>{var d,u;return(u=(d=t.current)==null?void 0:d.ownerDocument)!=null?u:document},getWindow:()=>{var d,u;return(u=(d=t.current)==null?void 0:d.ownerDocument.defaultView)!=null?u:window}},[n]),a=!r||!n;return s.jsxs(q.Provider,{value:i,children:[o,a&&s.jsx("span",{id:"__chakra_env",hidden:!0,ref:t})]})}N.displayName="EnvironmentProvider";var se=e=>{const{children:o,colorModeManager:n,portalZIndex:r,resetScope:t,resetCSS:i=!0,theme:a={},environment:d,cssVarsRoot:u,disableEnvironment:p,disableGlobalStyle:y}=e,b=s.jsx(N,{environment:d,disabled:p,children:o});return s.jsx(D,{theme:a,cssVarsRoot:u,children:s.jsxs(O,{colorModeManager:n,options:a.config,children:[i?s.jsx(J,{scope:t}):s.jsx(B,{}),!y&&s.jsx(F,{}),r?s.jsx(G,{zIndex:r,children:b}):b]})})},le=e=>function({children:n,theme:r=e,toastOptions:t,...i}){return s.jsxs(se,{theme:r,...i,children:[s.jsx(W,{value:t==null?void 0:t.defaultOptions,children:n}),s.jsx(K,{...t})]})},de=le(j);const ue=()=>l.useMemo(()=>({colorScheme:"dark",fontFamily:"'Inter Variable', sans-serif",components:{ScrollArea:{defaultProps:{scrollbarSize:10},styles:{scrollbar:{"&:hover":{backgroundColor:"var(--invokeai-colors-baseAlpha-300)"}},thumb:{backgroundColor:"var(--invokeai-colors-baseAlpha-300)"}}}}}),[]),ce=L("@@invokeai-color-mode");function he({children:e}){const{i18n:o}=H(),n=o.dir(),r=l.useMemo(()=>ie({...U,direction:n}),[n]);l.useEffect(()=>{document.body.dir=n},[n]);const t=ue();return s.jsx(Z,{theme:t,children:s.jsx(de,{theme:r,colorModeManager:ce,toastOptions:Y,children:e})})}const ve=l.memo(he);export{ve as default};

pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
   "fastapi~=0.103.2",
   "fastapi-events~=0.9.1",
   "huggingface-hub~=0.16.4",
+  "imohash",
   "invisible-watermark~=0.2.0",            # needed to install SDXL base and refiner using their repo_ids
   "matplotlib",                            # needed for plotting of Penner easing functions
   "mediapipe",                             # needed for "mediapipeface" controlnet model
@@ -136,6 +137,7 @@ dependencies = [
 "invokeai-node-web" = "invokeai.app.api_app:invoke_api"
 "invokeai-import-images" = "invokeai.frontend.install.import_images:main"
 "invokeai-db-maintenance" = "invokeai.backend.util.db_maintenance:main"
+"invokeai-migrate-models-to-db" = "invokeai.backend.model_manager.migrate_to_db:main"
 
 [project.urls]
 "Homepage" = "https://invoke-ai.github.io/InvokeAI/"

tests/app/services/model_records/test_model_records_sql.py

@@ -0,0 +1,267 @@
+"""
+Test the refactored model config classes.
+"""
+
+from hashlib import sha256
+
+import pytest
+
+from invokeai.app.services.config import InvokeAIAppConfig
+from invokeai.app.services.model_records import (
+    DuplicateModelException,
+    ModelRecordServiceBase,
+    ModelRecordServiceSQL,
+    UnknownModelException,
+)
+from invokeai.app.services.shared.sqlite import SqliteDatabase
+from invokeai.backend.model_manager.config import (
+    BaseModelType,
+    MainCheckpointConfig,
+    MainDiffusersConfig,
+    ModelType,
+    TextualInversionConfig,
+    VaeDiffusersConfig,
+)
+from invokeai.backend.util.logging import InvokeAILogger
+
+
+@pytest.fixture
+def store(datadir) -> ModelRecordServiceBase:
+    config = InvokeAIAppConfig(root=datadir)
+    logger = InvokeAILogger.get_logger(config=config)
+    db = SqliteDatabase(config, logger)
+    return ModelRecordServiceSQL(db)
+
+
+def example_config() -> TextualInversionConfig:
+    return TextualInversionConfig(
+        path="/tmp/pokemon.bin",
+        name="old name",
+        base=BaseModelType("sd-1"),
+        type=ModelType("embedding"),
+        format="embedding_file",
+        original_hash="ABC123",
+    )
+
+
+def test_type(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", config)
+    config1 = store.get_model("key1")
+    assert type(config1) == TextualInversionConfig
+
+
+def test_add(store: ModelRecordServiceBase):
+    raw = {
+        "path": "/tmp/foo.ckpt",
+        "name": "model1",
+        "base": BaseModelType("sd-1"),
+        "type": "main",
+        "config": "/tmp/foo.yaml",
+        "variant": "normal",
+        "format": "checkpoint",
+        "original_hash": "111222333444",
+    }
+    store.add_model("key1", raw)
+    config1 = store.get_model("key1")
+    assert config1 is not None
+    assert type(config1) == MainCheckpointConfig
+    assert config1.base == BaseModelType("sd-1")
+    assert config1.name == "model1"
+    assert config1.original_hash == "111222333444"
+    assert config1.current_hash is None
+
+
+def test_dup(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", example_config())
+    with pytest.raises(DuplicateModelException):
+        store.add_model("key1", config)
+    with pytest.raises(DuplicateModelException):
+        store.add_model("key2", config)
+
+
+def test_update(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", config)
+    config = store.get_model("key1")
+    assert config.name == "old name"
+
+    config.name = "new name"
+    store.update_model("key1", config)
+    new_config = store.get_model("key1")
+    assert new_config.name == "new name"
+
+
+def test_rename(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", config)
+    config = store.get_model("key1")
+    assert config.name == "old name"
+
+    store.rename_model("key1", "new name")
+    new_config = store.get_model("key1")
+    assert new_config.name == "new name"
+
+
+def test_unknown_key(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", config)
+    with pytest.raises(UnknownModelException):
+        store.update_model("unknown_key", config)
+
+
+def test_delete(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", config)
+    config = store.get_model("key1")
+    store.del_model("key1")
+    with pytest.raises(UnknownModelException):
+        config = store.get_model("key1")
+
+
+def test_exists(store: ModelRecordServiceBase):
+    config = example_config()
+    store.add_model("key1", config)
+    assert store.exists("key1")
+    assert not store.exists("key2")
+
+
+def test_filter(store: ModelRecordServiceBase):
+    config1 = MainDiffusersConfig(
+        path="/tmp/config1",
+        name="config1",
+        base=BaseModelType("sd-1"),
+        type=ModelType("main"),
+        original_hash="CONFIG1HASH",
+    )
+    config2 = MainDiffusersConfig(
+        path="/tmp/config2",
+        name="config2",
+        base=BaseModelType("sd-1"),
+        type=ModelType("main"),
+        original_hash="CONFIG2HASH",
+    )
+    config3 = VaeDiffusersConfig(
+        path="/tmp/config3",
+        name="config3",
+        base=BaseModelType("sd-2"),
+        type=ModelType("vae"),
+        original_hash="CONFIG3HASH",
+    )
+    for c in config1, config2, config3:
+        store.add_model(sha256(c.name.encode("utf-8")).hexdigest(), c)
+    matches = store.search_by_attr(model_type=ModelType("main"))
+    assert len(matches) == 2
+    assert matches[0].name in {"config1", "config2"}
+
+    matches = store.search_by_attr(model_type=ModelType("vae"))
+    assert len(matches) == 1
+    assert matches[0].name == "config3"
+    assert matches[0].key == sha256("config3".encode("utf-8")).hexdigest()
+    assert isinstance(matches[0].type, ModelType)  # This tests that we get proper enums back
+
+    matches = store.search_by_hash("CONFIG1HASH")
+    assert len(matches) == 1
+    assert matches[0].original_hash == "CONFIG1HASH"
+
+    matches = store.all_models()
+    assert len(matches) == 3
+
+
+def test_unique(store: ModelRecordServiceBase):
+    config1 = MainDiffusersConfig(
+        path="/tmp/config1",
+        base=BaseModelType("sd-1"),
+        type=ModelType("main"),
+        name="nonuniquename",
+        original_hash="CONFIG1HASH",
+    )
+    config2 = MainDiffusersConfig(
+        path="/tmp/config2",
+        base=BaseModelType("sd-2"),
+        type=ModelType("main"),
+        name="nonuniquename",
+        original_hash="CONFIG1HASH",
+    )
+    config3 = VaeDiffusersConfig(
+        path="/tmp/config3",
+        base=BaseModelType("sd-2"),
+        type=ModelType("vae"),
+        name="nonuniquename",
+        original_hash="CONFIG1HASH",
+    )
+    config4 = MainDiffusersConfig(
+        path="/tmp/config4",
+        base=BaseModelType("sd-1"),
+        type=ModelType("main"),
+        name="nonuniquename",
+        original_hash="CONFIG1HASH",
+    )
+    # config1, config2 and config3 are compatible because they have unique combos
+    # of name, type and base
+    for c in config1, config2, config3:
+        store.add_model(sha256(c.path.encode("utf-8")).hexdigest(), c)
+
+    # config4 clashes with config1 and should raise an integrity error
+    with pytest.raises(DuplicateModelException):
+        store.add_model(sha256(c.path.encode("utf-8")).hexdigest(), config4)
+
+
+def test_filter_2(store: ModelRecordServiceBase):
+    config1 = MainDiffusersConfig(
+        path="/tmp/config1",
+        name="config1",
+        base=BaseModelType("sd-1"),
+        type=ModelType("main"),
+        original_hash="CONFIG1HASH",
+    )
+    config2 = MainDiffusersConfig(
+        path="/tmp/config2",
+        name="config2",
+        base=BaseModelType("sd-1"),
+        type=ModelType("main"),
+        original_hash="CONFIG2HASH",
+    )
+    config3 = MainDiffusersConfig(
+        path="/tmp/config3",
+        name="dup_name1",
+        base=BaseModelType("sd-2"),
+        type=ModelType("main"),
+        original_hash="CONFIG3HASH",
+    )
+    config4 = MainDiffusersConfig(
+        path="/tmp/config4",
+        name="dup_name1",
+        base=BaseModelType("sdxl"),
+        type=ModelType("main"),
+        original_hash="CONFIG3HASH",
+    )
+    config5 = VaeDiffusersConfig(
+        path="/tmp/config5",
+        name="dup_name1",
+        base=BaseModelType("sd-1"),
+        type=ModelType("vae"),
+        original_hash="CONFIG3HASH",
+    )
+    for c in config1, config2, config3, config4, config5:
+        store.add_model(sha256(c.path.encode("utf-8")).hexdigest(), c)
+
+    matches = store.search_by_attr(
+        model_type=ModelType("main"),
+        model_name="dup_name1",
+    )
+    assert len(matches) == 2
+
+    matches = store.search_by_attr(
+        base_model=BaseModelType("sd-1"),
+        model_type=ModelType("main"),
+    )
+    assert len(matches) == 2
+
+    matches = store.search_by_attr(
+        base_model=BaseModelType("sd-1"),
+        model_type=ModelType("vae"),
+        model_name="dup_name1",
+    )
+    assert len(matches) == 1

tests/nodes/test_graph_execution_state.py

@@ -68,6 +68,7 @@ def mock_services() -> InvocationServices:
         latents=None,  # type: ignore
         logger=logging,  # type: ignore
         model_manager=None,  # type: ignore
+        model_records=None,  # type: ignore
         names=None,  # type: ignore
         performance_statistics=InvocationStatsService(),
         processor=DefaultInvocationProcessor(),

tests/nodes/test_invoker.py

@@ -73,6 +73,7 @@ def mock_services() -> InvocationServices:
         latents=None,  # type: ignore
         logger=logging,  # type: ignore
         model_manager=None,  # type: ignore
+        model_records=None,  # type: ignore
         names=None,  # type: ignore
         performance_statistics=InvocationStatsService(),
         processor=DefaultInvocationProcessor(),