make session_list api return a raw dict rather than pydantic object

## What type of PR is this? (check all applicable)

- [ ] Refactor
- [ ] Feature
- [x] Bug Fix
- [ ] Optimization
- [ ] Documentation Update
- [ ] Community Node Submission


## Have you discussed this change with the InvokeAI team?
- [ ] Yes
- [ ] No, because:

      
## Have you updated all relevant documentation?
- [ ] Yes
- [ ] No


## Description


## Related Tickets & Documents

<!--
For pull requests that relate or close an issue, please include them
below. 

For example having the text: "closes #1234" would connect the current
pull
request to issue 1234.  And when we merge the pull request, Github will
automatically close the issue.
-->

- Related Issue #
- Closes #

## QA Instructions, Screenshots, Recordings

<!-- 
Please provide steps on how to test changes, any hardware or 
software specifications as well as any other pertinent information. 
-->

## Added/updated tests?

- [ ] Yes
- [ ] No : _please replace this line with details on why tests
      have not been included_

## [optional] Are there any post deployment tasks we need to perform?
Should be removed when added in diffusers
https://github.com/huggingface/diffusers/pull/4599

.github/workflows/lint-frontend.yml

@@ -2,8 +2,6 @@ name: Lint frontend
 
 on:
   pull_request:
-    paths:
-      - 'invokeai/frontend/web/**'
     types:
       - 'ready_for_review'
       - 'opened'
@@ -11,8 +9,6 @@ on:
   push:
     branches:
       - 'main'
-    paths:
-      - 'invokeai/frontend/web/**'
   merge_group:
   workflow_dispatch:
 

.github/workflows/style-checks.yml

@@ -1,13 +1,14 @@
-name: Black  # TODO: add isort and flake8 later
+name: style checks
+# just formatting for now
+# TODO: add isort and flake8 later
 
 on:
-  pull_request: {}
+  pull_request:
   push:
-    branches: master
-    tags: "*"
+    branches: main
 
 jobs:
-  test:
+  black:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -19,8 +20,7 @@ jobs:
 
       - name: Install dependencies with pip
         run: |
-          pip install --upgrade pip wheel
-          pip install .[test]
+          pip install black
 
       # - run: isort --check-only .
       - run: black --check .

.github/workflows/test-invoke-pip-skip.yml

@@ -1,50 +0,0 @@
-name: Test invoke.py pip
-
-# This is a dummy stand-in for the actual tests
-# we don't need to run python tests on non-Python changes
-# But PRs require passing tests to be mergeable
-
-on:
-  pull_request:
-    paths:
-      - '**'
-      - '!pyproject.toml'
-      - '!invokeai/**'
-      - '!tests/**'
-      - 'invokeai/frontend/web/**'
-  merge_group:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  matrix:
-    if: github.event.pull_request.draft == false
-    strategy:
-      matrix:
-        python-version:
-          - '3.10'
-        pytorch:
-          - linux-cuda-11_7
-          - linux-rocm-5_2
-          - linux-cpu
-          - macos-default
-          - windows-cpu
-        include:
-          - pytorch: linux-cuda-11_7
-            os: ubuntu-22.04
-          - pytorch: linux-rocm-5_2
-            os: ubuntu-22.04
-          - pytorch: linux-cpu
-            os: ubuntu-22.04
-          - pytorch: macos-default
-            os: macOS-12
-          - pytorch: windows-cpu
-            os: windows-2022
-    name: ${{ matrix.pytorch }} on ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: skip
-        run: echo "no build required"

.github/workflows/test-invoke-pip.yml

@@ -3,16 +3,7 @@ on:
   push:
     branches:
       - 'main'
-    paths:
-      - 'pyproject.toml'
-      - 'invokeai/**'
-      - '!invokeai/frontend/web/**'
   pull_request:
-    paths:
-      - 'pyproject.toml'
-      - 'invokeai/**'
-      - 'tests/**'
-      - '!invokeai/frontend/web/**'
     types:
       - 'ready_for_review'
       - 'opened'
@@ -65,10 +56,23 @@ jobs:
         id: checkout-sources
         uses: actions/checkout@v3
 
+      - name: Check for changed python files
+        id: changed-files
+        uses: tj-actions/changed-files@v37
+        with:
+          files_yaml: |
+            python:
+              - 'pyproject.toml'
+              - 'invokeai/**'
+              - '!invokeai/frontend/web/**'
+              - 'tests/**'
+
       - name: set test prompt to main branch validation
+        if: steps.changed-files.outputs.python_any_changed == 'true'
         run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }}
 
       - name: setup python
+        if: steps.changed-files.outputs.python_any_changed == 'true'
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
@@ -76,6 +80,7 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: install invokeai
+        if: steps.changed-files.outputs.python_any_changed == 'true'
         env:
           PIP_EXTRA_INDEX_URL: ${{ matrix.extra-index-url }}
         run: >
@@ -83,6 +88,7 @@ jobs:
           --editable=".[test]"
 
       - name: run pytest
+        if: steps.changed-files.outputs.python_any_changed == 'true'
         id: run-pytest
         run: pytest
 

README.md

@@ -161,7 +161,7 @@ the command `npm install -g yarn` if needed)
     _For Windows/Linux with an NVIDIA GPU:_
 
     ```terminal
-    pip install "InvokeAI[xformers]" --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu117
+    pip install "InvokeAI[xformers]" --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu118
     ```
 
     _For Linux with an AMD GPU:_
@@ -184,8 +184,9 @@ the command `npm install -g yarn` if needed)
 6. Configure InvokeAI and install a starting set of image generation models (you only need to do this once):
 
     ```terminal
-    invokeai-configure
+    invokeai-configure --root .
     ```
+	Don't miss the dot at the end!
 
 7. Launch the web server (do it every time you run InvokeAI):
 
@@ -193,15 +194,9 @@ the command `npm install -g yarn` if needed)
     invokeai-web
     ```
 
-8. Build Node.js assets
+8. Point your browser to http://localhost:9090 to bring up the web interface.
 
-  ```terminal
-  cd invokeai/frontend/web/
-  yarn vite build
-  ```
-
-9. Point your browser to http://localhost:9090 to bring up the web interface.
-10. Type `banana sushi` in the box on the top left and click `Invoke`.
+9. Type `banana sushi` in the box on the top left and click `Invoke`.
 
 Be sure to activate the virtual environment each time before re-launching InvokeAI,
 using `source .venv/bin/activate` or `.venv\Scripts\activate`.
@@ -311,13 +306,30 @@ InvokeAI. The second will prepare the 2.3 directory for use with 3.0.
 You may now launch the WebUI in the usual way, by selecting option [1]
 from the launcher script
 
-#### Migration Caveats
+#### Migrating Images
 
 The migration script will migrate your invokeai settings and models,
 including textual inversion models, LoRAs and merges that you may have
 installed previously. However it does **not** migrate the generated
-images stored in your 2.3-format outputs directory. You will need to
-manually import selected images into the 3.0 gallery via drag-and-drop.
+images stored in your 2.3-format outputs directory. To do this, you 
+need to run an additional step:
+
+1. From a working InvokeAI 3.0 root directory, start the launcher and
+enter menu option [8] to open the "developer's console".
+
+2. At the developer's console command line, type the command:
+
+```bash
+invokeai-import-images
+```
+
+3. This will lead you through the process of confirming the desired
+   source and destination for the imported images. The images will
+   appear in the gallery board of your choice, and contain the
+   original prompt, model name, and other parameters used to generate
+   the image.
+   
+(Many kudos to **techjedi** for contributing this script.)
 
 ## Hardware Requirements
 

docker/docker-entrypoint.sh

@@ -29,8 +29,8 @@ configure() {
         echo "To reconfigure InvokeAI, delete the above file."
         echo "======================================================================"
     else
-        mkdir -p ${INVOKEAI_ROOT}
-        chown --recursive ${USER} ${INVOKEAI_ROOT}
+        mkdir -p "${INVOKEAI_ROOT}"
+        chown --recursive ${USER} "${INVOKEAI_ROOT}"
         gosu ${USER} invokeai-configure --yes --default_only
     fi
 }
@@ -50,16 +50,16 @@ fi
 if [[ -v "PUBLIC_KEY" ]] && [[ ! -d "${HOME}/.ssh" ]]; then
     apt-get update
     apt-get install -y openssh-server
-    pushd $HOME
+    pushd "$HOME"
     mkdir -p .ssh
-    echo ${PUBLIC_KEY} > .ssh/authorized_keys
+    echo "${PUBLIC_KEY}" > .ssh/authorized_keys
     chmod -R 700 .ssh
     popd
     service ssh start
 fi
 
 
-cd ${INVOKEAI_ROOT}
+cd "${INVOKEAI_ROOT}"
 
 # Run the CMD as the Container User (not root).
 exec gosu ${USER} "$@"

docs/contributing/contribution_guides/development.md

@@ -16,7 +16,7 @@ If you don't feel ready to make a code contribution yet, no problem! You can als
 There are two paths to making a development contribution: 
 
 1. Choosing an open issue to address. Open issues can be found in the [Issues](https://github.com/invoke-ai/InvokeAI/issues?q=is%3Aissue+is%3Aopen) section of the InvokeAI repository. These are tagged by the issue type (bug, enhancement, etc.) along with the “good first issues” tag denoting if they are suitable for first time contributors.
-    1. Additional items can be found on our roadmap <******************************link to roadmap>******************************. The roadmap is organized in terms of priority, and contains features of varying size and complexity. If there is an inflight item you’d like to help with, reach out to the contributor assigned to the item to see how you can help. 
+    1. Additional items can be found on our [roadmap](https://github.com/orgs/invoke-ai/projects/7). The roadmap is organized in terms of priority, and contains features of varying size and complexity. If there is an inflight item you’d like to help with, reach out to the contributor assigned to the item to see how you can help. 
 2. Opening a new issue or feature to add. **Please make sure you have searched through existing issues before creating new ones.**
 
 *Regardless of what you choose, please post in the  [#dev-chat](https://discord.com/channels/1020123559063990373/1049495067846524939) channel of the Discord before you start development in order to confirm that the issue or feature is aligned with the current direction of the project. We value our contributors time and effort and want to ensure that no one’s time is being misspent.*

docs/features/POSTPROCESS.md

@@ -4,35 +4,13 @@ title: Postprocessing
 
 # :material-image-edit: Postprocessing
 
-## Intro
-
-This extension provides the ability to restore faces and upscale images.
+This sections details the ability to improve faces and upscale images.
 
 ## Face Fixing
 
-The default face restoration module is GFPGAN. The default upscale is
-Real-ESRGAN. For an alternative face restoration module, see
-[CodeFormer Support](#codeformer-support) below.
+As of InvokeAI 3.0, the easiest way to improve faces created during image generation is through the Inpainting functionality of the Unified Canvas. Simply add the image containing the faces that you would like to improve to the canvas, mask the face to be improved and run the invocation. For best results, make sure to use an inpainting specific model; these are usually identified by the "-inpainting" term in the model name. 
 
-As of version 1.14, environment.yaml will install the Real-ESRGAN package into
-the standard install location for python packages, and will put GFPGAN into a
-subdirectory of "src" in the InvokeAI directory. Upscaling with Real-ESRGAN
-should "just work" without further intervention. Simply indicate the desired scale on
-the popup in the Web GUI.
-
-**GFPGAN** requires a series of downloadable model files to work. These are
-loaded when you run `invokeai-configure`. If GFPAN is failing with an
-error, please run the following from the InvokeAI directory:
-
-```bash
-invokeai-configure
-```
-
-If you do not run this script in advance, the GFPGAN module will attempt to
-download the models files the first time you try to perform facial
-reconstruction.
-
-### Upscaling
+## Upscaling
 
 Open the upscaling dialog by clicking on the "expand" icon located
 above the image display area in the Web UI:
@@ -41,82 +19,23 @@ above the image display area in the Web UI:
 ![upscale1](../assets/features/upscale-dialog.png)
 </figure>
 
-There are three different upscaling parameters that you can
-adjust. The first is the scale itself, either 2x or 4x.
+The default upscaling option is Real-ESRGAN x2 Plus, which will scale your image by a factor of two. This means upscaling a 512x512 image will result in a new 1024x1024 image.
 
-The second is the "Denoising Strength." Higher values will smooth out
-the image and remove digital chatter, but may lose fine detail at
-higher values.
+Other options are the x4 upscalers, which will scale your image by a factor of 4. 
 
-Third, "Upscale Strength" allows you to adjust how the You can set the
-scaling stength between `0` and `1.0` to control the intensity of the
-scaling. AI upscalers generally tend to smooth out texture details. If
-you wish to retain some of those for natural looking results, we
-recommend using values between `0.5 to 0.8`.
-
-[This figure](../assets/features/upscaling-montage.png) illustrates
-the effects of denoising and strength. The original image was 512x512,
-4x scaled to 2048x2048. The "original" version on the upper left was
-scaled using simple pixel averaging. The remainder use the ESRGAN
-upscaling algorithm at different levels of denoising and strength.
-
-<figure markdown>
-![upscaling](../assets/features/upscaling-montage.png){ width=720 }
-</figure>
-
-Both denoising and strength default to 0.75.
-
-### Face Restoration
-
-InvokeAI offers alternative two face restoration algorithms,
-[GFPGAN](https://github.com/TencentARC/GFPGAN) and
-[CodeFormer](https://huggingface.co/spaces/sczhou/CodeFormer). These
-algorithms improve the appearance of faces, particularly eyes and
-mouths. Issues with faces are less common with the latest set of
-Stable Diffusion models than with the original 1.4 release, but the
-restoration algorithms can still make a noticeable improvement in
-certain cases. You can also apply restoration to old photographs you
-upload.
-
-To access face restoration, click the "smiley face" icon in the
-toolbar above the InvokeAI image panel. You will be presented with a
-dialog that offers a choice between the two algorithm and sliders that
-allow you to adjust their parameters. Alternatively, you may open the
-left-hand accordion panel labeled "Face Restoration" and have the
-restoration algorithm of your choice applied to generated images
-automatically.
-
-
-Like upscaling, there are a number of parameters that adjust the face
-restoration output. GFPGAN has a single parameter, `strength`, which
-controls how much the algorithm is allowed to adjust the
-image. CodeFormer has two parameters, `strength`, and `fidelity`,
-which together control the quality of the output image as described in
-the [CodeFormer project
-page](https://shangchenzhou.com/projects/CodeFormer/). Default values
-are 0.75 for both parameters, which achieves a reasonable balance
-between changing the image too much and not enough.
-
-[This figure](../assets/features/restoration-montage.png) illustrates
-the effects of adjusting GFPGAN and CodeFormer parameters.
-
-<figure markdown>
-![upscaling](../assets/features/restoration-montage.png){ width=720 }
-</figure>
 
 !!! note
 
-    GFPGAN and Real-ESRGAN are both memory intensive. In order to avoid crashes and memory overloads
+    Real-ESRGAN is memory intensive. In order to avoid crashes and memory overloads
     during the Stable Diffusion process, these effects are applied after Stable Diffusion has completed
     its work.
 
     In single image generations, you will see the output right away but when you are using multiple
-    iterations, the images will first be generated and then upscaled and face restored after that
+    iterations, the images will first be generated and then upscaled after that
     process is complete. While the image generation is taking place, you will still be able to preview
     the base images.
 
 ## How to disable
 
-If, for some reason, you do not wish to load the GFPGAN and/or ESRGAN libraries,
-you can disable them on the invoke.py command line with the `--no_restore` and
-`--no_esrgan` options, respectively.
+If, for some reason, you do not wish to load the ESRGAN libraries,
+you can disable them on the invoke.py command line with the `--no_esrgan` options.

docs/features/index.md

@@ -4,6 +4,9 @@ title: Overview
 
 Here you can find the documentation for InvokeAI's various features.
 
+## The [Getting Started Guide](../help/gettingStartedWithAI) 
+A getting started guide for those new to AI image generation. 
+
 ## The Basics
 ### * The [Web User Interface](WEB.md)
 Guide to the Web interface. Also see the [WebUI Hotkeys Reference Guide](WEBUIHOTKEYS.md)
@@ -46,7 +49,7 @@ Personalize models by adding your own style or subjects.
 
 ## Other Features
 
-### * [The NSFW Checker](NSFW.md)
+### * [The NSFW Checker](WATERMARK+NSFW.md)
 Prevent InvokeAI from displaying unwanted racy images.
 
 ### * [Controlling Logging](LOGGING.md)

docs/help/gettingStartedWithAI.md

@@ -0,0 +1,95 @@
+# Getting Started with AI Image Generation
+
+New to image generation with AI? You’re in the right place! 
+
+This is a high level walkthrough of some of the concepts and terms you’ll see as you start using InvokeAI. Please note, this is not an exhaustive guide and may be out of date due to the rapidly changing nature of the space. 
+
+## Using InvokeAI
+
+### **Prompt Crafting**
+
+- Prompts are the basis of using InvokeAI, providing the models directions on what to generate. As a general rule of thumb, the more detailed your prompt is, the better your result will be.
+
+ *To get started, here’s an easy template to use for structuring your prompts:*
+
+- Subject, Style, Quality, Aesthetic
+    - **Subject:** What your image will be about. E.g. “a futuristic city with trains”,  “penguins floating on icebergs”, “friends sharing beers”
+    - **Style:** The style or medium in which your image will be in. E.g. “photograph”, “pencil sketch”, “oil paints”, or “pop art”, “cubism”, “abstract”
+    - **Quality:** A particular aspect or trait that you would like to see emphasized in your image. E.g. "award-winning", "featured in {relevant set of high quality works}", "professionally acclaimed". Many people often use "masterpiece".
+    - **Aesthetics:** The visual impact and design of the artwork. This can be colors, mood, lighting, setting, etc.
+- There are two prompt boxes: *Positive Prompt* & *Negative Prompt*.
+    - A **Positive** Prompt includes words you want the model to reference when creating an image.
+    - Negative Prompt is for anything you want the model to eliminate when creating an image. It doesn’t always interpret things exactly the way you would, but helps control the generation process. Always try to include a few terms - you can typically use lower quality image terms like “blurry” or “distorted” with good success.
+- Some examples prompts you can try on your own:
+    - A detailed oil painting of a tranquil forest at sunset with vibrant+ colors and soft, golden light filtering through the trees
+    - friends sharing beers in a busy city, realistic colored pencil sketch, twilight, masterpiece, bright, lively
+
+### Generation Workflows
+
+- Invoke offers a number of different workflows for interacting with models to produce images. Each is extremely powerful on its own, but together provide you an unparalleled way of producing high quality creative outputs that align with your vision.
+    - **Text to Image:** The text to image tab focuses on the key workflow of using a prompt to generate a new image. It includes other features that help control the generation process as well.
+    - **Image to Image:** With image to image, you provide an image as a reference (called the “initial image”), which provides more guidance around color and structure to the AI as it generates a new image. This is provided alongside the same features as Text to Image.
+    - **Unified Canvas:** The Unified Canvas is an advanced AI-first image editing tool that is easy to use, but hard to master. Drag an image onto the canvas from your gallery in order to regenerate certain elements, edit content or colors (known as inpainting), or extend the image with an exceptional degree of consistency and clarity (called outpainting).
+
+### Improving Image Quality
+
+- Fine tuning your prompt - the more specific you are, the closer the image will turn out to what is in your head!  Adding more details in the Positive Prompt or Negative Prompt can help add / remove pieces of your image to improve it - You can also use advanced techniques like upweighting and downweighting to control the influence of certain words. [Learn more here](https://invoke-ai.github.io/InvokeAI/features/PROMPTS/#prompt-syntax-features).
+    - **Tip: If you’re seeing poor results, try adding the things you don’t like about the image to your negative prompt may help. E.g. distorted, low quality, unrealistic, etc.**
+- Explore different models - Other models can produce different results due to the data they’ve been trained on. Each model has specific language and settings it works best with; a model’s documentation is your friend here.  Play around with some and see what works best for you!
+- Increasing Steps - The number of steps used controls how much time the model is given to produce an image, and depends on the “Scheduler” used. The schedule controls how each step is processed by the model. More steps tends to mean better results, but will take longer - We recommend at least 30 steps for most
+- Tweak and Iterate - Remember, it’s best to change one thing at a time so you know what is working and what isn't. Sometimes you just need to try a new image, and other times using a new prompt might be the ticket. For testing, consider turning off the “random” Seed - Using the same seed with the same settings will produce the same image, which makes it the perfect way to learn exactly what your changes are doing.
+- Explore Advanced Settings - InvokeAI has a full suite of tools available to allow you complete control over your image creation process - Check out our [docs if you want to learn more](https://invoke-ai.github.io/InvokeAI/features/).
+
+
+## Terms & Concepts
+
+If you're interested in learning more, check out [this presentation](https://docs.google.com/presentation/d/1IO78i8oEXFTZ5peuHHYkVF-Y3e2M6iM5tCnc-YBfcCM/edit?usp=sharing) from one of our maintainers (@lstein). 
+
+### Stable Diffusion
+
+Stable Diffusion is deep learning, text-to-image model that is the foundation of the capabilities found in InvokeAI. Since the release of Stable Diffusion, there have been many subsequent models created based on Stable Diffusion that are designed to generate specific types of images. 
+
+### Prompts
+
+Prompts provide the models directions on what to generate. As a general rule of thumb, the more detailed your prompt is, the better your result will be.
+
+### Models
+
+Models are the magic that power InvokeAI. These files represent the output of training a machine on understanding massive amounts of images - providing them with the capability to generate new images using just a text description of what you’d like to see. (Like Stable Diffusion!)
+
+Invoke offers a simple way to download several different models upon installation, but many more can be discovered online, including at ****. Each model can produce a unique style of output, based on the images it was trained on - Try out different models to see which best fits your creative vision!
+
+- *Models that contain “inpainting” in the name are designed for use with the inpainting feature of the Unified Canvas*
+
+### Scheduler
+
+Schedulers guide the process of removing noise (de-noising) from data. They determine:
+
+1. The number of steps to take to remove the noise.
+2. Whether the steps are random (stochastic) or predictable (deterministic).
+3. The specific method (algorithm) used for de-noising.
+
+Experimenting with different schedulers is recommended as each will produce different outputs!
+
+### Steps
+
+The number of de-noising steps each generation through. 
+
+Schedulers can be intricate and there's often a balance to strike between how quickly they can de-noise data and how well they can do it. It's typically advised to experiment with different schedulers to see which one gives the best results. There has been a lot written on the internet about different schedulers, as well as exploring what the right level of "steps" are for each. You can save generation time by reducing the number of steps used, but you'll want to make sure that you are satisfied with the quality of images produced!
+
+### Low-Rank Adaptations / LoRAs
+
+Low-Rank Adaptations (LoRAs) are like a smaller, more focused version of models, intended to focus on training a better understanding of how a specific character, style, or concept looks.
+
+### Textual Inversion Embeddings
+
+Textual Inversion Embeddings, like LoRAs, assist with more easily prompting for certain characters, styles, or concepts. However, embeddings are trained to update the relationship between a specific word (known as the “trigger”) and the intended output. 
+
+### ControlNet
+
+ControlNets are neural network models that are able to extract key features from an existing image and use these features to guide the output of the image generation model. 
+
+### VAE
+
+Variational auto-encoder (VAE) is a encode/decode model that translates the "latents" image produced during the image generation procees to the large pixel images that we see. 
+

docs/index.md

@@ -11,6 +11,33 @@ title: Home
   ```
 -->
 
+<!-- CSS styling -->
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6.2.1/css/fontawesome.min.css">
+<style>
+    .button {
+      width: 300px;
+      height: 50px;
+      background-color: #448AFF;
+      color: #fff;
+      font-size: 16px;
+      border: none;
+      cursor: pointer;
+      border-radius: 0.2rem;
+    }
+
+    .button-container {
+      display: grid;
+      grid-template-columns: repeat(3, 300px);
+      gap: 20px;
+    }
+
+    .button:hover {
+      background-color: #526CFE;
+    }
+</style>
+
+
+
 <div align="center" markdown>
 
 
@@ -70,61 +97,23 @@ image-to-image generator. It provides a streamlined process with various new
 features and options to aid the image generation process. It runs on Windows,
 Mac and Linux machines, and runs on GPU cards with as little as 4 GB of RAM.
 
-**Quick links**: [<a href="https://discord.gg/ZmtBAhwWhy">Discord Server</a>]
-[<a href="https://github.com/invoke-ai/InvokeAI/">Code and Downloads</a>] [<a
-href="https://github.com/invoke-ai/InvokeAI/issues">Bug Reports</a>] [<a
-href="https://github.com/invoke-ai/InvokeAI/discussions">Discussion, Ideas &
-Q&A</a>]
-
 <div align="center"><img src="assets/invoke-web-server-1.png" width=640></div>
 
-!!! note
+!!! Note
 
-    This fork is rapidly evolving. Please use the [Issues tab](https://github.com/invoke-ai/InvokeAI/issues) to report bugs and make feature requests. Be sure to use the provided templates. They will help aid diagnose issues faster.
+    This project is rapidly evolving. Please use the [Issues tab](https://github.com/invoke-ai/InvokeAI/issues) to report bugs and make feature requests. Be sure to use the provided templates as it will help aid response time.
 
-## :octicons-package-dependencies-24: Installation
+## :octicons-link-24: Quick Links
 
-This fork is supported across Linux, Windows and Macintosh. Linux users can use
-either an Nvidia-based card (with CUDA support) or an AMD card (using the ROCm
-driver).
-
-### [Installation Getting Started Guide](installation)
-#### **[Automated Installer](installation/010_INSTALL_AUTOMATED.md)**
-✅ This is the recommended installation method for first-time users. 
-#### [Manual Installation](installation/020_INSTALL_MANUAL.md)
-This method is recommended for experienced users and developers
-#### [Docker Installation](installation/040_INSTALL_DOCKER.md)
-This method is recommended for those familiar with running Docker containers
-### Other Installation Guides
-  - [PyPatchMatch](installation/060_INSTALL_PATCHMATCH.md)
-  - [XFormers](installation/070_INSTALL_XFORMERS.md)
-  - [CUDA and ROCm Drivers](installation/030_INSTALL_CUDA_AND_ROCM.md)
-  - [Installing New Models](installation/050_INSTALLING_MODELS.md)
-
-## :fontawesome-solid-computer: Hardware Requirements
-
-### :octicons-cpu-24: System
-
-You wil need one of the following:
-
-- :simple-nvidia: An NVIDIA-based graphics card with 4 GB or more VRAM memory.
-- :simple-amd: An AMD-based graphics card with 4 GB or more VRAM memory (Linux
-  only)
-- :fontawesome-brands-apple: An Apple computer with an M1 chip.
-
-We do **not recommend** the following video cards due to issues with their
-running in half-precision mode and having insufficient VRAM to render 512x512
-images in full-precision mode:
-
-- NVIDIA 10xx series cards such as the 1080ti
-- GTX 1650 series cards
-- GTX 1660 series cards
-
-### :fontawesome-solid-memory: Memory and Disk
-
-- At least 12 GB Main Memory RAM.
-- At least 18 GB of free disk space for the machine learning model, Python, and
-  all its dependencies.
+<div class="button-container">
+    <a href="installation/INSTALLATION"> <button class="button">Installation</button> </a>
+    <a href="features/"> <button class="button">Features</button> </a>
+    <a href="help/gettingStartedWithAI/"> <button class="button">Getting Started</button> </a>
+    <a href="contributing/CONTRIBUTING/"> <button class="button">Contributing</button> </a>
+    <a href="https://github.com/invoke-ai/InvokeAI/"> <button class="button">Code and Downloads</button> </a>
+    <a href="https://github.com/invoke-ai/InvokeAI/issues"> <button class="button">Bug Reports </button> </a>
+    <a href="https://discord.gg/ZmtBAhwWhy"> <button class="button"> Join the Discord Server!</button> </a>
+</div>
 
 
 ## :octicons-gift-24: InvokeAI Features
@@ -230,7 +219,7 @@ encouraged to do so.
 
 ## :octicons-person-24: Contributors
 
-This fork is a combined effort of various people from across the world.
+This software is a combined effort of various people from across the world.
 [Check out the list of all these amazing people](other/CONTRIBUTORS.md). We
 thank them for their time, hard work and effort.
 

docs/installation/010_INSTALL_AUTOMATED.md

@@ -264,7 +264,7 @@ experimental versions later.
 	  you can create several levels of subfolders and drop your models into
       whichever ones you want.
 	
-    - ***Autoimport FolderLICENSE***     
+    - ***LICENSE***     
 
     At the bottom of the screen you will see a checkbox for accepting
     the CreativeML Responsible AI Licenses. You need to accept the license
@@ -372,8 +372,71 @@ experimental versions later.
         Once InvokeAI is installed, do not move or remove this directory."
 
 
+<a name="troubleshooting"></a>
 ## Troubleshooting
 
+### _OSErrors on Windows while installing dependencies_
+
+During a zip file installation or an online update, installation stops
+with an error like this:
+
+![broken-dependency-screenshot](../assets/troubleshooting/broken-dependency.png){:width="800px"}
+
+This seems to happen particularly often with the `pydantic` and
+`numpy` packages. The most reliable solution requires several manual
+steps to complete installation.
+
+Open up a Powershell window and navigate to the `invokeai` directory
+created by the installer. Then give the following series of commands:
+
+```cmd
+rm .\.venv -r -force
+python -mvenv .venv
+.\.venv\Scripts\activate
+pip install invokeai
+invokeai-configure --yes --root .
+```
+
+If you see anything marked as an error during this process please stop
+and seek help on the Discord [installation support
+channel](https://discord.com/channels/1020123559063990373/1041391462190956654). A
+few warning messages are OK.
+
+If you are updating from a previous version, this should restore your
+system to a working state. If you are installing from scratch, there
+is one additional command to give:
+
+```cmd
+wget -O invoke.bat https://raw.githubusercontent.com/invoke-ai/InvokeAI/main/installer/templates/invoke.bat.in
+```
+
+This will create the `invoke.bat` script needed to launch InvokeAI and
+its related programs.
+
+
+### _Stable Diffusion XL Generation Fails after Trying to Load unet_
+
+InvokeAI is working in other respects, but when trying to generate
+images with Stable Diffusion XL you get a "Server Error". The text log
+in the launch window contains this log line above several more lines of
+error messages:
+
+```INFO --> Loading model:D:\LONG\PATH\TO\MODEL, type sdxl:main:unet```
+
+This failure mode occurs when there is a network glitch during
+downloading the very large SDXL model.
+
+To address this, first go to the Web Model Manager and delete the
+Stable-Diffusion-XL-base-1.X model. Then navigate to HuggingFace and
+manually download the .safetensors version of the model. The 1.0
+version is located at
+https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/tree/main
+and the file is named `sd_xl_base_1.0.safetensors`.
+
+Save this file to disk and then reenter the Model Manager. Navigate to
+Import Models->Add Model, then type (or drag-and-drop) the path to the
+.safetensors file. Press "Add Model".
+
 ### _Package dependency conflicts_
 
 If you have previously installed InvokeAI or another Stable Diffusion
@@ -408,7 +471,7 @@ Then type the following commands:
 
 === "NVIDIA System"
     ```bash
-    pip install torch torchvision --force-reinstall --extra-index-url https://download.pytorch.org/whl/cu117
+    pip install torch torchvision --force-reinstall --extra-index-url https://download.pytorch.org/whl/cu118
     pip install xformers
     ```
 

docs/installation/020_INSTALL_MANUAL.md

@@ -148,7 +148,7 @@ manager, please follow these steps:
     === "CUDA (NVidia)"
 
         ```bash
-        pip install "InvokeAI[xformers]" --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu117
+        pip install "InvokeAI[xformers]" --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu118
         ```
 
     === "ROCm (AMD)"
@@ -192,8 +192,10 @@ manager, please follow these steps:
     your outputs.
 
     ```terminal
-    invokeai-configure
+    invokeai-configure --root .
     ```
+	
+	Don't miss the dot at the end of the command!
 
     The script `invokeai-configure` will interactively guide you through the
     process of downloading and installing the weights files needed for InvokeAI.
@@ -225,12 +227,6 @@ manager, please follow these steps:
 
         !!! warning "Make sure that the virtual environment is activated, which should create `(.venv)` in front of your prompt!"
 
-        === "CLI"
-
-            ```bash
-            invokeai
-            ```
-
         === "local Webserver"
 
             ```bash
@@ -243,6 +239,12 @@ manager, please follow these steps:
             invokeai --web --host 0.0.0.0
             ```
 
+        === "CLI"
+
+            ```bash
+            invokeai
+            ```
+
         If you choose the run the web interface, point your browser at
         http://localhost:9090 in order to load the GUI.
 
@@ -310,7 +312,7 @@ installation protocol (important!)
 
     === "CUDA (NVidia)"
         ```bash
-        pip install -e .[xformers] --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu117
+        pip install -e .[xformers] --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu118
         ```
 
     === "ROCm (AMD)"
@@ -354,7 +356,7 @@ you can do so using this unsupported recipe:
 mkdir ~/invokeai
 conda create -n invokeai python=3.10
 conda activate invokeai
-pip install InvokeAI[xformers] --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu117
+pip install InvokeAI[xformers] --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu118
 invokeai-configure --root ~/invokeai
 invokeai --root ~/invokeai --web
 ```

docs/installation/030_INSTALL_CUDA_AND_ROCM.md

@@ -34,11 +34,11 @@ directly from NVIDIA. **Do not try to install Ubuntu's
 nvidia-cuda-toolkit package. It is out of date and will cause
 conflicts among the NVIDIA driver and binaries.**
 
-Go to [CUDA Toolkit 11.7
-Downloads](https://developer.nvidia.com/cuda-11-7-0-download-archive),
-and use the target selection wizard to choose your operating system,
-hardware platform, and preferred installation method (e.g. "local"
-versus "network").
+Go to [CUDA Toolkit
+Downloads](https://developer.nvidia.com/cuda-downloads), and use the
+target selection wizard to choose your operating system, hardware
+platform, and preferred installation method (e.g. "local" versus
+"network").
 
 This will provide you with a downloadable install file or, depending
 on your choices, a recipe for downloading and running a install shell
@@ -61,7 +61,7 @@ Runtime Site](https://developer.nvidia.com/nvidia-container-runtime)
 
 When installing torch and torchvision manually with `pip`, remember to provide
 the argument `--extra-index-url
-https://download.pytorch.org/whl/cu117` as described in the [Manual
+https://download.pytorch.org/whl/cu118` as described in the [Manual
 Installation Guide](020_INSTALL_MANUAL.md).
 
 ## :simple-amd: ROCm

docs/installation/050_INSTALLING_MODELS.md

@@ -124,7 +124,7 @@ installation. Examples:
 invokeai-model-install --list controlnet
 
 # (install the model at the indicated URL)
-invokeai-model-install --add http://civitai.com/2860
+invokeai-model-install --add https://civitai.com/api/download/models/128713
 
 # (delete the named model)
 invokeai-model-install --delete sd-1/main/analog-diffusion
@@ -170,4 +170,4 @@ elsewhere on disk and they will be autoimported. You can also create
 subfolders and organize them as you wish.
 
 The location of the autoimport directories are controlled by settings
-in `invokeai.yaml`. See [Configuration](../features/CONFIGURATION.md).
+in `invokeai.yaml`. See [Configuration](../features/CONFIGURATION.md).

docs/installation/070_INSTALL_XFORMERS.md

@@ -28,18 +28,21 @@ command line, then just be sure to activate it's virtual environment.
 Then run the following three commands:
 
 ```sh
-pip install xformers==0.0.16rc425
-pip install triton
+pip install xformers~=0.0.19
+pip install triton    # WON'T WORK ON WINDOWS
 python -m xformers.info output
 ```
 
 The first command installs `xformers`, the second installs the
 `triton` training accelerator, and the third prints out the `xformers`
-installation status. If all goes well, you'll see a report like the
+installation status. On Windows, please omit the `triton` package,
+which is not available on that platform.
+
+If all goes well, you'll see a report like the
 following:
 
 ```sh
-xFormers 0.0.16rc425
+xFormers 0.0.20
 memory_efficient_attention.cutlassF:               available
 memory_efficient_attention.cutlassB:               available
 memory_efficient_attention.flshattF:               available
@@ -48,22 +51,28 @@ memory_efficient_attention.smallkF:                available
 memory_efficient_attention.smallkB:                available
 memory_efficient_attention.tritonflashattF:        available
 memory_efficient_attention.tritonflashattB:        available
+indexing.scaled_index_addF:                        available
+indexing.scaled_index_addB:                        available
+indexing.index_select:                             available
+swiglu.dual_gemm_silu:                             available
+swiglu.gemm_fused_operand_sum:                     available
 swiglu.fused.p.cpp:                                available
 is_triton_available:                               True
 is_functorch_available:                            False
-pytorch.version:                                   1.13.1+cu117
+pytorch.version:                                   2.0.1+cu118
 pytorch.cuda:                                      available
-gpu.compute_capability:                            8.6
-gpu.name:                                          NVIDIA RTX A2000 12GB
+gpu.compute_capability:                            8.9
+gpu.name:                                          NVIDIA GeForce RTX 4070
 build.info:                                        available
-build.cuda_version:                                1107
-build.python_version:                              3.10.9
-build.torch_version:                               1.13.1+cu117
+build.cuda_version:                                1108
+build.python_version:                              3.10.11
+build.torch_version:                               2.0.1+cu118
 build.env.TORCH_CUDA_ARCH_LIST:                    5.0+PTX 6.0 6.1 7.0 7.5 8.0 8.6
 build.env.XFORMERS_BUILD_TYPE:                     Release
 build.env.XFORMERS_ENABLE_DEBUG_ASSERTIONS:        None
 build.env.NVCC_FLAGS:                              None
-build.env.XFORMERS_PACKAGE_FROM:                   wheel-v0.0.16rc425
+build.env.XFORMERS_PACKAGE_FROM:                   wheel-v0.0.20
+build.nvcc_version:                                11.8.89
 source.privacy:                                    open source
 ```
 
@@ -83,14 +92,14 @@ installed from source. These instructions were written for a system
 running Ubuntu 22.04, but other Linux distributions should be able to
 adapt this recipe.
 
-#### 1. Install CUDA Toolkit 11.7
+#### 1. Install CUDA Toolkit 11.8
 
 You will need the CUDA developer's toolkit in order to compile and
 install xFormers. **Do not try to install Ubuntu's nvidia-cuda-toolkit
 package.** It is out of date and will cause conflicts among the NVIDIA
 driver and binaries. Instead install the CUDA Toolkit package provided
-by NVIDIA itself. Go to [CUDA Toolkit 11.7
-Downloads](https://developer.nvidia.com/cuda-11-7-0-download-archive)
+by NVIDIA itself. Go to [CUDA Toolkit 11.8
+Downloads](https://developer.nvidia.com/cuda-11-8-0-download-archive)
 and use the target selection wizard to choose your platform and Linux
 distribution. Select an installer type of "runfile (local)" at the
 last step.
@@ -101,17 +110,17 @@ example, the install script recipe for Ubuntu 22.04 running on a
 x86_64 system is:
 
 ```
-wget https://developer.download.nvidia.com/compute/cuda/11.7.0/local_installers/cuda_11.7.0_515.43.04_linux.run
-sudo sh cuda_11.7.0_515.43.04_linux.run
+wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
+sudo sh cuda_11.8.0_520.61.05_linux.run
 ```
 
 Rather than cut-and-paste this example, We recommend that you walk
 through the toolkit wizard in order to get the most up to date
 installer for your system.
 
-#### 2. Confirm/Install pyTorch 1.13 with CUDA 11.7 support
+#### 2. Confirm/Install pyTorch 2.01 with CUDA 11.8 support
 
-If you are using InvokeAI 2.3 or higher, these will already be
+If you are using InvokeAI 3.0.2 or higher, these will already be
 installed. If not, you can check whether you have the needed libraries
 using a quick command. Activate the invokeai virtual environment,
 either by entering the "developer's console", or manually with a
@@ -124,7 +133,7 @@ Then run the command:
 python -c 'exec("import torch\nprint(torch.__version__)")'
 ```
 
-If it prints __1.13.1+cu117__ you're good. If not, you can install the
+If it prints __1.13.1+cu118__ you're good. If not, you can install the
 most up to date libraries with this command:
 
 ```sh

docs/installation/INSTALLATION.md

@@ -1,6 +1,4 @@
----
-title: Overview
----
+# Overview
 
 We offer several ways to install InvokeAI, each one suited to your
 experience and preferences. We suggest that everyone start by
@@ -15,6 +13,56 @@ See the [troubleshooting
 section](010_INSTALL_AUTOMATED.md#troubleshooting) of the automated
 install guide for frequently-encountered installation issues.
 
+This fork is supported across Linux, Windows and Macintosh. Linux users can use
+either an Nvidia-based card (with CUDA support) or an AMD card (using the ROCm
+driver).
+
+### [Installation Getting Started Guide](installation)
+#### **[Automated Installer](010_INSTALL_AUTOMATED.md)**
+✅ This is the recommended installation method for first-time users. 
+#### [Manual Installation](020_INSTALL_MANUAL.md)
+This method is recommended for experienced users and developers
+#### [Docker Installation](040_INSTALL_DOCKER.md)
+This method is recommended for those familiar with running Docker containers
+### Other Installation Guides
+  - [PyPatchMatch](060_INSTALL_PATCHMATCH.md)
+  - [XFormers](070_INSTALL_XFORMERS.md)
+  - [CUDA and ROCm Drivers](030_INSTALL_CUDA_AND_ROCM.md)
+  - [Installing New Models](050_INSTALLING_MODELS.md)
+
+## :fontawesome-solid-computer: Hardware Requirements
+
+### :octicons-cpu-24: System
+
+You wil need one of the following:
+
+- :simple-nvidia: An NVIDIA-based graphics card with 4 GB or more VRAM memory.
+- :simple-amd: An AMD-based graphics card with 4 GB or more VRAM memory (Linux
+  only)
+- :fontawesome-brands-apple: An Apple computer with an M1 chip.
+
+** SDXL 1.0 Requirements*
+To use SDXL, user must have one of the following: 
+- :simple-nvidia: An NVIDIA-based graphics card with 8 GB or more VRAM memory.
+- :simple-amd: An AMD-based graphics card with 16 GB or more VRAM memory (Linux
+  only)
+- :fontawesome-brands-apple: An Apple computer with an M1 chip.
+
+
+### :fontawesome-solid-memory: Memory and Disk
+
+- At least 12 GB Main Memory RAM.
+- At least 18 GB of free disk space for the machine learning model, Python, and
+  all its dependencies.
+
+We do **not recommend** the following video cards due to issues with their
+running in half-precision mode and having insufficient VRAM to render 512x512
+images in full-precision mode:
+
+- NVIDIA 10xx series cards such as the 1080ti
+- GTX 1650 series cards
+- GTX 1660 series cards
+
 ## Installation options
 
 1. [Automated Installer](010_INSTALL_AUTOMATED.md)

docs/nodes/communityNodes.md

@@ -14,23 +14,28 @@ The nodes linked below have been developed and contributed by members of the Inv
 
 ## List of Nodes
 
-### Face Mask
+### FaceTools
 
-**Description:** This node autodetects a face in the image using MediaPipe and masks it by making it transparent. Via outpainting you can swap faces with other faces, or invert the mask and swap things around the face with other things. Additionally, you can supply X and Y offset values to scale/change the shape of the mask for finer control. The node also outputs an all-white mask in the same dimensions as the input image. This is needed by the inpaint node (and unified canvas) for outpainting.
+**Description:** FaceTools is a collection of nodes created to manipulate faces as you would in Unified Canvas. It includes FaceMask, FaceOff, and FacePlace. FaceMask autodetects a face in the image using MediaPipe and creates a mask from it. FaceOff similarly detects a face, then takes the face off of the image by adding a square bounding box around it and cropping/scaling it. FacePlace puts the bounded face image from FaceOff back onto the original image. Using these nodes with other inpainting node(s), you can put new faces on existing things, put new things around existing faces, and work closer with a face as a bounded image. Additionally, you can supply X and Y offset values to scale/change the shape of the mask for finer control on FaceMask and FaceOff. See GitHub repository below for usage examples.
 
-**Node Link:** https://github.com/ymgenesis/InvokeAI/blob/facemaskmediapipe/invokeai/app/invocations/facemask.py
+**Node Link:** https://github.com/ymgenesis/FaceTools/
 
-**Example Node Graph:** https://www.mediafire.com/file/gohn5sb1bfp8use/21-July_2023-FaceMask.json/file
+**FaceMask Output Examples** 
 
-**Output Examples** 
+![5cc8abce-53b0-487a-b891-3bf94dcc8960](https://github.com/invoke-ai/InvokeAI/assets/25252829/43f36d24-1429-4ab1-bd06-a4bedfe0955e)
+![b920b710-1882-49a0-8d02-82dff2cca907](https://github.com/invoke-ai/InvokeAI/assets/25252829/7660c1ed-bf7d-4d0a-947f-1fc1679557ba)
+![71a91805-fda5-481c-b380-264665703133](https://github.com/invoke-ai/InvokeAI/assets/25252829/f8f6a2ee-2b68-4482-87da-b90221d5c3e2)
 
-![2e3168cb-af6a-475d-bfac-c7b7fd67b4c2](https://github.com/ymgenesis/InvokeAI/assets/25252829/a5ad7d44-2ada-4b3c-a56e-a21f8244a1ac)
-![2_annotated](https://github.com/ymgenesis/InvokeAI/assets/25252829/53416c8a-a23b-4d76-bb6d-3cfd776e0096)
-![2fe2150c-fd08-4e26-8c36-f0610bf441bb](https://github.com/ymgenesis/InvokeAI/assets/25252829/b0f7ecfe-f093-4147-a904-b9f131b41dc9)
-![831b6b98-4f0f-4360-93c8-69a9c1338cbe](https://github.com/ymgenesis/InvokeAI/assets/25252829/fc7b0622-e361-4155-8a76-082894d084f0)
+<hr>
+
+### Ideal Size
+
+**Description:** This node calculates an ideal image size for a first pass of a multi-pass upscaling. The aim is to avoid duplication that results from choosing a size larger than the model is capable of.
+
+**Node Link:** https://github.com/JPPhoto/ideal-size-node
 
 --------------------------------
-### Super Cool Node Template
+### Example Node Template
 
 **Description:** This node allows you to do super cool things with InvokeAI.
 
@@ -40,13 +45,9 @@ The nodes linked below have been developed and contributed by members of the Inv
 
 **Output Examples** 
 
-![Invoke AI](https://invoke-ai.github.io/InvokeAI/assets/invoke_ai_banner.png)
-
-### Ideal Size
-
-**Description:** This node calculates an ideal image size for a first pass of a multi-pass upscaling. The aim is to avoid duplication that results from choosing a size larger than the model is capable of.
-
-**Node Link:** https://github.com/JPPhoto/ideal-size-node
+![Example Image](https://invoke-ai.github.io/InvokeAI/assets/invoke_ai_banner.png){: style="height:115px;width:240px"}
 
 ## Help
 If you run into any issues with a node, please post in the [InvokeAI Discord](https://discord.gg/ZmtBAhwWhy). 
+
+

flake.lock

@@ -0,0 +1,25 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1690630721,
+        "narHash": "sha256-Y04onHyBQT4Erfr2fc82dbJTfXGYrf4V0ysLUYnPOP8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d2b52322f35597c62abf56de91b0236746b2a03d",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,91 @@
+# Important note: this flake does not attempt to create a fully isolated, 'pure'
+# Python environment for InvokeAI. Instead, it depends on local invocations of
+# virtualenv/pip to install the required (binary) packages, most importantly the
+# prebuilt binary pytorch packages with CUDA support.
+# ML Python packages with CUDA support, like pytorch, are notoriously expensive
+# to compile so it's purposefuly not what this flake does.
+
+{
+  description = "An (impure) flake to develop on InvokeAI.";
+
+  outputs = { self, nixpkgs }:
+    let
+      system = "x86_64-linux";
+      pkgs = import nixpkgs {
+        inherit system;
+        config.allowUnfree = true;
+      };
+
+      python = pkgs.python310;
+
+      mkShell = { dir, install }:
+        let
+          setupScript = pkgs.writeScript "setup-invokai" ''
+            # This must be sourced using 'source', not executed.
+            ${python}/bin/python -m venv ${dir}
+            ${dir}/bin/python -m pip install ${install}
+            # ${dir}/bin/python -c 'import torch; assert(torch.cuda.is_available())'
+            source ${dir}/bin/activate
+          '';
+        in
+        pkgs.mkShell rec {
+          buildInputs = with pkgs; [
+            # Backend: graphics, CUDA.
+            cudaPackages.cudnn
+            cudaPackages.cuda_nvrtc
+            cudatoolkit
+            pkgconfig
+            libconfig
+            cmake
+            blas
+            freeglut
+            glib
+            gperf
+            procps
+            libGL
+            libGLU
+            linuxPackages.nvidia_x11
+            python
+            (opencv4.override {
+              enableGtk3 = true;
+              enableFfmpeg = true;
+              enableCuda = true;
+              enableUnfree = true;
+            })
+            stdenv.cc
+            stdenv.cc.cc.lib
+            xorg.libX11
+            xorg.libXext
+            xorg.libXi
+            xorg.libXmu
+            xorg.libXrandr
+            xorg.libXv
+            zlib
+
+            # Pre-commit hooks.
+            black
+
+            # Frontend.
+            yarn
+            nodejs
+          ];
+          LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
+          CUDA_PATH = pkgs.cudatoolkit;
+          EXTRA_LDFLAGS = "-L${pkgs.linuxPackages.nvidia_x11}/lib";
+          shellHook = ''
+            if [[ -f "${dir}/bin/activate" ]]; then
+              source "${dir}/bin/activate"
+              echo "Using Python: $(which python)"
+            else
+              echo "Use 'source ${setupScript}' to set up the environment."
+            fi
+          '';
+        };
+    in
+    {
+      devShells.${system} = rec {
+        develop = mkShell { dir = "venv"; install = "-e '.[xformers]' --extra-index-url https://download.pytorch.org/whl/cu118"; };
+        default = develop;
+      };
+    };
+}

installer/lib/installer.py

@@ -13,7 +13,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Union
 
-SUPPORTED_PYTHON = ">=3.9.0,<3.11"
+SUPPORTED_PYTHON = ">=3.9.0,<=3.11.100"
 INSTALLER_REQS = ["rich", "semver", "requests", "plumbum", "prompt-toolkit"]
 BOOTSTRAP_VENV_PREFIX = "invokeai-installer-tmp"
 
@@ -149,7 +149,7 @@ class Installer:
         return venv_dir
 
     def install(
-        self, root: str = "~/invokeai-3", version: str = "latest", yes_to_all=False, find_links: Path = None
+        self, root: str = "~/invokeai", version: str = "latest", yes_to_all=False, find_links: Path = None
     ) -> None:
         """
         Install the InvokeAI application into the given runtime path
@@ -168,7 +168,8 @@ class Installer:
 
         messages.welcome()
 
-        self.dest = Path(root).expanduser().resolve() if yes_to_all else messages.dest_path(root)
+        default_path = os.environ.get("INVOKEAI_ROOT") or Path(root).expanduser().resolve()
+        self.dest = default_path if yes_to_all else messages.dest_path(root)
 
         # create the venv for the app
         self.venv = self.app_venv()
@@ -248,6 +249,9 @@ class InvokeAiInstance:
             pip[
                 "install",
                 "--require-virtualenv",
+                "numpy~=1.24.0",  # choose versions that won't be uninstalled during phase 2
+                "urllib3~=1.26.0",
+                "requests~=2.28.0",
                 "torch~=2.0.0",
                 "torchmetrics==0.11.4",
                 "torchvision>=0.14.1",
@@ -344,7 +348,7 @@ class InvokeAiInstance:
 
         introduction()
 
-        from invokeai.frontend.install import invokeai_configure
+        from invokeai.frontend.install.invokeai_configure import invokeai_configure
 
         # NOTE: currently the config script does its own arg parsing! this means the command-line switches
         # from the installer will also automatically propagate down to the config script.
@@ -451,7 +455,7 @@ def get_torch_source() -> (Union[str, None], str):
     device = graphical_accelerator()
 
     url = None
-    optional_modules = None
+    optional_modules = "[onnx]"
     if OS == "Linux":
         if device == "rocm":
             url = "https://download.pytorch.org/whl/rocm5.4.2"
@@ -459,8 +463,11 @@ def get_torch_source() -> (Union[str, None], str):
             url = "https://download.pytorch.org/whl/cpu"
 
     if device == "cuda":
-        url = "https://download.pytorch.org/whl/cu117"
-        optional_modules = "[xformers]"
+        url = "https://download.pytorch.org/whl/cu118"
+        optional_modules = "[xformers,onnx-cuda]"
+    if device == "cuda_and_dml":
+        url = "https://download.pytorch.org/whl/cu118"
+        optional_modules = "[xformers,onnx-directml]"
 
     # in all other cases, Torch wheels should be coming from PyPi as of Torch 1.13
 

installer/lib/main.py

@@ -3,6 +3,7 @@ InvokeAI Installer
 """
 
 import argparse
+import os
 from pathlib import Path
 from installer import Installer
 
@@ -15,7 +16,7 @@ if __name__ == "__main__":
         dest="root",
         type=str,
         help="Destination path for installation",
-        default="~/invokeai",
+        default=os.environ.get("INVOKEAI_ROOT") or "~/invokeai",
     )
     parser.add_argument(
         "-y",

installer/lib/messages.py

@@ -167,6 +167,10 @@ def graphical_accelerator():
         "an [gold1 b]NVIDIA[/] GPU (using CUDA™)",
         "cuda",
     )
+    nvidia_with_dml = (
+        "an [gold1 b]NVIDIA[/] GPU (using CUDA™, and DirectML™ for ONNX) -- ALPHA",
+        "cuda_and_dml",
+    )
     amd = (
         "an [gold1 b]AMD[/] GPU (using ROCm™)",
         "rocm",
@@ -181,7 +185,7 @@ def graphical_accelerator():
     )
 
     if OS == "Windows":
-        options = [nvidia, cpu]
+        options = [nvidia, nvidia_with_dml, cpu]
     if OS == "Linux":
         options = [nvidia, amd, cpu]
     elif OS == "Darwin":

installer/readme.txt

@@ -8,16 +8,13 @@ Preparations:
   to work. Instructions are given here:
   https://invoke-ai.github.io/InvokeAI/installation/INSTALL_AUTOMATED/
 
-  NOTE: At this time we do not recommend Python 3.11. We recommend 
-  Version 3.10.9, which has been extensively tested with InvokeAI.
-  
   Before you start the installer, please open up your system's command
   line window (Terminal or Command) and type the commands:
 
      python --version
 
   If all is well, it will print "Python 3.X.X", where the version number
-  is at least 3.9.1, and less than 3.11.
+  is at least 3.9.*, and not higher than 3.11.*.
 
   If this works, check the version of the Python package manager, pip:
 

installer/templates/invoke.bat.in

@@ -41,7 +41,7 @@ IF /I "%choice%" == "1" (
     python .venv\Scripts\invokeai-configure.exe --skip-sd-weight --skip-support-models
 ) ELSE IF /I "%choice%" == "7" (
     echo Running invokeai-configure...
-    python .venv\Scripts\invokeai-configure.exe --yes --default_only
+    python .venv\Scripts\invokeai-configure.exe --yes --skip-sd-weight
 ) ELSE IF /I "%choice%" == "8" (
     echo Developer Console
     echo Python command is:

installer/templates/invoke.sh.in

@@ -82,7 +82,7 @@ do_choice() {
     7)
         clear
         printf "Re-run the configure script to fix a broken install or to complete a major upgrade\n"
-        invokeai-configure --root ${INVOKEAI_ROOT} --yes --default_only
+        invokeai-configure --root ${INVOKEAI_ROOT} --yes --default_only --skip-sd-weights
         ;;
     8)
         clear

invokeai/app/api/dependencies.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
+from typing import Optional
 from logging import Logger
-import os
 from invokeai.app.services.board_image_record_storage import (
     SqliteBoardImageRecordStorage,
 )
@@ -29,6 +29,7 @@ from ..services.invoker import Invoker
 from ..services.processor import DefaultInvocationProcessor
 from ..services.sqlite import SqliteItemStorage
 from ..services.model_manager_service import ModelManagerService
+from ..services.invocation_stats import InvocationStatsService
 from .events import FastAPIEventService
 
 
@@ -54,7 +55,7 @@ logger = InvokeAILogger.getLogger()
 class ApiDependencies:
     """Contains and initializes all dependencies for the API"""
 
-    invoker: Invoker = None
+    invoker: Invoker
 
     @staticmethod
     def initialize(config: InvokeAIAppConfig, event_handler_id: int, logger: Logger = logger):
@@ -67,8 +68,9 @@ class ApiDependencies:
         output_folder = config.output_path
 
         # TODO: build a file/path manager?
-        db_location = config.db_path
-        db_location.parent.mkdir(parents=True, exist_ok=True)
+        db_path = config.db_path
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        db_location = str(db_path)
 
         graph_execution_manager = SqliteItemStorage[GraphExecutionState](
             filename=db_location, table_name="graph_executions"
@@ -127,6 +129,7 @@ class ApiDependencies:
             graph_execution_manager=graph_execution_manager,
             processor=DefaultInvocationProcessor(),
             configuration=config,
+            performance_statistics=InvocationStatsService(graph_execution_manager),
             logger=logger,
         )
 

invokeai/app/api/routers/board_images.py

@@ -1,24 +1,30 @@
-from fastapi import Body, HTTPException, Path, Query
+from fastapi import Body, HTTPException
 from fastapi.routing import APIRouter
-from invokeai.app.services.board_record_storage import BoardRecord, BoardChanges
-from invokeai.app.services.image_record_storage import OffsetPaginatedResults
-from invokeai.app.services.models.board_record import BoardDTO
-from invokeai.app.services.models.image_record import ImageDTO
+from pydantic import BaseModel, Field
 
 from ..dependencies import ApiDependencies
 
 board_images_router = APIRouter(prefix="/v1/board_images", tags=["boards"])
 
 
+class AddImagesToBoardResult(BaseModel):
+    board_id: str = Field(description="The id of the board the images were added to")
+    added_image_names: list[str] = Field(description="The image names that were added to the board")
+
+
+class RemoveImagesFromBoardResult(BaseModel):
+    removed_image_names: list[str] = Field(description="The image names that were removed from their board")
+
+
 @board_images_router.post(
     "/",
-    operation_id="create_board_image",
+    operation_id="add_image_to_board",
     responses={
         201: {"description": "The image was added to a board successfully"},
     },
     status_code=201,
 )
-async def create_board_image(
+async def add_image_to_board(
     board_id: str = Body(description="The id of the board to add to"),
     image_name: str = Body(description="The name of the image to add"),
 ):
@@ -29,26 +35,78 @@ async def create_board_image(
         )
         return result
     except Exception as e:
-        raise HTTPException(status_code=500, detail="Failed to add to board")
+        raise HTTPException(status_code=500, detail="Failed to add image to board")
 
 
 @board_images_router.delete(
     "/",
-    operation_id="remove_board_image",
+    operation_id="remove_image_from_board",
     responses={
         201: {"description": "The image was removed from the board successfully"},
     },
     status_code=201,
 )
-async def remove_board_image(
-    board_id: str = Body(description="The id of the board"),
-    image_name: str = Body(description="The name of the image to remove"),
+async def remove_image_from_board(
+    image_name: str = Body(description="The name of the image to remove", embed=True),
 ):
-    """Deletes a board_image"""
+    """Removes an image from its board, if it had one"""
     try:
-        result = ApiDependencies.invoker.services.board_images.remove_image_from_board(
-            board_id=board_id, image_name=image_name
-        )
+        result = ApiDependencies.invoker.services.board_images.remove_image_from_board(image_name=image_name)
         return result
     except Exception as e:
-        raise HTTPException(status_code=500, detail="Failed to update board")
+        raise HTTPException(status_code=500, detail="Failed to remove image from board")
+
+
+@board_images_router.post(
+    "/batch",
+    operation_id="add_images_to_board",
+    responses={
+        201: {"description": "Images were added to board successfully"},
+    },
+    status_code=201,
+    response_model=AddImagesToBoardResult,
+)
+async def add_images_to_board(
+    board_id: str = Body(description="The id of the board to add to"),
+    image_names: list[str] = Body(description="The names of the images to add", embed=True),
+) -> AddImagesToBoardResult:
+    """Adds a list of images to a board"""
+    try:
+        added_image_names: list[str] = []
+        for image_name in image_names:
+            try:
+                ApiDependencies.invoker.services.board_images.add_image_to_board(
+                    board_id=board_id, image_name=image_name
+                )
+                added_image_names.append(image_name)
+            except:
+                pass
+        return AddImagesToBoardResult(board_id=board_id, added_image_names=added_image_names)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Failed to add images to board")
+
+
+@board_images_router.post(
+    "/batch/delete",
+    operation_id="remove_images_from_board",
+    responses={
+        201: {"description": "Images were removed from board successfully"},
+    },
+    status_code=201,
+    response_model=RemoveImagesFromBoardResult,
+)
+async def remove_images_from_board(
+    image_names: list[str] = Body(description="The names of the images to remove", embed=True),
+) -> RemoveImagesFromBoardResult:
+    """Removes a list of images from their board, if they had one"""
+    try:
+        removed_image_names: list[str] = []
+        for image_name in image_names:
+            try:
+                ApiDependencies.invoker.services.board_images.remove_image_from_board(image_name=image_name)
+                removed_image_names.append(image_name)
+            except:
+                pass
+        return RemoveImagesFromBoardResult(removed_image_names=removed_image_names)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Failed to remove images from board")

invokeai/app/api/routers/images.py

@@ -1,31 +1,31 @@
 import io
 from typing import Optional
 
+from PIL import Image
 from fastapi import Body, HTTPException, Path, Query, Request, Response, UploadFile
 from fastapi.responses import FileResponse
 from fastapi.routing import APIRouter
-from PIL import Image
+from pydantic import BaseModel, Field
 
 from invokeai.app.invocations.metadata import ImageMetadata
 from invokeai.app.models.image import ImageCategory, ResourceOrigin
 from invokeai.app.services.image_record_storage import OffsetPaginatedResults
-from invokeai.app.services.item_storage import PaginatedResults
 from invokeai.app.services.models.image_record import (
     ImageDTO,
     ImageRecordChanges,
     ImageUrlsDTO,
 )
-
 from ..dependencies import ApiDependencies
 
 images_router = APIRouter(prefix="/v1/images", tags=["images"])
 
+
 # images are immutable; set a high max-age
 IMAGE_MAX_AGE = 31536000
 
 
 @images_router.post(
-    "/",
+    "/upload",
     operation_id="upload_image",
     responses={
         201: {"description": "The image was uploaded successfully"},
@@ -77,7 +77,7 @@ async def upload_image(
         raise HTTPException(status_code=500, detail="Failed to create image")
 
 
-@images_router.delete("/{image_name}", operation_id="delete_image")
+@images_router.delete("/i/{image_name}", operation_id="delete_image")
 async def delete_image(
     image_name: str = Path(description="The name of the image to delete"),
 ) -> None:
@@ -103,7 +103,7 @@ async def clear_intermediates() -> int:
 
 
 @images_router.patch(
-    "/{image_name}",
+    "/i/{image_name}",
     operation_id="update_image",
     response_model=ImageDTO,
 )
@@ -120,7 +120,7 @@ async def update_image(
 
 
 @images_router.get(
-    "/{image_name}",
+    "/i/{image_name}",
     operation_id="get_image_dto",
     response_model=ImageDTO,
 )
@@ -136,7 +136,7 @@ async def get_image_dto(
 
 
 @images_router.get(
-    "/{image_name}/metadata",
+    "/i/{image_name}/metadata",
     operation_id="get_image_metadata",
     response_model=ImageMetadata,
 )
@@ -151,8 +151,9 @@ async def get_image_metadata(
         raise HTTPException(status_code=404)
 
 
-@images_router.get(
-    "/{image_name}/full",
+@images_router.api_route(
+    "/i/{image_name}/full",
+    methods=["GET", "HEAD"],
     operation_id="get_image_full",
     response_class=Response,
     responses={
@@ -187,7 +188,7 @@ async def get_image_full(
 
 
 @images_router.get(
-    "/{image_name}/thumbnail",
+    "/i/{image_name}/thumbnail",
     operation_id="get_image_thumbnail",
     response_class=Response,
     responses={
@@ -216,7 +217,7 @@ async def get_image_thumbnail(
 
 
 @images_router.get(
-    "/{image_name}/urls",
+    "/i/{image_name}/urls",
     operation_id="get_image_urls",
     response_model=ImageUrlsDTO,
 )
@@ -265,3 +266,62 @@ async def list_image_dtos(
     )
 
     return image_dtos
+
+
+class DeleteImagesFromListResult(BaseModel):
+    deleted_images: list[str]
+
+
+@images_router.post("/delete", operation_id="delete_images_from_list", response_model=DeleteImagesFromListResult)
+async def delete_images_from_list(
+    image_names: list[str] = Body(description="The list of names of images to delete", embed=True),
+) -> DeleteImagesFromListResult:
+    try:
+        deleted_images: list[str] = []
+        for image_name in image_names:
+            try:
+                ApiDependencies.invoker.services.images.delete(image_name)
+                deleted_images.append(image_name)
+            except:
+                pass
+        return DeleteImagesFromListResult(deleted_images=deleted_images)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Failed to delete images")
+
+
+class ImagesUpdatedFromListResult(BaseModel):
+    updated_image_names: list[str] = Field(description="The image names that were updated")
+
+
+@images_router.post("/star", operation_id="star_images_in_list", response_model=ImagesUpdatedFromListResult)
+async def star_images_in_list(
+    image_names: list[str] = Body(description="The list of names of images to star", embed=True),
+) -> ImagesUpdatedFromListResult:
+    try:
+        updated_image_names: list[str] = []
+        for image_name in image_names:
+            try:
+                ApiDependencies.invoker.services.images.update(image_name, changes=ImageRecordChanges(starred=True))
+                updated_image_names.append(image_name)
+            except:
+                pass
+        return ImagesUpdatedFromListResult(updated_image_names=updated_image_names)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Failed to star images")
+
+
+@images_router.post("/unstar", operation_id="unstar_images_in_list", response_model=ImagesUpdatedFromListResult)
+async def unstar_images_in_list(
+    image_names: list[str] = Body(description="The list of names of images to unstar", embed=True),
+) -> ImagesUpdatedFromListResult:
+    try:
+        updated_image_names: list[str] = []
+        for image_name in image_names:
+            try:
+                ApiDependencies.invoker.services.images.update(image_name, changes=ImageRecordChanges(starred=False))
+                updated_image_names.append(image_name)
+            except:
+                pass
+        return ImagesUpdatedFromListResult(updated_image_names=updated_image_names)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Failed to unstar images")

invokeai/app/api/routers/models.py

@@ -104,8 +104,12 @@ async def update_model(
             ):  # model manager moved model path during rename - don't overwrite it
                 info.path = new_info.get("path")
 
+        # replace empty string values with None/null to avoid phenomenon of vae: ''
+        info_dict = info.dict()
+        info_dict = {x: info_dict[x] if info_dict[x] else None for x in info_dict.keys()}
+
         ApiDependencies.invoker.services.model_manager.update_model(
-            model_name=model_name, base_model=base_model, model_type=model_type, model_attributes=info.dict()
+            model_name=model_name, base_model=base_model, model_type=model_type, model_attributes=info_dict
         )
 
         model_raw = ApiDependencies.invoker.services.model_manager.list_model(

invokeai/app/api/routers/sessions.py

@@ -40,7 +40,7 @@ async def create_session(
 @session_router.get(
     "/",
     operation_id="list_sessions",
-    responses={200: {"model": PaginatedResults[GraphExecutionState]}},
+    responses={200: {"model": PaginatedResults[dict]}},
 )
 async def list_sessions(
     page: int = Query(default=0, description="The page of results to get"),

invokeai/app/api_app.py

@@ -38,7 +38,7 @@ import mimetypes
 from .api.dependencies import ApiDependencies
 from .api.routers import sessions, models, images, boards, board_images, app_info
 from .api.sockets import SocketIO
-from .invocations.baseinvocation import BaseInvocation
+from .invocations.baseinvocation import BaseInvocation, _InputField, _OutputField, UIConfigBase
 
 
 import torch
@@ -134,6 +134,11 @@ def custom_openapi():
         # This could break in some cases, figure out a better way to do it
         output_type_titles[schema_key] = output_schema["title"]
 
+    # Add Node Editor UI helper schemas
+    ui_config_schemas = schema([UIConfigBase, _InputField, _OutputField], ref_prefix="#/components/schemas/")
+    for schema_key, output_schema in ui_config_schemas["definitions"].items():
+        openapi_schema["components"]["schemas"][schema_key] = output_schema
+
     # Add a reference to the output type to additionalProperties of the invoker schema
     for invoker in all_invocations:
         invoker_name = invoker.__name__
@@ -211,7 +216,7 @@ def invoke_api():
     port = find_port(app_config.port)
     if port != app_config.port:
         logger.warn(f"Port {app_config.port} in use, using port {port}")
-        
+
     # Start our own event loop for eventing usage
     loop = asyncio.new_event_loop()
     config = uvicorn.Config(
@@ -229,7 +234,7 @@ def invoke_api():
         l.handlers.clear()
         for ch in logger.handlers:
             l.addHandler(ch)
-    
+
     loop.run_until_complete(server.serve())
 
 

invokeai/app/cli_app.py

@@ -37,6 +37,7 @@ from invokeai.app.services.image_record_storage import SqliteImageRecordStorage
 from invokeai.app.services.images import ImageService, ImageServiceDependencies
 from invokeai.app.services.resource_name import SimpleNameService
 from invokeai.app.services.urls import LocalUrlService
+from invokeai.app.services.invocation_stats import InvocationStatsService
 from .services.default_graphs import default_text_to_image_graph_id, create_system_graphs
 from .services.latent_storage import DiskLatentsStorage, ForwardCacheLatentsStorage
 
@@ -311,6 +312,7 @@ def invoke_cli():
         graph_library=SqliteItemStorage[LibraryGraph](filename=db_location, table_name="graphs"),
         graph_execution_manager=graph_execution_manager,
         processor=DefaultInvocationProcessor(),
+        performance_statistics=InvocationStatsService(graph_execution_manager),
         logger=logger,
         configuration=config,
     )

invokeai/app/invocations/baseinvocation.py

@@ -3,15 +3,366 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from enum import Enum
 from inspect import signature
-from typing import TYPE_CHECKING, Dict, List, Literal, TypedDict, get_args, get_type_hints
+from typing import (
+    TYPE_CHECKING,
+    AbstractSet,
+    Any,
+    Callable,
+    ClassVar,
+    Mapping,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+    get_type_hints,
+)
 
-from pydantic import BaseConfig, BaseModel, Field
+from pydantic import BaseModel, Field
+from pydantic.fields import Undefined
+from pydantic.typing import NoArgAnyCallable
 
 if TYPE_CHECKING:
     from ..services.invocation_services import InvocationServices
 
 
+class FieldDescriptions:
+    denoising_start = "When to start denoising, expressed a percentage of total steps"
+    denoising_end = "When to stop denoising, expressed a percentage of total steps"
+    cfg_scale = "Classifier-Free Guidance scale"
+    scheduler = "Scheduler to use during inference"
+    positive_cond = "Positive conditioning tensor"
+    negative_cond = "Negative conditioning tensor"
+    noise = "Noise tensor"
+    clip = "CLIP (tokenizer, text encoder, LoRAs) and skipped layer count"
+    unet = "UNet (scheduler, LoRAs)"
+    vae = "VAE"
+    cond = "Conditioning tensor"
+    controlnet_model = "ControlNet model to load"
+    vae_model = "VAE model to load"
+    lora_model = "LoRA model to load"
+    main_model = "Main model (UNet, VAE, CLIP) to load"
+    sdxl_main_model = "SDXL Main model (UNet, VAE, CLIP1, CLIP2) to load"
+    sdxl_refiner_model = "SDXL Refiner Main Modde (UNet, VAE, CLIP2) to load"
+    onnx_main_model = "ONNX Main model (UNet, VAE, CLIP) to load"
+    lora_weight = "The weight at which the LoRA is applied to each model"
+    compel_prompt = "Prompt to be parsed by Compel to create a conditioning tensor"
+    raw_prompt = "Raw prompt text (no parsing)"
+    sdxl_aesthetic = "The aesthetic score to apply to the conditioning tensor"
+    skipped_layers = "Number of layers to skip in text encoder"
+    seed = "Seed for random number generation"
+    steps = "Number of steps to run"
+    width = "Width of output (px)"
+    height = "Height of output (px)"
+    control = "ControlNet(s) to apply"
+    denoised_latents = "Denoised latents tensor"
+    latents = "Latents tensor"
+    strength = "Strength of denoising (proportional to steps)"
+    core_metadata = "Optional core metadata to be written to image"
+    interp_mode = "Interpolation mode"
+    torch_antialias = "Whether or not to apply antialiasing (bilinear or bicubic only)"
+    fp32 = "Whether or not to use full float32 precision"
+    precision = "Precision to use"
+    tiled = "Processing using overlapping tiles (reduce memory consumption)"
+    detect_res = "Pixel resolution for detection"
+    image_res = "Pixel resolution for output image"
+    safe_mode = "Whether or not to use safe mode"
+    scribble_mode = "Whether or not to use scribble mode"
+    scale_factor = "The factor by which to scale"
+    num_1 = "The first number"
+    num_2 = "The second number"
+    mask = "The mask to use for the operation"
+
+
+class Input(str, Enum):
+    """
+    The type of input a field accepts.
+    - `Input.Direct`: The field must have its value provided directly, when the invocation and field \
+      are instantiated.
+    - `Input.Connection`: The field must have its value provided by a connection.
+    - `Input.Any`: The field may have its value provided either directly or by a connection.
+    """
+
+    Connection = "connection"
+    Direct = "direct"
+    Any = "any"
+
+
+class UIType(str, Enum):
+    """
+    Type hints for the UI.
+    If a field should be provided a data type that does not exactly match the python type of the field, \
+    use this to provide the type that should be used instead. See the node development docs for detail \
+    on adding a new field type, which involves client-side changes.
+    """
+
+    # region Primitives
+    Integer = "integer"
+    Float = "float"
+    Boolean = "boolean"
+    String = "string"
+    Array = "array"
+    Image = "ImageField"
+    Latents = "LatentsField"
+    Conditioning = "ConditioningField"
+    Control = "ControlField"
+    Color = "ColorField"
+    ImageCollection = "ImageCollection"
+    ConditioningCollection = "ConditioningCollection"
+    ColorCollection = "ColorCollection"
+    LatentsCollection = "LatentsCollection"
+    IntegerCollection = "IntegerCollection"
+    FloatCollection = "FloatCollection"
+    StringCollection = "StringCollection"
+    BooleanCollection = "BooleanCollection"
+    # endregion
+
+    # region Models
+    MainModel = "MainModelField"
+    SDXLMainModel = "SDXLMainModelField"
+    SDXLRefinerModel = "SDXLRefinerModelField"
+    ONNXModel = "ONNXModelField"
+    VaeModel = "VaeModelField"
+    LoRAModel = "LoRAModelField"
+    ControlNetModel = "ControlNetModelField"
+    UNet = "UNetField"
+    Vae = "VaeField"
+    CLIP = "ClipField"
+    # endregion
+
+    # region Iterate/Collect
+    Collection = "Collection"
+    CollectionItem = "CollectionItem"
+    # endregion
+
+    # region Misc
+    FilePath = "FilePath"
+    Enum = "enum"
+    # endregion
+
+
+class UIComponent(str, Enum):
+    """
+    The type of UI component to use for a field, used to override the default components, which are \
+    inferred from the field type.
+    """
+
+    None_ = "none"
+    Textarea = "textarea"
+    Slider = "slider"
+
+
+class _InputField(BaseModel):
+    """
+    *DO NOT USE*
+    This helper class is used to tell the client about our custom field attributes via OpenAPI
+    schema generation, and Typescript type generation from that schema. It serves no functional
+    purpose in the backend.
+    """
+
+    input: Input
+    ui_hidden: bool
+    ui_type: Optional[UIType]
+    ui_component: Optional[UIComponent]
+
+
+class _OutputField(BaseModel):
+    """
+    *DO NOT USE*
+    This helper class is used to tell the client about our custom field attributes via OpenAPI
+    schema generation, and Typescript type generation from that schema. It serves no functional
+    purpose in the backend.
+    """
+
+    ui_hidden: bool
+    ui_type: Optional[UIType]
+
+
+def InputField(
+    *args: Any,
+    default: Any = Undefined,
+    default_factory: Optional[NoArgAnyCallable] = None,
+    alias: Optional[str] = None,
+    title: Optional[str] = None,
+    description: Optional[str] = None,
+    exclude: Optional[Union[AbstractSet[Union[int, str]], Mapping[Union[int, str], Any], Any]] = None,
+    include: Optional[Union[AbstractSet[Union[int, str]], Mapping[Union[int, str], Any], Any]] = None,
+    const: Optional[bool] = None,
+    gt: Optional[float] = None,
+    ge: Optional[float] = None,
+    lt: Optional[float] = None,
+    le: Optional[float] = None,
+    multiple_of: Optional[float] = None,
+    allow_inf_nan: Optional[bool] = None,
+    max_digits: Optional[int] = None,
+    decimal_places: Optional[int] = None,
+    min_items: Optional[int] = None,
+    max_items: Optional[int] = None,
+    unique_items: Optional[bool] = None,
+    min_length: Optional[int] = None,
+    max_length: Optional[int] = None,
+    allow_mutation: bool = True,
+    regex: Optional[str] = None,
+    discriminator: Optional[str] = None,
+    repr: bool = True,
+    input: Input = Input.Any,
+    ui_type: Optional[UIType] = None,
+    ui_component: Optional[UIComponent] = None,
+    ui_hidden: bool = False,
+    **kwargs: Any,
+) -> Any:
+    """
+    Creates an input field for an invocation.
+
+    This is a wrapper for Pydantic's [Field](https://docs.pydantic.dev/1.10/usage/schema/#field-customization) \
+    that adds a few extra parameters to support graph execution and the node editor UI.
+
+    :param Input input: [Input.Any] The kind of input this field requires. \
+      `Input.Direct` means a value must be provided on instantiation. \
+      `Input.Connection` means the value must be provided by a connection. \
+      `Input.Any` means either will do.
+
+    :param UIType ui_type: [None] Optionally provides an extra type hint for the UI. \
+      In some situations, the field's type is not enough to infer the correct UI type. \
+      For example, model selection fields should render a dropdown UI component to select a model. \
+      Internally, there is no difference between SD-1, SD-2 and SDXL model fields, they all use \
+      `MainModelField`. So to ensure the base-model-specific UI is rendered, you can use \
+      `UIType.SDXLMainModelField` to indicate that the field is an SDXL main model field.
+      
+    :param UIComponent ui_component: [None] Optionally specifies a specific component to use in the UI. \
+      The UI will always render a suitable component, but sometimes you want something different than the default. \
+      For example, a `string` field will default to a single-line input, but you may want a multi-line textarea instead. \
+      For this case, you could provide `UIComponent.Textarea`.
+
+    : param bool ui_hidden: [False] Specifies whether or not this field should be hidden in the UI.
+    """
+    return Field(
+        *args,
+        default=default,
+        default_factory=default_factory,
+        alias=alias,
+        title=title,
+        description=description,
+        exclude=exclude,
+        include=include,
+        const=const,
+        gt=gt,
+        ge=ge,
+        lt=lt,
+        le=le,
+        multiple_of=multiple_of,
+        allow_inf_nan=allow_inf_nan,
+        max_digits=max_digits,
+        decimal_places=decimal_places,
+        min_items=min_items,
+        max_items=max_items,
+        unique_items=unique_items,
+        min_length=min_length,
+        max_length=max_length,
+        allow_mutation=allow_mutation,
+        regex=regex,
+        discriminator=discriminator,
+        repr=repr,
+        input=input,
+        ui_type=ui_type,
+        ui_component=ui_component,
+        ui_hidden=ui_hidden,
+        **kwargs,
+    )
+
+
+def OutputField(
+    *args: Any,
+    default: Any = Undefined,
+    default_factory: Optional[NoArgAnyCallable] = None,
+    alias: Optional[str] = None,
+    title: Optional[str] = None,
+    description: Optional[str] = None,
+    exclude: Optional[Union[AbstractSet[Union[int, str]], Mapping[Union[int, str], Any], Any]] = None,
+    include: Optional[Union[AbstractSet[Union[int, str]], Mapping[Union[int, str], Any], Any]] = None,
+    const: Optional[bool] = None,
+    gt: Optional[float] = None,
+    ge: Optional[float] = None,
+    lt: Optional[float] = None,
+    le: Optional[float] = None,
+    multiple_of: Optional[float] = None,
+    allow_inf_nan: Optional[bool] = None,
+    max_digits: Optional[int] = None,
+    decimal_places: Optional[int] = None,
+    min_items: Optional[int] = None,
+    max_items: Optional[int] = None,
+    unique_items: Optional[bool] = None,
+    min_length: Optional[int] = None,
+    max_length: Optional[int] = None,
+    allow_mutation: bool = True,
+    regex: Optional[str] = None,
+    discriminator: Optional[str] = None,
+    repr: bool = True,
+    ui_type: Optional[UIType] = None,
+    ui_hidden: bool = False,
+    **kwargs: Any,
+) -> Any:
+    """
+    Creates an output field for an invocation output.
+
+    This is a wrapper for Pydantic's [Field](https://docs.pydantic.dev/1.10/usage/schema/#field-customization) \
+    that adds a few extra parameters to support graph execution and the node editor UI.
+
+    :param UIType ui_type: [None] Optionally provides an extra type hint for the UI. \
+      In some situations, the field's type is not enough to infer the correct UI type. \
+      For example, model selection fields should render a dropdown UI component to select a model. \
+      Internally, there is no difference between SD-1, SD-2 and SDXL model fields, they all use \
+      `MainModelField`. So to ensure the base-model-specific UI is rendered, you can use \
+      `UIType.SDXLMainModelField` to indicate that the field is an SDXL main model field.
+
+    : param bool ui_hidden: [False] Specifies whether or not this field should be hidden in the UI. \
+    """
+    return Field(
+        *args,
+        default=default,
+        default_factory=default_factory,
+        alias=alias,
+        title=title,
+        description=description,
+        exclude=exclude,
+        include=include,
+        const=const,
+        gt=gt,
+        ge=ge,
+        lt=lt,
+        le=le,
+        multiple_of=multiple_of,
+        allow_inf_nan=allow_inf_nan,
+        max_digits=max_digits,
+        decimal_places=decimal_places,
+        min_items=min_items,
+        max_items=max_items,
+        unique_items=unique_items,
+        min_length=min_length,
+        max_length=max_length,
+        allow_mutation=allow_mutation,
+        regex=regex,
+        discriminator=discriminator,
+        repr=repr,
+        ui_type=ui_type,
+        ui_hidden=ui_hidden,
+        **kwargs,
+    )
+
+
+class UIConfigBase(BaseModel):
+    """
+    Provides additional node configuration to the UI.
+    This is used internally by the @tags and @title decorator logic. You probably want to use those
+    decorators, though you may add this class to a node definition to specify the title and tags.
+    """
+
+    tags: Optional[list[str]] = Field(default_factory=None, description="The tags to display in the UI")
+    title: Optional[str] = Field(default=None, description="The display name of the node")
+
+
 class InvocationContext:
     services: InvocationServices
     graph_execution_state_id: str
@@ -39,6 +390,20 @@ class BaseInvocationOutput(BaseModel):
         return tuple(subclasses)
 
 
+class RequiredConnectionException(Exception):
+    """Raised when an field which requires a connection did not receive a value."""
+
+    def __init__(self, node_id: str, field_name: str):
+        super().__init__(f"Node {node_id} missing connections for field {field_name}")
+
+
+class MissingInputException(Exception):
+    """Raised when an field which requires some input, but did not receive a value."""
+
+    def __init__(self, node_id: str, field_name: str):
+        super().__init__(f"Node {node_id} missing value or connection for field {field_name}")
+
+
 class BaseInvocation(ABC, BaseModel):
     """A node to process inputs and produce outputs.
     May use dependency injection in __init__ to receive providers.
@@ -76,70 +441,81 @@ class BaseInvocation(ABC, BaseModel):
     def get_output_type(cls):
         return signature(cls.invoke).return_annotation
 
+    class Config:
+        @staticmethod
+        def schema_extra(schema: dict[str, Any], model_class: Type[BaseModel]) -> None:
+            uiconfig = getattr(model_class, "UIConfig", None)
+            if uiconfig and hasattr(uiconfig, "title"):
+                schema["title"] = uiconfig.title
+            if uiconfig and hasattr(uiconfig, "tags"):
+                schema["tags"] = uiconfig.tags
+
     @abstractmethod
     def invoke(self, context: InvocationContext) -> BaseInvocationOutput:
         """Invoke with provided context and return outputs."""
         pass
 
-    # fmt: off
-    id: str = Field(description="The id of this node. Must be unique among all nodes.")
-    is_intermediate: bool = Field(default=False, description="Whether or not this node is an intermediate node.")
-    # fmt: on
+    def __init__(self, **data):
+        # nodes may have required fields, that can accept input from connections
+        # on instantiation of the model, we need to exclude these from validation
+        restore = dict()
+        try:
+            field_names = list(self.__fields__.keys())
+            for field_name in field_names:
+                # if the field is required and may get its value from a connection, exclude it from validation
+                field = self.__fields__[field_name]
+                _input = field.field_info.extra.get("input", None)
+                if _input in [Input.Connection, Input.Any] and field.required:
+                    if field_name not in data:
+                        restore[field_name] = self.__fields__.pop(field_name)
+            # instantiate the node, which will validate the data
+            super().__init__(**data)
+        finally:
+            # restore the removed fields
+            for field_name, field in restore.items():
+                self.__fields__[field_name] = field
+
+    def invoke_internal(self, context: InvocationContext) -> BaseInvocationOutput:
+        for field_name, field in self.__fields__.items():
+            _input = field.field_info.extra.get("input", None)
+            if field.required and not hasattr(self, field_name):
+                if _input == Input.Connection:
+                    raise RequiredConnectionException(self.__fields__["type"].default, field_name)
+                elif _input == Input.Any:
+                    raise MissingInputException(self.__fields__["type"].default, field_name)
+        return self.invoke(context)
+
+    id: str = InputField(description="The id of this node. Must be unique among all nodes.")
+    is_intermediate: bool = InputField(
+        default=False, description="Whether or not this node is an intermediate node.", input=Input.Direct
+    )
+    UIConfig: ClassVar[Type[UIConfigBase]]
 
 
-# TODO: figure out a better way to provide these hints
-# TODO: when we can upgrade to python 3.11, we can use the`NotRequired` type instead of `total=False`
-class UIConfig(TypedDict, total=False):
-    type_hints: Dict[
-        str,
-        Literal[
-            "integer",
-            "float",
-            "boolean",
-            "string",
-            "enum",
-            "image",
-            "latents",
-            "model",
-            "control",
-            "image_collection",
-            "vae_model",
-            "lora_model",
-        ],
-    ]
-    tags: List[str]
-    title: str
+T = TypeVar("T", bound=BaseInvocation)
 
 
-class CustomisedSchemaExtra(TypedDict):
-    ui: UIConfig
+def title(title: str) -> Callable[[Type[T]], Type[T]]:
+    """Adds a title to the invocation. Use this to override the default title generation, which is based on the class name."""
+
+    def wrapper(cls: Type[T]) -> Type[T]:
+        uiconf_name = cls.__qualname__ + ".UIConfig"
+        if not hasattr(cls, "UIConfig") or cls.UIConfig.__qualname__ != uiconf_name:
+            cls.UIConfig = type(uiconf_name, (UIConfigBase,), dict())
+        cls.UIConfig.title = title
+        return cls
+
+    return wrapper
 
 
-class InvocationConfig(BaseConfig):
-    """Customizes pydantic's BaseModel.Config class for use by Invocations.
+def tags(*tags: str) -> Callable[[Type[T]], Type[T]]:
+    """Adds tags to the invocation. Use this to improve the streamline finding the invocation in the UI."""
 
-    Provide `schema_extra` a `ui` dict to add hints for generated UIs.
+    def wrapper(cls: Type[T]) -> Type[T]:
+        uiconf_name = cls.__qualname__ + ".UIConfig"
+        if not hasattr(cls, "UIConfig") or cls.UIConfig.__qualname__ != uiconf_name:
+            cls.UIConfig = type(uiconf_name, (UIConfigBase,), dict())
+        cls.UIConfig.tags = list(tags)
+        return cls
 
-    `tags`
-    - A list of strings, used to categorise invocations.
-
-    `type_hints`
-    - A dict of field types which override the types in the invocation definition.
-    - Each key should be the name of one of the invocation's fields.
-    - Each value should be one of the valid types:
-      - `integer`, `float`, `boolean`, `string`, `enum`, `image`, `latents`, `model`
-
-    ```python
-    class Config(InvocationConfig):
-      schema_extra = {
-          "ui": {
-              "tags": ["stable-diffusion", "image"],
-              "type_hints": {
-                  "initial_image": "image",
-              },
-          },
-      }
-    ```
-    """
-
-    schema_extra: CustomisedSchemaExtra
+    return wrapper

invokeai/app/invocations/collections.py

@@ -3,58 +3,25 @@
 from typing import Literal
 
 import numpy as np
-from pydantic import Field, validator
+from pydantic import validator
 
-from invokeai.app.models.image import ImageField
+from invokeai.app.invocations.primitives import ImageCollectionOutput, ImageField, IntegerCollectionOutput
 from invokeai.app.util.misc import SEED_MAX, get_random_seed
 
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext, UIConfig
-
-
-class IntCollectionOutput(BaseInvocationOutput):
-    """A collection of integers"""
-
-    type: Literal["int_collection"] = "int_collection"
-
-    # Outputs
-    collection: list[int] = Field(default=[], description="The int collection")
-
-
-class FloatCollectionOutput(BaseInvocationOutput):
-    """A collection of floats"""
-
-    type: Literal["float_collection"] = "float_collection"
-
-    # Outputs
-    collection: list[float] = Field(default=[], description="The float collection")
-
-
-class ImageCollectionOutput(BaseInvocationOutput):
-    """A collection of images"""
-
-    type: Literal["image_collection"] = "image_collection"
-
-    # Outputs
-    collection: list[ImageField] = Field(default=[], description="The output images")
-
-    class Config:
-        schema_extra = {"required": ["type", "collection"]}
+from .baseinvocation import BaseInvocation, InputField, InvocationContext, UIType, tags, title
 
 
+@title("Integer Range")
+@tags("collection", "integer", "range")
 class RangeInvocation(BaseInvocation):
     """Creates a range of numbers from start to stop with step"""
 
     type: Literal["range"] = "range"
 
     # Inputs
-    start: int = Field(default=0, description="The start of the range")
-    stop: int = Field(default=10, description="The stop of the range")
-    step: int = Field(default=1, description="The step of the range")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Range", "tags": ["range", "integer", "collection"]},
-        }
+    start: int = InputField(default=0, description="The start of the range")
+    stop: int = InputField(default=10, description="The stop of the range")
+    step: int = InputField(default=1, description="The step of the range")
 
     @validator("stop")
     def stop_gt_start(cls, v, values):
@@ -62,76 +29,44 @@ class RangeInvocation(BaseInvocation):
             raise ValueError("stop must be greater than start")
         return v
 
-    def invoke(self, context: InvocationContext) -> IntCollectionOutput:
-        return IntCollectionOutput(collection=list(range(self.start, self.stop, self.step)))
+    def invoke(self, context: InvocationContext) -> IntegerCollectionOutput:
+        return IntegerCollectionOutput(collection=list(range(self.start, self.stop, self.step)))
 
 
+@title("Integer Range of Size")
+@tags("range", "integer", "size", "collection")
 class RangeOfSizeInvocation(BaseInvocation):
     """Creates a range from start to start + size with step"""
 
     type: Literal["range_of_size"] = "range_of_size"
 
     # Inputs
-    start: int = Field(default=0, description="The start of the range")
-    size: int = Field(default=1, description="The number of values")
-    step: int = Field(default=1, description="The step of the range")
+    start: int = InputField(default=0, description="The start of the range")
+    size: int = InputField(default=1, description="The number of values")
+    step: int = InputField(default=1, description="The step of the range")
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Sized Range", "tags": ["range", "integer", "size", "collection"]},
-        }
-
-    def invoke(self, context: InvocationContext) -> IntCollectionOutput:
-        return IntCollectionOutput(collection=list(range(self.start, self.start + self.size, self.step)))
+    def invoke(self, context: InvocationContext) -> IntegerCollectionOutput:
+        return IntegerCollectionOutput(collection=list(range(self.start, self.start + self.size, self.step)))
 
 
+@title("Random Range")
+@tags("range", "integer", "random", "collection")
 class RandomRangeInvocation(BaseInvocation):
     """Creates a collection of random numbers"""
 
     type: Literal["random_range"] = "random_range"
 
     # Inputs
-    low: int = Field(default=0, description="The inclusive low value")
-    high: int = Field(default=np.iinfo(np.int32).max, description="The exclusive high value")
-    size: int = Field(default=1, description="The number of values to generate")
-    seed: int = Field(
+    low: int = InputField(default=0, description="The inclusive low value")
+    high: int = InputField(default=np.iinfo(np.int32).max, description="The exclusive high value")
+    size: int = InputField(default=1, description="The number of values to generate")
+    seed: int = InputField(
         ge=0,
         le=SEED_MAX,
         description="The seed for the RNG (omit for random)",
         default_factory=get_random_seed,
     )
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Random Range", "tags": ["range", "integer", "random", "collection"]},
-        }
-
-    def invoke(self, context: InvocationContext) -> IntCollectionOutput:
+    def invoke(self, context: InvocationContext) -> IntegerCollectionOutput:
         rng = np.random.default_rng(self.seed)
-        return IntCollectionOutput(collection=list(rng.integers(low=self.low, high=self.high, size=self.size)))
-
-
-class ImageCollectionInvocation(BaseInvocation):
-    """Load a collection of images and provide it as output."""
-
-    # fmt: off
-    type: Literal["image_collection"] = "image_collection"
-
-    # Inputs
-    images: list[ImageField] = Field(
-        default=[], description="The image collection to load"
-    )
-    # fmt: on
-
-    def invoke(self, context: InvocationContext) -> ImageCollectionOutput:
-        return ImageCollectionOutput(collection=self.images)
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "type_hints": {
-                    "title": "Image Collection",
-                    "images": "image_collection",
-                }
-            },
-        }
+        return IntegerCollectionOutput(collection=list(rng.integers(low=self.low, high=self.high, size=self.size)))

invokeai/app/invocations/compel.py

@@ -1,48 +1,40 @@
-from typing import Literal, Optional, Union, List, Annotated
-from pydantic import BaseModel, Field
 import re
+from dataclasses import dataclass
+from typing import List, Literal, Union
+
 import torch
 from compel import Compel, ReturnedEmbeddingsType
 from compel.prompt_parser import Blend, Conjunction, CrossAttentionControlSubstitute, FlattenedPrompt, Fragment
-from ...backend.util.devices import torch_dtype
-from ...backend.model_management import ModelType
-from ...backend.model_management.models import ModelNotFoundException
+from invokeai.app.invocations.primitives import ConditioningField, ConditioningOutput
+
+from invokeai.backend.stable_diffusion.diffusion.shared_invokeai_diffusion import (
+    BasicConditioningInfo,
+    SDXLConditioningInfo,
+)
+
+from ...backend.model_management import ModelPatcher, ModelType
 from ...backend.model_management.lora import ModelPatcher
+from ...backend.model_management.models import ModelNotFoundException
 from ...backend.stable_diffusion.diffusion import InvokeAIDiffuserComponent
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
+from ...backend.util.devices import torch_dtype
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    Input,
+    InputField,
+    InvocationContext,
+    OutputField,
+    UIComponent,
+    tags,
+    title,
+)
 from .model import ClipField
-from dataclasses import dataclass
-
-
-class ConditioningField(BaseModel):
-    conditioning_name: Optional[str] = Field(default=None, description="The name of conditioning data")
-
-    class Config:
-        schema_extra = {"required": ["conditioning_name"]}
-
-
-@dataclass
-class BasicConditioningInfo:
-    # type: Literal["basic_conditioning"] = "basic_conditioning"
-    embeds: torch.Tensor
-    extra_conditioning: Optional[InvokeAIDiffuserComponent.ExtraConditioningInfo]
-    # weight: float
-    # mode: ConditioningAlgo
-
-
-@dataclass
-class SDXLConditioningInfo(BasicConditioningInfo):
-    # type: Literal["sdxl_conditioning"] = "sdxl_conditioning"
-    pooled_embeds: torch.Tensor
-    add_time_ids: torch.Tensor
-
-
-ConditioningInfoType = Annotated[Union[BasicConditioningInfo, SDXLConditioningInfo], Field(discriminator="type")]
 
 
 @dataclass
 class ConditioningFieldData:
-    conditionings: List[Union[BasicConditioningInfo, SDXLConditioningInfo]]
+    conditionings: List[BasicConditioningInfo]
     # unconditioned: Optional[torch.Tensor]
 
 
@@ -52,32 +44,26 @@ class ConditioningFieldData:
 #    PerpNeg = "perp_neg"
 
 
-class CompelOutput(BaseInvocationOutput):
-    """Compel parser output"""
-
-    # fmt: off
-    type: Literal["compel_output"] = "compel_output"
-
-    conditioning: ConditioningField = Field(default=None, description="Conditioning")
-    # fmt: on
-
-
+@title("Compel Prompt")
+@tags("prompt", "compel")
 class CompelInvocation(BaseInvocation):
     """Parse prompt using compel package to conditioning."""
 
     type: Literal["compel"] = "compel"
 
-    prompt: str = Field(default="", description="Prompt")
-    clip: ClipField = Field(None, description="Clip to use")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Prompt (Compel)", "tags": ["prompt", "compel"], "type_hints": {"model": "model"}},
-        }
+    prompt: str = InputField(
+        default="",
+        description=FieldDescriptions.compel_prompt,
+        ui_component=UIComponent.Textarea,
+    )
+    clip: ClipField = InputField(
+        title="CLIP",
+        description=FieldDescriptions.clip,
+        input=Input.Connection,
+    )
 
     @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> CompelOutput:
+    def invoke(self, context: InvocationContext) -> ConditioningOutput:
         tokenizer_info = context.services.model_manager.get_model(
             **self.clip.tokenizer.dict(),
             context=context,
@@ -101,12 +87,15 @@ class CompelInvocation(BaseInvocation):
             name = trigger[1:-1]
             try:
                 ti_list.append(
-                    context.services.model_manager.get_model(
-                        model_name=name,
-                        base_model=self.clip.text_encoder.base_model,
-                        model_type=ModelType.TextualInversion,
-                        context=context,
-                    ).context.model
+                    (
+                        name,
+                        context.services.model_manager.get_model(
+                            model_name=name,
+                            base_model=self.clip.text_encoder.base_model,
+                            model_type=ModelType.TextualInversion,
+                            context=context,
+                        ).context.model,
+                    )
                 )
             except ModelNotFoundException:
                 # print(e)
@@ -157,7 +146,7 @@ class CompelInvocation(BaseInvocation):
         conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
         context.services.latents.save(conditioning_name, conditioning_data)
 
-        return CompelOutput(
+        return ConditioningOutput(
             conditioning=ConditioningField(
                 conditioning_name=conditioning_name,
             ),
@@ -165,7 +154,15 @@ class CompelInvocation(BaseInvocation):
 
 
 class SDXLPromptInvocationBase:
-    def run_clip_raw(self, context, clip_field, prompt, get_pooled):
+    def run_clip_compel(
+        self,
+        context: InvocationContext,
+        clip_field: ClipField,
+        prompt: str,
+        get_pooled: bool,
+        lora_prefix: str,
+        zero_on_empty: bool,
+    ):
         tokenizer_info = context.services.model_manager.get_model(
             **clip_field.tokenizer.dict(),
             context=context,
@@ -175,79 +172,21 @@ class SDXLPromptInvocationBase:
             context=context,
         )
 
-        def _lora_loader():
-            for lora in clip_field.loras:
-                lora_info = context.services.model_manager.get_model(**lora.dict(exclude={"weight"}), context=context)
-                yield (lora_info.context.model, lora.weight)
-                del lora_info
-            return
-
-        # loras = [(context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.clip.loras]
-
-        ti_list = []
-        for trigger in re.findall(r"<[a-zA-Z0-9., _-]+>", prompt):
-            name = trigger[1:-1]
-            try:
-                ti_list.append(
-                    context.services.model_manager.get_model(
-                        model_name=name,
-                        base_model=clip_field.text_encoder.base_model,
-                        model_type=ModelType.TextualInversion,
-                        context=context,
-                    ).context.model
-                )
-            except ModelNotFoundException:
-                # print(e)
-                # import traceback
-                # print(traceback.format_exc())
-                print(f'Warn: trigger: "{trigger}" not found')
-
-        with ModelPatcher.apply_lora_text_encoder(
-            text_encoder_info.context.model, _lora_loader()
-        ), ModelPatcher.apply_ti(tokenizer_info.context.model, text_encoder_info.context.model, ti_list) as (
-            tokenizer,
-            ti_manager,
-        ), ModelPatcher.apply_clip_skip(
-            text_encoder_info.context.model, clip_field.skipped_layers
-        ), text_encoder_info as text_encoder:
-            text_inputs = tokenizer(
-                prompt,
-                padding="max_length",
-                max_length=tokenizer.model_max_length,
-                truncation=True,
-                return_tensors="pt",
-            )
-            text_input_ids = text_inputs.input_ids
-            prompt_embeds = text_encoder(
-                text_input_ids.to(text_encoder.device),
-                output_hidden_states=True,
+        # return zero on empty
+        if prompt == "" and zero_on_empty:
+            cpu_text_encoder = text_encoder_info.context.model
+            c = torch.zeros(
+                (1, cpu_text_encoder.config.max_position_embeddings, cpu_text_encoder.config.hidden_size),
+                dtype=text_encoder_info.context.cache.precision,
             )
             if get_pooled:
-                c_pooled = prompt_embeds[0]
+                c_pooled = torch.zeros(
+                    (1, cpu_text_encoder.config.hidden_size),
+                    dtype=c.dtype,
+                )
             else:
                 c_pooled = None
-            c = prompt_embeds.hidden_states[-2]
-
-        del tokenizer
-        del text_encoder
-        del tokenizer_info
-        del text_encoder_info
-
-        c = c.detach().to("cpu")
-        if c_pooled is not None:
-            c_pooled = c_pooled.detach().to("cpu")
-
-        return c, c_pooled, None
-
-    def run_clip_compel(self, context, clip_field, prompt, get_pooled):
-        tokenizer_info = context.services.model_manager.get_model(
-            **clip_field.tokenizer.dict(),
-            context=context,
-        )
-        text_encoder_info = context.services.model_manager.get_model(
-            **clip_field.text_encoder.dict(),
-            context=context,
-        )
+            return c, c_pooled, None
 
         def _lora_loader():
             for lora in clip_field.loras:
@@ -263,12 +202,15 @@ class SDXLPromptInvocationBase:
             name = trigger[1:-1]
             try:
                 ti_list.append(
-                    context.services.model_manager.get_model(
-                        model_name=name,
-                        base_model=clip_field.text_encoder.base_model,
-                        model_type=ModelType.TextualInversion,
-                        context=context,
-                    ).context.model
+                    (
+                        name,
+                        context.services.model_manager.get_model(
+                            model_name=name,
+                            base_model=clip_field.text_encoder.base_model,
+                            model_type=ModelType.TextualInversion,
+                            context=context,
+                        ).context.model,
+                    )
                 )
             except ModelNotFoundException:
                 # print(e)
@@ -276,8 +218,8 @@ class SDXLPromptInvocationBase:
                 # print(traceback.format_exc())
                 print(f'Warn: trigger: "{trigger}" not found')
 
-        with ModelPatcher.apply_lora_text_encoder(
-            text_encoder_info.context.model, _lora_loader()
+        with ModelPatcher.apply_lora(
+            text_encoder_info.context.model, _lora_loader(), lora_prefix
         ), ModelPatcher.apply_ti(tokenizer_info.context.model, text_encoder_info.context.model, ti_list) as (
             tokenizer,
             ti_manager,
@@ -325,35 +267,37 @@ class SDXLPromptInvocationBase:
         return c, c_pooled, ec
 
 
+@title("SDXL Compel Prompt")
+@tags("sdxl", "compel", "prompt")
 class SDXLCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
     """Parse prompt using compel package to conditioning."""
 
     type: Literal["sdxl_compel_prompt"] = "sdxl_compel_prompt"
 
-    prompt: str = Field(default="", description="Prompt")
-    style: str = Field(default="", description="Style prompt")
-    original_width: int = Field(1024, description="")
-    original_height: int = Field(1024, description="")
-    crop_top: int = Field(0, description="")
-    crop_left: int = Field(0, description="")
-    target_width: int = Field(1024, description="")
-    target_height: int = Field(1024, description="")
-    clip: ClipField = Field(None, description="Clip to use")
-    clip2: ClipField = Field(None, description="Clip2 to use")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "SDXL Prompt (Compel)", "tags": ["prompt", "compel"], "type_hints": {"model": "model"}},
-        }
+    prompt: str = InputField(default="", description=FieldDescriptions.compel_prompt, ui_component=UIComponent.Textarea)
+    style: str = InputField(default="", description=FieldDescriptions.compel_prompt, ui_component=UIComponent.Textarea)
+    original_width: int = InputField(default=1024, description="")
+    original_height: int = InputField(default=1024, description="")
+    crop_top: int = InputField(default=0, description="")
+    crop_left: int = InputField(default=0, description="")
+    target_width: int = InputField(default=1024, description="")
+    target_height: int = InputField(default=1024, description="")
+    clip: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection)
+    clip2: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection)
 
     @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> CompelOutput:
-        c1, c1_pooled, ec1 = self.run_clip_compel(context, self.clip, self.prompt, False)
+    def invoke(self, context: InvocationContext) -> ConditioningOutput:
+        c1, c1_pooled, ec1 = self.run_clip_compel(
+            context, self.clip, self.prompt, False, "lora_te1_", zero_on_empty=True
+        )
         if self.style.strip() == "":
-            c2, c2_pooled, ec2 = self.run_clip_compel(context, self.clip2, self.prompt, True)
+            c2, c2_pooled, ec2 = self.run_clip_compel(
+                context, self.clip2, self.prompt, True, "lora_te2_", zero_on_empty=True
+            )
         else:
-            c2, c2_pooled, ec2 = self.run_clip_compel(context, self.clip2, self.style, True)
+            c2, c2_pooled, ec2 = self.run_clip_compel(
+                context, self.clip2, self.style, True, "lora_te2_", zero_on_empty=True
+            )
 
         original_size = (self.original_height, self.original_width)
         crop_coords = (self.crop_top, self.crop_left)
@@ -375,39 +319,34 @@ class SDXLCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
         conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
         context.services.latents.save(conditioning_name, conditioning_data)
 
-        return CompelOutput(
+        return ConditioningOutput(
             conditioning=ConditioningField(
                 conditioning_name=conditioning_name,
             ),
         )
 
 
+@title("SDXL Refiner Compel Prompt")
+@tags("sdxl", "compel", "prompt")
 class SDXLRefinerCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
     """Parse prompt using compel package to conditioning."""
 
     type: Literal["sdxl_refiner_compel_prompt"] = "sdxl_refiner_compel_prompt"
 
-    style: str = Field(default="", description="Style prompt")  # TODO: ?
-    original_width: int = Field(1024, description="")
-    original_height: int = Field(1024, description="")
-    crop_top: int = Field(0, description="")
-    crop_left: int = Field(0, description="")
-    aesthetic_score: float = Field(6.0, description="")
-    clip2: ClipField = Field(None, description="Clip to use")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "SDXL Refiner Prompt (Compel)",
-                "tags": ["prompt", "compel"],
-                "type_hints": {"model": "model"},
-            },
-        }
+    style: str = InputField(
+        default="", description=FieldDescriptions.compel_prompt, ui_component=UIComponent.Textarea
+    )  # TODO: ?
+    original_width: int = InputField(default=1024, description="")
+    original_height: int = InputField(default=1024, description="")
+    crop_top: int = InputField(default=0, description="")
+    crop_left: int = InputField(default=0, description="")
+    aesthetic_score: float = InputField(default=6.0, description=FieldDescriptions.sdxl_aesthetic)
+    clip2: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection)
 
     @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> CompelOutput:
-        c2, c2_pooled, ec2 = self.run_clip_compel(context, self.clip2, self.style, True)
+    def invoke(self, context: InvocationContext) -> ConditioningOutput:
+        # TODO: if there will appear lora for refiner - write proper prefix
+        c2, c2_pooled, ec2 = self.run_clip_compel(context, self.clip2, self.style, True, "<NONE>", zero_on_empty=False)
 
         original_size = (self.original_height, self.original_width)
         crop_coords = (self.crop_top, self.crop_left)
@@ -428,117 +367,7 @@ class SDXLRefinerCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase
         conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
         context.services.latents.save(conditioning_name, conditioning_data)
 
-        return CompelOutput(
-            conditioning=ConditioningField(
-                conditioning_name=conditioning_name,
-            ),
-        )
-
-
-class SDXLRawPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
-    """Pass unmodified prompt to conditioning without compel processing."""
-
-    type: Literal["sdxl_raw_prompt"] = "sdxl_raw_prompt"
-
-    prompt: str = Field(default="", description="Prompt")
-    style: str = Field(default="", description="Style prompt")
-    original_width: int = Field(1024, description="")
-    original_height: int = Field(1024, description="")
-    crop_top: int = Field(0, description="")
-    crop_left: int = Field(0, description="")
-    target_width: int = Field(1024, description="")
-    target_height: int = Field(1024, description="")
-    clip: ClipField = Field(None, description="Clip to use")
-    clip2: ClipField = Field(None, description="Clip2 to use")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "SDXL Prompt (Raw)", "tags": ["prompt", "compel"], "type_hints": {"model": "model"}},
-        }
-
-    @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> CompelOutput:
-        c1, c1_pooled, ec1 = self.run_clip_raw(context, self.clip, self.prompt, False)
-        if self.style.strip() == "":
-            c2, c2_pooled, ec2 = self.run_clip_raw(context, self.clip2, self.prompt, True)
-        else:
-            c2, c2_pooled, ec2 = self.run_clip_raw(context, self.clip2, self.style, True)
-
-        original_size = (self.original_height, self.original_width)
-        crop_coords = (self.crop_top, self.crop_left)
-        target_size = (self.target_height, self.target_width)
-
-        add_time_ids = torch.tensor([original_size + crop_coords + target_size])
-
-        conditioning_data = ConditioningFieldData(
-            conditionings=[
-                SDXLConditioningInfo(
-                    embeds=torch.cat([c1, c2], dim=-1),
-                    pooled_embeds=c2_pooled,
-                    add_time_ids=add_time_ids,
-                    extra_conditioning=ec1,
-                )
-            ]
-        )
-
-        conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
-        context.services.latents.save(conditioning_name, conditioning_data)
-
-        return CompelOutput(
-            conditioning=ConditioningField(
-                conditioning_name=conditioning_name,
-            ),
-        )
-
-
-class SDXLRefinerRawPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
-    """Parse prompt using compel package to conditioning."""
-
-    type: Literal["sdxl_refiner_raw_prompt"] = "sdxl_refiner_raw_prompt"
-
-    style: str = Field(default="", description="Style prompt")  # TODO: ?
-    original_width: int = Field(1024, description="")
-    original_height: int = Field(1024, description="")
-    crop_top: int = Field(0, description="")
-    crop_left: int = Field(0, description="")
-    aesthetic_score: float = Field(6.0, description="")
-    clip2: ClipField = Field(None, description="Clip to use")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "SDXL Refiner Prompt (Raw)",
-                "tags": ["prompt", "compel"],
-                "type_hints": {"model": "model"},
-            },
-        }
-
-    @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> CompelOutput:
-        c2, c2_pooled, ec2 = self.run_clip_raw(context, self.clip2, self.style, True)
-
-        original_size = (self.original_height, self.original_width)
-        crop_coords = (self.crop_top, self.crop_left)
-
-        add_time_ids = torch.tensor([original_size + crop_coords + (self.aesthetic_score,)])
-
-        conditioning_data = ConditioningFieldData(
-            conditionings=[
-                SDXLConditioningInfo(
-                    embeds=c2,
-                    pooled_embeds=c2_pooled,
-                    add_time_ids=add_time_ids,
-                    extra_conditioning=ec2,  # or None
-                )
-            ]
-        )
-
-        conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
-        context.services.latents.save(conditioning_name, conditioning_data)
-
-        return CompelOutput(
+        return ConditioningOutput(
             conditioning=ConditioningField(
                 conditioning_name=conditioning_name,
             ),
@@ -549,21 +378,18 @@ class ClipSkipInvocationOutput(BaseInvocationOutput):
     """Clip skip node output"""
 
     type: Literal["clip_skip_output"] = "clip_skip_output"
-    clip: ClipField = Field(None, description="Clip with skipped layers")
+    clip: ClipField = OutputField(default=None, description=FieldDescriptions.clip, title="CLIP")
 
 
+@title("CLIP Skip")
+@tags("clipskip", "clip", "skip")
 class ClipSkipInvocation(BaseInvocation):
     """Skip layers in clip text_encoder model."""
 
     type: Literal["clip_skip"] = "clip_skip"
 
-    clip: ClipField = Field(None, description="Clip to use")
-    skipped_layers: int = Field(0, description="Number of layers to skip in text_encoder")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "CLIP Skip", "tags": ["clip", "skip"]},
-        }
+    clip: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection, title="CLIP")
+    skipped_layers: int = InputField(default=0, description=FieldDescriptions.skipped_layers)
 
     def invoke(self, context: InvocationContext) -> ClipSkipInvocationOutput:
         self.clip.skipped_layers += self.skipped_layers

invokeai/app/invocations/controlnet_image_processors.py

@@ -26,79 +26,31 @@ from controlnet_aux.util import HWC3, ade_palette
 from PIL import Image
 from pydantic import BaseModel, Field, validator
 
+from invokeai.app.invocations.primitives import ImageField, ImageOutput
+
+
 from ...backend.model_management import BaseModelType, ModelType
-from ..models.image import ImageCategory, ImageField, ResourceOrigin
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
-from ..models.image import ImageOutput, PILInvocationConfig
+from ..models.image import ImageCategory, ResourceOrigin
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    InputField,
+    Input,
+    InvocationContext,
+    OutputField,
+    UIType,
+    tags,
+    title,
+)
 
-CONTROLNET_DEFAULT_MODELS = [
-    ###########################################
-    # lllyasviel sd v1.5, ControlNet v1.0 models
-    ##############################################
-    "lllyasviel/sd-controlnet-canny",
-    "lllyasviel/sd-controlnet-depth",
-    "lllyasviel/sd-controlnet-hed",
-    "lllyasviel/sd-controlnet-seg",
-    "lllyasviel/sd-controlnet-openpose",
-    "lllyasviel/sd-controlnet-scribble",
-    "lllyasviel/sd-controlnet-normal",
-    "lllyasviel/sd-controlnet-mlsd",
-    #############################################
-    # lllyasviel sd v1.5, ControlNet v1.1 models
-    #############################################
-    "lllyasviel/control_v11p_sd15_canny",
-    "lllyasviel/control_v11p_sd15_openpose",
-    "lllyasviel/control_v11p_sd15_seg",
-    # "lllyasviel/control_v11p_sd15_depth",  # broken
-    "lllyasviel/control_v11f1p_sd15_depth",
-    "lllyasviel/control_v11p_sd15_normalbae",
-    "lllyasviel/control_v11p_sd15_scribble",
-    "lllyasviel/control_v11p_sd15_mlsd",
-    "lllyasviel/control_v11p_sd15_softedge",
-    "lllyasviel/control_v11p_sd15s2_lineart_anime",
-    "lllyasviel/control_v11p_sd15_lineart",
-    "lllyasviel/control_v11p_sd15_inpaint",
-    # "lllyasviel/control_v11u_sd15_tile",
-    # problem (temporary?) with huffingface "lllyasviel/control_v11u_sd15_tile",
-    # so for now replace  "lllyasviel/control_v11f1e_sd15_tile",
-    "lllyasviel/control_v11e_sd15_shuffle",
-    "lllyasviel/control_v11e_sd15_ip2p",
-    "lllyasviel/control_v11f1e_sd15_tile",
-    #################################################
-    #  thibaud sd v2.1 models (ControlNet v1.0? or v1.1?
-    ##################################################
-    "thibaud/controlnet-sd21-openpose-diffusers",
-    "thibaud/controlnet-sd21-canny-diffusers",
-    "thibaud/controlnet-sd21-depth-diffusers",
-    "thibaud/controlnet-sd21-scribble-diffusers",
-    "thibaud/controlnet-sd21-hed-diffusers",
-    "thibaud/controlnet-sd21-zoedepth-diffusers",
-    "thibaud/controlnet-sd21-color-diffusers",
-    "thibaud/controlnet-sd21-openposev2-diffusers",
-    "thibaud/controlnet-sd21-lineart-diffusers",
-    "thibaud/controlnet-sd21-normalbae-diffusers",
-    "thibaud/controlnet-sd21-ade20k-diffusers",
-    ##############################################
-    #  ControlNetMediaPipeface, ControlNet v1.1
-    ##############################################
-    # ["CrucibleAI/ControlNetMediaPipeFace", "diffusion_sd15"],  # SD 1.5
-    #    diffusion_sd15 needs to be passed to from_pretrained() as subfolder arg
-    #    hacked t2l to split to model & subfolder if format is "model,subfolder"
-    "CrucibleAI/ControlNetMediaPipeFace,diffusion_sd15",  # SD 1.5
-    "CrucibleAI/ControlNetMediaPipeFace",  # SD 2.1?
-]
 
-CONTROLNET_NAME_VALUES = Literal[tuple(CONTROLNET_DEFAULT_MODELS)]
-CONTROLNET_MODE_VALUES = Literal[tuple(["balanced", "more_prompt", "more_control", "unbalanced"])]
+CONTROLNET_MODE_VALUES = Literal["balanced", "more_prompt", "more_control", "unbalanced"]
 CONTROLNET_RESIZE_VALUES = Literal[
-    tuple(
-        [
-            "just_resize",
-            "crop_resize",
-            "fill_resize",
-            "just_resize_simple",
-        ]
-    )
+    "just_resize",
+    "crop_resize",
+    "fill_resize",
+    "just_resize_simple",
 ]
 
 
@@ -110,9 +62,8 @@ class ControlNetModelField(BaseModel):
 
 
 class ControlField(BaseModel):
-    image: ImageField = Field(default=None, description="The control image")
-    control_model: Optional[ControlNetModelField] = Field(default=None, description="The ControlNet model to use")
-    # control_weight: Optional[float] = Field(default=1, description="weight given to controlnet")
+    image: ImageField = Field(description="The control image")
+    control_model: ControlNetModelField = Field(description="The ControlNet model to use")
     control_weight: Union[float, List[float]] = Field(default=1, description="The weight given to the ControlNet")
     begin_step_percent: float = Field(
         default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)"
@@ -135,60 +86,39 @@ class ControlField(BaseModel):
                 raise ValueError("Control weights must be within -1 to 2 range")
         return v
 
-    class Config:
-        schema_extra = {
-            "required": ["image", "control_model", "control_weight", "begin_step_percent", "end_step_percent"],
-            "ui": {
-                "type_hints": {
-                    "control_weight": "float",
-                    "control_model": "controlnet_model",
-                    # "control_weight": "number",
-                }
-            },
-        }
-
 
 class ControlOutput(BaseInvocationOutput):
     """node output for ControlNet info"""
 
-    # fmt: off
     type: Literal["control_output"] = "control_output"
-    control: ControlField = Field(default=None, description="The control info")
-    # fmt: on
+
+    # Outputs
+    control: ControlField = OutputField(description=FieldDescriptions.control)
 
 
+@title("ControlNet")
+@tags("controlnet")
 class ControlNetInvocation(BaseInvocation):
     """Collects ControlNet info to pass to other nodes"""
 
-    # fmt: off
     type: Literal["controlnet"] = "controlnet"
-    # Inputs
-    image: ImageField = Field(default=None, description="The control image")
-    control_model: ControlNetModelField = Field(default="lllyasviel/sd-controlnet-canny",
-                                                  description="control model used")
-    control_weight: Union[float, List[float]] = Field(default=1.0, description="The weight given to the ControlNet")
-    begin_step_percent: float = Field(default=0, ge=-1, le=2,
-                                      description="When the ControlNet is first applied (% of total steps)")
-    end_step_percent: float = Field(default=1, ge=0, le=1,
-                                    description="When the ControlNet is last applied (% of total steps)")
-    control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The control mode used")
-    resize_mode: CONTROLNET_RESIZE_VALUES = Field(default="just_resize", description="The resize mode used")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "ControlNet",
-                "tags": ["controlnet", "latents"],
-                "type_hints": {
-                    "model": "model",
-                    "control": "control",
-                    # "cfg_scale": "float",
-                    "cfg_scale": "number",
-                    "control_weight": "float",
-                },
-            },
-        }
+    # Inputs
+    image: ImageField = InputField(description="The control image")
+    control_model: ControlNetModelField = InputField(
+        default="lllyasviel/sd-controlnet-canny", description=FieldDescriptions.controlnet_model, input=Input.Direct
+    )
+    control_weight: Union[float, List[float]] = InputField(
+        default=1.0, description="The weight given to the ControlNet", ui_type=UIType.Float
+    )
+    begin_step_percent: float = InputField(
+        default=0, ge=-1, le=2, description="When the ControlNet is first applied (% of total steps)"
+    )
+    end_step_percent: float = InputField(
+        default=1, ge=0, le=1, description="When the ControlNet is last applied (% of total steps)"
+    )
+    control_mode: CONTROLNET_MODE_VALUES = InputField(default="balanced", description="The control mode used")
+    resize_mode: CONTROLNET_RESIZE_VALUES = InputField(default="just_resize", description="The resize mode used")
 
     def invoke(self, context: InvocationContext) -> ControlOutput:
         return ControlOutput(
@@ -204,19 +134,13 @@ class ControlNetInvocation(BaseInvocation):
         )
 
 
-class ImageProcessorInvocation(BaseInvocation, PILInvocationConfig):
+class ImageProcessorInvocation(BaseInvocation):
     """Base class for invocations that preprocess images for ControlNet"""
 
-    # fmt: off
     type: Literal["image_processor"] = "image_processor"
-    # Inputs
-    image: ImageField = Field(default=None, description="The image to process")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Image Processor", "tags": ["image", "processor"]},
-        }
+    # Inputs
+    image: ImageField = InputField(description="The image to process")
 
     def run_processor(self, image):
         # superclass just passes through image without processing
@@ -255,20 +179,20 @@ class ImageProcessorInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class CannyImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Canny Processor")
+@tags("controlnet", "canny")
+class CannyImageProcessorInvocation(ImageProcessorInvocation):
     """Canny edge detection for ControlNet"""
 
-    # fmt: off
     type: Literal["canny_image_processor"] = "canny_image_processor"
-    # Input
-    low_threshold: int = Field(default=100, ge=0, le=255, description="The low threshold of the Canny pixel gradient (0-255)")
-    high_threshold: int = Field(default=200, ge=0, le=255, description="The high threshold of the Canny pixel gradient (0-255)")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Canny Processor", "tags": ["controlnet", "canny", "image", "processor"]},
-        }
+    # Input
+    low_threshold: int = InputField(
+        default=100, ge=0, le=255, description="The low threshold of the Canny pixel gradient (0-255)"
+    )
+    high_threshold: int = InputField(
+        default=200, ge=0, le=255, description="The high threshold of the Canny pixel gradient (0-255)"
+    )
 
     def run_processor(self, image):
         canny_processor = CannyDetector()
@@ -276,23 +200,19 @@ class CannyImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfi
         return processed_image
 
 
-class HedImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("HED (softedge) Processor")
+@tags("controlnet", "hed", "softedge")
+class HedImageProcessorInvocation(ImageProcessorInvocation):
     """Applies HED edge detection to image"""
 
-    # fmt: off
     type: Literal["hed_image_processor"] = "hed_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    # safe not supported in controlnet_aux v0.0.3
-    # safe: bool = Field(default=False, description="whether to use safe mode")
-    scribble: bool = Field(default=False, description="Whether to use scribble mode")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Softedge(HED) Processor", "tags": ["controlnet", "softedge", "hed", "image", "processor"]},
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
+    # safe not supported in controlnet_aux v0.0.3
+    # safe: bool = InputField(default=False, description=FieldDescriptions.safe_mode)
+    scribble: bool = InputField(default=False, description=FieldDescriptions.scribble_mode)
 
     def run_processor(self, image):
         hed_processor = HEDdetector.from_pretrained("lllyasviel/Annotators")
@@ -307,21 +227,17 @@ class HedImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig)
         return processed_image
 
 
-class LineartImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Lineart Processor")
+@tags("controlnet", "lineart")
+class LineartImageProcessorInvocation(ImageProcessorInvocation):
     """Applies line art processing to image"""
 
-    # fmt: off
     type: Literal["lineart_image_processor"] = "lineart_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    coarse: bool = Field(default=False, description="Whether to use coarse mode")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Lineart Processor", "tags": ["controlnet", "lineart", "image", "processor"]},
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
+    coarse: bool = InputField(default=False, description="Whether to use coarse mode")
 
     def run_processor(self, image):
         lineart_processor = LineartDetector.from_pretrained("lllyasviel/Annotators")
@@ -331,23 +247,16 @@ class LineartImageProcessorInvocation(ImageProcessorInvocation, PILInvocationCon
         return processed_image
 
 
-class LineartAnimeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Lineart Anime Processor")
+@tags("controlnet", "lineart", "anime")
+class LineartAnimeImageProcessorInvocation(ImageProcessorInvocation):
     """Applies line art anime processing to image"""
 
-    # fmt: off
     type: Literal["lineart_anime_image_processor"] = "lineart_anime_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Lineart Anime Processor",
-                "tags": ["controlnet", "lineart", "anime", "image", "processor"],
-            },
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
 
     def run_processor(self, image):
         processor = LineartAnimeDetector.from_pretrained("lllyasviel/Annotators")
@@ -359,21 +268,17 @@ class LineartAnimeImageProcessorInvocation(ImageProcessorInvocation, PILInvocati
         return processed_image
 
 
-class OpenposeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Openpose Processor")
+@tags("controlnet", "openpose", "pose")
+class OpenposeImageProcessorInvocation(ImageProcessorInvocation):
     """Applies Openpose processing to image"""
 
-    # fmt: off
     type: Literal["openpose_image_processor"] = "openpose_image_processor"
-    # Inputs
-    hand_and_face: bool = Field(default=False, description="Whether to use hands and face mode")
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Openpose Processor", "tags": ["controlnet", "openpose", "image", "processor"]},
-        }
+    # Inputs
+    hand_and_face: bool = InputField(default=False, description="Whether to use hands and face mode")
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
 
     def run_processor(self, image):
         openpose_processor = OpenposeDetector.from_pretrained("lllyasviel/Annotators")
@@ -386,22 +291,18 @@ class OpenposeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationCo
         return processed_image
 
 
-class MidasDepthImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Midas (Depth) Processor")
+@tags("controlnet", "midas", "depth")
+class MidasDepthImageProcessorInvocation(ImageProcessorInvocation):
     """Applies Midas depth processing to image"""
 
-    # fmt: off
     type: Literal["midas_depth_image_processor"] = "midas_depth_image_processor"
-    # Inputs
-    a_mult: float = Field(default=2.0, ge=0, description="Midas parameter `a_mult` (a = a_mult * PI)")
-    bg_th: float = Field(default=0.1, ge=0, description="Midas parameter `bg_th`")
-    # depth_and_normal not supported in controlnet_aux v0.0.3
-    # depth_and_normal: bool = Field(default=False, description="whether to use depth and normal mode")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Midas (Depth) Processor", "tags": ["controlnet", "midas", "depth", "image", "processor"]},
-        }
+    # Inputs
+    a_mult: float = InputField(default=2.0, ge=0, description="Midas parameter `a_mult` (a = a_mult * PI)")
+    bg_th: float = InputField(default=0.1, ge=0, description="Midas parameter `bg_th`")
+    # depth_and_normal not supported in controlnet_aux v0.0.3
+    # depth_and_normal: bool = InputField(default=False, description="whether to use depth and normal mode")
 
     def run_processor(self, image):
         midas_processor = MidasDetector.from_pretrained("lllyasviel/Annotators")
@@ -415,20 +316,16 @@ class MidasDepthImageProcessorInvocation(ImageProcessorInvocation, PILInvocation
         return processed_image
 
 
-class NormalbaeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Normal BAE Processor")
+@tags("controlnet", "normal", "bae")
+class NormalbaeImageProcessorInvocation(ImageProcessorInvocation):
     """Applies NormalBae processing to image"""
 
-    # fmt: off
     type: Literal["normalbae_image_processor"] = "normalbae_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Normal BAE Processor", "tags": ["controlnet", "normal", "bae", "image", "processor"]},
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
 
     def run_processor(self, image):
         normalbae_processor = NormalBaeDetector.from_pretrained("lllyasviel/Annotators")
@@ -438,22 +335,18 @@ class NormalbaeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationC
         return processed_image
 
 
-class MlsdImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("MLSD Processor")
+@tags("controlnet", "mlsd")
+class MlsdImageProcessorInvocation(ImageProcessorInvocation):
     """Applies MLSD processing to image"""
 
-    # fmt: off
     type: Literal["mlsd_image_processor"] = "mlsd_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    thr_v: float = Field(default=0.1, ge=0, description="MLSD parameter `thr_v`")
-    thr_d: float = Field(default=0.1, ge=0, description="MLSD parameter `thr_d`")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "MLSD Processor", "tags": ["controlnet", "mlsd", "image", "processor"]},
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
+    thr_v: float = InputField(default=0.1, ge=0, description="MLSD parameter `thr_v`")
+    thr_d: float = InputField(default=0.1, ge=0, description="MLSD parameter `thr_d`")
 
     def run_processor(self, image):
         mlsd_processor = MLSDdetector.from_pretrained("lllyasviel/Annotators")
@@ -467,22 +360,18 @@ class MlsdImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig
         return processed_image
 
 
-class PidiImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("PIDI Processor")
+@tags("controlnet", "pidi")
+class PidiImageProcessorInvocation(ImageProcessorInvocation):
     """Applies PIDI processing to image"""
 
-    # fmt: off
     type: Literal["pidi_image_processor"] = "pidi_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    safe: bool = Field(default=False, description="Whether to use safe mode")
-    scribble: bool = Field(default=False, description="Whether to use scribble mode")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "PIDI Processor", "tags": ["controlnet", "pidi", "image", "processor"]},
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
+    safe: bool = InputField(default=False, description=FieldDescriptions.safe_mode)
+    scribble: bool = InputField(default=False, description=FieldDescriptions.scribble_mode)
 
     def run_processor(self, image):
         pidi_processor = PidiNetDetector.from_pretrained("lllyasviel/Annotators")
@@ -496,26 +385,19 @@ class PidiImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig
         return processed_image
 
 
-class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Content Shuffle Processor")
+@tags("controlnet", "contentshuffle")
+class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation):
     """Applies content shuffle processing to image"""
 
-    # fmt: off
     type: Literal["content_shuffle_image_processor"] = "content_shuffle_image_processor"
-    # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    h: Optional[int] = Field(default=512, ge=0, description="Content shuffle `h` parameter")
-    w: Optional[int] = Field(default=512, ge=0, description="Content shuffle `w` parameter")
-    f: Optional[int] = Field(default=256, ge=0, description="Content shuffle `f` parameter")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Content Shuffle Processor",
-                "tags": ["controlnet", "contentshuffle", "image", "processor"],
-            },
-        }
+    # Inputs
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
+    h: Optional[int] = InputField(default=512, ge=0, description="Content shuffle `h` parameter")
+    w: Optional[int] = InputField(default=512, ge=0, description="Content shuffle `w` parameter")
+    f: Optional[int] = InputField(default=256, ge=0, description="Content shuffle `f` parameter")
 
     def run_processor(self, image):
         content_shuffle_processor = ContentShuffleDetector()
@@ -531,17 +413,12 @@ class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation, PILInvoca
 
 
 # should work with controlnet_aux >= 0.0.4 and timm <= 0.6.13
-class ZoeDepthImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Zoe (Depth) Processor")
+@tags("controlnet", "zoe", "depth")
+class ZoeDepthImageProcessorInvocation(ImageProcessorInvocation):
     """Applies Zoe depth processing to image"""
 
-    # fmt: off
     type: Literal["zoe_depth_image_processor"] = "zoe_depth_image_processor"
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Zoe (Depth) Processor", "tags": ["controlnet", "zoe", "depth", "image", "processor"]},
-        }
 
     def run_processor(self, image):
         zoe_depth_processor = ZoeDetector.from_pretrained("lllyasviel/Annotators")
@@ -549,20 +426,16 @@ class ZoeDepthImageProcessorInvocation(ImageProcessorInvocation, PILInvocationCo
         return processed_image
 
 
-class MediapipeFaceProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Mediapipe Face Processor")
+@tags("controlnet", "mediapipe", "face")
+class MediapipeFaceProcessorInvocation(ImageProcessorInvocation):
     """Applies mediapipe face processing to image"""
 
-    # fmt: off
     type: Literal["mediapipe_face_processor"] = "mediapipe_face_processor"
-    # Inputs
-    max_faces: int = Field(default=1, ge=1, description="Maximum number of faces to detect")
-    min_confidence: float = Field(default=0.5, ge=0, le=1, description="Minimum confidence for face detection")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Mediapipe Processor", "tags": ["controlnet", "mediapipe", "image", "processor"]},
-        }
+    # Inputs
+    max_faces: int = InputField(default=1, ge=1, description="Maximum number of faces to detect")
+    min_confidence: float = InputField(default=0.5, ge=0, le=1, description="Minimum confidence for face detection")
 
     def run_processor(self, image):
         # MediaPipeFaceDetector throws an error if image has alpha channel
@@ -574,23 +447,19 @@ class MediapipeFaceProcessorInvocation(ImageProcessorInvocation, PILInvocationCo
         return processed_image
 
 
-class LeresImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Leres (Depth) Processor")
+@tags("controlnet", "leres", "depth")
+class LeresImageProcessorInvocation(ImageProcessorInvocation):
     """Applies leres processing to image"""
 
-    # fmt: off
     type: Literal["leres_image_processor"] = "leres_image_processor"
-    # Inputs
-    thr_a: float = Field(default=0, description="Leres parameter `thr_a`")
-    thr_b: float = Field(default=0, description="Leres parameter `thr_b`")
-    boost: bool = Field(default=False, description="Whether to use boost mode")
-    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
-    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Leres (Depth) Processor", "tags": ["controlnet", "leres", "depth", "image", "processor"]},
-        }
+    # Inputs
+    thr_a: float = InputField(default=0, description="Leres parameter `thr_a`")
+    thr_b: float = InputField(default=0, description="Leres parameter `thr_b`")
+    boost: bool = InputField(default=False, description="Whether to use boost mode")
+    detect_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.detect_res)
+    image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
 
     def run_processor(self, image):
         leres_processor = LeresDetector.from_pretrained("lllyasviel/Annotators")
@@ -605,21 +474,16 @@ class LeresImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfi
         return processed_image
 
 
-class TileResamplerProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
-    # fmt: off
-    type: Literal["tile_image_processor"] = "tile_image_processor"
-    # Inputs
-    #res: int = Field(default=512, ge=0, le=1024, description="The pixel resolution for each tile")
-    down_sampling_rate: float = Field(default=1.0, ge=1.0, le=8.0, description="Down sampling rate")
-    # fmt: on
+@title("Tile Resample Processor")
+@tags("controlnet", "tile")
+class TileResamplerProcessorInvocation(ImageProcessorInvocation):
+    """Tile resampler processor"""
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Tile Resample Processor",
-                "tags": ["controlnet", "tile", "resample", "image", "processor"],
-            },
-        }
+    type: Literal["tile_image_processor"] = "tile_image_processor"
+
+    # Inputs
+    # res: int = InputField(default=512, ge=0, le=1024, description="The pixel resolution for each tile")
+    down_sampling_rate: float = InputField(default=1.0, ge=1.0, le=8.0, description="Down sampling rate")
 
     # tile_resample copied from sd-webui-controlnet/scripts/processor.py
     def tile_resample(
@@ -648,20 +512,12 @@ class TileResamplerProcessorInvocation(ImageProcessorInvocation, PILInvocationCo
         return processed_image
 
 
-class SegmentAnythingProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+@title("Segment Anything Processor")
+@tags("controlnet", "segmentanything")
+class SegmentAnythingProcessorInvocation(ImageProcessorInvocation):
     """Applies segment anything processing to image"""
 
-    # fmt: off
     type: Literal["segment_anything_processor"] = "segment_anything_processor"
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Segment Anything Processor",
-                "tags": ["controlnet", "segment", "anything", "sam", "image", "processor"],
-            },
-        }
 
     def run_processor(self, image):
         # segment_anything_processor = SamDetector.from_pretrained("ybelkada/segment-anything", subfolder="checkpoints")

invokeai/app/invocations/cv.py

@@ -5,40 +5,22 @@ from typing import Literal
 import cv2 as cv
 import numpy
 from PIL import Image, ImageOps
-from pydantic import BaseModel, Field
+from invokeai.app.invocations.primitives import ImageField, ImageOutput
 
-from invokeai.app.models.image import ImageCategory, ImageField, ResourceOrigin
-from .baseinvocation import BaseInvocation, InvocationContext, InvocationConfig
-from .image import ImageOutput
+from invokeai.app.models.image import ImageCategory, ResourceOrigin
+from .baseinvocation import BaseInvocation, InputField, InvocationContext, tags, title
 
 
-class CvInvocationConfig(BaseModel):
-    """Helper class to provide all OpenCV invocations with additional config"""
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "tags": ["cv", "image"],
-            },
-        }
-
-
-class CvInpaintInvocation(BaseInvocation, CvInvocationConfig):
+@title("OpenCV Inpaint")
+@tags("opencv", "inpaint")
+class CvInpaintInvocation(BaseInvocation):
     """Simple inpaint using opencv."""
 
-    # fmt: off
     type: Literal["cv_inpaint"] = "cv_inpaint"
 
     # Inputs
-    image: ImageField = Field(default=None, description="The image to inpaint")
-    mask: ImageField = Field(default=None, description="The mask to use when inpainting")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "OpenCV Inpaint", "tags": ["opencv", "inpaint"]},
-        }
+    image: ImageField = InputField(description="The image to inpaint")
+    mask: ImageField = InputField(description="The mask to use when inpainting")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)

invokeai/app/invocations/generate.py

@@ -1,251 +0,0 @@
-# Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
-
-from functools import partial
-from typing import Literal, Optional, get_args
-
-import torch
-from pydantic import Field
-
-from invokeai.app.models.image import ColorField, ImageCategory, ImageField, ResourceOrigin
-from invokeai.app.util.misc import SEED_MAX, get_random_seed
-from invokeai.backend.generator.inpaint import infill_methods
-
-from ...backend.generator import Inpaint, InvokeAIGenerator
-from ...backend.stable_diffusion import PipelineIntermediateState
-from ..util.step_callback import stable_diffusion_step_callback
-from .baseinvocation import BaseInvocation, InvocationConfig, InvocationContext
-from .image import ImageOutput
-
-from ...backend.model_management.lora import ModelPatcher
-from ...backend.stable_diffusion.diffusers_pipeline import StableDiffusionGeneratorPipeline
-from .model import UNetField, VaeField
-from .compel import ConditioningField
-from contextlib import contextmanager, ExitStack, ContextDecorator
-
-SAMPLER_NAME_VALUES = Literal[tuple(InvokeAIGenerator.schedulers())]
-INFILL_METHODS = Literal[tuple(infill_methods())]
-DEFAULT_INFILL_METHOD = "patchmatch" if "patchmatch" in get_args(INFILL_METHODS) else "tile"
-
-
-from .latent import get_scheduler
-
-
-class OldModelContext(ContextDecorator):
-    model: StableDiffusionGeneratorPipeline
-
-    def __init__(self, model):
-        self.model = model
-
-    def __enter__(self):
-        return self.model
-
-    def __exit__(self, *exc):
-        return False
-
-
-class OldModelInfo:
-    name: str
-    hash: str
-    context: OldModelContext
-
-    def __init__(self, name: str, hash: str, model: StableDiffusionGeneratorPipeline):
-        self.name = name
-        self.hash = hash
-        self.context = OldModelContext(
-            model=model,
-        )
-
-
-class InpaintInvocation(BaseInvocation):
-    """Generates an image using inpaint."""
-
-    type: Literal["inpaint"] = "inpaint"
-
-    positive_conditioning: Optional[ConditioningField] = Field(description="Positive conditioning for generation")
-    negative_conditioning: Optional[ConditioningField] = Field(description="Negative conditioning for generation")
-    seed: int = Field(
-        ge=0, le=SEED_MAX, description="The seed to use (omit for random)", default_factory=get_random_seed
-    )
-    steps: int = Field(default=30, gt=0, description="The number of steps to use to generate the image")
-    width: int = Field(
-        default=512,
-        multiple_of=8,
-        gt=0,
-        description="The width of the resulting image",
-    )
-    height: int = Field(
-        default=512,
-        multiple_of=8,
-        gt=0,
-        description="The height of the resulting image",
-    )
-    cfg_scale: float = Field(
-        default=7.5,
-        ge=1,
-        description="The Classifier-Free Guidance, higher values may result in a result closer to the prompt",
-    )
-    scheduler: SAMPLER_NAME_VALUES = Field(default="euler", description="The scheduler to use")
-    unet: UNetField = Field(default=None, description="UNet model")
-    vae: VaeField = Field(default=None, description="Vae model")
-
-    # Inputs
-    image: Optional[ImageField] = Field(description="The input image")
-    strength: float = Field(default=0.75, gt=0, le=1, description="The strength of the original image")
-    fit: bool = Field(
-        default=True,
-        description="Whether or not the result should be fit to the aspect ratio of the input image",
-    )
-
-    # Inputs
-    mask: Optional[ImageField] = Field(description="The mask")
-    seam_size: int = Field(default=96, ge=1, description="The seam inpaint size (px)")
-    seam_blur: int = Field(default=16, ge=0, description="The seam inpaint blur radius (px)")
-    seam_strength: float = Field(default=0.75, gt=0, le=1, description="The seam inpaint strength")
-    seam_steps: int = Field(default=30, ge=1, description="The number of steps to use for seam inpaint")
-    tile_size: int = Field(default=32, ge=1, description="The tile infill method size (px)")
-    infill_method: INFILL_METHODS = Field(
-        default=DEFAULT_INFILL_METHOD,
-        description="The method used to infill empty regions (px)",
-    )
-    inpaint_width: Optional[int] = Field(
-        default=None,
-        multiple_of=8,
-        gt=0,
-        description="The width of the inpaint region (px)",
-    )
-    inpaint_height: Optional[int] = Field(
-        default=None,
-        multiple_of=8,
-        gt=0,
-        description="The height of the inpaint region (px)",
-    )
-    inpaint_fill: Optional[ColorField] = Field(
-        default=ColorField(r=127, g=127, b=127, a=255),
-        description="The solid infill method color",
-    )
-    inpaint_replace: float = Field(
-        default=0.0,
-        ge=0.0,
-        le=1.0,
-        description="The amount by which to replace masked areas with latent noise",
-    )
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"tags": ["stable-diffusion", "image"], "title": "Inpaint"},
-        }
-
-    def dispatch_progress(
-        self,
-        context: InvocationContext,
-        source_node_id: str,
-        intermediate_state: PipelineIntermediateState,
-    ) -> None:
-        stable_diffusion_step_callback(
-            context=context,
-            intermediate_state=intermediate_state,
-            node=self.dict(),
-            source_node_id=source_node_id,
-        )
-
-    def get_conditioning(self, context, unet):
-        positive_cond_data = context.services.latents.get(self.positive_conditioning.conditioning_name)
-        c = positive_cond_data.conditionings[0].embeds.to(device=unet.device, dtype=unet.dtype)
-        extra_conditioning_info = positive_cond_data.conditionings[0].extra_conditioning
-
-        negative_cond_data = context.services.latents.get(self.negative_conditioning.conditioning_name)
-        uc = negative_cond_data.conditionings[0].embeds.to(device=unet.device, dtype=unet.dtype)
-
-        return (uc, c, extra_conditioning_info)
-
-    @contextmanager
-    def load_model_old_way(self, context, scheduler):
-        def _lora_loader():
-            for lora in self.unet.loras:
-                lora_info = context.services.model_manager.get_model(
-                    **lora.dict(exclude={"weight"}),
-                    context=context,
-                )
-                yield (lora_info.context.model, lora.weight)
-                del lora_info
-            return
-
-        unet_info = context.services.model_manager.get_model(
-            **self.unet.unet.dict(),
-            context=context,
-        )
-        vae_info = context.services.model_manager.get_model(
-            **self.vae.vae.dict(),
-            context=context,
-        )
-
-        with vae_info as vae, ModelPatcher.apply_lora_unet(unet_info.context.model, _lora_loader()), unet_info as unet:
-            device = context.services.model_manager.mgr.cache.execution_device
-            dtype = context.services.model_manager.mgr.cache.precision
-
-            pipeline = StableDiffusionGeneratorPipeline(
-                vae=vae,
-                text_encoder=None,
-                tokenizer=None,
-                unet=unet,
-                scheduler=scheduler,
-                safety_checker=None,
-                feature_extractor=None,
-                requires_safety_checker=False,
-                precision="float16" if dtype == torch.float16 else "float32",
-                execution_device=device,
-            )
-
-            yield OldModelInfo(
-                name=self.unet.unet.model_name,
-                hash="<NO-HASH>",
-                model=pipeline,
-            )
-
-    def invoke(self, context: InvocationContext) -> ImageOutput:
-        image = None if self.image is None else context.services.images.get_pil_image(self.image.image_name)
-        mask = None if self.mask is None else context.services.images.get_pil_image(self.mask.image_name)
-
-        # Get the source node id (we are invoking the prepared node)
-        graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
-        source_node_id = graph_execution_state.prepared_source_mapping[self.id]
-
-        scheduler = get_scheduler(
-            context=context,
-            scheduler_info=self.unet.scheduler,
-            scheduler_name=self.scheduler,
-        )
-
-        with self.load_model_old_way(context, scheduler) as model:
-            conditioning = self.get_conditioning(context, model.context.model.unet)
-
-            outputs = Inpaint(model).generate(
-                conditioning=conditioning,
-                scheduler=scheduler,
-                init_image=image,
-                mask_image=mask,
-                step_callback=partial(self.dispatch_progress, context, source_node_id),
-                **self.dict(
-                    exclude={"positive_conditioning", "negative_conditioning", "scheduler", "image", "mask"}
-                ),  # Shorthand for passing all of the parameters above manually
-            )
-
-            # Outputs is an infinite iterator that will return a new InvokeAIGeneratorOutput object
-            # each time it is called. We only need the first one.
-            generator_output = next(outputs)
-
-            image_dto = context.services.images.create(
-                image=generator_output.image,
-                image_origin=ResourceOrigin.INTERNAL,
-                image_category=ImageCategory.GENERAL,
-                session_id=context.graph_execution_state_id,
-                node_id=self.id,
-                is_intermediate=self.is_intermediate,
-            )
-
-            return ImageOutput(
-                image=ImageField(image_name=image_dto.image_name),
-                width=image_dto.width,
-                height=image_dto.height,
-            )

invokeai/app/invocations/image.py

@@ -1,69 +1,31 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
+from pathlib import Path
 from typing import Literal, Optional
 
+import cv2
 import numpy
-from PIL import Image, ImageFilter, ImageOps, ImageChops
-from pydantic import Field
-from pathlib import Path
-from typing import Union
+from PIL import Image, ImageChops, ImageFilter, ImageOps
+
 from invokeai.app.invocations.metadata import CoreMetadata
-from ..models.image import (
-    ImageCategory,
-    ImageField,
-    ResourceOrigin,
-    PILInvocationConfig,
-    ImageOutput,
-    MaskOutput,
-)
-from .baseinvocation import (
-    BaseInvocation,
-    InvocationContext,
-    InvocationConfig,
-)
-from invokeai.backend.image_util.safety_checker import SafetyChecker
+from invokeai.app.invocations.primitives import ImageField, ImageOutput
 from invokeai.backend.image_util.invisible_watermark import InvisibleWatermark
+from invokeai.backend.image_util.safety_checker import SafetyChecker
+
+from ..models.image import ImageCategory, ResourceOrigin
+from .baseinvocation import BaseInvocation, FieldDescriptions, InputField, InvocationContext, tags, title
 
 
-class LoadImageInvocation(BaseInvocation):
-    """Load an image and provide it as output."""
-
-    # fmt: off
-    type: Literal["load_image"] = "load_image"
-
-    # Inputs
-    image: Optional[ImageField] = Field(
-        default=None, description="The image to load"
-    )
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Load Image", "tags": ["image", "load"]},
-        }
-
-    def invoke(self, context: InvocationContext) -> ImageOutput:
-        image = context.services.images.get_pil_image(self.image.image_name)
-
-        return ImageOutput(
-            image=ImageField(image_name=self.image.image_name),
-            width=image.width,
-            height=image.height,
-        )
-
-
+@title("Show Image")
+@tags("image")
 class ShowImageInvocation(BaseInvocation):
     """Displays a provided image, and passes it forward in the pipeline."""
 
+    # Metadata
     type: Literal["show_image"] = "show_image"
 
     # Inputs
-    image: Optional[ImageField] = Field(default=None, description="The image to show")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Show Image", "tags": ["image", "show"]},
-        }
+    image: ImageField = InputField(description="The image to show")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -79,24 +41,20 @@ class ShowImageInvocation(BaseInvocation):
         )
 
 
-class ImageCropInvocation(BaseInvocation, PILInvocationConfig):
+@title("Crop Image")
+@tags("image", "crop")
+class ImageCropInvocation(BaseInvocation):
     """Crops an image to a specified box. The box can be outside of the image."""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_crop"] = "img_crop"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to crop")
-    x:      int = Field(default=0, description="The left x coordinate of the crop rectangle")
-    y:      int = Field(default=0, description="The top y coordinate of the crop rectangle")
-    width:  int = Field(default=512, gt=0, description="The width of the crop rectangle")
-    height: int = Field(default=512, gt=0, description="The height of the crop rectangle")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Crop Image", "tags": ["image", "crop"]},
-        }
+    image: ImageField = InputField(description="The image to crop")
+    x: int = InputField(default=0, description="The left x coordinate of the crop rectangle")
+    y: int = InputField(default=0, description="The top y coordinate of the crop rectangle")
+    width: int = InputField(default=512, gt=0, description="The width of the crop rectangle")
+    height: int = InputField(default=512, gt=0, description="The height of the crop rectangle")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -120,31 +78,31 @@ class ImageCropInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class ImagePasteInvocation(BaseInvocation, PILInvocationConfig):
+@title("Paste Image")
+@tags("image", "paste")
+class ImagePasteInvocation(BaseInvocation):
     """Pastes an image into another image."""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_paste"] = "img_paste"
 
     # Inputs
-    base_image:     Optional[ImageField]  = Field(default=None, description="The base image")
-    image:          Optional[ImageField]  = Field(default=None, description="The image to paste")
-    mask: Optional[ImageField] = Field(default=None, description="The mask to use when pasting")
-    x:                     int = Field(default=0, description="The left x coordinate at which to paste the image")
-    y:                     int = Field(default=0, description="The top y coordinate at which to paste the image")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Paste Image", "tags": ["image", "paste"]},
-        }
+    base_image: ImageField = InputField(description="The base image")
+    image: ImageField = InputField(description="The image to paste")
+    mask: Optional[ImageField] = InputField(
+        default=None,
+        description="The mask to use when pasting",
+    )
+    x: int = InputField(default=0, description="The left x coordinate at which to paste the image")
+    y: int = InputField(default=0, description="The top y coordinate at which to paste the image")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         base_image = context.services.images.get_pil_image(self.base_image.image_name)
         image = context.services.images.get_pil_image(self.image.image_name)
-        mask = (
-            None if self.mask is None else ImageOps.invert(context.services.images.get_pil_image(self.mask.image_name))
-        )
+        mask = None
+        if self.mask is not None:
+            mask = context.services.images.get_pil_image(self.mask.image_name)
+            mask = ImageOps.invert(mask.convert("L"))
         # TODO: probably shouldn't invert mask here... should user be required to do it?
 
         min_x = min(0, self.x)
@@ -172,23 +130,19 @@ class ImagePasteInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class MaskFromAlphaInvocation(BaseInvocation, PILInvocationConfig):
+@title("Mask from Alpha")
+@tags("image", "mask")
+class MaskFromAlphaInvocation(BaseInvocation):
     """Extracts the alpha channel of an image as a mask."""
 
-    # fmt: off
+    # Metadata
     type: Literal["tomask"] = "tomask"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to create the mask from")
-    invert:      bool = Field(default=False, description="Whether or not to invert the mask")
-    # fmt: on
+    image: ImageField = InputField(description="The image to create the mask from")
+    invert: bool = InputField(default=False, description="Whether or not to invert the mask")
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Mask From Alpha", "tags": ["image", "mask", "alpha"]},
-        }
-
-    def invoke(self, context: InvocationContext) -> MaskOutput:
+    def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
 
         image_mask = image.split()[-1]
@@ -204,28 +158,24 @@ class MaskFromAlphaInvocation(BaseInvocation, PILInvocationConfig):
             is_intermediate=self.is_intermediate,
         )
 
-        return MaskOutput(
-            mask=ImageField(image_name=image_dto.image_name),
+        return ImageOutput(
+            image=ImageField(image_name=image_dto.image_name),
             width=image_dto.width,
             height=image_dto.height,
         )
 
 
-class ImageMultiplyInvocation(BaseInvocation, PILInvocationConfig):
+@title("Multiply Images")
+@tags("image", "multiply")
+class ImageMultiplyInvocation(BaseInvocation):
     """Multiplies two images together using `PIL.ImageChops.multiply()`."""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_mul"] = "img_mul"
 
     # Inputs
-    image1: Optional[ImageField]  = Field(default=None, description="The first image to multiply")
-    image2: Optional[ImageField]  = Field(default=None, description="The second image to multiply")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Multiply Images", "tags": ["image", "multiply"]},
-        }
+    image1: ImageField = InputField(description="The first image to multiply")
+    image2: ImageField = InputField(description="The second image to multiply")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image1 = context.services.images.get_pil_image(self.image1.image_name)
@@ -252,21 +202,17 @@ class ImageMultiplyInvocation(BaseInvocation, PILInvocationConfig):
 IMAGE_CHANNELS = Literal["A", "R", "G", "B"]
 
 
-class ImageChannelInvocation(BaseInvocation, PILInvocationConfig):
+@title("Extract Image Channel")
+@tags("image", "channel")
+class ImageChannelInvocation(BaseInvocation):
     """Gets a channel from an image."""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_chan"] = "img_chan"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to get the channel from")
-    channel: IMAGE_CHANNELS  = Field(default="A", description="The channel to get")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Image Channel", "tags": ["image", "channel"]},
-        }
+    image: ImageField = InputField(description="The image to get the channel from")
+    channel: IMAGE_CHANNELS = InputField(default="A", description="The channel to get")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -292,21 +238,17 @@ class ImageChannelInvocation(BaseInvocation, PILInvocationConfig):
 IMAGE_MODES = Literal["L", "RGB", "RGBA", "CMYK", "YCbCr", "LAB", "HSV", "I", "F"]
 
 
-class ImageConvertInvocation(BaseInvocation, PILInvocationConfig):
+@title("Convert Image Mode")
+@tags("image", "convert")
+class ImageConvertInvocation(BaseInvocation):
     """Converts an image to a different mode."""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_conv"] = "img_conv"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to convert")
-    mode: IMAGE_MODES  = Field(default="L", description="The mode to convert to")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Convert Image", "tags": ["image", "convert"]},
-        }
+    image: ImageField = InputField(description="The image to convert")
+    mode: IMAGE_MODES = InputField(default="L", description="The mode to convert to")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -329,22 +271,19 @@ class ImageConvertInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class ImageBlurInvocation(BaseInvocation, PILInvocationConfig):
+@title("Blur Image")
+@tags("image", "blur")
+class ImageBlurInvocation(BaseInvocation):
     """Blurs an image"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_blur"] = "img_blur"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to blur")
-    radius:     float = Field(default=8.0, ge=0, description="The blur radius")
-    blur_type: Literal["gaussian", "box"] = Field(default="gaussian", description="The type of blur")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Blur Image", "tags": ["image", "blur"]},
-        }
+    image: ImageField = InputField(description="The image to blur")
+    radius: float = InputField(default=8.0, ge=0, description="The blur radius")
+    # Metadata
+    blur_type: Literal["gaussian", "box"] = InputField(default="gaussian", description="The type of blur")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -390,23 +329,19 @@ PIL_RESAMPLING_MAP = {
 }
 
 
-class ImageResizeInvocation(BaseInvocation, PILInvocationConfig):
+@title("Resize Image")
+@tags("image", "resize")
+class ImageResizeInvocation(BaseInvocation):
     """Resizes an image to specific dimensions"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_resize"] = "img_resize"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to resize")
-    width:                         Union[int, None] = Field(ge=64, multiple_of=8, description="The width to resize to (px)")
-    height:                        Union[int, None] = Field(ge=64, multiple_of=8, description="The height to resize to (px)")
-    resample_mode:  PIL_RESAMPLING_MODES = Field(default="bicubic", description="The resampling mode")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Resize Image", "tags": ["image", "resize"]},
-        }
+    image: ImageField = InputField(description="The image to resize")
+    width: int = InputField(default=512, ge=64, multiple_of=8, description="The width to resize to (px)")
+    height: int = InputField(default=512, ge=64, multiple_of=8, description="The height to resize to (px)")
+    resample_mode: PIL_RESAMPLING_MODES = InputField(default="bicubic", description="The resampling mode")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -434,22 +369,22 @@ class ImageResizeInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class ImageScaleInvocation(BaseInvocation, PILInvocationConfig):
+@title("Scale Image")
+@tags("image", "scale")
+class ImageScaleInvocation(BaseInvocation):
     """Scales an image by a factor"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_scale"] = "img_scale"
 
     # Inputs
-    image:          Optional[ImageField] = Field(default=None, description="The image to scale")
-    scale_factor:        Optional[float] = Field(default=2.0, gt=0, description="The factor by which to scale the image")
-    resample_mode:  PIL_RESAMPLING_MODES = Field(default="bicubic", description="The resampling mode")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Scale Image", "tags": ["image", "scale"]},
-        }
+    image: ImageField = InputField(description="The image to scale")
+    scale_factor: float = InputField(
+        default=2.0,
+        gt=0,
+        description="The factor by which to scale the image",
+    )
+    resample_mode: PIL_RESAMPLING_MODES = InputField(default="bicubic", description="The resampling mode")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -479,28 +414,24 @@ class ImageScaleInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class ImageLerpInvocation(BaseInvocation, PILInvocationConfig):
+@title("Lerp Image")
+@tags("image", "lerp")
+class ImageLerpInvocation(BaseInvocation):
     """Linear interpolation of all pixels of an image"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_lerp"] = "img_lerp"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to lerp")
-    min: int = Field(default=0, ge=0, le=255, description="The minimum output value")
-    max: int = Field(default=255, ge=0, le=255, description="The maximum output value")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Image Linear Interpolation", "tags": ["image", "linear", "interpolation", "lerp"]},
-        }
+    image: ImageField = InputField(description="The image to lerp")
+    min: int = InputField(default=0, ge=0, le=255, description="The minimum output value")
+    max: int = InputField(default=255, ge=0, le=255, description="The maximum output value")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
 
         image_arr = numpy.asarray(image, dtype=numpy.float32) / 255
-        image_arr = image_arr * (self.max - self.min) + self.max
+        image_arr = image_arr * (self.max - self.min) + self.min
 
         lerp_image = Image.fromarray(numpy.uint8(image_arr))
 
@@ -520,25 +451,18 @@ class ImageLerpInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class ImageInverseLerpInvocation(BaseInvocation, PILInvocationConfig):
+@title("Inverse Lerp Image")
+@tags("image", "ilerp")
+class ImageInverseLerpInvocation(BaseInvocation):
     """Inverse linear interpolation of all pixels of an image"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_ilerp"] = "img_ilerp"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to lerp")
-    min: int = Field(default=0, ge=0, le=255, description="The minimum input value")
-    max: int = Field(default=255, ge=0, le=255, description="The maximum input value")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Image Inverse Linear Interpolation",
-                "tags": ["image", "linear", "interpolation", "inverse"],
-            },
-        }
+    image: ImageField = InputField(description="The image to lerp")
+    min: int = InputField(default=0, ge=0, le=255, description="The minimum input value")
+    max: int = InputField(default=255, ge=0, le=255, description="The maximum input value")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -564,21 +488,19 @@ class ImageInverseLerpInvocation(BaseInvocation, PILInvocationConfig):
         )
 
 
-class ImageNSFWBlurInvocation(BaseInvocation, PILInvocationConfig):
+@title("Blur NSFW Image")
+@tags("image", "nsfw")
+class ImageNSFWBlurInvocation(BaseInvocation):
     """Add blur to NSFW-flagged images"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_nsfw"] = "img_nsfw"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to check")
-    metadata: Optional[CoreMetadata] = Field(default=None, description="Optional core metadata to be written to the image")    
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Blur NSFW Images", "tags": ["image", "nsfw", "checker"]},
-        }
+    image: ImageField = InputField(description="The image to check")
+    metadata: Optional[CoreMetadata] = InputField(
+        default=None, description=FieldDescriptions.core_metadata, ui_hidden=True
+    )
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -615,22 +537,20 @@ class ImageNSFWBlurInvocation(BaseInvocation, PILInvocationConfig):
         return caution.resize((caution.width // 2, caution.height // 2))
 
 
-class ImageWatermarkInvocation(BaseInvocation, PILInvocationConfig):
+@title("Add Invisible Watermark")
+@tags("image", "watermark")
+class ImageWatermarkInvocation(BaseInvocation):
     """Add an invisible watermark to an image"""
 
-    # fmt: off
+    # Metadata
     type: Literal["img_watermark"] = "img_watermark"
 
     # Inputs
-    image: Optional[ImageField]  = Field(default=None, description="The image to check")
-    text: str = Field(default='InvokeAI', description="Watermark text")
-    metadata: Optional[CoreMetadata] = Field(default=None, description="Optional core metadata to be written to the image")    
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Add Invisible Watermark", "tags": ["image", "watermark", "invisible"]},
-        }
+    image: ImageField = InputField(description="The image to check")
+    text: str = InputField(default="InvokeAI", description="Watermark text")
+    metadata: Optional[CoreMetadata] = InputField(
+        default=None, description=FieldDescriptions.core_metadata, ui_hidden=True
+    )
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
@@ -650,3 +570,334 @@ class ImageWatermarkInvocation(BaseInvocation, PILInvocationConfig):
             width=image_dto.width,
             height=image_dto.height,
         )
+
+
+@title("Mask Edge")
+@tags("image", "mask", "inpaint")
+class MaskEdgeInvocation(BaseInvocation):
+    """Applies an edge mask to an image"""
+
+    type: Literal["mask_edge"] = "mask_edge"
+
+    # Inputs
+    image: ImageField = InputField(description="The image to apply the mask to")
+    edge_size: int = InputField(description="The size of the edge")
+    edge_blur: int = InputField(description="The amount of blur on the edge")
+    low_threshold: int = InputField(description="First threshold for the hysteresis procedure in Canny edge detection")
+    high_threshold: int = InputField(
+        description="Second threshold for the hysteresis procedure in Canny edge detection"
+    )
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        mask = context.services.images.get_pil_image(self.image.image_name)
+
+        npimg = numpy.asarray(mask, dtype=numpy.uint8)
+        npgradient = numpy.uint8(255 * (1.0 - numpy.floor(numpy.abs(0.5 - numpy.float32(npimg) / 255.0) * 2.0)))
+        npedge = cv2.Canny(npimg, threshold1=self.low_threshold, threshold2=self.high_threshold)
+        npmask = npgradient + npedge
+        npmask = cv2.dilate(npmask, numpy.ones((3, 3), numpy.uint8), iterations=int(self.edge_size / 2))
+
+        new_mask = Image.fromarray(npmask)
+
+        if self.edge_blur > 0:
+            new_mask = new_mask.filter(ImageFilter.BoxBlur(self.edge_blur))
+
+        new_mask = ImageOps.invert(new_mask)
+
+        image_dto = context.services.images.create(
+            image=new_mask,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.MASK,
+            node_id=self.id,
+            session_id=context.graph_execution_state_id,
+            is_intermediate=self.is_intermediate,
+        )
+
+        return ImageOutput(
+            image=ImageField(image_name=image_dto.image_name),
+            width=image_dto.width,
+            height=image_dto.height,
+        )
+
+
+@title("Combine Mask")
+@tags("image", "mask", "multiply")
+class MaskCombineInvocation(BaseInvocation):
+    """Combine two masks together by multiplying them using `PIL.ImageChops.multiply()`."""
+
+    type: Literal["mask_combine"] = "mask_combine"
+
+    # Inputs
+    mask1: ImageField = InputField(description="The first mask to combine")
+    mask2: ImageField = InputField(description="The second image to combine")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        mask1 = context.services.images.get_pil_image(self.mask1.image_name).convert("L")
+        mask2 = context.services.images.get_pil_image(self.mask2.image_name).convert("L")
+
+        combined_mask = ImageChops.multiply(mask1, mask2)
+
+        image_dto = context.services.images.create(
+            image=combined_mask,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            session_id=context.graph_execution_state_id,
+            is_intermediate=self.is_intermediate,
+        )
+
+        return ImageOutput(
+            image=ImageField(image_name=image_dto.image_name),
+            width=image_dto.width,
+            height=image_dto.height,
+        )
+
+
+@title("Color Correct")
+@tags("image", "color")
+class ColorCorrectInvocation(BaseInvocation):
+    """
+    Shifts the colors of a target image to match the reference image, optionally
+    using a mask to only color-correct certain regions of the target image.
+    """
+
+    type: Literal["color_correct"] = "color_correct"
+
+    # Inputs
+    image: ImageField = InputField(description="The image to color-correct")
+    reference: ImageField = InputField(description="Reference image for color-correction")
+    mask: Optional[ImageField] = InputField(default=None, description="Mask to use when applying color-correction")
+    mask_blur_radius: float = InputField(default=8, description="Mask blur radius")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        pil_init_mask = None
+        if self.mask is not None:
+            pil_init_mask = context.services.images.get_pil_image(self.mask.image_name).convert("L")
+
+        init_image = context.services.images.get_pil_image(self.reference.image_name)
+
+        result = context.services.images.get_pil_image(self.image.image_name).convert("RGBA")
+
+        # if init_image is None or init_mask is None:
+        #    return result
+
+        # Get the original alpha channel of the mask if there is one.
+        # Otherwise it is some other black/white image format ('1', 'L' or 'RGB')
+        # pil_init_mask = (
+        #    init_mask.getchannel("A")
+        #    if init_mask.mode == "RGBA"
+        #    else init_mask.convert("L")
+        # )
+        pil_init_image = init_image.convert("RGBA")  # Add an alpha channel if one doesn't exist
+
+        # Build an image with only visible pixels from source to use as reference for color-matching.
+        init_rgb_pixels = numpy.asarray(init_image.convert("RGB"), dtype=numpy.uint8)
+        init_a_pixels = numpy.asarray(pil_init_image.getchannel("A"), dtype=numpy.uint8)
+        init_mask_pixels = numpy.asarray(pil_init_mask, dtype=numpy.uint8)
+
+        # Get numpy version of result
+        np_image = numpy.asarray(result.convert("RGB"), dtype=numpy.uint8)
+
+        # Mask and calculate mean and standard deviation
+        mask_pixels = init_a_pixels * init_mask_pixels > 0
+        np_init_rgb_pixels_masked = init_rgb_pixels[mask_pixels, :]
+        np_image_masked = np_image[mask_pixels, :]
+
+        if np_init_rgb_pixels_masked.size > 0:
+            init_means = np_init_rgb_pixels_masked.mean(axis=0)
+            init_std = np_init_rgb_pixels_masked.std(axis=0)
+            gen_means = np_image_masked.mean(axis=0)
+            gen_std = np_image_masked.std(axis=0)
+
+            # Color correct
+            np_matched_result = np_image.copy()
+            np_matched_result[:, :, :] = (
+                (
+                    (
+                        (np_matched_result[:, :, :].astype(numpy.float32) - gen_means[None, None, :])
+                        / gen_std[None, None, :]
+                    )
+                    * init_std[None, None, :]
+                    + init_means[None, None, :]
+                )
+                .clip(0, 255)
+                .astype(numpy.uint8)
+            )
+            matched_result = Image.fromarray(np_matched_result, mode="RGB")
+        else:
+            matched_result = Image.fromarray(np_image, mode="RGB")
+
+        # Blur the mask out (into init image) by specified amount
+        if self.mask_blur_radius > 0:
+            nm = numpy.asarray(pil_init_mask, dtype=numpy.uint8)
+            nmd = cv2.erode(
+                nm,
+                kernel=numpy.ones((3, 3), dtype=numpy.uint8),
+                iterations=int(self.mask_blur_radius / 2),
+            )
+            pmd = Image.fromarray(nmd, mode="L")
+            blurred_init_mask = pmd.filter(ImageFilter.BoxBlur(self.mask_blur_radius))
+        else:
+            blurred_init_mask = pil_init_mask
+
+        multiplied_blurred_init_mask = ImageChops.multiply(blurred_init_mask, result.split()[-1])
+
+        # Paste original on color-corrected generation (using blurred mask)
+        matched_result.paste(init_image, (0, 0), mask=multiplied_blurred_init_mask)
+
+        image_dto = context.services.images.create(
+            image=matched_result,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            session_id=context.graph_execution_state_id,
+            is_intermediate=self.is_intermediate,
+        )
+
+        return ImageOutput(
+            image=ImageField(image_name=image_dto.image_name),
+            width=image_dto.width,
+            height=image_dto.height,
+        )
+
+
+@title("Image Hue Adjustment")
+@tags("image", "hue", "hsl")
+class ImageHueAdjustmentInvocation(BaseInvocation):
+    """Adjusts the Hue of an image."""
+
+    type: Literal["img_hue_adjust"] = "img_hue_adjust"
+
+    # Inputs
+    image: ImageField = InputField(description="The image to adjust")
+    hue: int = InputField(default=0, description="The degrees by which to rotate the hue, 0-360")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        pil_image = context.services.images.get_pil_image(self.image.image_name)
+
+        # Convert image to HSV color space
+        hsv_image = numpy.array(pil_image.convert("HSV"))
+
+        # Convert hue from 0..360 to 0..256
+        hue = int(256 * ((self.hue % 360) / 360))
+
+        # Increment each hue and wrap around at 255
+        hsv_image[:, :, 0] = (hsv_image[:, :, 0] + hue) % 256
+
+        # Convert back to PIL format and to original color mode
+        pil_image = Image.fromarray(hsv_image, mode="HSV").convert("RGBA")
+
+        image_dto = context.services.images.create(
+            image=pil_image,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            is_intermediate=self.is_intermediate,
+            session_id=context.graph_execution_state_id,
+        )
+
+        return ImageOutput(
+            image=ImageField(
+                image_name=image_dto.image_name,
+            ),
+            width=image_dto.width,
+            height=image_dto.height,
+        )
+
+
+@title("Image Luminosity Adjustment")
+@tags("image", "luminosity", "hsl")
+class ImageLuminosityAdjustmentInvocation(BaseInvocation):
+    """Adjusts the Luminosity (Value) of an image."""
+
+    type: Literal["img_luminosity_adjust"] = "img_luminosity_adjust"
+
+    # Inputs
+    image: ImageField = InputField(description="The image to adjust")
+    luminosity: float = InputField(
+        default=1.0, ge=0, le=1, description="The factor by which to adjust the luminosity (value)"
+    )
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        pil_image = context.services.images.get_pil_image(self.image.image_name)
+
+        # Convert PIL image to OpenCV format (numpy array), note color channel
+        # ordering is changed from RGB to BGR
+        image = numpy.array(pil_image.convert("RGB"))[:, :, ::-1]
+
+        # Convert image to HSV color space
+        hsv_image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
+
+        # Adjust the luminosity (value)
+        hsv_image[:, :, 2] = numpy.clip(hsv_image[:, :, 2] * self.luminosity, 0, 255)
+
+        # Convert image back to BGR color space
+        image = cv2.cvtColor(hsv_image, cv2.COLOR_HSV2BGR)
+
+        # Convert back to PIL format and to original color mode
+        pil_image = Image.fromarray(image[:, :, ::-1], "RGB").convert("RGBA")
+
+        image_dto = context.services.images.create(
+            image=pil_image,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            is_intermediate=self.is_intermediate,
+            session_id=context.graph_execution_state_id,
+        )
+
+        return ImageOutput(
+            image=ImageField(
+                image_name=image_dto.image_name,
+            ),
+            width=image_dto.width,
+            height=image_dto.height,
+        )
+
+
+@title("Image Saturation Adjustment")
+@tags("image", "saturation", "hsl")
+class ImageSaturationAdjustmentInvocation(BaseInvocation):
+    """Adjusts the Saturation of an image."""
+
+    type: Literal["img_saturation_adjust"] = "img_saturation_adjust"
+
+    # Inputs
+    image: ImageField = InputField(description="The image to adjust")
+    saturation: float = InputField(default=1.0, ge=0, le=1, description="The factor by which to adjust the saturation")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        pil_image = context.services.images.get_pil_image(self.image.image_name)
+
+        # Convert PIL image to OpenCV format (numpy array), note color channel
+        # ordering is changed from RGB to BGR
+        image = numpy.array(pil_image.convert("RGB"))[:, :, ::-1]
+
+        # Convert image to HSV color space
+        hsv_image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
+
+        # Adjust the saturation
+        hsv_image[:, :, 1] = numpy.clip(hsv_image[:, :, 1] * self.saturation, 0, 255)
+
+        # Convert image back to BGR color space
+        image = cv2.cvtColor(hsv_image, cv2.COLOR_HSV2BGR)
+
+        # Convert back to PIL format and to original color mode
+        pil_image = Image.fromarray(image[:, :, ::-1], "RGB").convert("RGBA")
+
+        image_dto = context.services.images.create(
+            image=pil_image,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            is_intermediate=self.is_intermediate,
+            session_id=context.graph_execution_state_id,
+        )
+
+        return ImageOutput(
+            image=ImageField(
+                image_name=image_dto.image_name,
+            ),
+            width=image_dto.width,
+            height=image_dto.height,
+        )

invokeai/app/invocations/infill.py

@@ -5,18 +5,13 @@ from typing import Literal, Optional, get_args
 import numpy as np
 import math
 from PIL import Image, ImageOps
-from pydantic import Field
+from invokeai.app.invocations.primitives import ImageField, ImageOutput, ColorField
 
-from invokeai.app.invocations.image import ImageOutput
 from invokeai.app.util.misc import SEED_MAX, get_random_seed
 from invokeai.backend.image_util.patchmatch import PatchMatch
 
-from ..models.image import ColorField, ImageCategory, ImageField, ResourceOrigin
-from .baseinvocation import (
-    BaseInvocation,
-    InvocationConfig,
-    InvocationContext,
-)
+from ..models.image import ImageCategory, ResourceOrigin
+from .baseinvocation import BaseInvocation, InputField, InvocationContext, title, tags
 
 
 def infill_methods() -> list[str]:
@@ -114,21 +109,20 @@ def tile_fill_missing(im: Image.Image, tile_size: int = 16, seed: Optional[int]
     return si
 
 
+@title("Solid Color Infill")
+@tags("image", "inpaint")
 class InfillColorInvocation(BaseInvocation):
     """Infills transparent areas of an image with a solid color"""
 
     type: Literal["infill_rgba"] = "infill_rgba"
-    image: Optional[ImageField] = Field(default=None, description="The image to infill")
-    color: ColorField = Field(
+
+    # Inputs
+    image: ImageField = InputField(description="The image to infill")
+    color: ColorField = InputField(
         default=ColorField(r=127, g=127, b=127, a=255),
         description="The color to use to infill",
     )
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Color Infill", "tags": ["image", "inpaint", "color", "infill"]},
-        }
-
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
 
@@ -153,25 +147,23 @@ class InfillColorInvocation(BaseInvocation):
         )
 
 
+@title("Tile Infill")
+@tags("image", "inpaint")
 class InfillTileInvocation(BaseInvocation):
     """Infills transparent areas of an image with tiles of the image"""
 
     type: Literal["infill_tile"] = "infill_tile"
 
-    image: Optional[ImageField] = Field(default=None, description="The image to infill")
-    tile_size: int = Field(default=32, ge=1, description="The tile size (px)")
-    seed: int = Field(
+    # Input
+    image: ImageField = InputField(description="The image to infill")
+    tile_size: int = InputField(default=32, ge=1, description="The tile size (px)")
+    seed: int = InputField(
         ge=0,
         le=SEED_MAX,
         description="The seed to use for tile generation (omit for random)",
         default_factory=get_random_seed,
     )
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Tile Infill", "tags": ["image", "inpaint", "tile", "infill"]},
-        }
-
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)
 
@@ -194,17 +186,15 @@ class InfillTileInvocation(BaseInvocation):
         )
 
 
+@title("PatchMatch Infill")
+@tags("image", "inpaint")
 class InfillPatchMatchInvocation(BaseInvocation):
     """Infills transparent areas of an image using the PatchMatch algorithm"""
 
     type: Literal["infill_patchmatch"] = "infill_patchmatch"
 
-    image: Optional[ImageField] = Field(default=None, description="The image to infill")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Patch Match Infill", "tags": ["image", "inpaint", "patchmatch", "infill"]},
-        }
+    # Inputs
+    image: ImageField = InputField(description="The image to infill")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)

invokeai/app/invocations/latent.py

@@ -5,15 +5,32 @@ from typing import List, Literal, Optional, Union
 
 import einops
 import torch
-from diffusers import ControlNetModel
+import torchvision.transforms as T
 from diffusers.image_processor import VaeImageProcessor
+from diffusers.models.attention_processor import (
+    AttnProcessor2_0,
+    LoRAAttnProcessor2_0,
+    LoRAXFormersAttnProcessor,
+    XFormersAttnProcessor,
+)
+from diffusers.schedulers import DPMSolverSDEScheduler
 from diffusers.schedulers import SchedulerMixin as Scheduler
 from pydantic import BaseModel, Field, validator
+from torchvision.transforms.functional import resize as tv_resize
 
 from invokeai.app.invocations.metadata import CoreMetadata
+from invokeai.app.invocations.primitives import (
+    ImageField,
+    ImageOutput,
+    LatentsField,
+    LatentsOutput,
+    build_latents_output,
+)
+from invokeai.app.util.controlnet_utils import prepare_control_image
 from invokeai.app.util.step_callback import stable_diffusion_step_callback
 from invokeai.backend.model_management.models import ModelType, SilenceWarnings
 
+from ...backend.model_management import BaseModelType, ModelPatcher
 from ...backend.model_management.lora import ModelPatcher
 from ...backend.stable_diffusion import PipelineIntermediateState
 from ...backend.stable_diffusion.diffusers_pipeline import (
@@ -24,56 +41,27 @@ from ...backend.stable_diffusion.diffusers_pipeline import (
 )
 from ...backend.stable_diffusion.diffusion.shared_invokeai_diffusion import PostprocessingSettings
 from ...backend.stable_diffusion.schedulers import SCHEDULER_MAP
-from ...backend.util.devices import choose_torch_device, torch_dtype, choose_precision
-from ..models.image import ImageCategory, ImageField, ResourceOrigin
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
+from ...backend.util.devices import choose_precision, choose_torch_device
+from ..models.image import ImageCategory, ResourceOrigin
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    Input,
+    InputField,
+    InvocationContext,
+    OutputField,
+    UIType,
+    tags,
+    title,
+)
 from .compel import ConditioningField
 from .controlnet_image_processors import ControlField
-from .image import ImageOutput
 from .model import ModelInfo, UNetField, VaeField
-from invokeai.app.util.controlnet_utils import prepare_control_image
-
-from diffusers.models.attention_processor import (
-    AttnProcessor2_0,
-    LoRAAttnProcessor2_0,
-    LoRAXFormersAttnProcessor,
-    XFormersAttnProcessor,
-)
-
 
 DEFAULT_PRECISION = choose_precision(choose_torch_device())
 
 
-class LatentsField(BaseModel):
-    """A latents field used for passing latents between invocations"""
-
-    latents_name: Optional[str] = Field(default=None, description="The name of the latents")
-
-    class Config:
-        schema_extra = {"required": ["latents_name"]}
-
-
-class LatentsOutput(BaseInvocationOutput):
-    """Base class for invocations that output latents"""
-
-    # fmt: off
-    type: Literal["latents_output"] = "latents_output"
-
-    # Inputs
-    latents: LatentsField          = Field(default=None, description="The output latents")
-    width:                     int = Field(description="The width of the latents in pixels")
-    height:                    int = Field(description="The height of the latents in pixels")
-    # fmt: on
-
-
-def build_latents_output(latents_name: str, latents: torch.Tensor):
-    return LatentsOutput(
-        latents=LatentsField(latents_name=latents_name),
-        width=latents.size()[3] * 8,
-        height=latents.size()[2] * 8,
-    )
-
-
 SAMPLER_NAME_VALUES = Literal[tuple(list(SCHEDULER_MAP.keys()))]
 
 
@@ -81,6 +69,7 @@ def get_scheduler(
     context: InvocationContext,
     scheduler_info: ModelInfo,
     scheduler_name: str,
+    seed: int,
 ) -> Scheduler:
     scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP["ddim"])
     orig_scheduler_info = context.services.model_manager.get_model(
@@ -97,6 +86,11 @@ def get_scheduler(
         **scheduler_extra_config,
         "_backup": scheduler_config,
     }
+
+    # make dpmpp_sde reproducable(seed can be passed only in initializer)
+    if scheduler_class is DPMSolverSDEScheduler:
+        scheduler_config["noise_sampler_seed"] = seed
+
     scheduler = scheduler_class.from_config(scheduler_config)
 
     # hack copied over from generate.py
@@ -105,25 +99,37 @@ def get_scheduler(
     return scheduler
 
 
-# Text to image
-class TextToLatentsInvocation(BaseInvocation):
-    """Generates latents from conditionings."""
+@title("Denoise Latents")
+@tags("latents", "denoise", "txt2img", "t2i", "t2l", "img2img", "i2i", "l2l")
+class DenoiseLatentsInvocation(BaseInvocation):
+    """Denoises noisy latents to decodable images"""
 
-    type: Literal["t2l"] = "t2l"
+    type: Literal["denoise_latents"] = "denoise_latents"
 
     # Inputs
-    # fmt: off
-    positive_conditioning: Optional[ConditioningField] = Field(description="Positive conditioning for generation")
-    negative_conditioning: Optional[ConditioningField] = Field(description="Negative conditioning for generation")
-    noise: Optional[LatentsField] = Field(description="The noise to use")
-    steps:       int = Field(default=10, gt=0, description="The number of steps to use to generate the image")
-    cfg_scale: Union[float, List[float]] = Field(default=7.5, ge=1, description="The Classifier-Free Guidance, higher values may result in a result closer to the prompt", )
-    scheduler: SAMPLER_NAME_VALUES = Field(default="euler", description="The scheduler to use" )
-    unet: UNetField = Field(default=None, description="UNet submodel")
-    control: Union[ControlField, list[ControlField]] = Field(default=None, description="The control to use")
-    # seamless:   bool = Field(default=False, description="Whether or not to generate an image that can tile without seams", )
-    # seamless_axes: str = Field(default="", description="The axes to tile the image on, 'x' and/or 'y'")
-    # fmt: on
+    positive_conditioning: ConditioningField = InputField(
+        description=FieldDescriptions.positive_cond, input=Input.Connection
+    )
+    negative_conditioning: ConditioningField = InputField(
+        description=FieldDescriptions.negative_cond, input=Input.Connection
+    )
+    noise: Optional[LatentsField] = InputField(description=FieldDescriptions.noise, input=Input.Connection)
+    steps: int = InputField(default=10, gt=0, description=FieldDescriptions.steps)
+    cfg_scale: Union[float, List[float]] = InputField(
+        default=7.5, ge=1, description=FieldDescriptions.cfg_scale, ui_type=UIType.Float
+    )
+    denoising_start: float = InputField(default=0.0, ge=0, le=1, description=FieldDescriptions.denoising_start)
+    denoising_end: float = InputField(default=1.0, ge=0, le=1, description=FieldDescriptions.denoising_end)
+    scheduler: SAMPLER_NAME_VALUES = InputField(default="euler", description=FieldDescriptions.scheduler)
+    unet: UNetField = InputField(description=FieldDescriptions.unet, input=Input.Connection)
+    control: Union[ControlField, list[ControlField]] = InputField(
+        default=None, description=FieldDescriptions.control, input=Input.Connection
+    )
+    latents: Optional[LatentsField] = InputField(description=FieldDescriptions.latents, input=Input.Connection)
+    mask: Optional[ImageField] = InputField(
+        default=None,
+        description=FieldDescriptions.mask,
+    )
 
     @validator("cfg_scale")
     def ge_one(cls, v):
@@ -137,33 +143,20 @@ class TextToLatentsInvocation(BaseInvocation):
                 raise ValueError("cfg_scale must be greater than 1")
         return v
 
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Text To Latents",
-                "tags": ["latents"],
-                "type_hints": {
-                    "model": "model",
-                    "control": "control",
-                    # "cfg_scale": "float",
-                    "cfg_scale": "number",
-                },
-            },
-        }
-
     # TODO: pass this an emitter method or something? or a session for dispatching?
     def dispatch_progress(
         self,
         context: InvocationContext,
         source_node_id: str,
         intermediate_state: PipelineIntermediateState,
+        base_model: BaseModelType,
     ) -> None:
         stable_diffusion_step_callback(
             context=context,
             intermediate_state=intermediate_state,
             node=self.dict(),
             source_node_id=source_node_id,
+            base_model=base_model,
         )
 
     def get_conditioning_data(
@@ -171,13 +164,14 @@ class TextToLatentsInvocation(BaseInvocation):
         context: InvocationContext,
         scheduler,
         unet,
+        seed,
     ) -> ConditioningData:
         positive_cond_data = context.services.latents.get(self.positive_conditioning.conditioning_name)
-        c = positive_cond_data.conditionings[0].embeds.to(device=unet.device, dtype=unet.dtype)
-        extra_conditioning_info = positive_cond_data.conditionings[0].extra_conditioning
+        c = positive_cond_data.conditionings[0].to(device=unet.device, dtype=unet.dtype)
+        extra_conditioning_info = c.extra_conditioning
 
         negative_cond_data = context.services.latents.get(self.negative_conditioning.conditioning_name)
-        uc = negative_cond_data.conditionings[0].embeds.to(device=unet.device, dtype=unet.dtype)
+        uc = negative_cond_data.conditionings[0].to(device=unet.device, dtype=unet.dtype)
 
         conditioning_data = ConditioningData(
             unconditioned_embeddings=uc,
@@ -197,7 +191,8 @@ class TextToLatentsInvocation(BaseInvocation):
             # for ddim scheduler
             eta=0.0,  # ddim_eta
             # for ancestral and sde schedulers
-            generator=torch.Generator(device=unet.device).manual_seed(0),
+            # flip all bits to have noise different from initial
+            generator=torch.Generator(device=unet.device).manual_seed(seed ^ 0xFFFFFFFF),
         )
         return conditioning_data
 
@@ -230,7 +225,6 @@ class TextToLatentsInvocation(BaseInvocation):
             safety_checker=None,
             feature_extractor=None,
             requires_safety_checker=False,
-            precision="float16" if unet.dtype == torch.float16 else "float32",
         )
 
     def prep_control_data(
@@ -309,110 +303,83 @@ class TextToLatentsInvocation(BaseInvocation):
                 # MultiControlNetModel has been refactored out, just need list[ControlNetData]
         return control_data
 
-    @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> LatentsOutput:
-        with SilenceWarnings():
-            noise = context.services.latents.get(self.noise.latents_name)
+    # original idea by https://github.com/AmericanPresidentJimmyCarter
+    # TODO: research more for second order schedulers timesteps
+    def init_scheduler(self, scheduler, device, steps, denoising_start, denoising_end):
+        num_inference_steps = steps
+        if scheduler.config.get("cpu_only", False):
+            scheduler.set_timesteps(num_inference_steps, device="cpu")
+            timesteps = scheduler.timesteps.to(device=device)
+        else:
+            scheduler.set_timesteps(num_inference_steps, device=device)
+            timesteps = scheduler.timesteps
 
-            # Get the source node id (we are invoking the prepared node)
-            graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
-            source_node_id = graph_execution_state.prepared_source_mapping[self.id]
+        # apply denoising_start
+        t_start_val = int(round(scheduler.config.num_train_timesteps * (1 - denoising_start)))
+        t_start_idx = len(list(filter(lambda ts: ts >= t_start_val, timesteps)))
+        timesteps = timesteps[t_start_idx:]
+        if scheduler.order == 2 and t_start_idx > 0:
+            timesteps = timesteps[1:]
 
-            def step_callback(state: PipelineIntermediateState):
-                self.dispatch_progress(context, source_node_id, state)
+        # save start timestep to apply noise
+        init_timestep = timesteps[:1]
 
-            def _lora_loader():
-                for lora in self.unet.loras:
-                    lora_info = context.services.model_manager.get_model(
-                        **lora.dict(exclude={"weight"}),
-                        context=context,
-                    )
-                    yield (lora_info.context.model, lora.weight)
-                    del lora_info
-                return
+        # apply denoising_end
+        t_end_val = int(round(scheduler.config.num_train_timesteps * (1 - denoising_end)))
+        t_end_idx = len(list(filter(lambda ts: ts >= t_end_val, timesteps)))
+        if scheduler.order == 2 and t_end_idx > 0:
+            t_end_idx += 1
+        timesteps = timesteps[:t_end_idx]
 
-            unet_info = context.services.model_manager.get_model(
-                **self.unet.unet.dict(),
-                context=context,
-            )
-            with ExitStack() as exit_stack, ModelPatcher.apply_lora_unet(
-                unet_info.context.model, _lora_loader()
-            ), unet_info as unet:
-                noise = noise.to(device=unet.device, dtype=unet.dtype)
+        # calculate step count based on scheduler order
+        num_inference_steps = len(timesteps)
+        if scheduler.order == 2:
+            num_inference_steps += num_inference_steps % 2
+            num_inference_steps = num_inference_steps // 2
 
-                scheduler = get_scheduler(
-                    context=context,
-                    scheduler_info=self.unet.scheduler,
-                    scheduler_name=self.scheduler,
-                )
+        return num_inference_steps, timesteps, init_timestep
 
-                pipeline = self.create_pipeline(unet, scheduler)
-                conditioning_data = self.get_conditioning_data(context, scheduler, unet)
+    def prep_mask_tensor(self, mask, context, lantents):
+        if mask is None:
+            return None
 
-                control_data = self.prep_control_data(
-                    model=pipeline,
-                    context=context,
-                    control_input=self.control,
-                    latents_shape=noise.shape,
-                    # do_classifier_free_guidance=(self.cfg_scale >= 1.0))
-                    do_classifier_free_guidance=True,
-                    exit_stack=exit_stack,
-                )
-
-                # TODO: Verify the noise is the right size
-                result_latents, result_attention_map_saver = pipeline.latents_from_embeddings(
-                    latents=torch.zeros_like(noise, dtype=torch_dtype(unet.device)),
-                    noise=noise,
-                    num_inference_steps=self.steps,
-                    conditioning_data=conditioning_data,
-                    control_data=control_data,  # list[ControlNetData]
-                    callback=step_callback,
-                )
-
-            # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699
-            result_latents = result_latents.to("cpu")
-            torch.cuda.empty_cache()
-
-            name = f"{context.graph_execution_state_id}__{self.id}"
-            context.services.latents.save(name, result_latents)
-            return build_latents_output(latents_name=name, latents=result_latents)
-
-
-class LatentsToLatentsInvocation(TextToLatentsInvocation):
-    """Generates latents using latents as base image."""
-
-    type: Literal["l2l"] = "l2l"
-
-    # Inputs
-    latents: Optional[LatentsField] = Field(description="The latents to use as a base image")
-    strength: float = Field(default=0.7, ge=0, le=1, description="The strength of the latents to use")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Latent To Latents",
-                "tags": ["latents"],
-                "type_hints": {
-                    "model": "model",
-                    "control": "control",
-                    "cfg_scale": "number",
-                },
-            },
-        }
+        mask_image = context.services.images.get_pil_image(mask.image_name)
+        if mask_image.mode != "L":
+            # FIXME: why do we get passed an RGB image here? We can only use single-channel.
+            mask_image = mask_image.convert("L")
+        mask_tensor = image_resized_to_grid_as_tensor(mask_image, normalize=False)
+        if mask_tensor.dim() == 3:
+            mask_tensor = mask_tensor.unsqueeze(0)
+        mask_tensor = tv_resize(mask_tensor, lantents.shape[-2:], T.InterpolationMode.BILINEAR)
+        return 1 - mask_tensor
 
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         with SilenceWarnings():  # this quenches NSFW nag from diffusers
-            noise = context.services.latents.get(self.noise.latents_name)
-            latent = context.services.latents.get(self.latents.latents_name)
+            seed = None
+            noise = None
+            if self.noise is not None:
+                noise = context.services.latents.get(self.noise.latents_name)
+                seed = self.noise.seed
+
+            if self.latents is not None:
+                latents = context.services.latents.get(self.latents.latents_name)
+                if seed is None:
+                    seed = self.latents.seed
+            else:
+                latents = torch.zeros_like(noise)
+
+            if seed is None:
+                seed = 0
+
+            mask = self.prep_mask_tensor(self.mask, context, latents)
 
             # Get the source node id (we are invoking the prepared node)
             graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
             source_node_id = graph_execution_state.prepared_source_mapping[self.id]
 
             def step_callback(state: PipelineIntermediateState):
-                self.dispatch_progress(context, source_node_id, state)
+                self.dispatch_progress(context, source_node_id, state, self.unet.unet.base_model)
 
             def _lora_loader():
                 for lora in self.unet.loras:
@@ -431,44 +398,48 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
             with ExitStack() as exit_stack, ModelPatcher.apply_lora_unet(
                 unet_info.context.model, _lora_loader()
             ), unet_info as unet:
-                noise = noise.to(device=unet.device, dtype=unet.dtype)
-                latent = latent.to(device=unet.device, dtype=unet.dtype)
+                latents = latents.to(device=unet.device, dtype=unet.dtype)
+                if noise is not None:
+                    noise = noise.to(device=unet.device, dtype=unet.dtype)
+                if mask is not None:
+                    mask = mask.to(device=unet.device, dtype=unet.dtype)
 
                 scheduler = get_scheduler(
                     context=context,
                     scheduler_info=self.unet.scheduler,
                     scheduler_name=self.scheduler,
+                    seed=seed,
                 )
 
                 pipeline = self.create_pipeline(unet, scheduler)
-                conditioning_data = self.get_conditioning_data(context, scheduler, unet)
+                conditioning_data = self.get_conditioning_data(context, scheduler, unet, seed)
 
                 control_data = self.prep_control_data(
                     model=pipeline,
                     context=context,
                     control_input=self.control,
-                    latents_shape=noise.shape,
+                    latents_shape=latents.shape,
                     # do_classifier_free_guidance=(self.cfg_scale >= 1.0))
                     do_classifier_free_guidance=True,
                     exit_stack=exit_stack,
                 )
 
-                # TODO: Verify the noise is the right size
-                initial_latents = (
-                    latent if self.strength < 1.0 else torch.zeros_like(latent, device=unet.device, dtype=latent.dtype)
-                )
-
-                timesteps, _ = pipeline.get_img2img_timesteps(
-                    self.steps,
-                    self.strength,
+                num_inference_steps, timesteps, init_timestep = self.init_scheduler(
+                    scheduler,
                     device=unet.device,
+                    steps=self.steps,
+                    denoising_start=self.denoising_start,
+                    denoising_end=self.denoising_end,
                 )
 
                 result_latents, result_attention_map_saver = pipeline.latents_from_embeddings(
-                    latents=initial_latents,
+                    latents=latents,
                     timesteps=timesteps,
+                    init_timestep=init_timestep,
                     noise=noise,
-                    num_inference_steps=self.steps,
+                    seed=seed,
+                    mask=mask,
+                    num_inference_steps=num_inference_steps,
                     conditioning_data=conditioning_data,
                     control_data=control_data,  # list[ControlNetData]
                     callback=step_callback,
@@ -480,32 +451,32 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
 
             name = f"{context.graph_execution_state_id}__{self.id}"
             context.services.latents.save(name, result_latents)
-        return build_latents_output(latents_name=name, latents=result_latents)
+        return build_latents_output(latents_name=name, latents=result_latents, seed=seed)
 
 
-# Latent to image
+@title("Latents to Image")
+@tags("latents", "image", "vae")
 class LatentsToImageInvocation(BaseInvocation):
     """Generates an image from latents."""
 
     type: Literal["l2i"] = "l2i"
 
     # Inputs
-    latents: Optional[LatentsField] = Field(description="The latents to generate an image from")
-    vae: VaeField = Field(default=None, description="Vae submodel")
-    tiled: bool = Field(default=False, description="Decode latents by overlaping tiles (less memory consumption)")
-    fp32: bool = Field(DEFAULT_PRECISION == "float32", description="Decode in full precision")
-    metadata: Optional[CoreMetadata] = Field(
-        default=None, description="Optional core metadata to be written to the image"
+    latents: LatentsField = InputField(
+        description=FieldDescriptions.latents,
+        input=Input.Connection,
+    )
+    vae: VaeField = InputField(
+        description=FieldDescriptions.vae,
+        input=Input.Connection,
+    )
+    tiled: bool = InputField(default=False, description=FieldDescriptions.tiled)
+    fp32: bool = InputField(default=DEFAULT_PRECISION == "float32", description=FieldDescriptions.fp32)
+    metadata: CoreMetadata = InputField(
+        default=None,
+        description=FieldDescriptions.core_metadata,
+        ui_hidden=True,
     )
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Latents To Image",
-                "tags": ["latents", "image"],
-            },
-        }
 
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> ImageOutput:
@@ -583,24 +554,30 @@ class LatentsToImageInvocation(BaseInvocation):
 LATENTS_INTERPOLATION_MODE = Literal["nearest", "linear", "bilinear", "bicubic", "trilinear", "area", "nearest-exact"]
 
 
+@title("Resize Latents")
+@tags("latents", "resize")
 class ResizeLatentsInvocation(BaseInvocation):
     """Resizes latents to explicit width/height (in pixels). Provided dimensions are floor-divided by 8."""
 
     type: Literal["lresize"] = "lresize"
 
     # Inputs
-    latents: Optional[LatentsField] = Field(description="The latents to resize")
-    width: Union[int, None] = Field(default=512, ge=64, multiple_of=8, description="The width to resize to (px)")
-    height: Union[int, None] = Field(default=512, ge=64, multiple_of=8, description="The height to resize to (px)")
-    mode: LATENTS_INTERPOLATION_MODE = Field(default="bilinear", description="The interpolation mode")
-    antialias: bool = Field(
-        default=False, description="Whether or not to antialias (applied in bilinear and bicubic modes only)"
+    latents: LatentsField = InputField(
+        description=FieldDescriptions.latents,
+        input=Input.Connection,
     )
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Resize Latents", "tags": ["latents", "resize"]},
-        }
+    width: int = InputField(
+        ge=64,
+        multiple_of=8,
+        description=FieldDescriptions.width,
+    )
+    height: int = InputField(
+        ge=64,
+        multiple_of=8,
+        description=FieldDescriptions.width,
+    )
+    mode: LATENTS_INTERPOLATION_MODE = InputField(default="bilinear", description=FieldDescriptions.interp_mode)
+    antialias: bool = InputField(default=False, description=FieldDescriptions.torch_antialias)
 
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         latents = context.services.latents.get(self.latents.latents_name)
@@ -622,26 +599,24 @@ class ResizeLatentsInvocation(BaseInvocation):
         name = f"{context.graph_execution_state_id}__{self.id}"
         # context.services.latents.set(name, resized_latents)
         context.services.latents.save(name, resized_latents)
-        return build_latents_output(latents_name=name, latents=resized_latents)
+        return build_latents_output(latents_name=name, latents=resized_latents, seed=self.latents.seed)
 
 
+@title("Scale Latents")
+@tags("latents", "resize")
 class ScaleLatentsInvocation(BaseInvocation):
     """Scales latents by a given factor."""
 
     type: Literal["lscale"] = "lscale"
 
     # Inputs
-    latents: Optional[LatentsField] = Field(description="The latents to scale")
-    scale_factor: float = Field(gt=0, description="The factor by which to scale the latents")
-    mode: LATENTS_INTERPOLATION_MODE = Field(default="bilinear", description="The interpolation mode")
-    antialias: bool = Field(
-        default=False, description="Whether or not to antialias (applied in bilinear and bicubic modes only)"
+    latents: LatentsField = InputField(
+        description=FieldDescriptions.latents,
+        input=Input.Connection,
     )
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Scale Latents", "tags": ["latents", "scale"]},
-        }
+    scale_factor: float = InputField(gt=0, description=FieldDescriptions.scale_factor)
+    mode: LATENTS_INTERPOLATION_MODE = InputField(default="bilinear", description=FieldDescriptions.interp_mode)
+    antialias: bool = InputField(default=False, description=FieldDescriptions.torch_antialias)
 
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         latents = context.services.latents.get(self.latents.latents_name)
@@ -664,25 +639,26 @@ class ScaleLatentsInvocation(BaseInvocation):
         name = f"{context.graph_execution_state_id}__{self.id}"
         # context.services.latents.set(name, resized_latents)
         context.services.latents.save(name, resized_latents)
-        return build_latents_output(latents_name=name, latents=resized_latents)
+        return build_latents_output(latents_name=name, latents=resized_latents, seed=self.latents.seed)
 
 
+@title("Image to Latents")
+@tags("latents", "image", "vae")
 class ImageToLatentsInvocation(BaseInvocation):
     """Encodes an image into latents."""
 
     type: Literal["i2l"] = "i2l"
 
     # Inputs
-    image: Optional[ImageField] = Field(description="The image to encode")
-    vae: VaeField = Field(default=None, description="Vae submodel")
-    tiled: bool = Field(default=False, description="Encode latents by overlaping tiles(less memory consumption)")
-    fp32: bool = Field(DEFAULT_PRECISION == "float32", description="Decode in full precision")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Image To Latents", "tags": ["latents", "image"]},
-        }
+    image: ImageField = InputField(
+        description="The image to encode",
+    )
+    vae: VaeField = InputField(
+        description=FieldDescriptions.vae,
+        input=Input.Connection,
+    )
+    tiled: bool = InputField(default=False, description=FieldDescriptions.tiled)
+    fp32: bool = InputField(default=DEFAULT_PRECISION == "float32", description=FieldDescriptions.fp32)
 
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
@@ -745,4 +721,4 @@ class ImageToLatentsInvocation(BaseInvocation):
         name = f"{context.graph_execution_state_id}__{self.id}"
         latents = latents.to("cpu")
         context.services.latents.save(name, latents)
-        return build_latents_output(latents_name=name, latents=latents)
+        return build_latents_output(latents_name=name, latents=latents, seed=None)

invokeai/app/invocations/math.py

@@ -2,134 +2,83 @@
 
 from typing import Literal
 
-from pydantic import BaseModel, Field
 import numpy as np
 
-from .baseinvocation import (
-    BaseInvocation,
-    BaseInvocationOutput,
-    InvocationContext,
-    InvocationConfig,
-)
+from invokeai.app.invocations.primitives import IntegerOutput
+
+from .baseinvocation import BaseInvocation, FieldDescriptions, InputField, InvocationContext, tags, title
 
 
-class MathInvocationConfig(BaseModel):
-    """Helper class to provide all math invocations with additional config"""
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "tags": ["math"],
-            }
-        }
-
-
-class IntOutput(BaseInvocationOutput):
-    """An integer output"""
-
-    # fmt: off
-    type: Literal["int_output"] = "int_output"
-    a: int = Field(default=None, description="The output integer")
-    # fmt: on
-
-
-class FloatOutput(BaseInvocationOutput):
-    """A float output"""
-
-    # fmt: off
-    type: Literal["float_output"] = "float_output"
-    param: float = Field(default=None, description="The output float")
-    # fmt: on
-
-
-class AddInvocation(BaseInvocation, MathInvocationConfig):
+@title("Add Integers")
+@tags("math")
+class AddInvocation(BaseInvocation):
     """Adds two numbers"""
 
-    # fmt: off
     type: Literal["add"] = "add"
-    a: int = Field(default=0, description="The first number")
-    b: int = Field(default=0, description="The second number")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Add", "tags": ["math", "add"]},
-        }
+    # Inputs
+    a: int = InputField(default=0, description=FieldDescriptions.num_1)
+    b: int = InputField(default=0, description=FieldDescriptions.num_2)
 
-    def invoke(self, context: InvocationContext) -> IntOutput:
-        return IntOutput(a=self.a + self.b)
+    def invoke(self, context: InvocationContext) -> IntegerOutput:
+        return IntegerOutput(a=self.a + self.b)
 
 
-class SubtractInvocation(BaseInvocation, MathInvocationConfig):
+@title("Subtract Integers")
+@tags("math")
+class SubtractInvocation(BaseInvocation):
     """Subtracts two numbers"""
 
-    # fmt: off
     type: Literal["sub"] = "sub"
-    a: int = Field(default=0, description="The first number")
-    b: int = Field(default=0, description="The second number")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Subtract", "tags": ["math", "subtract"]},
-        }
+    # Inputs
+    a: int = InputField(default=0, description=FieldDescriptions.num_1)
+    b: int = InputField(default=0, description=FieldDescriptions.num_2)
 
-    def invoke(self, context: InvocationContext) -> IntOutput:
-        return IntOutput(a=self.a - self.b)
+    def invoke(self, context: InvocationContext) -> IntegerOutput:
+        return IntegerOutput(a=self.a - self.b)
 
 
-class MultiplyInvocation(BaseInvocation, MathInvocationConfig):
+@title("Multiply Integers")
+@tags("math")
+class MultiplyInvocation(BaseInvocation):
     """Multiplies two numbers"""
 
-    # fmt: off
     type: Literal["mul"] = "mul"
-    a: int = Field(default=0, description="The first number")
-    b: int = Field(default=0, description="The second number")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Multiply", "tags": ["math", "multiply"]},
-        }
+    # Inputs
+    a: int = InputField(default=0, description=FieldDescriptions.num_1)
+    b: int = InputField(default=0, description=FieldDescriptions.num_2)
 
-    def invoke(self, context: InvocationContext) -> IntOutput:
-        return IntOutput(a=self.a * self.b)
+    def invoke(self, context: InvocationContext) -> IntegerOutput:
+        return IntegerOutput(a=self.a * self.b)
 
 
-class DivideInvocation(BaseInvocation, MathInvocationConfig):
+@title("Divide Integers")
+@tags("math")
+class DivideInvocation(BaseInvocation):
     """Divides two numbers"""
 
-    # fmt: off
     type: Literal["div"] = "div"
-    a: int = Field(default=0, description="The first number")
-    b: int = Field(default=0, description="The second number")
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Divide", "tags": ["math", "divide"]},
-        }
+    # Inputs
+    a: int = InputField(default=0, description=FieldDescriptions.num_1)
+    b: int = InputField(default=0, description=FieldDescriptions.num_2)
 
-    def invoke(self, context: InvocationContext) -> IntOutput:
-        return IntOutput(a=int(self.a / self.b))
+    def invoke(self, context: InvocationContext) -> IntegerOutput:
+        return IntegerOutput(a=int(self.a / self.b))
 
 
+@title("Random Integer")
+@tags("math")
 class RandomIntInvocation(BaseInvocation):
     """Outputs a single random integer."""
 
-    # fmt: off
     type: Literal["rand_int"] = "rand_int"
-    low: int = Field(default=0, description="The inclusive low value")
-    high: int = Field(
-        default=np.iinfo(np.int32).max, description="The exclusive high value"
-    )
-    # fmt: on
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Random Integer", "tags": ["math", "random", "integer"]},
-        }
+    # Inputs
+    low: int = InputField(default=0, description="The inclusive low value")
+    high: int = InputField(default=np.iinfo(np.int32).max, description="The exclusive high value")
 
-    def invoke(self, context: InvocationContext) -> IntOutput:
-        return IntOutput(a=np.random.randint(self.low, self.high))
+    def invoke(self, context: InvocationContext) -> IntegerOutput:
+        return IntegerOutput(a=np.random.randint(self.low, self.high))

invokeai/app/invocations/metadata.py

@@ -1,27 +1,34 @@
-from typing import Literal, Optional, Union
+from typing import Literal, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 from invokeai.app.invocations.baseinvocation import (
     BaseInvocation,
     BaseInvocationOutput,
-    InvocationConfig,
+    InputField,
     InvocationContext,
+    OutputField,
+    tags,
+    title,
 )
 from invokeai.app.invocations.controlnet_image_processors import ControlField
 from invokeai.app.invocations.model import LoRAModelField, MainModelField, VAEModelField
+from invokeai.app.util.model_exclude_null import BaseModelExcludeNull
+
+from ...version import __version__
 
 
-class LoRAMetadataField(BaseModel):
+class LoRAMetadataField(BaseModelExcludeNull):
     """LoRA metadata for an image generated in InvokeAI."""
 
     lora: LoRAModelField = Field(description="The LoRA model")
     weight: float = Field(description="The weight of the LoRA model")
 
 
-class CoreMetadata(BaseModel):
+class CoreMetadata(BaseModelExcludeNull):
     """Core generation metadata for an image generated in InvokeAI."""
 
+    app_version: str = Field(default=__version__, description="The version of InvokeAI used to generate this image")
     generation_mode: str = Field(
         description="The generation mode that output this image",
     )
@@ -40,37 +47,40 @@ class CoreMetadata(BaseModel):
     model: MainModelField = Field(description="The main model used for inference")
     controlnets: list[ControlField] = Field(description="The ControlNets used for inference")
     loras: list[LoRAMetadataField] = Field(description="The LoRAs used for inference")
-    vae: Union[VAEModelField, None] = Field(
+    vae: Optional[VAEModelField] = Field(
         default=None,
         description="The VAE used for decoding, if the main model's default was not used",
     )
 
     # Latents-to-Latents
-    strength: Union[float, None] = Field(
+    strength: Optional[float] = Field(
         default=None,
         description="The strength used for latents-to-latents",
     )
-    init_image: Union[str, None] = Field(default=None, description="The name of the initial image")
+    init_image: Optional[str] = Field(default=None, description="The name of the initial image")
 
     # SDXL
-    positive_style_prompt: Union[str, None] = Field(default=None, description="The positive style prompt parameter")
-    negative_style_prompt: Union[str, None] = Field(default=None, description="The negative style prompt parameter")
+    positive_style_prompt: Optional[str] = Field(default=None, description="The positive style prompt parameter")
+    negative_style_prompt: Optional[str] = Field(default=None, description="The negative style prompt parameter")
 
     # SDXL Refiner
-    refiner_model: Union[MainModelField, None] = Field(default=None, description="The SDXL Refiner model used")
-    refiner_cfg_scale: Union[float, None] = Field(
+    refiner_model: Optional[MainModelField] = Field(default=None, description="The SDXL Refiner model used")
+    refiner_cfg_scale: Optional[float] = Field(
         default=None,
         description="The classifier-free guidance scale parameter used for the refiner",
     )
-    refiner_steps: Union[int, None] = Field(default=None, description="The number of steps used for the refiner")
-    refiner_scheduler: Union[str, None] = Field(default=None, description="The scheduler used for the refiner")
-    refiner_aesthetic_store: Union[float, None] = Field(
+    refiner_steps: Optional[int] = Field(default=None, description="The number of steps used for the refiner")
+    refiner_scheduler: Optional[str] = Field(default=None, description="The scheduler used for the refiner")
+    refiner_positive_aesthetic_store: Optional[float] = Field(
         default=None, description="The aesthetic score used for the refiner"
     )
-    refiner_start: Union[float, None] = Field(default=None, description="The start value used for refiner denoising")
+    refiner_negative_aesthetic_store: Optional[float] = Field(
+        default=None, description="The aesthetic score used for the refiner"
+    )
+    refiner_start: Optional[float] = Field(default=None, description="The start value used for refiner denoising")
 
 
-class ImageMetadata(BaseModel):
+class ImageMetadata(BaseModelExcludeNull):
     """An image's generation metadata"""
 
     metadata: Optional[dict] = Field(
@@ -85,66 +95,86 @@ class MetadataAccumulatorOutput(BaseInvocationOutput):
 
     type: Literal["metadata_accumulator_output"] = "metadata_accumulator_output"
 
-    metadata: CoreMetadata = Field(description="The core metadata for the image")
+    metadata: CoreMetadata = OutputField(description="The core metadata for the image")
 
 
+@title("Metadata Accumulator")
+@tags("metadata")
 class MetadataAccumulatorInvocation(BaseInvocation):
     """Outputs a Core Metadata Object"""
 
     type: Literal["metadata_accumulator"] = "metadata_accumulator"
 
-    generation_mode: str = Field(
+    generation_mode: str = InputField(
         description="The generation mode that output this image",
     )
-    positive_prompt: str = Field(description="The positive prompt parameter")
-    negative_prompt: str = Field(description="The negative prompt parameter")
-    width: int = Field(description="The width parameter")
-    height: int = Field(description="The height parameter")
-    seed: int = Field(description="The seed used for noise generation")
-    rand_device: str = Field(description="The device used for random number generation")
-    cfg_scale: float = Field(description="The classifier-free guidance scale parameter")
-    steps: int = Field(description="The number of steps used for inference")
-    scheduler: str = Field(description="The scheduler used for inference")
-    clip_skip: int = Field(
+    positive_prompt: str = InputField(description="The positive prompt parameter")
+    negative_prompt: str = InputField(description="The negative prompt parameter")
+    width: int = InputField(description="The width parameter")
+    height: int = InputField(description="The height parameter")
+    seed: int = InputField(description="The seed used for noise generation")
+    rand_device: str = InputField(description="The device used for random number generation")
+    cfg_scale: float = InputField(description="The classifier-free guidance scale parameter")
+    steps: int = InputField(description="The number of steps used for inference")
+    scheduler: str = InputField(description="The scheduler used for inference")
+    clip_skip: int = InputField(
         description="The number of skipped CLIP layers",
     )
-    model: MainModelField = Field(description="The main model used for inference")
-    controlnets: list[ControlField] = Field(description="The ControlNets used for inference")
-    loras: list[LoRAMetadataField] = Field(description="The LoRAs used for inference")
-    strength: Union[float, None] = Field(
+    model: MainModelField = InputField(description="The main model used for inference")
+    controlnets: list[ControlField] = InputField(description="The ControlNets used for inference")
+    loras: list[LoRAMetadataField] = InputField(description="The LoRAs used for inference")
+    strength: Optional[float] = InputField(
         default=None,
         description="The strength used for latents-to-latents",
     )
-    init_image: Union[str, None] = Field(default=None, description="The name of the initial image")
-    vae: Union[VAEModelField, None] = Field(
+    init_image: Optional[str] = InputField(
+        default=None,
+        description="The name of the initial image",
+    )
+    vae: Optional[VAEModelField] = InputField(
         default=None,
         description="The VAE used for decoding, if the main model's default was not used",
     )
 
     # SDXL
-    positive_style_prompt: Union[str, None] = Field(default=None, description="The positive style prompt parameter")
-    negative_style_prompt: Union[str, None] = Field(default=None, description="The negative style prompt parameter")
+    positive_style_prompt: Optional[str] = InputField(
+        default=None,
+        description="The positive style prompt parameter",
+    )
+    negative_style_prompt: Optional[str] = InputField(
+        default=None,
+        description="The negative style prompt parameter",
+    )
 
     # SDXL Refiner
-    refiner_model: Union[MainModelField, None] = Field(default=None, description="The SDXL Refiner model used")
-    refiner_cfg_scale: Union[float, None] = Field(
+    refiner_model: Optional[MainModelField] = InputField(
+        default=None,
+        description="The SDXL Refiner model used",
+    )
+    refiner_cfg_scale: Optional[float] = InputField(
         default=None,
         description="The classifier-free guidance scale parameter used for the refiner",
     )
-    refiner_steps: Union[int, None] = Field(default=None, description="The number of steps used for the refiner")
-    refiner_scheduler: Union[str, None] = Field(default=None, description="The scheduler used for the refiner")
-    refiner_aesthetic_store: Union[float, None] = Field(
-        default=None, description="The aesthetic score used for the refiner"
+    refiner_steps: Optional[int] = InputField(
+        default=None,
+        description="The number of steps used for the refiner",
+    )
+    refiner_scheduler: Optional[str] = InputField(
+        default=None,
+        description="The scheduler used for the refiner",
+    )
+    refiner_positive_aesthetic_store: Optional[float] = InputField(
+        default=None,
+        description="The aesthetic score used for the refiner",
+    )
+    refiner_negative_aesthetic_store: Optional[float] = InputField(
+        default=None,
+        description="The aesthetic score used for the refiner",
+    )
+    refiner_start: Optional[float] = InputField(
+        default=None,
+        description="The start value used for refiner denoising",
     )
-    refiner_start: Union[float, None] = Field(default=None, description="The start value used for refiner denoising")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Metadata Accumulator",
-                "tags": ["image", "metadata", "generation"],
-            },
-        }
 
     def invoke(self, context: InvocationContext) -> MetadataAccumulatorOutput:
         """Collects and outputs a CoreMetadata object"""

invokeai/app/invocations/model.py

@@ -4,7 +4,18 @@ from typing import List, Literal, Optional, Union
 from pydantic import BaseModel, Field
 
 from ...backend.model_management import BaseModelType, ModelType, SubModelType
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    InputField,
+    Input,
+    InvocationContext,
+    OutputField,
+    UIType,
+    tags,
+    title,
+)
 
 
 class ModelInfo(BaseModel):
@@ -39,13 +50,11 @@ class VaeField(BaseModel):
 class ModelLoaderOutput(BaseInvocationOutput):
     """Model loader output"""
 
-    # fmt: off
     type: Literal["model_loader_output"] = "model_loader_output"
 
-    unet: UNetField = Field(default=None, description="UNet submodel")
-    clip: ClipField = Field(default=None, description="Tokenizer and text_encoder submodels")
-    vae: VaeField = Field(default=None, description="Vae submodel")
-    # fmt: on
+    unet: UNetField = OutputField(description=FieldDescriptions.unet, title="UNet")
+    clip: ClipField = OutputField(description=FieldDescriptions.clip, title="CLIP")
+    vae: VaeField = OutputField(description=FieldDescriptions.vae, title="VAE")
 
 
 class MainModelField(BaseModel):
@@ -53,6 +62,7 @@ class MainModelField(BaseModel):
 
     model_name: str = Field(description="Name of the model")
     base_model: BaseModelType = Field(description="Base model")
+    model_type: ModelType = Field(description="Model Type")
 
 
 class LoRAModelField(BaseModel):
@@ -62,24 +72,17 @@ class LoRAModelField(BaseModel):
     base_model: BaseModelType = Field(description="Base model")
 
 
+@title("Main Model Loader")
+@tags("model")
 class MainModelLoaderInvocation(BaseInvocation):
     """Loads a main model, outputting its submodels."""
 
     type: Literal["main_model_loader"] = "main_model_loader"
 
-    model: MainModelField = Field(description="The model to load")
+    # Inputs
+    model: MainModelField = InputField(description=FieldDescriptions.main_model, input=Input.Direct)
     # TODO: precision?
 
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Model Loader",
-                "tags": ["model", "loader"],
-                "type_hints": {"model": "model"},
-            },
-        }
-
     def invoke(self, context: InvocationContext) -> ModelLoaderOutput:
         base_model = self.model.base_model
         model_name = self.model.model_name
@@ -154,22 +157,6 @@ class MainModelLoaderInvocation(BaseInvocation):
                 loras=[],
                 skipped_layers=0,
             ),
-            clip2=ClipField(
-                tokenizer=ModelInfo(
-                    model_name=model_name,
-                    base_model=base_model,
-                    model_type=model_type,
-                    submodel=SubModelType.Tokenizer2,
-                ),
-                text_encoder=ModelInfo(
-                    model_name=model_name,
-                    base_model=base_model,
-                    model_type=model_type,
-                    submodel=SubModelType.TextEncoder2,
-                ),
-                loras=[],
-                skipped_layers=0,
-            ),
             vae=VaeField(
                 vae=ModelInfo(
                     model_name=model_name,
@@ -187,30 +174,27 @@ class LoraLoaderOutput(BaseInvocationOutput):
     # fmt: off
     type: Literal["lora_loader_output"] = "lora_loader_output"
 
-    unet: Optional[UNetField] = Field(default=None, description="UNet submodel")
-    clip: Optional[ClipField] = Field(default=None, description="Tokenizer and text_encoder submodels")
+    unet: Optional[UNetField] = OutputField(default=None, description=FieldDescriptions.unet, title="UNet")
+    clip: Optional[ClipField] = OutputField(default=None, description=FieldDescriptions.clip, title="CLIP")
     # fmt: on
 
 
+@title("LoRA Loader")
+@tags("lora", "model")
 class LoraLoaderInvocation(BaseInvocation):
     """Apply selected lora to unet and text_encoder."""
 
     type: Literal["lora_loader"] = "lora_loader"
 
-    lora: Union[LoRAModelField, None] = Field(default=None, description="Lora model name")
-    weight: float = Field(default=0.75, description="With what weight to apply lora")
-
-    unet: Optional[UNetField] = Field(description="UNet model for applying lora")
-    clip: Optional[ClipField] = Field(description="Clip model for applying lora")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Lora Loader",
-                "tags": ["lora", "loader"],
-                "type_hints": {"lora": "lora_model"},
-            },
-        }
+    # Inputs
+    lora: LoRAModelField = InputField(description=FieldDescriptions.lora_model, input=Input.Direct, title="LoRA")
+    weight: float = InputField(default=0.75, description=FieldDescriptions.lora_weight)
+    unet: Optional[UNetField] = InputField(
+        default=None, description=FieldDescriptions.unet, input=Input.Connection, title="UNet"
+    )
+    clip: Optional[ClipField] = InputField(
+        default=None, description=FieldDescriptions.clip, input=Input.Connection, title="CLIP"
+    )
 
     def invoke(self, context: InvocationContext) -> LoraLoaderOutput:
         if self.lora is None:
@@ -261,6 +245,101 @@ class LoraLoaderInvocation(BaseInvocation):
         return output
 
 
+class SDXLLoraLoaderOutput(BaseInvocationOutput):
+    """SDXL LoRA Loader Output"""
+
+    # fmt: off
+    type: Literal["sdxl_lora_loader_output"] = "sdxl_lora_loader_output"
+
+    unet: Optional[UNetField] = OutputField(default=None, description=FieldDescriptions.unet, title="UNet")
+    clip: Optional[ClipField] = OutputField(default=None, description=FieldDescriptions.clip, title="CLIP 1")
+    clip2: Optional[ClipField] = OutputField(default=None, description=FieldDescriptions.clip, title="CLIP 2")
+    # fmt: on
+
+
+@title("SDXL LoRA Loader")
+@tags("sdxl", "lora", "model")
+class SDXLLoraLoaderInvocation(BaseInvocation):
+    """Apply selected lora to unet and text_encoder."""
+
+    type: Literal["sdxl_lora_loader"] = "sdxl_lora_loader"
+
+    lora: LoRAModelField = InputField(description=FieldDescriptions.lora_model, input=Input.Direct, title="LoRA")
+    weight: float = Field(default=0.75, description=FieldDescriptions.lora_weight)
+    unet: Optional[UNetField] = Field(
+        default=None, description=FieldDescriptions.unet, input=Input.Connection, title="UNET"
+    )
+    clip: Optional[ClipField] = Field(
+        default=None, description=FieldDescriptions.clip, input=Input.Connection, title="CLIP 1"
+    )
+    clip2: Optional[ClipField] = Field(
+        default=None, description=FieldDescriptions.clip, input=Input.Connection, title="CLIP 2"
+    )
+
+    def invoke(self, context: InvocationContext) -> SDXLLoraLoaderOutput:
+        if self.lora is None:
+            raise Exception("No LoRA provided")
+
+        base_model = self.lora.base_model
+        lora_name = self.lora.model_name
+
+        if not context.services.model_manager.model_exists(
+            base_model=base_model,
+            model_name=lora_name,
+            model_type=ModelType.Lora,
+        ):
+            raise Exception(f"Unknown lora name: {lora_name}!")
+
+        if self.unet is not None and any(lora.model_name == lora_name for lora in self.unet.loras):
+            raise Exception(f'Lora "{lora_name}" already applied to unet')
+
+        if self.clip is not None and any(lora.model_name == lora_name for lora in self.clip.loras):
+            raise Exception(f'Lora "{lora_name}" already applied to clip')
+
+        if self.clip2 is not None and any(lora.model_name == lora_name for lora in self.clip2.loras):
+            raise Exception(f'Lora "{lora_name}" already applied to clip2')
+
+        output = SDXLLoraLoaderOutput()
+
+        if self.unet is not None:
+            output.unet = copy.deepcopy(self.unet)
+            output.unet.loras.append(
+                LoraInfo(
+                    base_model=base_model,
+                    model_name=lora_name,
+                    model_type=ModelType.Lora,
+                    submodel=None,
+                    weight=self.weight,
+                )
+            )
+
+        if self.clip is not None:
+            output.clip = copy.deepcopy(self.clip)
+            output.clip.loras.append(
+                LoraInfo(
+                    base_model=base_model,
+                    model_name=lora_name,
+                    model_type=ModelType.Lora,
+                    submodel=None,
+                    weight=self.weight,
+                )
+            )
+
+        if self.clip2 is not None:
+            output.clip2 = copy.deepcopy(self.clip2)
+            output.clip2.loras.append(
+                LoraInfo(
+                    base_model=base_model,
+                    model_name=lora_name,
+                    model_type=ModelType.Lora,
+                    submodel=None,
+                    weight=self.weight,
+                )
+            )
+
+        return output
+
+
 class VAEModelField(BaseModel):
     """Vae model field"""
 
@@ -271,29 +350,23 @@ class VAEModelField(BaseModel):
 class VaeLoaderOutput(BaseInvocationOutput):
     """Model loader output"""
 
-    # fmt: off
     type: Literal["vae_loader_output"] = "vae_loader_output"
 
-    vae: VaeField = Field(default=None, description="Vae model")
-    # fmt: on
+    # Outputs
+    vae: VaeField = OutputField(description=FieldDescriptions.vae, title="VAE")
 
 
+@title("VAE Loader")
+@tags("vae", "model")
 class VaeLoaderInvocation(BaseInvocation):
     """Loads a VAE model, outputting a VaeLoaderOutput"""
 
     type: Literal["vae_loader"] = "vae_loader"
 
-    vae_model: VAEModelField = Field(description="The VAE to load")
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "VAE Loader",
-                "tags": ["vae", "loader"],
-                "type_hints": {"vae_model": "vae_model"},
-            },
-        }
+    # Inputs
+    vae_model: VAEModelField = InputField(
+        description=FieldDescriptions.vae_model, input=Input.Direct, ui_type=UIType.VaeModel, title="VAE"
+    )
 
     def invoke(self, context: InvocationContext) -> VaeLoaderOutput:
         base_model = self.vae_model.base_model

invokeai/app/invocations/noise.py

@@ -1,19 +1,24 @@
 # Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654) & the InvokeAI Team
 
-import math
 from typing import Literal
 
-from pydantic import Field, validator
 import torch
-from invokeai.app.invocations.latent import LatentsField
+from pydantic import validator
 
+from invokeai.app.invocations.latent import LatentsField
 from invokeai.app.util.misc import SEED_MAX, get_random_seed
+
 from ...backend.util.devices import choose_torch_device, torch_dtype
 from .baseinvocation import (
     BaseInvocation,
     BaseInvocationOutput,
-    InvocationConfig,
+    FieldDescriptions,
+    InputField,
     InvocationContext,
+    OutputField,
+    UIType,
+    tags,
+    title,
 )
 
 """
@@ -61,62 +66,53 @@ Nodes
 class NoiseOutput(BaseInvocationOutput):
     """Invocation noise output"""
 
-    # fmt: off
-    type:  Literal["noise_output"] = "noise_output"
+    type: Literal["noise_output"] = "noise_output"
 
     # Inputs
-    noise: LatentsField            = Field(default=None, description="The output noise")
-    width:                     int = Field(description="The width of the noise in pixels")
-    height:                    int = Field(description="The height of the noise in pixels")
-    # fmt: on
+    noise: LatentsField = OutputField(default=None, description=FieldDescriptions.noise)
+    width: int = OutputField(description=FieldDescriptions.width)
+    height: int = OutputField(description=FieldDescriptions.height)
 
 
-def build_noise_output(latents_name: str, latents: torch.Tensor):
+def build_noise_output(latents_name: str, latents: torch.Tensor, seed: int):
     return NoiseOutput(
-        noise=LatentsField(latents_name=latents_name),
+        noise=LatentsField(latents_name=latents_name, seed=seed),
         width=latents.size()[3] * 8,
         height=latents.size()[2] * 8,
     )
 
 
+@title("Noise")
+@tags("latents", "noise")
 class NoiseInvocation(BaseInvocation):
     """Generates latent noise."""
 
     type: Literal["noise"] = "noise"
 
     # Inputs
-    seed: int = Field(
+    seed: int = InputField(
         ge=0,
         le=SEED_MAX,
-        description="The seed to use",
+        description=FieldDescriptions.seed,
         default_factory=get_random_seed,
     )
-    width: int = Field(
+    width: int = InputField(
         default=512,
         multiple_of=8,
         gt=0,
-        description="The width of the resulting noise",
+        description=FieldDescriptions.width,
     )
-    height: int = Field(
+    height: int = InputField(
         default=512,
         multiple_of=8,
         gt=0,
-        description="The height of the resulting noise",
+        description=FieldDescriptions.height,
     )
-    use_cpu: bool = Field(
+    use_cpu: bool = InputField(
         default=True,
         description="Use CPU for noise generation (for reproducible results across platforms)",
     )
 
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "Noise",
-                "tags": ["latents", "noise"],
-            },
-        }
-
     @validator("seed", pre=True)
     def modulo_seed(cls, v):
         """Returns the seed modulo (SEED_MAX + 1) to ensure it is within the valid range."""
@@ -132,4 +128,4 @@ class NoiseInvocation(BaseInvocation):
         )
         name = f"{context.graph_execution_state_id}__{self.id}"
         context.services.latents.save(name, noise)
-        return build_noise_output(latents_name=name, latents=noise)
+        return build_noise_output(latents_name=name, latents=noise, seed=self.seed)

invokeai/app/invocations/onnx.py

@@ -0,0 +1,512 @@
+# Copyright (c) 2023 Borisov Sergey (https://github.com/StAlKeR7779)
+
+import inspect
+import re
+from contextlib import ExitStack
+from typing import List, Literal, Optional, Union
+
+import numpy as np
+import torch
+from diffusers import ControlNetModel, DPMSolverMultistepScheduler
+from diffusers.image_processor import VaeImageProcessor
+from diffusers.schedulers import SchedulerMixin as Scheduler
+from pydantic import BaseModel, Field, validator
+from tqdm import tqdm
+
+from invokeai.app.invocations.metadata import CoreMetadata
+from invokeai.app.invocations.primitives import ConditioningField, ConditioningOutput, ImageField, ImageOutput
+from invokeai.app.util.step_callback import stable_diffusion_step_callback
+from invokeai.backend import BaseModelType, ModelType, SubModelType
+
+from ...backend.model_management import ONNXModelPatcher
+from ...backend.stable_diffusion import PipelineIntermediateState
+from ...backend.util import choose_torch_device
+from ..models.image import ImageCategory, ResourceOrigin
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    InputField,
+    Input,
+    InvocationContext,
+    OutputField,
+    UIComponent,
+    UIType,
+    tags,
+    title,
+)
+from .controlnet_image_processors import ControlField
+from .latent import SAMPLER_NAME_VALUES, LatentsField, LatentsOutput, build_latents_output, get_scheduler
+from .model import ClipField, ModelInfo, UNetField, VaeField
+
+ORT_TO_NP_TYPE = {
+    "tensor(bool)": np.bool_,
+    "tensor(int8)": np.int8,
+    "tensor(uint8)": np.uint8,
+    "tensor(int16)": np.int16,
+    "tensor(uint16)": np.uint16,
+    "tensor(int32)": np.int32,
+    "tensor(uint32)": np.uint32,
+    "tensor(int64)": np.int64,
+    "tensor(uint64)": np.uint64,
+    "tensor(float16)": np.float16,
+    "tensor(float)": np.float32,
+    "tensor(double)": np.float64,
+}
+
+PRECISION_VALUES = Literal[tuple(list(ORT_TO_NP_TYPE.keys()))]
+
+
+@title("ONNX Prompt (Raw)")
+@tags("onnx", "prompt")
+class ONNXPromptInvocation(BaseInvocation):
+    type: Literal["prompt_onnx"] = "prompt_onnx"
+
+    prompt: str = InputField(default="", description=FieldDescriptions.raw_prompt, ui_component=UIComponent.Textarea)
+    clip: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection)
+
+    def invoke(self, context: InvocationContext) -> ConditioningOutput:
+        tokenizer_info = context.services.model_manager.get_model(
+            **self.clip.tokenizer.dict(),
+        )
+        text_encoder_info = context.services.model_manager.get_model(
+            **self.clip.text_encoder.dict(),
+        )
+        with tokenizer_info as orig_tokenizer, text_encoder_info as text_encoder, ExitStack() as stack:
+            loras = [
+                (context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight)
+                for lora in self.clip.loras
+            ]
+
+            ti_list = []
+            for trigger in re.findall(r"<[a-zA-Z0-9., _-]+>", self.prompt):
+                name = trigger[1:-1]
+                try:
+                    ti_list.append(
+                        (
+                            name,
+                            context.services.model_manager.get_model(
+                                model_name=name,
+                                base_model=self.clip.text_encoder.base_model,
+                                model_type=ModelType.TextualInversion,
+                            ).context.model,
+                        )
+                    )
+                except Exception:
+                    # print(e)
+                    # import traceback
+                    # print(traceback.format_exc())
+                    print(f'Warn: trigger: "{trigger}" not found')
+            if loras or ti_list:
+                text_encoder.release_session()
+            with ONNXModelPatcher.apply_lora_text_encoder(text_encoder, loras), ONNXModelPatcher.apply_ti(
+                orig_tokenizer, text_encoder, ti_list
+            ) as (tokenizer, ti_manager):
+                text_encoder.create_session()
+
+                # copy from
+                # https://github.com/huggingface/diffusers/blob/3ebbaf7c96801271f9e6c21400033b6aa5ffcf29/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py#L153
+                text_inputs = tokenizer(
+                    self.prompt,
+                    padding="max_length",
+                    max_length=tokenizer.model_max_length,
+                    truncation=True,
+                    return_tensors="np",
+                )
+                text_input_ids = text_inputs.input_ids
+                """
+                untruncated_ids = tokenizer(prompt, padding="max_length", return_tensors="np").input_ids
+
+                if not np.array_equal(text_input_ids, untruncated_ids):
+                    removed_text = self.tokenizer.batch_decode(
+                        untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
+                    )
+                    logger.warning(
+                        "The following part of your input was truncated because CLIP can only handle sequences up to"
+                        f" {self.tokenizer.model_max_length} tokens: {removed_text}"
+                    )
+                """
+
+                prompt_embeds = text_encoder(input_ids=text_input_ids.astype(np.int32))[0]
+
+        conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
+
+        # TODO: hacky but works ;D maybe rename latents somehow?
+        context.services.latents.save(conditioning_name, (prompt_embeds, None))
+
+        return ConditioningOutput(
+            conditioning=ConditioningField(
+                conditioning_name=conditioning_name,
+            ),
+        )
+
+
+# Text to image
+@title("ONNX Text to Latents")
+@tags("latents", "inference", "txt2img", "onnx")
+class ONNXTextToLatentsInvocation(BaseInvocation):
+    """Generates latents from conditionings."""
+
+    type: Literal["t2l_onnx"] = "t2l_onnx"
+
+    # Inputs
+    positive_conditioning: ConditioningField = InputField(
+        description=FieldDescriptions.positive_cond,
+        input=Input.Connection,
+    )
+    negative_conditioning: ConditioningField = InputField(
+        description=FieldDescriptions.negative_cond,
+        input=Input.Connection,
+    )
+    noise: LatentsField = InputField(
+        description=FieldDescriptions.noise,
+        input=Input.Connection,
+    )
+    steps: int = InputField(default=10, gt=0, description=FieldDescriptions.steps)
+    cfg_scale: Union[float, List[float]] = InputField(
+        default=7.5,
+        ge=1,
+        description=FieldDescriptions.cfg_scale,
+        ui_type=UIType.Float,
+    )
+    scheduler: SAMPLER_NAME_VALUES = InputField(
+        default="euler", description=FieldDescriptions.scheduler, input=Input.Direct
+    )
+    precision: PRECISION_VALUES = InputField(default="tensor(float16)", description=FieldDescriptions.precision)
+    unet: UNetField = InputField(
+        description=FieldDescriptions.unet,
+        input=Input.Connection,
+    )
+    control: Optional[Union[ControlField, list[ControlField]]] = InputField(
+        default=None,
+        description=FieldDescriptions.control,
+        ui_type=UIType.Control,
+    )
+    # seamless:   bool = InputField(default=False, description="Whether or not to generate an image that can tile without seams", )
+    # seamless_axes: str = InputField(default="", description="The axes to tile the image on, 'x' and/or 'y'")
+
+    @validator("cfg_scale")
+    def ge_one(cls, v):
+        """validate that all cfg_scale values are >= 1"""
+        if isinstance(v, list):
+            for i in v:
+                if i < 1:
+                    raise ValueError("cfg_scale must be greater than 1")
+        else:
+            if v < 1:
+                raise ValueError("cfg_scale must be greater than 1")
+        return v
+
+    # based on
+    # https://github.com/huggingface/diffusers/blob/3ebbaf7c96801271f9e6c21400033b6aa5ffcf29/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py#L375
+    def invoke(self, context: InvocationContext) -> LatentsOutput:
+        c, _ = context.services.latents.get(self.positive_conditioning.conditioning_name)
+        uc, _ = context.services.latents.get(self.negative_conditioning.conditioning_name)
+        graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
+        source_node_id = graph_execution_state.prepared_source_mapping[self.id]
+        if isinstance(c, torch.Tensor):
+            c = c.cpu().numpy()
+        if isinstance(uc, torch.Tensor):
+            uc = uc.cpu().numpy()
+        device = torch.device(choose_torch_device())
+        prompt_embeds = np.concatenate([uc, c])
+
+        latents = context.services.latents.get(self.noise.latents_name)
+        if isinstance(latents, torch.Tensor):
+            latents = latents.cpu().numpy()
+
+        # TODO: better execution device handling
+        latents = latents.astype(ORT_TO_NP_TYPE[self.precision])
+
+        # get the initial random noise unless the user supplied it
+        do_classifier_free_guidance = True
+        # latents_dtype = prompt_embeds.dtype
+        # latents_shape = (batch_size * num_images_per_prompt, 4, height // 8, width // 8)
+        # if latents.shape != latents_shape:
+        #    raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}")
+
+        scheduler = get_scheduler(
+            context=context,
+            scheduler_info=self.unet.scheduler,
+            scheduler_name=self.scheduler,
+            seed=0,  # TODO: refactor this node
+        )
+
+        def torch2numpy(latent: torch.Tensor):
+            return latent.cpu().numpy()
+
+        def numpy2torch(latent, device):
+            return torch.from_numpy(latent).to(device)
+
+        def dispatch_progress(
+            self, context: InvocationContext, source_node_id: str, intermediate_state: PipelineIntermediateState
+        ) -> None:
+            stable_diffusion_step_callback(
+                context=context,
+                intermediate_state=intermediate_state,
+                node=self.dict(),
+                source_node_id=source_node_id,
+            )
+
+        scheduler.set_timesteps(self.steps)
+        latents = latents * np.float64(scheduler.init_noise_sigma)
+
+        extra_step_kwargs = dict()
+        if "eta" in set(inspect.signature(scheduler.step).parameters.keys()):
+            extra_step_kwargs.update(
+                eta=0.0,
+            )
+
+        unet_info = context.services.model_manager.get_model(**self.unet.unet.dict())
+
+        with unet_info as unet, ExitStack() as stack:
+            # loras = [(stack.enter_context(context.services.model_manager.get_model(**lora.dict(exclude={"weight"}))), lora.weight) for lora in self.unet.loras]
+            loras = [
+                (context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight)
+                for lora in self.unet.loras
+            ]
+
+            if loras:
+                unet.release_session()
+            with ONNXModelPatcher.apply_lora_unet(unet, loras):
+                # TODO:
+                _, _, h, w = latents.shape
+                unet.create_session(h, w)
+
+                timestep_dtype = next(
+                    (input.type for input in unet.session.get_inputs() if input.name == "timestep"), "tensor(float16)"
+                )
+                timestep_dtype = ORT_TO_NP_TYPE[timestep_dtype]
+                for i in tqdm(range(len(scheduler.timesteps))):
+                    t = scheduler.timesteps[i]
+                    # expand the latents if we are doing classifier free guidance
+                    latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents
+                    latent_model_input = scheduler.scale_model_input(numpy2torch(latent_model_input, device), t)
+                    latent_model_input = latent_model_input.cpu().numpy()
+
+                    # predict the noise residual
+                    timestep = np.array([t], dtype=timestep_dtype)
+                    noise_pred = unet(sample=latent_model_input, timestep=timestep, encoder_hidden_states=prompt_embeds)
+                    noise_pred = noise_pred[0]
+
+                    # perform guidance
+                    if do_classifier_free_guidance:
+                        noise_pred_uncond, noise_pred_text = np.split(noise_pred, 2)
+                        noise_pred = noise_pred_uncond + self.cfg_scale * (noise_pred_text - noise_pred_uncond)
+
+                    # compute the previous noisy sample x_t -> x_t-1
+                    scheduler_output = scheduler.step(
+                        numpy2torch(noise_pred, device), t, numpy2torch(latents, device), **extra_step_kwargs
+                    )
+                    latents = torch2numpy(scheduler_output.prev_sample)
+
+                    state = PipelineIntermediateState(
+                        run_id="test", step=i, timestep=timestep, latents=scheduler_output.prev_sample
+                    )
+                    dispatch_progress(self, context=context, source_node_id=source_node_id, intermediate_state=state)
+
+                    # call the callback, if provided
+                    # if callback is not None and i % callback_steps == 0:
+                    #    callback(i, t, latents)
+
+        torch.cuda.empty_cache()
+
+        name = f"{context.graph_execution_state_id}__{self.id}"
+        context.services.latents.save(name, latents)
+        return build_latents_output(latents_name=name, latents=torch.from_numpy(latents))
+
+
+# Latent to image
+@title("ONNX Latents to Image")
+@tags("latents", "image", "vae", "onnx")
+class ONNXLatentsToImageInvocation(BaseInvocation):
+    """Generates an image from latents."""
+
+    type: Literal["l2i_onnx"] = "l2i_onnx"
+
+    # Inputs
+    latents: LatentsField = InputField(
+        description=FieldDescriptions.denoised_latents,
+        input=Input.Connection,
+    )
+    vae: VaeField = InputField(
+        description=FieldDescriptions.vae,
+        input=Input.Connection,
+    )
+    metadata: Optional[CoreMetadata] = InputField(
+        default=None,
+        description=FieldDescriptions.core_metadata,
+        ui_hidden=True,
+    )
+    # tiled: bool = InputField(default=False, description="Decode latents by overlaping tiles(less memory consumption)")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        latents = context.services.latents.get(self.latents.latents_name)
+
+        if self.vae.vae.submodel != SubModelType.VaeDecoder:
+            raise Exception(f"Expected vae_decoder, found: {self.vae.vae.model_type}")
+
+        vae_info = context.services.model_manager.get_model(
+            **self.vae.vae.dict(),
+        )
+
+        # clear memory as vae decode can request a lot
+        torch.cuda.empty_cache()
+
+        with vae_info as vae:
+            vae.create_session()
+
+            # copied from
+            # https://github.com/huggingface/diffusers/blob/3ebbaf7c96801271f9e6c21400033b6aa5ffcf29/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py#L427
+            latents = 1 / 0.18215 * latents
+            # image = self.vae_decoder(latent_sample=latents)[0]
+            # it seems likes there is a strange result for using half-precision vae decoder if batchsize>1
+            image = np.concatenate([vae(latent_sample=latents[i : i + 1])[0] for i in range(latents.shape[0])])
+
+            image = np.clip(image / 2 + 0.5, 0, 1)
+            image = image.transpose((0, 2, 3, 1))
+            image = VaeImageProcessor.numpy_to_pil(image)[0]
+
+        torch.cuda.empty_cache()
+
+        image_dto = context.services.images.create(
+            image=image,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            session_id=context.graph_execution_state_id,
+            is_intermediate=self.is_intermediate,
+            metadata=self.metadata.dict() if self.metadata else None,
+        )
+
+        return ImageOutput(
+            image=ImageField(image_name=image_dto.image_name),
+            width=image_dto.width,
+            height=image_dto.height,
+        )
+
+
+class ONNXModelLoaderOutput(BaseInvocationOutput):
+    """Model loader output"""
+
+    # fmt: off
+    type: Literal["model_loader_output_onnx"] = "model_loader_output_onnx"
+
+    unet: UNetField = OutputField(default=None, description=FieldDescriptions.unet, title="UNet")
+    clip: ClipField = OutputField(default=None, description=FieldDescriptions.clip, title="CLIP")
+    vae_decoder: VaeField = OutputField(default=None, description=FieldDescriptions.vae, title="VAE Decoder")
+    vae_encoder: VaeField = OutputField(default=None, description=FieldDescriptions.vae, title="VAE Encoder")
+    # fmt: on
+
+
+class OnnxModelField(BaseModel):
+    """Onnx model field"""
+
+    model_name: str = Field(description="Name of the model")
+    base_model: BaseModelType = Field(description="Base model")
+    model_type: ModelType = Field(description="Model Type")
+
+
+@title("ONNX Model Loader")
+@tags("onnx", "model")
+class OnnxModelLoaderInvocation(BaseInvocation):
+    """Loads a main model, outputting its submodels."""
+
+    type: Literal["onnx_model_loader"] = "onnx_model_loader"
+
+    # Inputs
+    model: OnnxModelField = InputField(
+        description=FieldDescriptions.onnx_main_model, input=Input.Direct, ui_type=UIType.ONNXModel
+    )
+
+    def invoke(self, context: InvocationContext) -> ONNXModelLoaderOutput:
+        base_model = self.model.base_model
+        model_name = self.model.model_name
+        model_type = ModelType.ONNX
+
+        # TODO: not found exceptions
+        if not context.services.model_manager.model_exists(
+            model_name=model_name,
+            base_model=base_model,
+            model_type=model_type,
+        ):
+            raise Exception(f"Unknown {base_model} {model_type} model: {model_name}")
+
+        """
+        if not context.services.model_manager.model_exists(
+            model_name=self.model_name,
+            model_type=SDModelType.Diffusers,
+            submodel=SDModelType.Tokenizer,
+        ):
+            raise Exception(
+                f"Failed to find tokenizer submodel in {self.model_name}! Check if model corrupted"
+            )
+
+        if not context.services.model_manager.model_exists(
+            model_name=self.model_name,
+            model_type=SDModelType.Diffusers,
+            submodel=SDModelType.TextEncoder,
+        ):
+            raise Exception(
+                f"Failed to find text_encoder submodel in {self.model_name}! Check if model corrupted"
+            )
+
+        if not context.services.model_manager.model_exists(
+            model_name=self.model_name,
+            model_type=SDModelType.Diffusers,
+            submodel=SDModelType.UNet,
+        ):
+            raise Exception(
+                f"Failed to find unet submodel from {self.model_name}! Check if model corrupted"
+            )
+        """
+
+        return ONNXModelLoaderOutput(
+            unet=UNetField(
+                unet=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                    submodel=SubModelType.UNet,
+                ),
+                scheduler=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                    submodel=SubModelType.Scheduler,
+                ),
+                loras=[],
+            ),
+            clip=ClipField(
+                tokenizer=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                    submodel=SubModelType.Tokenizer,
+                ),
+                text_encoder=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                    submodel=SubModelType.TextEncoder,
+                ),
+                loras=[],
+                skipped_layers=0,
+            ),
+            vae_decoder=VaeField(
+                vae=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                    submodel=SubModelType.VaeDecoder,
+                ),
+            ),
+            vae_encoder=VaeField(
+                vae=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                    submodel=SubModelType.VaeEncoder,
+                ),
+            ),
+        )

invokeai/app/invocations/param_easing.py

@@ -1,73 +1,64 @@
 import io
-from typing import Literal, Optional, Any
+from typing import Literal, Optional
 
-# from PIL.Image import Image
-import PIL.Image
-from matplotlib.ticker import MaxNLocator
-from matplotlib.figure import Figure
-
-from pydantic import BaseModel, Field
-import numpy as np
 import matplotlib.pyplot as plt
+import numpy as np
 
+import PIL.Image
 from easing_functions import (
-    LinearInOut,
-    QuadEaseInOut,
-    QuadEaseIn,
-    QuadEaseOut,
-    CubicEaseInOut,
-    CubicEaseIn,
-    CubicEaseOut,
-    QuarticEaseInOut,
-    QuarticEaseIn,
-    QuarticEaseOut,
-    QuinticEaseInOut,
-    QuinticEaseIn,
-    QuinticEaseOut,
-    SineEaseInOut,
-    SineEaseIn,
-    SineEaseOut,
-    CircularEaseIn,
-    CircularEaseInOut,
-    CircularEaseOut,
-    ExponentialEaseInOut,
-    ExponentialEaseIn,
-    ExponentialEaseOut,
-    ElasticEaseIn,
-    ElasticEaseInOut,
-    ElasticEaseOut,
     BackEaseIn,
     BackEaseInOut,
     BackEaseOut,
     BounceEaseIn,
     BounceEaseInOut,
     BounceEaseOut,
+    CircularEaseIn,
+    CircularEaseInOut,
+    CircularEaseOut,
+    CubicEaseIn,
+    CubicEaseInOut,
+    CubicEaseOut,
+    ElasticEaseIn,
+    ElasticEaseInOut,
+    ElasticEaseOut,
+    ExponentialEaseIn,
+    ExponentialEaseInOut,
+    ExponentialEaseOut,
+    LinearInOut,
+    QuadEaseIn,
+    QuadEaseInOut,
+    QuadEaseOut,
+    QuarticEaseIn,
+    QuarticEaseInOut,
+    QuarticEaseOut,
+    QuinticEaseIn,
+    QuinticEaseInOut,
+    QuinticEaseOut,
+    SineEaseIn,
+    SineEaseInOut,
+    SineEaseOut,
 )
+from matplotlib.figure import Figure
+from matplotlib.ticker import MaxNLocator
+from pydantic import BaseModel, Field
+
+from invokeai.app.invocations.primitives import FloatCollectionOutput
 
-from .baseinvocation import (
-    BaseInvocation,
-    BaseInvocationOutput,
-    InvocationContext,
-    InvocationConfig,
-)
 from ...backend.util.logging import InvokeAILogger
-from .collections import FloatCollectionOutput
+from .baseinvocation import BaseInvocation, InputField, InvocationContext, tags, title
 
 
+@title("Float Range")
+@tags("math", "range")
 class FloatLinearRangeInvocation(BaseInvocation):
     """Creates a range"""
 
     type: Literal["float_range"] = "float_range"
 
     # Inputs
-    start: float = Field(default=5, description="The first value of the range")
-    stop: float = Field(default=10, description="The last value of the range")
-    steps: int = Field(default=30, description="number of values to interpolate over (including start and stop)")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Linear Range (Float)", "tags": ["math", "float", "linear", "range"]},
-        }
+    start: float = InputField(default=5, description="The first value of the range")
+    stop: float = InputField(default=10, description="The last value of the range")
+    steps: int = InputField(default=30, description="number of values to interpolate over (including start and stop)")
 
     def invoke(self, context: InvocationContext) -> FloatCollectionOutput:
         param_list = list(np.linspace(self.start, self.stop, self.steps))
@@ -108,37 +99,32 @@ EASING_FUNCTIONS_MAP = {
     "BounceInOut": BounceEaseInOut,
 }
 
-EASING_FUNCTION_KEYS: Any = Literal[tuple(list(EASING_FUNCTIONS_MAP.keys()))]
+EASING_FUNCTION_KEYS = Literal[tuple(list(EASING_FUNCTIONS_MAP.keys()))]
 
 
 # actually I think for now could just use CollectionOutput (which is list[Any]
+@title("Step Param Easing")
+@tags("step", "easing")
 class StepParamEasingInvocation(BaseInvocation):
     """Experimental per-step parameter easing for denoising steps"""
 
     type: Literal["step_param_easing"] = "step_param_easing"
 
     # Inputs
-    # fmt: off
-    easing: EASING_FUNCTION_KEYS = Field(default="Linear", description="The easing function to use")
-    num_steps: int = Field(default=20, description="number of denoising steps")
-    start_value: float = Field(default=0.0, description="easing starting value")
-    end_value: float = Field(default=1.0, description="easing ending value")
-    start_step_percent: float = Field(default=0.0, description="fraction of steps at which to start easing")
-    end_step_percent: float = Field(default=1.0, description="fraction of steps after which to end easing")
+    easing: EASING_FUNCTION_KEYS = InputField(default="Linear", description="The easing function to use")
+    num_steps: int = InputField(default=20, description="number of denoising steps")
+    start_value: float = InputField(default=0.0, description="easing starting value")
+    end_value: float = InputField(default=1.0, description="easing ending value")
+    start_step_percent: float = InputField(default=0.0, description="fraction of steps at which to start easing")
+    end_step_percent: float = InputField(default=1.0, description="fraction of steps after which to end easing")
     # if None, then start_value is used prior to easing start
-    pre_start_value: Optional[float] = Field(default=None, description="value before easing start")
+    pre_start_value: Optional[float] = InputField(default=None, description="value before easing start")
     # if None, then end value is used prior to easing end
-    post_end_value: Optional[float] = Field(default=None, description="value after easing end")
-    mirror: bool = Field(default=False, description="include mirror of easing function")
+    post_end_value: Optional[float] = InputField(default=None, description="value after easing end")
+    mirror: bool = InputField(default=False, description="include mirror of easing function")
     # FIXME: add alt_mirror option (alternative to default or mirror), or remove entirely
-    # alt_mirror: bool = Field(default=False, description="alternative mirroring by dual easing")
-    show_easing_plot: bool = Field(default=False, description="show easing plot")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Param Easing By Step", "tags": ["param", "step", "easing"]},
-        }
+    # alt_mirror: bool = InputField(default=False, description="alternative mirroring by dual easing")
+    show_easing_plot: bool = InputField(default=False, description="show easing plot")
 
     def invoke(self, context: InvocationContext) -> FloatCollectionOutput:
         log_diagnostics = False

invokeai/app/invocations/params.py

@@ -1,66 +0,0 @@
-# Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654)
-
-from typing import Literal
-
-from pydantic import Field
-
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
-from .math import FloatOutput, IntOutput
-
-# Pass-through parameter nodes - used by subgraphs
-
-
-class ParamIntInvocation(BaseInvocation):
-    """An integer parameter"""
-
-    # fmt: off
-    type: Literal["param_int"] = "param_int"
-    a: int = Field(default=0, description="The integer value")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"tags": ["param", "integer"], "title": "Integer Parameter"},
-        }
-
-    def invoke(self, context: InvocationContext) -> IntOutput:
-        return IntOutput(a=self.a)
-
-
-class ParamFloatInvocation(BaseInvocation):
-    """A float parameter"""
-
-    # fmt: off
-    type: Literal["param_float"] = "param_float"
-    param: float = Field(default=0.0, description="The float value")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"tags": ["param", "float"], "title": "Float Parameter"},
-        }
-
-    def invoke(self, context: InvocationContext) -> FloatOutput:
-        return FloatOutput(param=self.param)
-
-
-class StringOutput(BaseInvocationOutput):
-    """A string output"""
-
-    type: Literal["string_output"] = "string_output"
-    text: str = Field(default=None, description="The output string")
-
-
-class ParamStringInvocation(BaseInvocation):
-    """A string parameter"""
-
-    type: Literal["param_string"] = "param_string"
-    text: str = Field(default="", description="The string value")
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"tags": ["param", "string"], "title": "String Parameter"},
-        }
-
-    def invoke(self, context: InvocationContext) -> StringOutput:
-        return StringOutput(text=self.text)

invokeai/app/invocations/primitives.py

@@ -0,0 +1,494 @@
+# Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654)
+
+from typing import Literal, Optional, Tuple, Union
+from anyio import Condition
+
+from pydantic import BaseModel, Field
+import torch
+
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    Input,
+    InputField,
+    InvocationContext,
+    OutputField,
+    UIComponent,
+    UIType,
+    tags,
+    title,
+)
+
+"""
+Primitives: Boolean, Integer, Float, String, Image, Latents, Conditioning, Color
+- primitive nodes
+- primitive outputs
+- primitive collection outputs
+"""
+
+# region Boolean
+
+
+class BooleanOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single boolean"""
+
+    type: Literal["boolean_output"] = "boolean_output"
+    a: bool = OutputField(description="The output boolean")
+
+
+class BooleanCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of booleans"""
+
+    type: Literal["boolean_collection_output"] = "boolean_collection_output"
+
+    # Outputs
+    collection: list[bool] = OutputField(
+        default_factory=list, description="The output boolean collection", ui_type=UIType.BooleanCollection
+    )
+
+
+@title("Boolean Primitive")
+@tags("primitives", "boolean")
+class BooleanInvocation(BaseInvocation):
+    """A boolean primitive value"""
+
+    type: Literal["boolean"] = "boolean"
+
+    # Inputs
+    a: bool = InputField(default=False, description="The boolean value")
+
+    def invoke(self, context: InvocationContext) -> BooleanOutput:
+        return BooleanOutput(a=self.a)
+
+
+@title("Boolean Primitive Collection")
+@tags("primitives", "boolean", "collection")
+class BooleanCollectionInvocation(BaseInvocation):
+    """A collection of boolean primitive values"""
+
+    type: Literal["boolean_collection"] = "boolean_collection"
+
+    # Inputs
+    collection: list[bool] = InputField(
+        default=False, description="The collection of boolean values", ui_type=UIType.BooleanCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> BooleanCollectionOutput:
+        return BooleanCollectionOutput(collection=self.collection)
+
+
+# endregion
+
+# region Integer
+
+
+class IntegerOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single integer"""
+
+    type: Literal["integer_output"] = "integer_output"
+    a: int = OutputField(description="The output integer")
+
+
+class IntegerCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of integers"""
+
+    type: Literal["integer_collection_output"] = "integer_collection_output"
+
+    # Outputs
+    collection: list[int] = OutputField(
+        default_factory=list, description="The int collection", ui_type=UIType.IntegerCollection
+    )
+
+
+@title("Integer Primitive")
+@tags("primitives", "integer")
+class IntegerInvocation(BaseInvocation):
+    """An integer primitive value"""
+
+    type: Literal["integer"] = "integer"
+
+    # Inputs
+    a: int = InputField(default=0, description="The integer value")
+
+    def invoke(self, context: InvocationContext) -> IntegerOutput:
+        return IntegerOutput(a=self.a)
+
+
+@title("Integer Primitive Collection")
+@tags("primitives", "integer", "collection")
+class IntegerCollectionInvocation(BaseInvocation):
+    """A collection of integer primitive values"""
+
+    type: Literal["integer_collection"] = "integer_collection"
+
+    # Inputs
+    collection: list[int] = InputField(
+        default=0, description="The collection of integer values", ui_type=UIType.IntegerCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> IntegerCollectionOutput:
+        return IntegerCollectionOutput(collection=self.collection)
+
+
+# endregion
+
+# region Float
+
+
+class FloatOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single float"""
+
+    type: Literal["float_output"] = "float_output"
+    a: float = OutputField(description="The output float")
+
+
+class FloatCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of floats"""
+
+    type: Literal["float_collection_output"] = "float_collection_output"
+
+    # Outputs
+    collection: list[float] = OutputField(
+        default_factory=list, description="The float collection", ui_type=UIType.FloatCollection
+    )
+
+
+@title("Float Primitive")
+@tags("primitives", "float")
+class FloatInvocation(BaseInvocation):
+    """A float primitive value"""
+
+    type: Literal["float"] = "float"
+
+    # Inputs
+    param: float = InputField(default=0.0, description="The float value")
+
+    def invoke(self, context: InvocationContext) -> FloatOutput:
+        return FloatOutput(a=self.param)
+
+
+@title("Float Primitive Collection")
+@tags("primitives", "float", "collection")
+class FloatCollectionInvocation(BaseInvocation):
+    """A collection of float primitive values"""
+
+    type: Literal["float_collection"] = "float_collection"
+
+    # Inputs
+    collection: list[float] = InputField(
+        default=0, description="The collection of float values", ui_type=UIType.FloatCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> FloatCollectionOutput:
+        return FloatCollectionOutput(collection=self.collection)
+
+
+# endregion
+
+# region String
+
+
+class StringOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single string"""
+
+    type: Literal["string_output"] = "string_output"
+    text: str = OutputField(description="The output string")
+
+
+class StringCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of strings"""
+
+    type: Literal["string_collection_output"] = "string_collection_output"
+
+    # Outputs
+    collection: list[str] = OutputField(
+        default_factory=list, description="The output strings", ui_type=UIType.StringCollection
+    )
+
+
+@title("String Primitive")
+@tags("primitives", "string")
+class StringInvocation(BaseInvocation):
+    """A string primitive value"""
+
+    type: Literal["string"] = "string"
+
+    # Inputs
+    text: str = InputField(default="", description="The string value", ui_component=UIComponent.Textarea)
+
+    def invoke(self, context: InvocationContext) -> StringOutput:
+        return StringOutput(text=self.text)
+
+
+@title("String Primitive Collection")
+@tags("primitives", "string", "collection")
+class StringCollectionInvocation(BaseInvocation):
+    """A collection of string primitive values"""
+
+    type: Literal["string_collection"] = "string_collection"
+
+    # Inputs
+    collection: list[str] = InputField(
+        default=0, description="The collection of string values", ui_type=UIType.StringCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> StringCollectionOutput:
+        return StringCollectionOutput(collection=self.collection)
+
+
+# endregion
+
+# region Image
+
+
+class ImageField(BaseModel):
+    """An image primitive field"""
+
+    image_name: str = Field(description="The name of the image")
+
+
+class ImageOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single image"""
+
+    type: Literal["image_output"] = "image_output"
+    image: ImageField = OutputField(description="The output image")
+    width: int = OutputField(description="The width of the image in pixels")
+    height: int = OutputField(description="The height of the image in pixels")
+
+
+class ImageCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of images"""
+
+    type: Literal["image_collection_output"] = "image_collection_output"
+
+    # Outputs
+    collection: list[ImageField] = OutputField(
+        default_factory=list, description="The output images", ui_type=UIType.ImageCollection
+    )
+
+
+@title("Image Primitive")
+@tags("primitives", "image")
+class ImageInvocation(BaseInvocation):
+    """An image primitive value"""
+
+    # Metadata
+    type: Literal["image"] = "image"
+
+    # Inputs
+    image: ImageField = InputField(description="The image to load")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.services.images.get_pil_image(self.image.image_name)
+
+        return ImageOutput(
+            image=ImageField(image_name=self.image.image_name),
+            width=image.width,
+            height=image.height,
+        )
+
+
+@title("Image Primitive Collection")
+@tags("primitives", "image", "collection")
+class ImageCollectionInvocation(BaseInvocation):
+    """A collection of image primitive values"""
+
+    type: Literal["image_collection"] = "image_collection"
+
+    # Inputs
+    collection: list[ImageField] = InputField(
+        default=0, description="The collection of image values", ui_type=UIType.ImageCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> ImageCollectionOutput:
+        return ImageCollectionOutput(collection=self.collection)
+
+
+# endregion
+
+# region Latents
+
+
+class LatentsField(BaseModel):
+    """A latents tensor primitive field"""
+
+    latents_name: str = Field(description="The name of the latents")
+    seed: Optional[int] = Field(default=None, description="Seed used to generate this latents")
+
+
+class LatentsOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single latents tensor"""
+
+    type: Literal["latents_output"] = "latents_output"
+
+    latents: LatentsField = OutputField(
+        description=FieldDescriptions.latents,
+    )
+    width: int = OutputField(description=FieldDescriptions.width)
+    height: int = OutputField(description=FieldDescriptions.height)
+
+
+class LatentsCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of latents tensors"""
+
+    type: Literal["latents_collection_output"] = "latents_collection_output"
+
+    collection: list[LatentsField] = OutputField(
+        default_factory=list,
+        description=FieldDescriptions.latents,
+        ui_type=UIType.LatentsCollection,
+    )
+
+
+@title("Latents Primitive")
+@tags("primitives", "latents")
+class LatentsInvocation(BaseInvocation):
+    """A latents tensor primitive value"""
+
+    type: Literal["latents"] = "latents"
+
+    # Inputs
+    latents: LatentsField = InputField(description="The latents tensor", input=Input.Connection)
+
+    def invoke(self, context: InvocationContext) -> LatentsOutput:
+        latents = context.services.latents.get(self.latents.latents_name)
+
+        return build_latents_output(self.latents.latents_name, latents)
+
+
+@title("Latents Primitive Collection")
+@tags("primitives", "latents", "collection")
+class LatentsCollectionInvocation(BaseInvocation):
+    """A collection of latents tensor primitive values"""
+
+    type: Literal["latents_collection"] = "latents_collection"
+
+    # Inputs
+    collection: list[LatentsField] = InputField(
+        default=0, description="The collection of latents tensors", ui_type=UIType.LatentsCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> LatentsCollectionOutput:
+        return LatentsCollectionOutput(collection=self.collection)
+
+
+def build_latents_output(latents_name: str, latents: torch.Tensor, seed: Optional[int] = None):
+    return LatentsOutput(
+        latents=LatentsField(latents_name=latents_name, seed=seed),
+        width=latents.size()[3] * 8,
+        height=latents.size()[2] * 8,
+    )
+
+
+# endregion
+
+# region Color
+
+
+class ColorField(BaseModel):
+    """A color primitive field"""
+
+    r: int = Field(ge=0, le=255, description="The red component")
+    g: int = Field(ge=0, le=255, description="The green component")
+    b: int = Field(ge=0, le=255, description="The blue component")
+    a: int = Field(ge=0, le=255, description="The alpha component")
+
+    def tuple(self) -> Tuple[int, int, int, int]:
+        return (self.r, self.g, self.b, self.a)
+
+
+class ColorOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single color"""
+
+    type: Literal["color_output"] = "color_output"
+    color: ColorField = OutputField(description="The output color")
+
+
+class ColorCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of colors"""
+
+    type: Literal["color_collection_output"] = "color_collection_output"
+
+    # Outputs
+    collection: list[ColorField] = OutputField(
+        default_factory=list, description="The output colors", ui_type=UIType.ColorCollection
+    )
+
+
+@title("Color Primitive")
+@tags("primitives", "color")
+class ColorInvocation(BaseInvocation):
+    """A color primitive value"""
+
+    type: Literal["color"] = "color"
+
+    # Inputs
+    color: ColorField = InputField(default=ColorField(r=0, g=0, b=0, a=255), description="The color value")
+
+    def invoke(self, context: InvocationContext) -> ColorOutput:
+        return ColorOutput(color=self.color)
+
+
+# endregion
+
+# region Conditioning
+
+
+class ConditioningField(BaseModel):
+    """A conditioning tensor primitive value"""
+
+    conditioning_name: str = Field(description="The name of conditioning tensor")
+
+
+class ConditioningOutput(BaseInvocationOutput):
+    """Base class for nodes that output a single conditioning tensor"""
+
+    type: Literal["conditioning_output"] = "conditioning_output"
+
+    conditioning: ConditioningField = OutputField(description=FieldDescriptions.cond)
+
+
+class ConditioningCollectionOutput(BaseInvocationOutput):
+    """Base class for nodes that output a collection of conditioning tensors"""
+
+    type: Literal["conditioning_collection_output"] = "conditioning_collection_output"
+
+    # Outputs
+    collection: list[ConditioningField] = OutputField(
+        default_factory=list,
+        description="The output conditioning tensors",
+        ui_type=UIType.ConditioningCollection,
+    )
+
+
+@title("Conditioning Primitive")
+@tags("primitives", "conditioning")
+class ConditioningInvocation(BaseInvocation):
+    """A conditioning tensor primitive value"""
+
+    type: Literal["conditioning"] = "conditioning"
+
+    conditioning: ConditioningField = InputField(description=FieldDescriptions.cond, input=Input.Connection)
+
+    def invoke(self, context: InvocationContext) -> ConditioningOutput:
+        return ConditioningOutput(conditioning=self.conditioning)
+
+
+@title("Conditioning Primitive Collection")
+@tags("primitives", "conditioning", "collection")
+class ConditioningCollectionInvocation(BaseInvocation):
+    """A collection of conditioning tensor primitive values"""
+
+    type: Literal["conditioning_collection"] = "conditioning_collection"
+
+    # Inputs
+    collection: list[ConditioningField] = InputField(
+        default=0, description="The collection of conditioning tensors", ui_type=UIType.ConditioningCollection
+    )
+
+    def invoke(self, context: InvocationContext) -> ConditioningCollectionOutput:
+        return ConditioningCollectionOutput(collection=self.collection)
+
+
+# endregion

invokeai/app/invocations/prompt.py

@@ -1,59 +1,28 @@
 from os.path import exists
-from typing import Literal, Optional
+from typing import Literal, Optional, Union
 
 import numpy as np
-from pydantic import Field, validator
+from dynamicprompts.generators import CombinatorialPromptGenerator, RandomPromptGenerator
+from pydantic import validator
 
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
-from dynamicprompts.generators import RandomPromptGenerator, CombinatorialPromptGenerator
-
-
-class PromptOutput(BaseInvocationOutput):
-    """Base class for invocations that output a prompt"""
-
-    # fmt: off
-    type: Literal["prompt"] = "prompt"
-
-    prompt: str = Field(default=None, description="The output prompt")
-    # fmt: on
-
-    class Config:
-        schema_extra = {
-            "required": [
-                "type",
-                "prompt",
-            ]
-        }
-
-
-class PromptCollectionOutput(BaseInvocationOutput):
-    """Base class for invocations that output a collection of prompts"""
-
-    # fmt: off
-    type: Literal["prompt_collection_output"] = "prompt_collection_output"
-
-    prompt_collection: list[str] = Field(description="The output prompt collection")
-    count: int = Field(description="The size of the prompt collection")
-    # fmt: on
-
-    class Config:
-        schema_extra = {"required": ["type", "prompt_collection", "count"]}
+from invokeai.app.invocations.primitives import StringCollectionOutput
+
+from .baseinvocation import BaseInvocation, InputField, InvocationContext, UIComponent, UIType, tags, title
 
 
+@title("Dynamic Prompt")
+@tags("prompt", "collection")
 class DynamicPromptInvocation(BaseInvocation):
     """Parses a prompt using adieyal/dynamicprompts' random or combinatorial generator"""
 
     type: Literal["dynamic_prompt"] = "dynamic_prompt"
-    prompt: str = Field(description="The prompt to parse with dynamicprompts")
-    max_prompts: int = Field(default=1, description="The number of prompts to generate")
-    combinatorial: bool = Field(default=False, description="Whether to use the combinatorial generator")
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Dynamic Prompt", "tags": ["prompt", "dynamic"]},
-        }
+    # Inputs
+    prompt: str = InputField(description="The prompt to parse with dynamicprompts", ui_component=UIComponent.Textarea)
+    max_prompts: int = InputField(default=1, description="The number of prompts to generate")
+    combinatorial: bool = InputField(default=False, description="Whether to use the combinatorial generator")
 
-    def invoke(self, context: InvocationContext) -> PromptCollectionOutput:
+    def invoke(self, context: InvocationContext) -> StringCollectionOutput:
         if self.combinatorial:
             generator = CombinatorialPromptGenerator()
             prompts = generator.generate(self.prompt, max_prompts=self.max_prompts)
@@ -61,27 +30,26 @@ class DynamicPromptInvocation(BaseInvocation):
             generator = RandomPromptGenerator()
             prompts = generator.generate(self.prompt, num_images=self.max_prompts)
 
-        return PromptCollectionOutput(prompt_collection=prompts, count=len(prompts))
+        return StringCollectionOutput(collection=prompts)
 
 
+@title("Prompts from File")
+@tags("prompt", "file")
 class PromptsFromFileInvocation(BaseInvocation):
     """Loads prompts from a text file"""
 
-    # fmt: off
-    type: Literal['prompt_from_file'] = 'prompt_from_file'
+    type: Literal["prompt_from_file"] = "prompt_from_file"
 
     # Inputs
-    file_path: str = Field(description="Path to prompt text file")
-    pre_prompt: Optional[str] = Field(description="String to prepend to each prompt")
-    post_prompt: Optional[str] = Field(description="String to append to each prompt")
-    start_line: int = Field(default=1, ge=1, description="Line in the file to start start from")
-    max_prompts: int = Field(default=1, ge=0, description="Max lines to read from file (0=all)")
-    # fmt: on
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Prompts From File", "tags": ["prompt", "file"]},
-        }
+    file_path: str = InputField(description="Path to prompt text file", ui_type=UIType.FilePath)
+    pre_prompt: Optional[str] = InputField(
+        default=None, description="String to prepend to each prompt", ui_component=UIComponent.Textarea
+    )
+    post_prompt: Optional[str] = InputField(
+        default=None, description="String to append to each prompt", ui_component=UIComponent.Textarea
+    )
+    start_line: int = InputField(default=1, ge=1, description="Line in the file to start start from")
+    max_prompts: int = InputField(default=1, ge=0, description="Max lines to read from file (0=all)")
 
     @validator("file_path")
     def file_path_exists(cls, v):
@@ -89,7 +57,14 @@ class PromptsFromFileInvocation(BaseInvocation):
             raise ValueError(FileNotFoundError)
         return v
 
-    def promptsFromFile(self, file_path: str, pre_prompt: str, post_prompt: str, start_line: int, max_prompts: int):
+    def promptsFromFile(
+        self,
+        file_path: str,
+        pre_prompt: Union[str, None],
+        post_prompt: Union[str, None],
+        start_line: int,
+        max_prompts: int,
+    ):
         prompts = []
         start_line -= 1
         end_line = start_line + max_prompts
@@ -103,8 +78,8 @@ class PromptsFromFileInvocation(BaseInvocation):
                     break
         return prompts
 
-    def invoke(self, context: InvocationContext) -> PromptCollectionOutput:
+    def invoke(self, context: InvocationContext) -> StringCollectionOutput:
         prompts = self.promptsFromFile(
             self.file_path, self.pre_prompt, self.post_prompt, self.start_line, self.max_prompts
         )
-        return PromptCollectionOutput(prompt_collection=prompts, count=len(prompts))
+        return StringCollectionOutput(collection=prompts)

invokeai/app/invocations/sdxl.py

@@ -1,62 +1,55 @@
-import torch
-import inspect
-from tqdm import tqdm
-from typing import List, Literal, Optional, Union
-
-from pydantic import Field, validator
+from typing import Literal
 
 from ...backend.model_management import ModelType, SubModelType
-from invokeai.app.util.step_callback import stable_diffusion_xl_step_callback
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationConfig, InvocationContext
-
-from .model import UNetField, ClipField, VaeField, MainModelField, ModelInfo
-from .compel import ConditioningField
-from .latent import LatentsField, SAMPLER_NAME_VALUES, LatentsOutput, get_scheduler, build_latents_output
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    FieldDescriptions,
+    Input,
+    InputField,
+    InvocationContext,
+    OutputField,
+    UIType,
+    tags,
+    title,
+)
+from .model import ClipField, MainModelField, ModelInfo, UNetField, VaeField
 
 
 class SDXLModelLoaderOutput(BaseInvocationOutput):
     """SDXL base model loader output"""
 
-    # fmt: off
     type: Literal["sdxl_model_loader_output"] = "sdxl_model_loader_output"
 
-    unet: UNetField = Field(default=None, description="UNet submodel")
-    clip: ClipField = Field(default=None, description="Tokenizer and text_encoder submodels")
-    clip2: ClipField = Field(default=None, description="Tokenizer and text_encoder submodels")
-    vae: VaeField = Field(default=None, description="Vae submodel")
-    # fmt: on
+    unet: UNetField = OutputField(description=FieldDescriptions.unet, title="UNet")
+    clip: ClipField = OutputField(description=FieldDescriptions.clip, title="CLIP 1")
+    clip2: ClipField = OutputField(description=FieldDescriptions.clip, title="CLIP 2")
+    vae: VaeField = OutputField(description=FieldDescriptions.vae, title="VAE")
 
 
 class SDXLRefinerModelLoaderOutput(BaseInvocationOutput):
     """SDXL refiner model loader output"""
 
-    # fmt: off
     type: Literal["sdxl_refiner_model_loader_output"] = "sdxl_refiner_model_loader_output"
-    unet: UNetField = Field(default=None, description="UNet submodel")
-    clip2: ClipField = Field(default=None, description="Tokenizer and text_encoder submodels")
-    vae: VaeField = Field(default=None, description="Vae submodel")
-    # fmt: on
-    # fmt: on
+
+    unet: UNetField = OutputField(description=FieldDescriptions.unet, title="UNet")
+    clip2: ClipField = OutputField(description=FieldDescriptions.clip, title="CLIP 2")
+    vae: VaeField = OutputField(description=FieldDescriptions.vae, title="VAE")
 
 
+@title("SDXL Main Model Loader")
+@tags("model", "sdxl")
 class SDXLModelLoaderInvocation(BaseInvocation):
     """Loads an sdxl base model, outputting its submodels."""
 
     type: Literal["sdxl_model_loader"] = "sdxl_model_loader"
 
-    model: MainModelField = Field(description="The model to load")
+    # Inputs
+    model: MainModelField = InputField(
+        description=FieldDescriptions.sdxl_main_model, input=Input.Direct, ui_type=UIType.SDXLMainModel
+    )
     # TODO: precision?
 
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "SDXL Model Loader",
-                "tags": ["model", "loader", "sdxl"],
-                "type_hints": {"model": "model"},
-            },
-        }
-
     def invoke(self, context: InvocationContext) -> SDXLModelLoaderOutput:
         base_model = self.model.base_model
         model_name = self.model.model_name
@@ -129,24 +122,21 @@ class SDXLModelLoaderInvocation(BaseInvocation):
         )
 
 
+@title("SDXL Refiner Model Loader")
+@tags("model", "sdxl", "refiner")
 class SDXLRefinerModelLoaderInvocation(BaseInvocation):
     """Loads an sdxl refiner model, outputting its submodels."""
 
     type: Literal["sdxl_refiner_model_loader"] = "sdxl_refiner_model_loader"
 
-    model: MainModelField = Field(description="The model to load")
+    # Inputs
+    model: MainModelField = InputField(
+        description=FieldDescriptions.sdxl_refiner_model,
+        input=Input.Direct,
+        ui_type=UIType.SDXLRefinerModel,
+    )
     # TODO: precision?
 
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "SDXL Refiner Model Loader",
-                "tags": ["model", "loader", "sdxl_refiner"],
-                "type_hints": {"model": "refiner_model"},
-            },
-        }
-
     def invoke(self, context: InvocationContext) -> SDXLRefinerModelLoaderOutput:
         base_model = self.model.base_model
         model_name = self.model.model_name
@@ -201,504 +191,3 @@ class SDXLRefinerModelLoaderInvocation(BaseInvocation):
                 ),
             ),
         )
-
-
-# Text to image
-class SDXLTextToLatentsInvocation(BaseInvocation):
-    """Generates latents from conditionings."""
-
-    type: Literal["t2l_sdxl"] = "t2l_sdxl"
-
-    # Inputs
-    # fmt: off
-    positive_conditioning: Optional[ConditioningField] = Field(description="Positive conditioning for generation")
-    negative_conditioning: Optional[ConditioningField] = Field(description="Negative conditioning for generation")
-    noise: Optional[LatentsField] = Field(description="The noise to use")
-    steps:       int = Field(default=10, gt=0, description="The number of steps to use to generate the image")
-    cfg_scale: Union[float, List[float]] = Field(default=7.5, ge=1, description="The Classifier-Free Guidance, higher values may result in a result closer to the prompt", )
-    scheduler: SAMPLER_NAME_VALUES = Field(default="euler", description="The scheduler to use" )
-    unet: UNetField = Field(default=None, description="UNet submodel")
-    denoising_end: float = Field(default=1.0, gt=0, le=1, description="")
-    # control: Union[ControlField, list[ControlField]] = Field(default=None, description="The control to use")
-    # seamless:   bool = Field(default=False, description="Whether or not to generate an image that can tile without seams", )
-    # seamless_axes: str = Field(default="", description="The axes to tile the image on, 'x' and/or 'y'")
-    # fmt: on
-
-    @validator("cfg_scale")
-    def ge_one(cls, v):
-        """validate that all cfg_scale values are >= 1"""
-        if isinstance(v, list):
-            for i in v:
-                if i < 1:
-                    raise ValueError("cfg_scale must be greater than 1")
-        else:
-            if v < 1:
-                raise ValueError("cfg_scale must be greater than 1")
-        return v
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "SDXL Text To Latents",
-                "tags": ["latents"],
-                "type_hints": {
-                    "model": "model",
-                    # "cfg_scale": "float",
-                    "cfg_scale": "number",
-                },
-            },
-        }
-
-    def dispatch_progress(
-        self,
-        context: InvocationContext,
-        source_node_id: str,
-        sample,
-        step,
-        total_steps,
-    ) -> None:
-        stable_diffusion_xl_step_callback(
-            context=context,
-            node=self.dict(),
-            source_node_id=source_node_id,
-            sample=sample,
-            step=step,
-            total_steps=total_steps,
-        )
-
-    # based on
-    # https://github.com/huggingface/diffusers/blob/3ebbaf7c96801271f9e6c21400033b6aa5ffcf29/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py#L375
-    @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> LatentsOutput:
-        graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
-        source_node_id = graph_execution_state.prepared_source_mapping[self.id]
-        latents = context.services.latents.get(self.noise.latents_name)
-
-        positive_cond_data = context.services.latents.get(self.positive_conditioning.conditioning_name)
-        prompt_embeds = positive_cond_data.conditionings[0].embeds
-        pooled_prompt_embeds = positive_cond_data.conditionings[0].pooled_embeds
-        add_time_ids = positive_cond_data.conditionings[0].add_time_ids
-
-        negative_cond_data = context.services.latents.get(self.negative_conditioning.conditioning_name)
-        negative_prompt_embeds = negative_cond_data.conditionings[0].embeds
-        negative_pooled_prompt_embeds = negative_cond_data.conditionings[0].pooled_embeds
-        add_neg_time_ids = negative_cond_data.conditionings[0].add_time_ids
-
-        scheduler = get_scheduler(
-            context=context,
-            scheduler_info=self.unet.scheduler,
-            scheduler_name=self.scheduler,
-        )
-
-        num_inference_steps = self.steps
-        scheduler.set_timesteps(num_inference_steps)
-        timesteps = scheduler.timesteps
-
-        latents = latents * scheduler.init_noise_sigma
-
-        unet_info = context.services.model_manager.get_model(**self.unet.unet.dict(), context=context)
-        do_classifier_free_guidance = True
-        cross_attention_kwargs = None
-        with unet_info as unet:
-            extra_step_kwargs = dict()
-            if "eta" in set(inspect.signature(scheduler.step).parameters.keys()):
-                extra_step_kwargs.update(
-                    eta=0.0,
-                )
-            if "generator" in set(inspect.signature(scheduler.step).parameters.keys()):
-                extra_step_kwargs.update(
-                    generator=torch.Generator(device=unet.device).manual_seed(0),
-                )
-
-            num_warmup_steps = len(timesteps) - self.steps * scheduler.order
-
-            # apply denoising_end
-            skipped_final_steps = int(round((1 - self.denoising_end) * self.steps))
-            num_inference_steps = num_inference_steps - skipped_final_steps
-            timesteps = timesteps[: num_warmup_steps + scheduler.order * num_inference_steps]
-
-            if not context.services.configuration.sequential_guidance:
-                prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
-                add_text_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0)
-                add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0)
-
-                prompt_embeds = prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_text_embeds = add_text_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_time_ids = add_time_ids.to(device=unet.device, dtype=unet.dtype)
-                latents = latents.to(device=unet.device, dtype=unet.dtype)
-
-                with tqdm(total=num_inference_steps) as progress_bar:
-                    for i, t in enumerate(timesteps):
-                        # expand the latents if we are doing classifier free guidance
-                        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
-
-                        latent_model_input = scheduler.scale_model_input(latent_model_input, t)
-
-                        # predict the noise residual
-                        added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
-                        noise_pred = unet(
-                            latent_model_input,
-                            t,
-                            encoder_hidden_states=prompt_embeds,
-                            cross_attention_kwargs=cross_attention_kwargs,
-                            added_cond_kwargs=added_cond_kwargs,
-                            return_dict=False,
-                        )[0]
-
-                        # perform guidance
-                        if do_classifier_free_guidance:
-                            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
-                            noise_pred = noise_pred_uncond + self.cfg_scale * (noise_pred_text - noise_pred_uncond)
-                            # del noise_pred_uncond
-                            # del noise_pred_text
-
-                        # if do_classifier_free_guidance and guidance_rescale > 0.0:
-                        #    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
-                        #    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
-
-                        # compute the previous noisy sample x_t -> x_t-1
-                        latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
-
-                        # call the callback, if provided
-                        if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0):
-                            progress_bar.update()
-                            self.dispatch_progress(context, source_node_id, latents, i, num_inference_steps)
-                            # if callback is not None and i % callback_steps == 0:
-                            #    callback(i, t, latents)
-            else:
-                negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                negative_prompt_embeds = negative_prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_neg_time_ids = add_neg_time_ids.to(device=unet.device, dtype=unet.dtype)
-                pooled_prompt_embeds = pooled_prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                prompt_embeds = prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_time_ids = add_time_ids.to(device=unet.device, dtype=unet.dtype)
-                latents = latents.to(device=unet.device, dtype=unet.dtype)
-
-                with tqdm(total=num_inference_steps) as progress_bar:
-                    for i, t in enumerate(timesteps):
-                        # expand the latents if we are doing classifier free guidance
-                        # latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
-
-                        latent_model_input = scheduler.scale_model_input(latents, t)
-
-                        # import gc
-                        # gc.collect()
-                        # torch.cuda.empty_cache()
-
-                        # predict the noise residual
-
-                        added_cond_kwargs = {"text_embeds": negative_pooled_prompt_embeds, "time_ids": add_neg_time_ids}
-                        noise_pred_uncond = unet(
-                            latent_model_input,
-                            t,
-                            encoder_hidden_states=negative_prompt_embeds,
-                            cross_attention_kwargs=cross_attention_kwargs,
-                            added_cond_kwargs=added_cond_kwargs,
-                            return_dict=False,
-                        )[0]
-
-                        added_cond_kwargs = {"text_embeds": pooled_prompt_embeds, "time_ids": add_time_ids}
-                        noise_pred_text = unet(
-                            latent_model_input,
-                            t,
-                            encoder_hidden_states=prompt_embeds,
-                            cross_attention_kwargs=cross_attention_kwargs,
-                            added_cond_kwargs=added_cond_kwargs,
-                            return_dict=False,
-                        )[0]
-
-                        # perform guidance
-                        noise_pred = noise_pred_uncond + self.cfg_scale * (noise_pred_text - noise_pred_uncond)
-
-                        # del noise_pred_text
-                        # del noise_pred_uncond
-                        # import gc
-                        # gc.collect()
-                        # torch.cuda.empty_cache()
-
-                        # if do_classifier_free_guidance and guidance_rescale > 0.0:
-                        #    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
-                        #    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
-
-                        # compute the previous noisy sample x_t -> x_t-1
-                        latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
-
-                        # del noise_pred
-                        # import gc
-                        # gc.collect()
-                        # torch.cuda.empty_cache()
-
-                        # call the callback, if provided
-                        if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0):
-                            progress_bar.update()
-                            self.dispatch_progress(context, source_node_id, latents, i, num_inference_steps)
-                            # if callback is not None and i % callback_steps == 0:
-                            #    callback(i, t, latents)
-
-        #################
-
-        latents = latents.to("cpu")
-        torch.cuda.empty_cache()
-
-        name = f"{context.graph_execution_state_id}__{self.id}"
-        context.services.latents.save(name, latents)
-        return build_latents_output(latents_name=name, latents=latents)
-
-
-class SDXLLatentsToLatentsInvocation(BaseInvocation):
-    """Generates latents from conditionings."""
-
-    type: Literal["l2l_sdxl"] = "l2l_sdxl"
-
-    # Inputs
-    # fmt: off
-    positive_conditioning: Optional[ConditioningField] = Field(description="Positive conditioning for generation")
-    negative_conditioning: Optional[ConditioningField] = Field(description="Negative conditioning for generation")
-    noise: Optional[LatentsField] = Field(description="The noise to use")
-    steps:       int = Field(default=10, gt=0, description="The number of steps to use to generate the image")
-    cfg_scale: Union[float, List[float]] = Field(default=7.5, ge=1, description="The Classifier-Free Guidance, higher values may result in a result closer to the prompt", )
-    scheduler: SAMPLER_NAME_VALUES = Field(default="euler", description="The scheduler to use" )
-    unet: UNetField = Field(default=None, description="UNet submodel")
-    latents: Optional[LatentsField] = Field(description="Initial latents")
-
-    denoising_start: float = Field(default=0.0, ge=0, le=1, description="")
-    denoising_end: float = Field(default=1.0, ge=0, le=1, description="")
-
-    # control: Union[ControlField, list[ControlField]] = Field(default=None, description="The control to use")
-    # seamless:   bool = Field(default=False, description="Whether or not to generate an image that can tile without seams", )
-    # seamless_axes: str = Field(default="", description="The axes to tile the image on, 'x' and/or 'y'")
-    # fmt: on
-
-    @validator("cfg_scale")
-    def ge_one(cls, v):
-        """validate that all cfg_scale values are >= 1"""
-        if isinstance(v, list):
-            for i in v:
-                if i < 1:
-                    raise ValueError("cfg_scale must be greater than 1")
-        else:
-            if v < 1:
-                raise ValueError("cfg_scale must be greater than 1")
-        return v
-
-    # Schema customisation
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "title": "SDXL Latents to Latents",
-                "tags": ["latents"],
-                "type_hints": {
-                    "model": "model",
-                    # "cfg_scale": "float",
-                    "cfg_scale": "number",
-                },
-            },
-        }
-
-    def dispatch_progress(
-        self,
-        context: InvocationContext,
-        source_node_id: str,
-        sample,
-        step,
-        total_steps,
-    ) -> None:
-        stable_diffusion_xl_step_callback(
-            context=context,
-            node=self.dict(),
-            source_node_id=source_node_id,
-            sample=sample,
-            step=step,
-            total_steps=total_steps,
-        )
-
-    # based on
-    # https://github.com/huggingface/diffusers/blob/3ebbaf7c96801271f9e6c21400033b6aa5ffcf29/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py#L375
-    @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> LatentsOutput:
-        graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
-        source_node_id = graph_execution_state.prepared_source_mapping[self.id]
-        latents = context.services.latents.get(self.latents.latents_name)
-
-        positive_cond_data = context.services.latents.get(self.positive_conditioning.conditioning_name)
-        prompt_embeds = positive_cond_data.conditionings[0].embeds
-        pooled_prompt_embeds = positive_cond_data.conditionings[0].pooled_embeds
-        add_time_ids = positive_cond_data.conditionings[0].add_time_ids
-
-        negative_cond_data = context.services.latents.get(self.negative_conditioning.conditioning_name)
-        negative_prompt_embeds = negative_cond_data.conditionings[0].embeds
-        negative_pooled_prompt_embeds = negative_cond_data.conditionings[0].pooled_embeds
-        add_neg_time_ids = negative_cond_data.conditionings[0].add_time_ids
-
-        scheduler = get_scheduler(
-            context=context,
-            scheduler_info=self.unet.scheduler,
-            scheduler_name=self.scheduler,
-        )
-
-        # apply denoising_start
-        num_inference_steps = self.steps
-        scheduler.set_timesteps(num_inference_steps)
-
-        t_start = int(round(self.denoising_start * num_inference_steps))
-        timesteps = scheduler.timesteps[t_start * scheduler.order :]
-        num_inference_steps = num_inference_steps - t_start
-
-        # apply noise(if provided)
-        if self.noise is not None and timesteps.shape[0] > 0:
-            noise = context.services.latents.get(self.noise.latents_name)
-            latents = scheduler.add_noise(latents, noise, timesteps[:1])
-            del noise
-
-        unet_info = context.services.model_manager.get_model(
-            **self.unet.unet.dict(),
-            context=context,
-        )
-        do_classifier_free_guidance = True
-        cross_attention_kwargs = None
-        with unet_info as unet:
-            # apply scheduler extra args
-            extra_step_kwargs = dict()
-            if "eta" in set(inspect.signature(scheduler.step).parameters.keys()):
-                extra_step_kwargs.update(
-                    eta=0.0,
-                )
-            if "generator" in set(inspect.signature(scheduler.step).parameters.keys()):
-                extra_step_kwargs.update(
-                    generator=torch.Generator(device=unet.device).manual_seed(0),
-                )
-
-            num_warmup_steps = max(len(timesteps) - num_inference_steps * scheduler.order, 0)
-
-            # apply denoising_end
-            skipped_final_steps = int(round((1 - self.denoising_end) * self.steps))
-            num_inference_steps = num_inference_steps - skipped_final_steps
-            timesteps = timesteps[: num_warmup_steps + scheduler.order * num_inference_steps]
-
-            if not context.services.configuration.sequential_guidance:
-                prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
-                add_text_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0)
-                add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0)
-
-                prompt_embeds = prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_text_embeds = add_text_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_time_ids = add_time_ids.to(device=unet.device, dtype=unet.dtype)
-                latents = latents.to(device=unet.device, dtype=unet.dtype)
-
-                with tqdm(total=num_inference_steps) as progress_bar:
-                    for i, t in enumerate(timesteps):
-                        # expand the latents if we are doing classifier free guidance
-                        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
-
-                        latent_model_input = scheduler.scale_model_input(latent_model_input, t)
-
-                        # predict the noise residual
-                        added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
-                        noise_pred = unet(
-                            latent_model_input,
-                            t,
-                            encoder_hidden_states=prompt_embeds,
-                            cross_attention_kwargs=cross_attention_kwargs,
-                            added_cond_kwargs=added_cond_kwargs,
-                            return_dict=False,
-                        )[0]
-
-                        # perform guidance
-                        if do_classifier_free_guidance:
-                            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
-                            noise_pred = noise_pred_uncond + self.cfg_scale * (noise_pred_text - noise_pred_uncond)
-                            # del noise_pred_uncond
-                            # del noise_pred_text
-
-                        # if do_classifier_free_guidance and guidance_rescale > 0.0:
-                        #    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
-                        #    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
-
-                        # compute the previous noisy sample x_t -> x_t-1
-                        latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
-
-                        # call the callback, if provided
-                        if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0):
-                            progress_bar.update()
-                            self.dispatch_progress(context, source_node_id, latents, i, num_inference_steps)
-                            # if callback is not None and i % callback_steps == 0:
-                            #    callback(i, t, latents)
-            else:
-                negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                negative_prompt_embeds = negative_prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_neg_time_ids = add_neg_time_ids.to(device=unet.device, dtype=unet.dtype)
-                pooled_prompt_embeds = pooled_prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                prompt_embeds = prompt_embeds.to(device=unet.device, dtype=unet.dtype)
-                add_time_ids = add_time_ids.to(device=unet.device, dtype=unet.dtype)
-                latents = latents.to(device=unet.device, dtype=unet.dtype)
-
-                with tqdm(total=num_inference_steps) as progress_bar:
-                    for i, t in enumerate(timesteps):
-                        # expand the latents if we are doing classifier free guidance
-                        # latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
-
-                        latent_model_input = scheduler.scale_model_input(latents, t)
-
-                        # import gc
-                        # gc.collect()
-                        # torch.cuda.empty_cache()
-
-                        # predict the noise residual
-
-                        added_cond_kwargs = {"text_embeds": negative_pooled_prompt_embeds, "time_ids": add_time_ids}
-                        noise_pred_uncond = unet(
-                            latent_model_input,
-                            t,
-                            encoder_hidden_states=negative_prompt_embeds,
-                            cross_attention_kwargs=cross_attention_kwargs,
-                            added_cond_kwargs=added_cond_kwargs,
-                            return_dict=False,
-                        )[0]
-
-                        added_cond_kwargs = {"text_embeds": pooled_prompt_embeds, "time_ids": add_time_ids}
-                        noise_pred_text = unet(
-                            latent_model_input,
-                            t,
-                            encoder_hidden_states=prompt_embeds,
-                            cross_attention_kwargs=cross_attention_kwargs,
-                            added_cond_kwargs=added_cond_kwargs,
-                            return_dict=False,
-                        )[0]
-
-                        # perform guidance
-                        noise_pred = noise_pred_uncond + self.cfg_scale * (noise_pred_text - noise_pred_uncond)
-
-                        # del noise_pred_text
-                        # del noise_pred_uncond
-                        # import gc
-                        # gc.collect()
-                        # torch.cuda.empty_cache()
-
-                        # if do_classifier_free_guidance and guidance_rescale > 0.0:
-                        #    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
-                        #    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
-
-                        # compute the previous noisy sample x_t -> x_t-1
-                        latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
-
-                        # del noise_pred
-                        # import gc
-                        # gc.collect()
-                        # torch.cuda.empty_cache()
-
-                        # call the callback, if provided
-                        if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0):
-                            progress_bar.update()
-                            self.dispatch_progress(context, source_node_id, latents, i, num_inference_steps)
-                            # if callback is not None and i % callback_steps == 0:
-                            #    callback(i, t, latents)
-
-        #################
-
-        latents = latents.to("cpu")
-        torch.cuda.empty_cache()
-
-        name = f"{context.graph_execution_state_id}__{self.id}"
-        context.services.latents.save(name, latents)
-        return build_latents_output(latents_name=name, latents=latents)

invokeai/app/invocations/upscale.py

@@ -6,13 +6,12 @@ import cv2 as cv
 import numpy as np
 from basicsr.archs.rrdbnet_arch import RRDBNet
 from PIL import Image
-from pydantic import Field
 from realesrgan import RealESRGANer
+from invokeai.app.invocations.primitives import ImageField, ImageOutput
 
-from invokeai.app.models.image import ImageCategory, ImageField, ResourceOrigin
+from invokeai.app.models.image import ImageCategory, ResourceOrigin
 
-from .baseinvocation import BaseInvocation, InvocationConfig, InvocationContext
-from .image import ImageOutput
+from .baseinvocation import BaseInvocation, InputField, InvocationContext, title, tags
 
 # TODO: Populate this from disk?
 # TODO: Use model manager to load?
@@ -24,17 +23,16 @@ ESRGAN_MODELS = Literal[
 ]
 
 
+@title("Upscale (RealESRGAN)")
+@tags("esrgan", "upscale")
 class ESRGANInvocation(BaseInvocation):
     """Upscales an image using RealESRGAN."""
 
     type: Literal["esrgan"] = "esrgan"
-    image: Union[ImageField, None] = Field(default=None, description="The input image")
-    model_name: ESRGAN_MODELS = Field(default="RealESRGAN_x4plus.pth", description="The Real-ESRGAN model to use")
 
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {"title": "Upscale (RealESRGAN)", "tags": ["image", "upscale", "realesrgan"]},
-        }
+    # Inputs
+    image: ImageField = InputField(description="The input image")
+    model_name: ESRGAN_MODELS = InputField(default="RealESRGAN_x4plus.pth", description="The Real-ESRGAN model to use")
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.services.images.get_pil_image(self.image.image_name)

invokeai/app/models/image.py

@@ -1,31 +1,8 @@
 from enum import Enum
-from typing import Optional, Tuple, Literal
+
 from pydantic import BaseModel, Field
 
 from invokeai.app.util.metaenum import MetaEnum
-from ..invocations.baseinvocation import (
-    BaseInvocationOutput,
-    InvocationConfig,
-)
-
-
-class ImageField(BaseModel):
-    """An image field used for passing image objects between invocations"""
-
-    image_name: Optional[str] = Field(default=None, description="The name of the image")
-
-    class Config:
-        schema_extra = {"required": ["image_name"]}
-
-
-class ColorField(BaseModel):
-    r: int = Field(ge=0, le=255, description="The red component")
-    g: int = Field(ge=0, le=255, description="The green component")
-    b: int = Field(ge=0, le=255, description="The blue component")
-    a: int = Field(ge=0, le=255, description="The alpha component")
-
-    def tuple(self) -> Tuple[int, int, int, int]:
-        return (self.r, self.g, self.b, self.a)
 
 
 class ProgressImage(BaseModel):
@@ -36,50 +13,6 @@ class ProgressImage(BaseModel):
     dataURL: str = Field(description="The image data as a b64 data URL")
 
 
-class PILInvocationConfig(BaseModel):
-    """Helper class to provide all PIL invocations with additional config"""
-
-    class Config(InvocationConfig):
-        schema_extra = {
-            "ui": {
-                "tags": ["PIL", "image"],
-            },
-        }
-
-
-class ImageOutput(BaseInvocationOutput):
-    """Base class for invocations that output an image"""
-
-    # fmt: off
-    type: Literal["image_output"] = "image_output"
-    image:      ImageField = Field(default=None, description="The output image")
-    width:             int = Field(description="The width of the image in pixels")
-    height:            int = Field(description="The height of the image in pixels")
-    # fmt: on
-
-    class Config:
-        schema_extra = {"required": ["type", "image", "width", "height"]}
-
-
-class MaskOutput(BaseInvocationOutput):
-    """Base class for invocations that output a mask"""
-
-    # fmt: off
-    type: Literal["mask"] = "mask"
-    mask:      ImageField = Field(default=None, description="The output mask")
-    width:            int = Field(description="The width of the mask in pixels")
-    height:           int = Field(description="The height of the mask in pixels")
-    # fmt: on
-
-    class Config:
-        schema_extra = {
-            "required": [
-                "type",
-                "mask",
-            ]
-        }
-
-
 class ResourceOrigin(str, Enum, metaclass=MetaEnum):
     """The origin of a resource (eg image).
 

invokeai/app/services/board_image_record_storage.py

@@ -25,7 +25,6 @@ class BoardImageRecordStorageBase(ABC):
     @abstractmethod
     def remove_image_from_board(
         self,
-        board_id: str,
         image_name: str,
     ) -> None:
         """Removes an image from a board."""
@@ -154,7 +153,6 @@ class SqliteBoardImageRecordStorage(BoardImageRecordStorageBase):
 
     def remove_image_from_board(
         self,
-        board_id: str,
         image_name: str,
     ) -> None:
         try:
@@ -162,9 +160,9 @@ class SqliteBoardImageRecordStorage(BoardImageRecordStorageBase):
             self._cursor.execute(
                 """--sql
                 DELETE FROM board_images
-                WHERE board_id = ? AND image_name = ?;
+                WHERE image_name = ?;
                 """,
-                (board_id, image_name),
+                (image_name,),
             )
             self._conn.commit()
         except sqlite3.Error as e:

invokeai/app/services/board_images.py

@@ -31,7 +31,6 @@ class BoardImagesServiceABC(ABC):
     @abstractmethod
     def remove_image_from_board(
         self,
-        board_id: str,
         image_name: str,
     ) -> None:
         """Removes an image from a board."""
@@ -93,10 +92,9 @@ class BoardImagesService(BoardImagesServiceABC):
 
     def remove_image_from_board(
         self,
-        board_id: str,
         image_name: str,
     ) -> None:
-        self._services.board_image_records.remove_image_from_board(board_id, image_name)
+        self._services.board_image_records.remove_image_from_board(image_name)
 
     def get_all_board_image_names_for_board(
         self,

invokeai/app/services/config.py

@@ -24,11 +24,10 @@ InvokeAI:
     sequential_guidance: false
     precision: float16
     max_cache_size: 6
-    max_vram_cache_size: 2.7
+    max_vram_cache_size: 0.5
     always_use_cpu: false
     free_gpu_mem: false
   Features:
-    restore: true
     esrgan: true
     patchmatch: true
     internet_available: true
@@ -165,15 +164,15 @@ import pydoc
 import os
 import sys
 from argparse import ArgumentParser
-from omegaconf import OmegaConf, DictConfig
+from omegaconf import OmegaConf, DictConfig, ListConfig
 from pathlib import Path
 from pydantic import BaseSettings, Field, parse_obj_as
 from typing import ClassVar, Dict, List, Set, Literal, Union, get_origin, get_type_hints, get_args
 
 INIT_FILE = Path("invokeai.yaml")
-MODEL_CORE = Path("models/core")
 DB_FILE = Path("invokeai.db")
 LEGACY_INIT_FILE = Path("invokeai.init")
+DEFAULT_MAX_VRAM = 0.5
 
 
 class InvokeAISettings(BaseSettings):
@@ -190,7 +189,12 @@ class InvokeAISettings(BaseSettings):
         opt = parser.parse_args(argv)
         for name in self.__fields__:
             if name not in self._excluded():
-                setattr(self, name, getattr(opt, name))
+                value = getattr(opt, name)
+                if isinstance(value, ListConfig):
+                    value = list(value)
+                elif isinstance(value, DictConfig):
+                    value = dict(value)
+                setattr(self, name, value)
 
     def to_yaml(self) -> str:
         """
@@ -283,14 +287,10 @@ class InvokeAISettings(BaseSettings):
         return [
             "type",
             "initconf",
-            "gpu_mem_reserved",
-            "max_loaded_models",
             "version",
             "from_file",
             "model",
-            "restore",
             "root",
-            "nsfw_checker",
         ]
 
     class Config:
@@ -356,8 +356,8 @@ class InvokeAISettings(BaseSettings):
 def _find_root() -> Path:
     venv = Path(os.environ.get("VIRTUAL_ENV") or ".")
     if os.environ.get("INVOKEAI_ROOT"):
-        root = Path(os.environ.get("INVOKEAI_ROOT")).resolve()
-    elif any([(venv.parent / x).exists() for x in [INIT_FILE, LEGACY_INIT_FILE, MODEL_CORE]]):
+        root = Path(os.environ["INVOKEAI_ROOT"])
+    elif any([(venv.parent / x).exists() for x in [INIT_FILE, LEGACY_INIT_FILE]]):
         root = (venv.parent).resolve()
     else:
         root = Path("~/invokeai").expanduser().resolve()
@@ -389,21 +389,17 @@ class InvokeAIAppConfig(InvokeAISettings):
     internet_available  : bool = Field(default=True, description="If true, attempt to download models on the fly; otherwise only use local models", category='Features')
     log_tokenization    : bool = Field(default=False, description="Enable logging of parsed prompt tokens.", category='Features')
     patchmatch          : bool = Field(default=True, description="Enable/disable patchmatch inpaint code", category='Features')
-    restore             : bool = Field(default=True, description="Enable/disable face restoration code (DEPRECATED)", category='DEPRECATED')
 
     always_use_cpu      : bool = Field(default=False, description="If true, use the CPU for rendering even if a GPU is available.", category='Memory/Performance')
     free_gpu_mem        : bool = Field(default=False, description="If true, purge model from GPU after each generation.", category='Memory/Performance')
-    max_loaded_models   : int = Field(default=3, gt=0, description="(DEPRECATED: use max_cache_size) Maximum number of models to keep in memory for rapid switching", category='DEPRECATED')
     max_cache_size      : float = Field(default=6.0, gt=0, description="Maximum memory amount used by model cache for rapid switching", category='Memory/Performance')
     max_vram_cache_size : float = Field(default=2.75, ge=0, description="Amount of VRAM reserved for model storage", category='Memory/Performance')
-    gpu_mem_reserved    : float = Field(default=2.75, ge=0, description="DEPRECATED: use max_vram_cache_size. Amount of VRAM reserved for model storage", category='DEPRECATED')
-    nsfw_checker        : bool = Field(default=True, description="DEPRECATED: use Web settings to enable/disable", category='DEPRECATED')
     precision           : Literal[tuple(['auto','float16','float32','autocast'])] = Field(default='auto',description='Floating point precision', category='Memory/Performance')
     sequential_guidance : bool = Field(default=False, description="Whether to calculate guidance in serial instead of in parallel, lowering memory requirements", category='Memory/Performance')
     xformers_enabled    : bool = Field(default=True, description="Enable/disable memory-efficient attention", category='Memory/Performance')
     tiled_decode        : bool = Field(default=False, description="Whether to enable tiled VAE decode (reduces memory consumption with some performance penalty)", category='Memory/Performance')
 
-    root                : Path = Field(default=_find_root(), description='InvokeAI runtime root directory', category='Paths')
+    root                : Path = Field(default=None, description='InvokeAI runtime root directory', category='Paths')
     autoimport_dir      : Path = Field(default='autoimport', description='Path to a directory of models files to be imported on startup.', category='Paths')
     lora_dir            : Path = Field(default=None, description='Path to a directory of LoRA/LyCORIS models to be imported on startup.', category='Paths')
     embedding_dir       : Path = Field(default=None, description='Path to a directory of Textual Inversion embeddings to be imported on startup.', category='Paths')
@@ -415,8 +411,7 @@ class InvokeAIAppConfig(InvokeAISettings):
     outdir              : Path = Field(default='outputs', description='Default folder for output images', category='Paths')
     from_file           : Path = Field(default=None, description='Take command input from the indicated file (command-line client only)', category='Paths')
     use_memory_db       : bool = Field(default=False, description='Use in-memory database for storing image metadata', category='Paths')
-
-    model               : str = Field(default='stable-diffusion-1.5', description='Initial model name', category='Models')
+    ignore_missing_core_models : bool = Field(default=False, description='Ignore missing models in models/core/convert', category='Features')
 
     log_handlers        : List[str] = Field(default=["console"], description='Log handler. Valid options are "console", "file=<path>", "syslog=path|address:host:port", "http=<url>"', category="Logging")
     # note - would be better to read the log_format values from logging.py, but this creates circular dependencies issues
@@ -426,6 +421,9 @@ class InvokeAIAppConfig(InvokeAISettings):
     version             : bool = Field(default=False, description="Show InvokeAI version and exit", category="Other")
     # fmt: on
 
+    class Config:
+        validate_assignment = True
+
     def parse_args(self, argv: List[str] = None, conf: DictConfig = None, clobber=False):
         """
         Update settings with contents of init file, environment, and
@@ -472,9 +470,11 @@ class InvokeAIAppConfig(InvokeAISettings):
         Path to the runtime root directory
         """
         if self.root:
-            return Path(self.root).expanduser().absolute()
+            root = Path(self.root).expanduser().absolute()
         else:
-            return self.find_root()
+            root = self.find_root().expanduser().absolute()
+        self.root = root  # insulate ourselves from relative paths that may change
+        return root
 
     @property
     def root_dir(self) -> Path:

invokeai/app/services/default_graphs.py

@@ -1,8 +1,8 @@
-from ..invocations.latent import LatentsToImageInvocation, TextToLatentsInvocation
+from ..invocations.latent import LatentsToImageInvocation, DenoiseLatentsInvocation
 from ..invocations.image import ImageNSFWBlurInvocation
 from ..invocations.noise import NoiseInvocation
 from ..invocations.compel import CompelInvocation
-from ..invocations.params import ParamIntInvocation
+from ..invocations.primitives import IntegerInvocation
 from .graph import Edge, EdgeConnection, ExposedNodeInput, ExposedNodeOutput, Graph, LibraryGraph
 from .item_storage import ItemStorageABC
 
@@ -17,13 +17,13 @@ def create_text_to_image() -> LibraryGraph:
         description="Converts text to an image",
         graph=Graph(
             nodes={
-                "width": ParamIntInvocation(id="width", a=512),
-                "height": ParamIntInvocation(id="height", a=512),
-                "seed": ParamIntInvocation(id="seed", a=-1),
+                "width": IntegerInvocation(id="width", a=512),
+                "height": IntegerInvocation(id="height", a=512),
+                "seed": IntegerInvocation(id="seed", a=-1),
                 "3": NoiseInvocation(id="3"),
                 "4": CompelInvocation(id="4"),
                 "5": CompelInvocation(id="5"),
-                "6": TextToLatentsInvocation(id="6"),
+                "6": DenoiseLatentsInvocation(id="6"),
                 "7": LatentsToImageInvocation(id="7"),
                 "8": ImageNSFWBlurInvocation(id="8"),
             },

invokeai/app/services/events.py

@@ -35,6 +35,7 @@ class EventServiceBase:
         source_node_id: str,
         progress_image: Optional[ProgressImage],
         step: int,
+        order: int,
         total_steps: int,
     ) -> None:
         """Emitted when there is generation progress"""
@@ -46,6 +47,7 @@ class EventServiceBase:
                 source_node_id=source_node_id,
                 progress_image=progress_image.dict() if progress_image is not None else None,
                 step=step,
+                order=order,
                 total_steps=total_steps,
             ),
         )

invokeai/app/services/graph.py

@@ -3,16 +3,7 @@
 import copy
 import itertools
 import uuid
-from typing import (
-    Annotated,
-    Any,
-    Literal,
-    Optional,
-    Union,
-    get_args,
-    get_origin,
-    get_type_hints,
-)
+from typing import Annotated, Any, Literal, Optional, Union, get_args, get_origin, get_type_hints
 
 import networkx as nx
 from pydantic import BaseModel, root_validator, validator
@@ -22,7 +13,11 @@ from ..invocations import *
 from ..invocations.baseinvocation import (
     BaseInvocation,
     BaseInvocationOutput,
+    Input,
+    InputField,
     InvocationContext,
+    OutputField,
+    UIType,
 )
 
 # in 3.10 this would be "from types import NoneType"
@@ -183,15 +178,9 @@ class IterateInvocationOutput(BaseInvocationOutput):
 
     type: Literal["iterate_output"] = "iterate_output"
 
-    item: Any = Field(description="The item being iterated over")
-
-    class Config:
-        schema_extra = {
-            "required": [
-                "type",
-                "item",
-            ]
-        }
+    item: Any = OutputField(
+        description="The item being iterated over", title="Collection Item", ui_type=UIType.CollectionItem
+    )
 
 
 # TODO: Fill this out and move to invocations
@@ -200,8 +189,10 @@ class IterateInvocation(BaseInvocation):
 
     type: Literal["iterate"] = "iterate"
 
-    collection: list[Any] = Field(description="The list of items to iterate over", default_factory=list)
-    index: int = Field(description="The index, will be provided on executed iterators", default=0)
+    collection: list[Any] = InputField(
+        description="The list of items to iterate over", default_factory=list, ui_type=UIType.Collection
+    )
+    index: int = InputField(description="The index, will be provided on executed iterators", default=0, ui_hidden=True)
 
     def invoke(self, context: InvocationContext) -> IterateInvocationOutput:
         """Produces the outputs as values"""
@@ -211,15 +202,9 @@ class IterateInvocation(BaseInvocation):
 class CollectInvocationOutput(BaseInvocationOutput):
     type: Literal["collect_output"] = "collect_output"
 
-    collection: list[Any] = Field(description="The collection of input items")
-
-    class Config:
-        schema_extra = {
-            "required": [
-                "type",
-                "collection",
-            ]
-        }
+    collection: list[Any] = OutputField(
+        description="The collection of input items", title="Collection", ui_type=UIType.Collection
+    )
 
 
 class CollectInvocation(BaseInvocation):
@@ -227,13 +212,14 @@ class CollectInvocation(BaseInvocation):
 
     type: Literal["collect"] = "collect"
 
-    item: Any = Field(
+    item: Any = InputField(
         description="The item to collect (all inputs must be of the same type)",
-        default=None,
+        ui_type=UIType.CollectionItem,
+        title="Collection Item",
+        input=Input.Connection,
     )
-    collection: list[Any] = Field(
-        description="The collection, will be provided on execution",
-        default_factory=list,
+    collection: list[Any] = InputField(
+        description="The collection, will be provided on execution", default_factory=list, ui_hidden=True
     )
 
     def invoke(self, context: InvocationContext) -> CollectInvocationOutput:

invokeai/app/services/image_record_storage.py

@@ -67,6 +67,7 @@ IMAGE_DTO_COLS = ", ".join(
                 "created_at",
                 "updated_at",
                 "deleted_at",
+                "starred",
             ],
         )
     )
@@ -139,6 +140,7 @@ class ImageRecordStorageBase(ABC):
         node_id: Optional[str],
         metadata: Optional[dict],
         is_intermediate: bool = False,
+        starred: bool = False,
     ) -> datetime:
         """Saves an image record."""
         pass
@@ -200,6 +202,16 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
             """
         )
 
+        self._cursor.execute("PRAGMA table_info(images)")
+        columns = [column[1] for column in self._cursor.fetchall()]
+
+        if "starred" not in columns:
+            self._cursor.execute(
+                """--sql
+                ALTER TABLE images ADD COLUMN starred BOOLEAN DEFAULT FALSE;
+                """
+            )
+
         # Create the `images` table indices.
         self._cursor.execute(
             """--sql
@@ -222,6 +234,12 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
             """
         )
 
+        self._cursor.execute(
+            """--sql
+            CREATE INDEX IF NOT EXISTS idx_images_starred ON images(starred);
+            """
+        )
+
         # Add trigger for `updated_at`.
         self._cursor.execute(
             """--sql
@@ -321,6 +339,17 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
                     (changes.is_intermediate, image_name),
                 )
 
+            # Change the image's `starred`` state
+            if changes.starred is not None:
+                self._cursor.execute(
+                    f"""--sql
+                    UPDATE images
+                    SET starred = ?
+                    WHERE image_name = ?;
+                    """,
+                    (changes.starred, image_name),
+                )
+
             self._conn.commit()
         except sqlite3.Error as e:
             self._conn.rollback()
@@ -397,7 +426,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
                 query_params.append(board_id)
 
             query_pagination = """--sql
-            ORDER BY images.created_at DESC LIMIT ? OFFSET ?
+            ORDER BY images.starred DESC, images.created_at DESC LIMIT ? OFFSET ?
             """
 
             # Final images query with pagination
@@ -500,6 +529,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
         node_id: Optional[str],
         metadata: Optional[dict],
         is_intermediate: bool = False,
+        starred: bool = False,
     ) -> datetime:
         try:
             metadata_json = None if metadata is None else json.dumps(metadata)
@@ -515,9 +545,10 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
                     node_id,
                     session_id,
                     metadata,
-                    is_intermediate
+                    is_intermediate,
+                    starred
                     )
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
                 """,
                 (
                     image_name,
@@ -529,6 +560,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
                     session_id,
                     metadata_json,
                     is_intermediate,
+                    starred,
                 ),
             )
             self._conn.commit()

invokeai/app/services/images.py

@@ -289,9 +289,10 @@ class ImageService(ImageServiceABC):
     def get_metadata(self, image_name: str) -> Optional[ImageMetadata]:
         try:
             image_record = self._services.image_records.get(image_name)
+            metadata = self._services.image_records.get_metadata(image_name)
 
             if not image_record.session_id:
-                return ImageMetadata()
+                return ImageMetadata(metadata=metadata)
 
             session_raw = self._services.graph_execution_manager.get_raw(image_record.session_id)
             graph = None
@@ -303,7 +304,6 @@ class ImageService(ImageServiceABC):
                     self._services.logger.warn(f"Failed to parse session graph: {e}")
                     graph = None
 
-            metadata = self._services.image_records.get_metadata(image_name)
             return ImageMetadata(graph=graph, metadata=metadata)
         except ImageRecordNotFoundException:
             self._services.logger.error("Image record not found")

invokeai/app/services/invocation_services.py

@@ -32,6 +32,7 @@ class InvocationServices:
     logger: "Logger"
     model_manager: "ModelManagerServiceBase"
     processor: "InvocationProcessorABC"
+    performance_statistics: "InvocationStatsServiceBase"
     queue: "InvocationQueueABC"
 
     def __init__(
@@ -47,6 +48,7 @@ class InvocationServices:
         logger: "Logger",
         model_manager: "ModelManagerServiceBase",
         processor: "InvocationProcessorABC",
+        performance_statistics: "InvocationStatsServiceBase",
         queue: "InvocationQueueABC",
     ):
         self.board_images = board_images
@@ -61,4 +63,5 @@ class InvocationServices:
         self.logger = logger
         self.model_manager = model_manager
         self.processor = processor
+        self.performance_statistics = performance_statistics
         self.queue = queue

invokeai/app/services/invocation_stats.py

@@ -0,0 +1,305 @@
+# Copyright 2023 Lincoln D. Stein <lincoln.stein@gmail.com>
+"""Utility to collect execution time and GPU usage stats on invocations in flight"""
+
+"""
+Usage:
+
+statistics = InvocationStatsService(graph_execution_manager)
+with statistics.collect_stats(invocation, graph_execution_state.id):
+      ... execute graphs...
+statistics.log_stats()
+
+Typical output:
+[2023-08-02 18:03:04,507]::[InvokeAI]::INFO --> Graph stats: c7764585-9c68-4d9d-a199-55e8186790f3
+[2023-08-02 18:03:04,507]::[InvokeAI]::INFO --> Node                 Calls  Seconds  VRAM Used
+[2023-08-02 18:03:04,507]::[InvokeAI]::INFO --> main_model_loader        1   0.005s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> clip_skip                1   0.004s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> compel                   2   0.512s     0.26G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> rand_int                 1   0.001s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> range_of_size            1   0.001s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> iterate                  1   0.001s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> metadata_accumulator     1   0.002s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> noise                    1   0.002s     0.01G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> t2l                      1   3.541s     1.93G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> l2i                      1   0.679s     0.58G
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> TOTAL GRAPH EXECUTION TIME:  4.749s
+[2023-08-02 18:03:04,508]::[InvokeAI]::INFO --> Current VRAM utilization 0.01G
+
+The abstract base class for this class is InvocationStatsServiceBase. An implementing class which
+writes to the system log is stored in InvocationServices.performance_statistics.
+"""
+
+import psutil
+import time
+from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager
+from dataclasses import dataclass, field
+from typing import Dict
+
+import torch
+
+import invokeai.backend.util.logging as logger
+
+from ..invocations.baseinvocation import BaseInvocation
+from .graph import GraphExecutionState
+from .item_storage import ItemStorageABC
+from .model_manager_service import ModelManagerService
+from invokeai.backend.model_management.model_cache import CacheStats
+
+# size of GIG in bytes
+GIG = 1073741824
+
+
+class InvocationStatsServiceBase(ABC):
+    "Abstract base class for recording node memory/time performance statistics"
+
+    @abstractmethod
+    def __init__(self, graph_execution_manager: ItemStorageABC["GraphExecutionState"]):
+        """
+        Initialize the InvocationStatsService and reset counters to zero
+        :param graph_execution_manager: Graph execution manager for this session
+        """
+        pass
+
+    @abstractmethod
+    def collect_stats(
+        self,
+        invocation: BaseInvocation,
+        graph_execution_state_id: str,
+    ) -> AbstractContextManager:
+        """
+        Return a context object that will capture the statistics on the execution
+        of invocaation. Use with: to place around the part of the code that executes the invocation.
+        :param invocation: BaseInvocation object from the current graph.
+        :param graph_execution_state: GraphExecutionState object from the current session.
+        """
+        pass
+
+    @abstractmethod
+    def reset_stats(self, graph_execution_state_id: str):
+        """
+        Reset all statistics for the indicated graph
+        :param graph_execution_state_id
+        """
+        pass
+
+    @abstractmethod
+    def reset_all_stats(self):
+        """Zero all statistics"""
+        pass
+
+    @abstractmethod
+    def update_invocation_stats(
+        self,
+        graph_id: str,
+        invocation_type: str,
+        time_used: float,
+        vram_used: float,
+        ram_used: float,
+        ram_changed: float,
+    ):
+        """
+        Add timing information on execution of a node. Usually
+        used internally.
+        :param graph_id: ID of the graph that is currently executing
+        :param invocation_type: String literal type of the node
+        :param time_used: Time used by node's exection (sec)
+        :param vram_used: Maximum VRAM used during exection (GB)
+        :param ram_used: Current RAM available (GB)
+        :param ram_changed: Change in RAM usage over course of the run (GB)
+        """
+        pass
+
+    @abstractmethod
+    def log_stats(self):
+        """
+        Write out the accumulated statistics to the log or somewhere else.
+        """
+        pass
+
+
+@dataclass
+class NodeStats:
+    """Class for tracking execution stats of an invocation node"""
+
+    calls: int = 0
+    time_used: float = 0.0  # seconds
+    max_vram: float = 0.0  # GB
+    cache_hits: int = 0
+    cache_misses: int = 0
+    cache_high_watermark: int = 0
+
+
+@dataclass
+class NodeLog:
+    """Class for tracking node usage"""
+
+    # {node_type => NodeStats}
+    nodes: Dict[str, NodeStats] = field(default_factory=dict)
+
+
+class InvocationStatsService(InvocationStatsServiceBase):
+    """Accumulate performance information about a running graph. Collects time spent in each node,
+    as well as the maximum and current VRAM utilisation for CUDA systems"""
+
+    def __init__(self, graph_execution_manager: ItemStorageABC["GraphExecutionState"]):
+        self.graph_execution_manager = graph_execution_manager
+        # {graph_id => NodeLog}
+        self._stats: Dict[str, NodeLog] = {}
+        self._cache_stats: Dict[str, CacheStats] = {}
+        self.ram_used: float = 0.0
+        self.ram_changed: float = 0.0
+
+    class StatsContext:
+        """Context manager for collecting statistics."""
+
+        invocation: BaseInvocation = None
+        collector: "InvocationStatsServiceBase" = None
+        graph_id: str = None
+        start_time: int = 0
+        ram_used: int = 0
+        model_manager: ModelManagerService = None
+
+        def __init__(
+            self,
+            invocation: BaseInvocation,
+            graph_id: str,
+            model_manager: ModelManagerService,
+            collector: "InvocationStatsServiceBase",
+        ):
+            """Initialize statistics for this run."""
+            self.invocation = invocation
+            self.collector = collector
+            self.graph_id = graph_id
+            self.start_time = 0
+            self.ram_used = 0
+            self.model_manager = model_manager
+
+        def __enter__(self):
+            self.start_time = time.time()
+            if torch.cuda.is_available():
+                torch.cuda.reset_peak_memory_stats()
+            self.ram_used = psutil.Process().memory_info().rss
+            if self.model_manager:
+                self.model_manager.collect_cache_stats(self.collector._cache_stats[self.graph_id])
+
+        def __exit__(self, *args):
+            """Called on exit from the context."""
+            ram_used = psutil.Process().memory_info().rss
+            self.collector.update_mem_stats(
+                ram_used=ram_used / GIG,
+                ram_changed=(ram_used - self.ram_used) / GIG,
+            )
+            self.collector.update_invocation_stats(
+                graph_id=self.graph_id,
+                invocation_type=self.invocation.type,
+                time_used=time.time() - self.start_time,
+                vram_used=torch.cuda.max_memory_allocated() / GIG if torch.cuda.is_available() else 0.0,
+            )
+
+    def collect_stats(
+        self,
+        invocation: BaseInvocation,
+        graph_execution_state_id: str,
+        model_manager: ModelManagerService,
+    ) -> StatsContext:
+        """
+        Return a context object that will capture the statistics.
+        :param invocation: BaseInvocation object from the current graph.
+        :param graph_execution_state: GraphExecutionState object from the current session.
+        """
+        if not self._stats.get(graph_execution_state_id):  # first time we're seeing this
+            self._stats[graph_execution_state_id] = NodeLog()
+            self._cache_stats[graph_execution_state_id] = CacheStats()
+        return self.StatsContext(invocation, graph_execution_state_id, model_manager, self)
+
+    def reset_all_stats(self):
+        """Zero all statistics"""
+        self._stats = {}
+
+    def reset_stats(self, graph_execution_id: str):
+        """Zero the statistics for the indicated graph."""
+        try:
+            self._stats.pop(graph_execution_id)
+        except KeyError:
+            logger.warning(f"Attempted to clear statistics for unknown graph {graph_execution_id}")
+
+    def update_mem_stats(
+        self,
+        ram_used: float,
+        ram_changed: float,
+    ):
+        """
+        Update the collector with RAM memory usage info.
+
+        :param ram_used: How much RAM is currently in use.
+        :param ram_changed: How much RAM changed since last generation.
+        """
+        self.ram_used = ram_used
+        self.ram_changed = ram_changed
+
+    def update_invocation_stats(
+        self,
+        graph_id: str,
+        invocation_type: str,
+        time_used: float,
+        vram_used: float,
+    ):
+        """
+        Add timing information on execution of a node. Usually
+        used internally.
+        :param graph_id: ID of the graph that is currently executing
+        :param invocation_type: String literal type of the node
+        :param time_used: Time used by node's exection (sec)
+        :param vram_used: Maximum VRAM used during exection (GB)
+        :param ram_used: Current RAM available (GB)
+        :param ram_changed: Change in RAM usage over course of the run (GB)
+        """
+        if not self._stats[graph_id].nodes.get(invocation_type):
+            self._stats[graph_id].nodes[invocation_type] = NodeStats()
+        stats = self._stats[graph_id].nodes[invocation_type]
+        stats.calls += 1
+        stats.time_used += time_used
+        stats.max_vram = max(stats.max_vram, vram_used)
+
+    def log_stats(self):
+        """
+        Send the statistics to the system logger at the info level.
+        Stats will only be printed when the execution of the graph
+        is complete.
+        """
+        completed = set()
+        for graph_id, node_log in self._stats.items():
+            current_graph_state = self.graph_execution_manager.get(graph_id)
+            if not current_graph_state.is_complete():
+                continue
+
+            total_time = 0
+            logger.info(f"Graph stats: {graph_id}")
+            logger.info(f"{'Node':>30} {'Calls':>7}{'Seconds':>9} {'VRAM Used':>10}")
+            for node_type, stats in self._stats[graph_id].nodes.items():
+                logger.info(f"{node_type:>30}  {stats.calls:>4}   {stats.time_used:7.3f}s     {stats.max_vram:4.3f}G")
+                total_time += stats.time_used
+
+            cache_stats = self._cache_stats[graph_id]
+            hwm = cache_stats.high_watermark / GIG
+            tot = cache_stats.cache_size / GIG
+            loaded = sum([v for v in cache_stats.loaded_model_sizes.values()]) / GIG
+
+            logger.info(f"TOTAL GRAPH EXECUTION TIME:  {total_time:7.3f}s")
+            logger.info("RAM used by InvokeAI process: " + "%4.2fG" % self.ram_used + f" ({self.ram_changed:+5.3f}G)")
+            logger.info(f"RAM used to load models: {loaded:4.2f}G")
+            if torch.cuda.is_available():
+                logger.info("VRAM in use: " + "%4.3fG" % (torch.cuda.memory_allocated() / GIG))
+            logger.info("RAM cache statistics:")
+            logger.info(f"   Model cache hits: {cache_stats.hits}")
+            logger.info(f"   Model cache misses: {cache_stats.misses}")
+            logger.info(f"   Models cached: {cache_stats.in_cache}")
+            logger.info(f"   Models cleared from cache: {cache_stats.cleared}")
+            logger.info(f"   Cache high water mark: {hwm:4.2f}/{tot:4.2f}G")
+
+            completed.add(graph_id)
+
+        for graph_id in completed:
+            del self._stats[graph_id]
+            del self._cache_stats[graph_id]

invokeai/app/services/model_manager_service.py

@@ -3,9 +3,10 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from logging import Logger
 from pathlib import Path
 from pydantic import Field
-from typing import Optional, Union, Callable, List, Tuple, TYPE_CHECKING
+from typing import Literal, Optional, Union, Callable, List, Tuple, TYPE_CHECKING
 from types import ModuleType
 
 from invokeai.backend.model_management import (
@@ -21,6 +22,7 @@ from invokeai.backend.model_management import (
     ModelNotFoundException,
 )
 from invokeai.backend.model_management.model_search import FindModels
+from invokeai.backend.model_management.model_cache import CacheStats
 
 import torch
 from invokeai.app.models.exceptions import CanceledException
@@ -193,7 +195,7 @@ class ModelManagerServiceBase(ABC):
         self,
         model_name: str,
         base_model: BaseModelType,
-        model_type: Union[ModelType.Main, ModelType.Vae],
+        model_type: Literal[ModelType.Main, ModelType.Vae],
     ) -> AddModelResult:
         """
         Convert a checkpoint file into a diffusers folder, deleting the cached
@@ -275,6 +277,13 @@ class ModelManagerServiceBase(ABC):
         """
         pass
 
+    @abstractmethod
+    def collect_cache_stats(self, cache_stats: CacheStats):
+        """
+        Reset model cache statistics for graph with graph_id.
+        """
+        pass
+
     @abstractmethod
     def commit(self, conf_file: Optional[Path] = None) -> None:
         """
@@ -292,7 +301,7 @@ class ModelManagerService(ModelManagerServiceBase):
     def __init__(
         self,
         config: InvokeAIAppConfig,
-        logger: ModuleType,
+        logger: Logger,
     ):
         """
         Initialize with the path to the models.yaml config file.
@@ -396,7 +405,7 @@ class ModelManagerService(ModelManagerServiceBase):
             model_type,
         )
 
-    def model_info(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> dict:
+    def model_info(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> Union[dict, None]:
         """
         Given a model name returns a dict-like (OmegaConf) object describing it.
         """
@@ -416,7 +425,7 @@ class ModelManagerService(ModelManagerServiceBase):
         """
         return self.mgr.list_models(base_model, model_type)
 
-    def list_model(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> dict:
+    def list_model(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> Union[dict, None]:
         """
         Return information about the model using the same format as list_models()
         """
@@ -429,7 +438,7 @@ class ModelManagerService(ModelManagerServiceBase):
         model_type: ModelType,
         model_attributes: dict,
         clobber: bool = False,
-    ) -> None:
+    ) -> AddModelResult:
         """
         Update the named model with a dictionary of attributes. Will fail with an
         assertion error if the name already exists. Pass clobber=True to overwrite.
@@ -478,7 +487,7 @@ class ModelManagerService(ModelManagerServiceBase):
         self,
         model_name: str,
         base_model: BaseModelType,
-        model_type: Union[ModelType.Main, ModelType.Vae],
+        model_type: Literal[ModelType.Main, ModelType.Vae],
         convert_dest_directory: Optional[Path] = Field(
             default=None, description="Optional directory location for merged model"
         ),
@@ -499,6 +508,12 @@ class ModelManagerService(ModelManagerServiceBase):
         self.logger.debug(f"convert model {model_name}")
         return self.mgr.convert_model(model_name, base_model, model_type, convert_dest_directory)
 
+    def collect_cache_stats(self, cache_stats: CacheStats):
+        """
+        Reset model cache statistics for graph with graph_id.
+        """
+        self.mgr.cache.stats = cache_stats
+
     def commit(self, conf_file: Optional[Path] = None):
         """
         Write current configuration out to the indicated file.
@@ -573,9 +588,9 @@ class ModelManagerService(ModelManagerServiceBase):
             default=None, description="Base model shared by all models to be merged"
         ),
         merged_model_name: str = Field(default=None, description="Name of destination model after merging"),
-        alpha: Optional[float] = 0.5,
+        alpha: float = 0.5,
         interp: Optional[MergeInterpolationMethod] = None,
-        force: Optional[bool] = False,
+        force: bool = False,
         merge_dest_directory: Optional[Path] = Field(
             default=None, description="Optional directory location for merged model"
         ),
@@ -633,8 +648,8 @@ class ModelManagerService(ModelManagerServiceBase):
         model_name: str,
         base_model: BaseModelType,
         model_type: ModelType,
-        new_name: str = None,
-        new_base: BaseModelType = None,
+        new_name: Optional[str] = None,
+        new_base: Optional[BaseModelType] = None,
     ):
         """
         Rename the indicated model. Can provide a new name and/or a new base.

invokeai/app/services/models/board_image.py

@@ -0,0 +1,8 @@
+from pydantic import Field
+
+from invokeai.app.util.model_exclude_null import BaseModelExcludeNull
+
+
+class BoardImage(BaseModelExcludeNull):
+    board_id: str = Field(description="The id of the board")
+    image_name: str = Field(description="The name of the image")

invokeai/app/services/models/board_record.py

@@ -1,10 +1,11 @@
 from typing import Optional, Union
 from datetime import datetime
-from pydantic import BaseModel, Extra, Field, StrictBool, StrictStr
+from pydantic import Field
 from invokeai.app.util.misc import get_iso_timestamp
+from invokeai.app.util.model_exclude_null import BaseModelExcludeNull
 
 
-class BoardRecord(BaseModel):
+class BoardRecord(BaseModelExcludeNull):
     """Deserialized board record."""
 
     board_id: str = Field(description="The unique ID of the board.")

invokeai/app/services/models/image_record.py

@@ -1,13 +1,14 @@
 import datetime
 from typing import Optional, Union
 
-from pydantic import BaseModel, Extra, Field, StrictBool, StrictStr
+from pydantic import Extra, Field, StrictBool, StrictStr
 
 from invokeai.app.models.image import ImageCategory, ResourceOrigin
 from invokeai.app.util.misc import get_iso_timestamp
+from invokeai.app.util.model_exclude_null import BaseModelExcludeNull
 
 
-class ImageRecord(BaseModel):
+class ImageRecord(BaseModelExcludeNull):
     """Deserialized image record without metadata."""
 
     image_name: str = Field(description="The unique name of the image.")
@@ -38,15 +39,18 @@ class ImageRecord(BaseModel):
         description="The node ID that generated this image, if it is a generated image.",
     )
     """The node ID that generated this image, if it is a generated image."""
+    starred: bool = Field(description="Whether this image is starred.")
+    """Whether this image is starred."""
 
 
-class ImageRecordChanges(BaseModel, extra=Extra.forbid):
+class ImageRecordChanges(BaseModelExcludeNull, extra=Extra.forbid):
     """A set of changes to apply to an image record.
 
     Only limited changes are valid:
       - `image_category`: change the category of an image
       - `session_id`: change the session associated with an image
       - `is_intermediate`: change the image's `is_intermediate` flag
+      - `starred`: change whether the image is starred
     """
 
     image_category: Optional[ImageCategory] = Field(description="The image's new category.")
@@ -58,9 +62,11 @@ class ImageRecordChanges(BaseModel, extra=Extra.forbid):
     """The image's new session ID."""
     is_intermediate: Optional[StrictBool] = Field(default=None, description="The image's new `is_intermediate` flag.")
     """The image's new `is_intermediate` flag."""
+    starred: Optional[StrictBool] = Field(default=None, description="The image's new `starred` state")
+    """The image's new `starred` state."""
 
 
-class ImageUrlsDTO(BaseModel):
+class ImageUrlsDTO(BaseModelExcludeNull):
     """The URLs for an image and its thumbnail."""
 
     image_name: str = Field(description="The unique name of the image.")
@@ -76,11 +82,15 @@ class ImageDTO(ImageRecord, ImageUrlsDTO):
 
     board_id: Optional[str] = Field(description="The id of the board the image belongs to, if one exists.")
     """The id of the board the image belongs to, if one exists."""
+
     pass
 
 
 def image_record_to_dto(
-    image_record: ImageRecord, image_url: str, thumbnail_url: str, board_id: Optional[str]
+    image_record: ImageRecord,
+    image_url: str,
+    thumbnail_url: str,
+    board_id: Optional[str],
 ) -> ImageDTO:
     """Converts an image record to an image DTO."""
     return ImageDTO(
@@ -108,6 +118,7 @@ def deserialize_image_record(image_dict: dict) -> ImageRecord:
     updated_at = image_dict.get("updated_at", get_iso_timestamp())
     deleted_at = image_dict.get("deleted_at", get_iso_timestamp())
     is_intermediate = image_dict.get("is_intermediate", False)
+    starred = image_dict.get("starred", False)
 
     return ImageRecord(
         image_name=image_name,
@@ -121,4 +132,5 @@ def deserialize_image_record(image_dict: dict) -> ImageRecord:
         updated_at=updated_at,
         deleted_at=deleted_at,
         is_intermediate=is_intermediate,
+        starred=starred,
     )

invokeai/app/services/processor.py

@@ -1,14 +1,15 @@
 import time
 import traceback
-from threading import Event, Thread, BoundedSemaphore
-
-from ..invocations.baseinvocation import InvocationContext
-from .invocation_queue import InvocationQueueItem
-from .invoker import InvocationProcessorABC, Invoker
-from ..models.exceptions import CanceledException
+from threading import BoundedSemaphore, Event, Thread
 
 import invokeai.backend.util.logging as logger
 
+from ..invocations.baseinvocation import InvocationContext
+from ..models.exceptions import CanceledException
+from .invocation_queue import InvocationQueueItem
+from .invocation_stats import InvocationStatsServiceBase
+from .invoker import InvocationProcessorABC, Invoker
+
 
 class DefaultInvocationProcessor(InvocationProcessorABC):
     __invoker_thread: Thread
@@ -35,6 +36,8 @@ class DefaultInvocationProcessor(InvocationProcessorABC):
     def __process(self, stop_event: Event):
         try:
             self.__threadLimit.acquire()
+            statistics: InvocationStatsServiceBase = self.__invoker.services.performance_statistics
+
             while not stop_event.is_set():
                 try:
                     queue_item: InvocationQueueItem = self.__invoker.services.queue.get()
@@ -83,35 +86,43 @@ class DefaultInvocationProcessor(InvocationProcessorABC):
 
                 # Invoke
                 try:
-                    outputs = invocation.invoke(
-                        InvocationContext(
-                            services=self.__invoker.services,
-                            graph_execution_state_id=graph_execution_state.id,
+                    graph_id = graph_execution_state.id
+                    model_manager = self.__invoker.services.model_manager
+                    with statistics.collect_stats(invocation, graph_id, model_manager):
+                        # use the internal invoke_internal(), which wraps the node's invoke() method in
+                        # this accomodates nodes which require a value, but get it only from a
+                        # connection
+                        outputs = invocation.invoke_internal(
+                            InvocationContext(
+                                services=self.__invoker.services,
+                                graph_execution_state_id=graph_execution_state.id,
+                            )
                         )
-                    )
 
-                    # Check queue to see if this is canceled, and skip if so
-                    if self.__invoker.services.queue.is_canceled(graph_execution_state.id):
-                        continue
+                        # Check queue to see if this is canceled, and skip if so
+                        if self.__invoker.services.queue.is_canceled(graph_execution_state.id):
+                            continue
 
-                    # Save outputs and history
-                    graph_execution_state.complete(invocation.id, outputs)
+                        # Save outputs and history
+                        graph_execution_state.complete(invocation.id, outputs)
 
-                    # Save the state changes
-                    self.__invoker.services.graph_execution_manager.set(graph_execution_state)
+                        # Save the state changes
+                        self.__invoker.services.graph_execution_manager.set(graph_execution_state)
 
-                    # Send complete event
-                    self.__invoker.services.events.emit_invocation_complete(
-                        graph_execution_state_id=graph_execution_state.id,
-                        node=invocation.dict(),
-                        source_node_id=source_node_id,
-                        result=outputs.dict(),
-                    )
+                        # Send complete event
+                        self.__invoker.services.events.emit_invocation_complete(
+                            graph_execution_state_id=graph_execution_state.id,
+                            node=invocation.dict(),
+                            source_node_id=source_node_id,
+                            result=outputs.dict(),
+                        )
+                    statistics.log_stats()
 
                 except KeyboardInterrupt:
                     pass
 
                 except CanceledException:
+                    statistics.reset_stats(graph_execution_state.id)
                     pass
 
                 except Exception as e:
@@ -133,7 +144,7 @@ class DefaultInvocationProcessor(InvocationProcessorABC):
                         error_type=e.__class__.__name__,
                         error=error,
                     )
-
+                    statistics.reset_stats(graph_execution_state.id)
                     pass
 
                 # Check queue to see if this is canceled, and skip if so

invokeai/app/services/sqlite.py

@@ -1,6 +1,7 @@
 import sqlite3
+import json
 from threading import Lock
-from typing import Generic, Optional, TypeVar, get_args
+from typing import Generic, Optional, TypeVar, Union, get_args
 
 from pydantic import BaseModel, parse_raw_as
 
@@ -49,7 +50,8 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
 
     def _parse_item(self, item: str) -> T:
         item_type = get_args(self.__orig_class__)[0]
-        return parse_raw_as(item_type, item)
+        parsed = parse_raw_as(item_type, item)
+        return parsed
 
     def set(self, item: T):
         try:
@@ -98,7 +100,7 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
             self._lock.release()
         self._on_deleted(id)
 
-    def list(self, page: int = 0, per_page: int = 10) -> PaginatedResults[T]:
+    def list(self, page: int = 0, per_page: int = 10) -> PaginatedResults[dict]:
         try:
             self._lock.acquire()
             self._cursor.execute(
@@ -107,7 +109,7 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
             )
             result = self._cursor.fetchall()
 
-            items = list(map(lambda r: self._parse_item(r[0]), result))
+            items = [json.loads(r[0]) for r in result]
 
             self._cursor.execute(f"""SELECT count(*) FROM {self._table_name};""")
             count = self._cursor.fetchone()[0]
@@ -116,9 +118,9 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
 
         pageCount = int(count / per_page) + 1
 
-        return PaginatedResults[T](items=items, page=page, pages=pageCount, per_page=per_page, total=count)
+        return PaginatedResults[dict](items=items, page=page, pages=pageCount, per_page=per_page, total=count)
 
-    def search(self, query: str, page: int = 0, per_page: int = 10) -> PaginatedResults[T]:
+    def search(self, query: str, page: int = 0, per_page: int = 10) -> PaginatedResults[dict]:
         try:
             self._lock.acquire()
             self._cursor.execute(
@@ -127,7 +129,7 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
             )
             result = self._cursor.fetchall()
 
-            items = list(map(lambda r: self._parse_item(r[0]), result))
+            items = [json.loads(r[0]) for r in result]
 
             self._cursor.execute(
                 f"""SELECT count(*) FROM {self._table_name} WHERE item LIKE ?;""",
@@ -139,4 +141,4 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
 
         pageCount = int(count / per_page) + 1
 
-        return PaginatedResults[T](items=items, page=page, pages=pageCount, per_page=per_page, total=count)
+        return PaginatedResults[dict](items=items, page=page, pages=pageCount, per_page=per_page, total=count)

invokeai/app/services/urls.py

@@ -20,6 +20,6 @@ class LocalUrlService(UrlServiceBase):
 
         # These paths are determined by the routes in invokeai/app/api/routers/images.py
         if thumbnail:
-            return f"{self._base_url}/images/{image_basename}/thumbnail"
+            return f"{self._base_url}/images/i/{image_basename}/thumbnail"
 
-        return f"{self._base_url}/images/{image_basename}/full"
+        return f"{self._base_url}/images/i/{image_basename}/full"

invokeai/app/util/misc.py

@@ -18,5 +18,5 @@ SEED_MAX = np.iinfo(np.uint32).max
 
 
 def get_random_seed():
-    rng = np.random.default_rng(seed=0)
+    rng = np.random.default_rng(seed=None)
     return int(rng.integers(0, SEED_MAX))

invokeai/app/util/model_exclude_null.py

@@ -0,0 +1,23 @@
+from typing import Any
+from pydantic import BaseModel
+
+
+"""
+We want to exclude null values from objects that make their way to the client.
+
+Unfortunately there is no built-in way to do this in pydantic, so we need to override the default
+dict method to do this.
+
+From https://github.com/tiangolo/fastapi/discussions/8882#discussioncomment-5154541
+"""
+
+
+class BaseModelExcludeNull(BaseModel):
+    def dict(self, *args, **kwargs) -> dict[str, Any]:
+        """
+        Override the default dict method to exclude None values in the response
+        """
+        kwargs.pop("exclude_none", None)
+        return super().dict(*args, exclude_none=True, **kwargs)
+
+    pass

invokeai/app/util/step_callback.py

@@ -4,9 +4,9 @@ from invokeai.app.models.exceptions import CanceledException
 from invokeai.app.models.image import ProgressImage
 from ..invocations.baseinvocation import InvocationContext
 from ...backend.util.util import image_to_dataURL
-from ...backend.generator.base import Generator
 from ...backend.stable_diffusion import PipelineIntermediateState
 from invokeai.app.services.config import InvokeAIAppConfig
+from ...backend.model_management.models import BaseModelType
 
 
 def sample_to_lowres_estimated_image(samples, latent_rgb_factors, smooth_matrix=None):
@@ -29,6 +29,7 @@ def stable_diffusion_step_callback(
     intermediate_state: PipelineIntermediateState,
     node: dict,
     source_node_id: str,
+    base_model: BaseModelType,
 ):
     if context.services.queue.is_canceled(context.graph_execution_state_id):
         raise CanceledException
@@ -56,23 +57,50 @@ def stable_diffusion_step_callback(
 
     # TODO: only output a preview image when requested
 
-    # origingally adapted from code by @erucipe and @keturn here:
-    # https://discuss.huggingface.co/t/decoding-latents-to-rgb-without-upscaling/23204/7
+    if base_model in [BaseModelType.StableDiffusionXL, BaseModelType.StableDiffusionXLRefiner]:
+        # fast latents preview matrix for sdxl
+        # generated by @StAlKeR7779
+        sdxl_latent_rgb_factors = torch.tensor(
+            [
+                #   R        G        B
+                [0.3816, 0.4930, 0.5320],
+                [-0.3753, 0.1631, 0.1739],
+                [0.1770, 0.3588, -0.2048],
+                [-0.4350, -0.2644, -0.4289],
+            ],
+            dtype=sample.dtype,
+            device=sample.device,
+        )
 
-    # these updated numbers for v1.5 are from @torridgristle
-    v1_5_latent_rgb_factors = torch.tensor(
-        [
-            #    R        G        B
-            [0.3444, 0.1385, 0.0670],  # L1
-            [0.1247, 0.4027, 0.1494],  # L2
-            [-0.3192, 0.2513, 0.2103],  # L3
-            [-0.1307, -0.1874, -0.7445],  # L4
-        ],
-        dtype=sample.dtype,
-        device=sample.device,
-    )
+        sdxl_smooth_matrix = torch.tensor(
+            [
+                [0.0358, 0.0964, 0.0358],
+                [0.0964, 0.4711, 0.0964],
+                [0.0358, 0.0964, 0.0358],
+            ],
+            dtype=sample.dtype,
+            device=sample.device,
+        )
 
-    image = sample_to_lowres_estimated_image(sample, v1_5_latent_rgb_factors)
+        image = sample_to_lowres_estimated_image(sample, sdxl_latent_rgb_factors, sdxl_smooth_matrix)
+    else:
+        # origingally adapted from code by @erucipe and @keturn here:
+        # https://discuss.huggingface.co/t/decoding-latents-to-rgb-without-upscaling/23204/7
+
+        # these updated numbers for v1.5 are from @torridgristle
+        v1_5_latent_rgb_factors = torch.tensor(
+            [
+                #    R        G        B
+                [0.3444, 0.1385, 0.0670],  # L1
+                [0.1247, 0.4027, 0.1494],  # L2
+                [-0.3192, 0.2513, 0.2103],  # L3
+                [-0.1307, -0.1874, -0.7445],  # L4
+            ],
+            dtype=sample.dtype,
+            device=sample.device,
+        )
+
+        image = sample_to_lowres_estimated_image(sample, v1_5_latent_rgb_factors)
 
     (width, height) = image.size
     width *= 8
@@ -86,59 +114,6 @@ def stable_diffusion_step_callback(
         source_node_id=source_node_id,
         progress_image=ProgressImage(width=width, height=height, dataURL=dataURL),
         step=intermediate_state.step,
-        total_steps=node["steps"],
-    )
-
-
-def stable_diffusion_xl_step_callback(
-    context: InvocationContext,
-    node: dict,
-    source_node_id: str,
-    sample,
-    step,
-    total_steps,
-):
-    if context.services.queue.is_canceled(context.graph_execution_state_id):
-        raise CanceledException
-
-    sdxl_latent_rgb_factors = torch.tensor(
-        [
-            #   R        G        B
-            [0.3816, 0.4930, 0.5320],
-            [-0.3753, 0.1631, 0.1739],
-            [0.1770, 0.3588, -0.2048],
-            [-0.4350, -0.2644, -0.4289],
-        ],
-        dtype=sample.dtype,
-        device=sample.device,
-    )
-
-    sdxl_smooth_matrix = torch.tensor(
-        [
-            # [ 0.0478,  0.1285,  0.0478],
-            # [ 0.1285,  0.2948,  0.1285],
-            # [ 0.0478,  0.1285,  0.0478],
-            [0.0358, 0.0964, 0.0358],
-            [0.0964, 0.4711, 0.0964],
-            [0.0358, 0.0964, 0.0358],
-        ],
-        dtype=sample.dtype,
-        device=sample.device,
-    )
-
-    image = sample_to_lowres_estimated_image(sample, sdxl_latent_rgb_factors, sdxl_smooth_matrix)
-
-    (width, height) = image.size
-    width *= 8
-    height *= 8
-
-    dataURL = image_to_dataURL(image, image_format="JPEG")
-
-    context.services.events.emit_generator_progress(
-        graph_execution_state_id=context.graph_execution_state_id,
-        node=node,
-        source_node_id=source_node_id,
-        progress_image=ProgressImage(width=width, height=height, dataURL=dataURL),
-        step=step,
-        total_steps=total_steps,
+        order=intermediate_state.order,
+        total_steps=intermediate_state.total_steps,
     )

invokeai/backend/__init__.py

@@ -1,6 +1,5 @@
 """
 Initialization file for invokeai.backend
 """
-from .generator import InvokeAIGeneratorBasicParams, InvokeAIGenerator, InvokeAIGeneratorOutput, Img2Img, Inpaint
 from .model_management import ModelManager, ModelCache, BaseModelType, ModelType, SubModelType, ModelInfo
 from .model_management.models import SilenceWarnings

invokeai/backend/generator/__init__.py

@@ -1,12 +0,0 @@
-"""
-Initialization file for the invokeai.generator package
-"""
-from .base import (
-    InvokeAIGenerator,
-    InvokeAIGeneratorBasicParams,
-    InvokeAIGeneratorOutput,
-    Img2Img,
-    Inpaint,
-    Generator,
-)
-from .inpaint import infill_methods

invokeai/backend/generator/base.py

@@ -1,559 +0,0 @@
-"""
-Base class for invokeai.backend.generator.*
-including img2img, txt2img, and inpaint
-"""
-from __future__ import annotations
-
-import itertools
-import dataclasses
-import diffusers
-import os
-import random
-import traceback
-from abc import ABCMeta
-from argparse import Namespace
-from contextlib import nullcontext
-
-import cv2
-import numpy as np
-import torch
-from PIL import Image, ImageChops, ImageFilter
-from accelerate.utils import set_seed
-from diffusers import DiffusionPipeline
-from tqdm import trange
-from typing import Callable, List, Iterator, Optional, Type, Union
-from dataclasses import dataclass, field
-from diffusers.schedulers import SchedulerMixin as Scheduler
-
-import invokeai.backend.util.logging as logger
-from ..image_util import configure_model_padding
-from ..util.util import rand_perlin_2d
-from ..stable_diffusion.diffusers_pipeline import StableDiffusionGeneratorPipeline
-from ..stable_diffusion.schedulers import SCHEDULER_MAP
-
-downsampling = 8
-
-
-@dataclass
-class InvokeAIGeneratorBasicParams:
-    seed: Optional[int] = None
-    width: int = 512
-    height: int = 512
-    cfg_scale: float = 7.5
-    steps: int = 20
-    ddim_eta: float = 0.0
-    scheduler: str = "ddim"
-    precision: str = "float16"
-    perlin: float = 0.0
-    threshold: float = 0.0
-    seamless: bool = False
-    seamless_axes: List[str] = field(default_factory=lambda: ["x", "y"])
-    h_symmetry_time_pct: Optional[float] = None
-    v_symmetry_time_pct: Optional[float] = None
-    variation_amount: float = 0.0
-    with_variations: list = field(default_factory=list)
-
-
-@dataclass
-class InvokeAIGeneratorOutput:
-    """
-    InvokeAIGeneratorOutput is a dataclass that contains the outputs of a generation
-    operation, including the image, its seed, the model name used to generate the image
-    and the model hash, as well as all the generate() parameters that went into
-    generating the image (in .params, also available as attributes)
-    """
-
-    image: Image.Image
-    seed: int
-    model_hash: str
-    attention_maps_images: List[Image.Image]
-    params: Namespace
-
-
-# we are interposing a wrapper around the original Generator classes so that
-# old code that calls Generate will continue to work.
-class InvokeAIGenerator(metaclass=ABCMeta):
-    def __init__(
-        self,
-        model_info: dict,
-        params: InvokeAIGeneratorBasicParams = InvokeAIGeneratorBasicParams(),
-        **kwargs,
-    ):
-        self.model_info = model_info
-        self.params = params
-        self.kwargs = kwargs
-
-    def generate(
-        self,
-        conditioning: tuple,
-        scheduler,
-        callback: Optional[Callable] = None,
-        step_callback: Optional[Callable] = None,
-        iterations: int = 1,
-        **keyword_args,
-    ) -> Iterator[InvokeAIGeneratorOutput]:
-        """
-        Return an iterator across the indicated number of generations.
-        Each time the iterator is called it will return an InvokeAIGeneratorOutput
-        object. Use like this:
-
-           outputs = txt2img.generate(prompt='banana sushi', iterations=5)
-           for result in outputs:
-               print(result.image, result.seed)
-
-        In the typical case of wanting to get just a single image, iterations
-        defaults to 1 and do:
-
-           output = next(txt2img.generate(prompt='banana sushi')
-
-        Pass None to get an infinite iterator.
-
-           outputs = txt2img.generate(prompt='banana sushi', iterations=None)
-           for o in outputs:
-               print(o.image, o.seed)
-
-        """
-        generator_args = dataclasses.asdict(self.params)
-        generator_args.update(keyword_args)
-
-        model_info = self.model_info
-        model_name = model_info.name
-        model_hash = model_info.hash
-        with model_info.context as model:
-            gen_class = self._generator_class()
-            generator = gen_class(model, self.params.precision, **self.kwargs)
-            if self.params.variation_amount > 0:
-                generator.set_variation(
-                    generator_args.get("seed"),
-                    generator_args.get("variation_amount"),
-                    generator_args.get("with_variations"),
-                )
-
-            if isinstance(model, DiffusionPipeline):
-                for component in [model.unet, model.vae]:
-                    configure_model_padding(
-                        component, generator_args.get("seamless", False), generator_args.get("seamless_axes")
-                    )
-            else:
-                configure_model_padding(
-                    model, generator_args.get("seamless", False), generator_args.get("seamless_axes")
-                )
-
-            iteration_count = range(iterations) if iterations else itertools.count(start=0, step=1)
-            for i in iteration_count:
-                results = generator.generate(
-                    conditioning=conditioning,
-                    step_callback=step_callback,
-                    sampler=scheduler,
-                    **generator_args,
-                )
-                output = InvokeAIGeneratorOutput(
-                    image=results[0][0],
-                    seed=results[0][1],
-                    attention_maps_images=results[0][2],
-                    model_hash=model_hash,
-                    params=Namespace(model_name=model_name, **generator_args),
-                )
-                if callback:
-                    callback(output)
-            yield output
-
-    @classmethod
-    def schedulers(self) -> List[str]:
-        """
-        Return list of all the schedulers that we currently handle.
-        """
-        return list(SCHEDULER_MAP.keys())
-
-    def load_generator(self, model: StableDiffusionGeneratorPipeline, generator_class: Type[Generator]):
-        return generator_class(model, self.params.precision)
-
-    @classmethod
-    def _generator_class(cls) -> Type[Generator]:
-        """
-        In derived classes return the name of the generator to apply.
-        If you don't override will return the name of the derived
-        class, which nicely parallels the generator class names.
-        """
-        return Generator
-
-
-# ------------------------------------
-class Img2Img(InvokeAIGenerator):
-    def generate(
-        self, init_image: Union[Image.Image, torch.FloatTensor], strength: float = 0.75, **keyword_args
-    ) -> Iterator[InvokeAIGeneratorOutput]:
-        return super().generate(init_image=init_image, strength=strength, **keyword_args)
-
-    @classmethod
-    def _generator_class(cls):
-        from .img2img import Img2Img
-
-        return Img2Img
-
-
-# ------------------------------------
-# Takes all the arguments of Img2Img and adds the mask image and the seam/infill stuff
-class Inpaint(Img2Img):
-    def generate(
-        self,
-        mask_image: Union[Image.Image, torch.FloatTensor],
-        # Seam settings - when 0, doesn't fill seam
-        seam_size: int = 96,
-        seam_blur: int = 16,
-        seam_strength: float = 0.7,
-        seam_steps: int = 30,
-        tile_size: int = 32,
-        inpaint_replace=False,
-        infill_method=None,
-        inpaint_width=None,
-        inpaint_height=None,
-        inpaint_fill: tuple(int) = (0x7F, 0x7F, 0x7F, 0xFF),
-        **keyword_args,
-    ) -> Iterator[InvokeAIGeneratorOutput]:
-        return super().generate(
-            mask_image=mask_image,
-            seam_size=seam_size,
-            seam_blur=seam_blur,
-            seam_strength=seam_strength,
-            seam_steps=seam_steps,
-            tile_size=tile_size,
-            inpaint_replace=inpaint_replace,
-            infill_method=infill_method,
-            inpaint_width=inpaint_width,
-            inpaint_height=inpaint_height,
-            inpaint_fill=inpaint_fill,
-            **keyword_args,
-        )
-
-    @classmethod
-    def _generator_class(cls):
-        from .inpaint import Inpaint
-
-        return Inpaint
-
-
-class Generator:
-    downsampling_factor: int
-    latent_channels: int
-    precision: str
-    model: DiffusionPipeline
-
-    def __init__(self, model: DiffusionPipeline, precision: str, **kwargs):
-        self.model = model
-        self.precision = precision
-        self.seed = None
-        self.latent_channels = model.unet.config.in_channels
-        self.downsampling_factor = downsampling  # BUG: should come from model or config
-        self.perlin = 0.0
-        self.threshold = 0
-        self.variation_amount = 0
-        self.with_variations = []
-        self.use_mps_noise = False
-        self.free_gpu_mem = None
-
-    # this is going to be overridden in img2img.py, txt2img.py and inpaint.py
-    def get_make_image(self, **kwargs):
-        """
-        Returns a function returning an image derived from the prompt and the initial image
-        Return value depends on the seed at the time you call it
-        """
-        raise NotImplementedError("image_iterator() must be implemented in a descendent class")
-
-    def set_variation(self, seed, variation_amount, with_variations):
-        self.seed = seed
-        self.variation_amount = variation_amount
-        self.with_variations = with_variations
-
-    def generate(
-        self,
-        width,
-        height,
-        sampler,
-        init_image=None,
-        iterations=1,
-        seed=None,
-        image_callback=None,
-        step_callback=None,
-        threshold=0.0,
-        perlin=0.0,
-        h_symmetry_time_pct=None,
-        v_symmetry_time_pct=None,
-        free_gpu_mem: bool = False,
-        **kwargs,
-    ):
-        scope = nullcontext
-        self.free_gpu_mem = free_gpu_mem
-        attention_maps_images = []
-        attention_maps_callback = lambda saver: attention_maps_images.append(saver.get_stacked_maps_image())
-        make_image = self.get_make_image(
-            sampler=sampler,
-            init_image=init_image,
-            width=width,
-            height=height,
-            step_callback=step_callback,
-            threshold=threshold,
-            perlin=perlin,
-            h_symmetry_time_pct=h_symmetry_time_pct,
-            v_symmetry_time_pct=v_symmetry_time_pct,
-            attention_maps_callback=attention_maps_callback,
-            **kwargs,
-        )
-        results = []
-        seed = seed if seed is not None and seed >= 0 else self.new_seed()
-        first_seed = seed
-        seed, initial_noise = self.generate_initial_noise(seed, width, height)
-
-        # There used to be an additional self.model.ema_scope() here, but it breaks
-        # the inpaint-1.5 model. Not sure what it did.... ?
-        with scope(self.model.device.type):
-            for n in trange(iterations, desc="Generating"):
-                x_T = None
-                if self.variation_amount > 0:
-                    set_seed(seed)
-                    target_noise = self.get_noise(width, height)
-                    x_T = self.slerp(self.variation_amount, initial_noise, target_noise)
-                elif initial_noise is not None:
-                    # i.e. we specified particular variations
-                    x_T = initial_noise
-                else:
-                    set_seed(seed)
-                    try:
-                        x_T = self.get_noise(width, height)
-                    except:
-                        logger.error("An error occurred while getting initial noise")
-                        print(traceback.format_exc())
-
-                # Pass on the seed in case a layer beneath us needs to generate noise on its own.
-                image = make_image(x_T, seed)
-
-                results.append([image, seed, attention_maps_images])
-
-                if image_callback is not None:
-                    attention_maps_image = None if len(attention_maps_images) == 0 else attention_maps_images[-1]
-                    image_callback(
-                        image,
-                        seed,
-                        first_seed=first_seed,
-                        attention_maps_image=attention_maps_image,
-                    )
-
-                seed = self.new_seed()
-
-                # Free up memory from the last generation.
-                clear_cuda_cache = kwargs["clear_cuda_cache"] if "clear_cuda_cache" in kwargs else None
-                if clear_cuda_cache is not None:
-                    clear_cuda_cache()
-
-        return results
-
-    def sample_to_image(self, samples) -> Image.Image:
-        """
-        Given samples returned from a sampler, converts
-        it into a PIL Image
-        """
-        with torch.inference_mode():
-            image = self.model.decode_latents(samples)
-        return self.model.numpy_to_pil(image)[0]
-
-    def repaste_and_color_correct(
-        self,
-        result: Image.Image,
-        init_image: Image.Image,
-        init_mask: Image.Image,
-        mask_blur_radius: int = 8,
-    ) -> Image.Image:
-        if init_image is None or init_mask is None:
-            return result
-
-        # Get the original alpha channel of the mask if there is one.
-        # Otherwise it is some other black/white image format ('1', 'L' or 'RGB')
-        pil_init_mask = init_mask.getchannel("A") if init_mask.mode == "RGBA" else init_mask.convert("L")
-        pil_init_image = init_image.convert("RGBA")  # Add an alpha channel if one doesn't exist
-
-        # Build an image with only visible pixels from source to use as reference for color-matching.
-        init_rgb_pixels = np.asarray(init_image.convert("RGB"), dtype=np.uint8)
-        init_a_pixels = np.asarray(pil_init_image.getchannel("A"), dtype=np.uint8)
-        init_mask_pixels = np.asarray(pil_init_mask, dtype=np.uint8)
-
-        # Get numpy version of result
-        np_image = np.asarray(result, dtype=np.uint8)
-
-        # Mask and calculate mean and standard deviation
-        mask_pixels = init_a_pixels * init_mask_pixels > 0
-        np_init_rgb_pixels_masked = init_rgb_pixels[mask_pixels, :]
-        np_image_masked = np_image[mask_pixels, :]
-
-        if np_init_rgb_pixels_masked.size > 0:
-            init_means = np_init_rgb_pixels_masked.mean(axis=0)
-            init_std = np_init_rgb_pixels_masked.std(axis=0)
-            gen_means = np_image_masked.mean(axis=0)
-            gen_std = np_image_masked.std(axis=0)
-
-            # Color correct
-            np_matched_result = np_image.copy()
-            np_matched_result[:, :, :] = (
-                (
-                    (
-                        (np_matched_result[:, :, :].astype(np.float32) - gen_means[None, None, :])
-                        / gen_std[None, None, :]
-                    )
-                    * init_std[None, None, :]
-                    + init_means[None, None, :]
-                )
-                .clip(0, 255)
-                .astype(np.uint8)
-            )
-            matched_result = Image.fromarray(np_matched_result, mode="RGB")
-        else:
-            matched_result = Image.fromarray(np_image, mode="RGB")
-
-        # Blur the mask out (into init image) by specified amount
-        if mask_blur_radius > 0:
-            nm = np.asarray(pil_init_mask, dtype=np.uint8)
-            nmd = cv2.erode(
-                nm,
-                kernel=np.ones((3, 3), dtype=np.uint8),
-                iterations=int(mask_blur_radius / 2),
-            )
-            pmd = Image.fromarray(nmd, mode="L")
-            blurred_init_mask = pmd.filter(ImageFilter.BoxBlur(mask_blur_radius))
-        else:
-            blurred_init_mask = pil_init_mask
-
-        multiplied_blurred_init_mask = ImageChops.multiply(blurred_init_mask, self.pil_image.split()[-1])
-
-        # Paste original on color-corrected generation (using blurred mask)
-        matched_result.paste(init_image, (0, 0), mask=multiplied_blurred_init_mask)
-        return matched_result
-
-    @staticmethod
-    def sample_to_lowres_estimated_image(samples):
-        # origingally adapted from code by @erucipe and @keturn here:
-        # https://discuss.huggingface.co/t/decoding-latents-to-rgb-without-upscaling/23204/7
-
-        # these updated numbers for v1.5 are from @torridgristle
-        v1_5_latent_rgb_factors = torch.tensor(
-            [
-                #    R        G        B
-                [0.3444, 0.1385, 0.0670],  # L1
-                [0.1247, 0.4027, 0.1494],  # L2
-                [-0.3192, 0.2513, 0.2103],  # L3
-                [-0.1307, -0.1874, -0.7445],  # L4
-            ],
-            dtype=samples.dtype,
-            device=samples.device,
-        )
-
-        latent_image = samples[0].permute(1, 2, 0) @ v1_5_latent_rgb_factors
-        latents_ubyte = (
-            ((latent_image + 1) / 2).clamp(0, 1).mul(0xFF).byte()  # change scale from -1..1 to 0..1  # to 0..255
-        ).cpu()
-
-        return Image.fromarray(latents_ubyte.numpy())
-
-    def generate_initial_noise(self, seed, width, height):
-        initial_noise = None
-        if self.variation_amount > 0 or len(self.with_variations) > 0:
-            # use fixed initial noise plus random noise per iteration
-            set_seed(seed)
-            initial_noise = self.get_noise(width, height)
-            for v_seed, v_weight in self.with_variations:
-                seed = v_seed
-                set_seed(seed)
-                next_noise = self.get_noise(width, height)
-                initial_noise = self.slerp(v_weight, initial_noise, next_noise)
-            if self.variation_amount > 0:
-                random.seed()  # reset RNG to an actually random state, so we can get a random seed for variations
-                seed = random.randrange(0, np.iinfo(np.uint32).max)
-        return (seed, initial_noise)
-
-    def get_perlin_noise(self, width, height):
-        fixdevice = "cpu" if (self.model.device.type == "mps") else self.model.device
-        # limit noise to only the diffusion image channels, not the mask channels
-        input_channels = min(self.latent_channels, 4)
-        # round up to the nearest block of 8
-        temp_width = int((width + 7) / 8) * 8
-        temp_height = int((height + 7) / 8) * 8
-        noise = torch.stack(
-            [
-                rand_perlin_2d((temp_height, temp_width), (8, 8), device=self.model.device).to(fixdevice)
-                for _ in range(input_channels)
-            ],
-            dim=0,
-        ).to(self.model.device)
-        return noise[0:4, 0:height, 0:width]
-
-    def new_seed(self):
-        self.seed = random.randrange(0, np.iinfo(np.uint32).max)
-        return self.seed
-
-    def slerp(self, t, v0, v1, DOT_THRESHOLD=0.9995):
-        """
-        Spherical linear interpolation
-        Args:
-            t (float/np.ndarray): Float value between 0.0 and 1.0
-            v0 (np.ndarray): Starting vector
-            v1 (np.ndarray): Final vector
-            DOT_THRESHOLD (float): Threshold for considering the two vectors as
-                                colineal. Not recommended to alter this.
-        Returns:
-            v2 (np.ndarray): Interpolation vector between v0 and v1
-        """
-        inputs_are_torch = False
-        if not isinstance(v0, np.ndarray):
-            inputs_are_torch = True
-            v0 = v0.detach().cpu().numpy()
-        if not isinstance(v1, np.ndarray):
-            inputs_are_torch = True
-            v1 = v1.detach().cpu().numpy()
-
-        dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1)))
-        if np.abs(dot) > DOT_THRESHOLD:
-            v2 = (1 - t) * v0 + t * v1
-        else:
-            theta_0 = np.arccos(dot)
-            sin_theta_0 = np.sin(theta_0)
-            theta_t = theta_0 * t
-            sin_theta_t = np.sin(theta_t)
-            s0 = np.sin(theta_0 - theta_t) / sin_theta_0
-            s1 = sin_theta_t / sin_theta_0
-            v2 = s0 * v0 + s1 * v1
-
-        if inputs_are_torch:
-            v2 = torch.from_numpy(v2).to(self.model.device)
-
-        return v2
-
-    # this is a handy routine for debugging use. Given a generated sample,
-    # convert it into a PNG image and store it at the indicated path
-    def save_sample(self, sample, filepath):
-        image = self.sample_to_image(sample)
-        dirname = os.path.dirname(filepath) or "."
-        if not os.path.exists(dirname):
-            logger.info(f"creating directory {dirname}")
-            os.makedirs(dirname, exist_ok=True)
-        image.save(filepath, "PNG")
-
-    def torch_dtype(self) -> torch.dtype:
-        return torch.float16 if self.precision == "float16" else torch.float32
-
-    # returns a tensor filled with random numbers from a normal distribution
-    def get_noise(self, width, height):
-        device = self.model.device
-        # limit noise to only the diffusion image channels, not the mask channels
-        input_channels = min(self.latent_channels, 4)
-        x = torch.randn(
-            [
-                1,
-                input_channels,
-                height // self.downsampling_factor,
-                width // self.downsampling_factor,
-            ],
-            dtype=self.torch_dtype(),
-            device=device,
-        )
-        if self.perlin > 0.0:
-            perlin_noise = self.get_perlin_noise(width // self.downsampling_factor, height // self.downsampling_factor)
-            x = (1 - self.perlin) * x + self.perlin * perlin_noise
-        return x

invokeai/backend/generator/img2img.py

@@ -1,92 +0,0 @@
-"""
-invokeai.backend.generator.img2img descends from .generator
-"""
-from typing import Optional
-
-import torch
-from accelerate.utils import set_seed
-from diffusers import logging
-
-from ..stable_diffusion import (
-    ConditioningData,
-    PostprocessingSettings,
-    StableDiffusionGeneratorPipeline,
-)
-from .base import Generator
-
-
-class Img2Img(Generator):
-    def __init__(self, model, precision):
-        super().__init__(model, precision)
-        self.init_latent = None  # by get_noise()
-
-    def get_make_image(
-        self,
-        sampler,
-        steps,
-        cfg_scale,
-        ddim_eta,
-        conditioning,
-        init_image,
-        strength,
-        step_callback=None,
-        threshold=0.0,
-        warmup=0.2,
-        perlin=0.0,
-        h_symmetry_time_pct=None,
-        v_symmetry_time_pct=None,
-        attention_maps_callback=None,
-        **kwargs,
-    ):
-        """
-        Returns a function returning an image derived from the prompt and the initial image
-        Return value depends on the seed at the time you call it.
-        """
-        self.perlin = perlin
-
-        # noinspection PyTypeChecker
-        pipeline: StableDiffusionGeneratorPipeline = self.model
-        pipeline.scheduler = sampler
-
-        uc, c, extra_conditioning_info = conditioning
-        conditioning_data = ConditioningData(
-            uc,
-            c,
-            cfg_scale,
-            extra_conditioning_info,
-            postprocessing_settings=PostprocessingSettings(
-                threshold=threshold,
-                warmup=warmup,
-                h_symmetry_time_pct=h_symmetry_time_pct,
-                v_symmetry_time_pct=v_symmetry_time_pct,
-            ),
-        ).add_scheduler_args_if_applicable(pipeline.scheduler, eta=ddim_eta)
-
-        def make_image(x_T: torch.Tensor, seed: int):
-            # FIXME: use x_T for initial seeded noise
-            # We're not at the moment because the pipeline automatically resizes init_image if
-            # necessary, which the x_T input might not match.
-            # In the meantime, reset the seed prior to generating pipeline output so we at least get the same result.
-            logging.set_verbosity_error()  # quench safety check warnings
-            pipeline_output = pipeline.img2img_from_embeddings(
-                init_image,
-                strength,
-                steps,
-                conditioning_data,
-                noise_func=self.get_noise_like,
-                callback=step_callback,
-                seed=seed,
-            )
-            if pipeline_output.attention_map_saver is not None and attention_maps_callback is not None:
-                attention_maps_callback(pipeline_output.attention_map_saver)
-            return pipeline.numpy_to_pil(pipeline_output.images)[0]
-
-        return make_image
-
-    def get_noise_like(self, like: torch.Tensor):
-        device = like.device
-        x = torch.randn_like(like, device=device)
-        if self.perlin > 0.0:
-            shape = like.shape
-            x = (1 - self.perlin) * x + self.perlin * self.get_perlin_noise(shape[3], shape[2])
-        return x

invokeai/backend/generator/inpaint.py

@@ -1,379 +0,0 @@
-"""
-invokeai.backend.generator.inpaint descends from .generator
-"""
-from __future__ import annotations
-
-import math
-from typing import Tuple, Union, Optional
-
-import cv2
-import numpy as np
-import torch
-from PIL import Image, ImageChops, ImageFilter, ImageOps
-
-from ..image_util import PatchMatch, debug_image
-from ..stable_diffusion.diffusers_pipeline import (
-    ConditioningData,
-    StableDiffusionGeneratorPipeline,
-    image_resized_to_grid_as_tensor,
-)
-from .img2img import Img2Img
-
-
-def infill_methods() -> list[str]:
-    methods = [
-        "tile",
-        "solid",
-    ]
-    if PatchMatch.patchmatch_available():
-        methods.insert(0, "patchmatch")
-    return methods
-
-
-class Inpaint(Img2Img):
-    def __init__(self, model, precision):
-        self.inpaint_height = 0
-        self.inpaint_width = 0
-        self.enable_image_debugging = False
-        self.init_latent = None
-        self.pil_image = None
-        self.pil_mask = None
-        self.mask_blur_radius = 0
-        self.infill_method = None
-        super().__init__(model, precision)
-
-    # Outpaint support code
-    def get_tile_images(self, image: np.ndarray, width=8, height=8):
-        _nrows, _ncols, depth = image.shape
-        _strides = image.strides
-
-        nrows, _m = divmod(_nrows, height)
-        ncols, _n = divmod(_ncols, width)
-        if _m != 0 or _n != 0:
-            return None
-
-        return np.lib.stride_tricks.as_strided(
-            np.ravel(image),
-            shape=(nrows, ncols, height, width, depth),
-            strides=(height * _strides[0], width * _strides[1], *_strides),
-            writeable=False,
-        )
-
-    def infill_patchmatch(self, im: Image.Image) -> Image.Image:
-        if im.mode != "RGBA":
-            return im
-
-        # Skip patchmatch if patchmatch isn't available
-        if not PatchMatch.patchmatch_available():
-            return im
-
-        # Patchmatch (note, we may want to expose patch_size? Increasing it significantly impacts performance though)
-        im_patched_np = PatchMatch.inpaint(im.convert("RGB"), ImageOps.invert(im.split()[-1]), patch_size=3)
-        im_patched = Image.fromarray(im_patched_np, mode="RGB")
-        return im_patched
-
-    def tile_fill_missing(self, im: Image.Image, tile_size: int = 16, seed: Optional[int] = None) -> Image.Image:
-        # Only fill if there's an alpha layer
-        if im.mode != "RGBA":
-            return im
-
-        a = np.asarray(im, dtype=np.uint8)
-
-        tile_size_tuple = (tile_size, tile_size)
-
-        # Get the image as tiles of a specified size
-        tiles = self.get_tile_images(a, *tile_size_tuple).copy()
-
-        # Get the mask as tiles
-        tiles_mask = tiles[:, :, :, :, 3]
-
-        # Find any mask tiles with any fully transparent pixels (we will be replacing these later)
-        tmask_shape = tiles_mask.shape
-        tiles_mask = tiles_mask.reshape(math.prod(tiles_mask.shape))
-        n, ny = (math.prod(tmask_shape[0:2])), math.prod(tmask_shape[2:])
-        tiles_mask = tiles_mask > 0
-        tiles_mask = tiles_mask.reshape((n, ny)).all(axis=1)
-
-        # Get RGB tiles in single array and filter by the mask
-        tshape = tiles.shape
-        tiles_all = tiles.reshape((math.prod(tiles.shape[0:2]), *tiles.shape[2:]))
-        filtered_tiles = tiles_all[tiles_mask]
-
-        if len(filtered_tiles) == 0:
-            return im
-
-        # Find all invalid tiles and replace with a random valid tile
-        replace_count = (tiles_mask == False).sum()
-        rng = np.random.default_rng(seed=seed)
-        tiles_all[np.logical_not(tiles_mask)] = filtered_tiles[
-            rng.choice(filtered_tiles.shape[0], replace_count), :, :, :
-        ]
-
-        # Convert back to an image
-        tiles_all = tiles_all.reshape(tshape)
-        tiles_all = tiles_all.swapaxes(1, 2)
-        st = tiles_all.reshape(
-            (
-                math.prod(tiles_all.shape[0:2]),
-                math.prod(tiles_all.shape[2:4]),
-                tiles_all.shape[4],
-            )
-        )
-        si = Image.fromarray(st, mode="RGBA")
-
-        return si
-
-    def mask_edge(self, mask: Image.Image, edge_size: int, edge_blur: int) -> Image.Image:
-        npimg = np.asarray(mask, dtype=np.uint8)
-
-        # Detect any partially transparent regions
-        npgradient = np.uint8(255 * (1.0 - np.floor(np.abs(0.5 - np.float32(npimg) / 255.0) * 2.0)))
-
-        # Detect hard edges
-        npedge = cv2.Canny(npimg, threshold1=100, threshold2=200)
-
-        # Combine
-        npmask = npgradient + npedge
-
-        # Expand
-        npmask = cv2.dilate(npmask, np.ones((3, 3), np.uint8), iterations=int(edge_size / 2))
-
-        new_mask = Image.fromarray(npmask)
-
-        if edge_blur > 0:
-            new_mask = new_mask.filter(ImageFilter.BoxBlur(edge_blur))
-
-        return ImageOps.invert(new_mask)
-
-    def seam_paint(
-        self,
-        im: Image.Image,
-        seam_size: int,
-        seam_blur: int,
-        seed,
-        steps,
-        cfg_scale,
-        ddim_eta,
-        conditioning,
-        strength,
-        noise,
-        infill_method,
-        step_callback,
-    ) -> Image.Image:
-        hard_mask = self.pil_image.split()[-1].copy()
-        mask = self.mask_edge(hard_mask, seam_size, seam_blur)
-
-        make_image = self.get_make_image(
-            steps,
-            cfg_scale,
-            ddim_eta,
-            conditioning,
-            init_image=im.copy().convert("RGBA"),
-            mask_image=mask,
-            strength=strength,
-            mask_blur_radius=0,
-            seam_size=0,
-            step_callback=step_callback,
-            inpaint_width=im.width,
-            inpaint_height=im.height,
-            infill_method=infill_method,
-        )
-
-        seam_noise = self.get_noise(im.width, im.height)
-
-        result = make_image(seam_noise, seed=None)
-
-        return result
-
-    @torch.no_grad()
-    def get_make_image(
-        self,
-        steps,
-        cfg_scale,
-        ddim_eta,
-        conditioning,
-        init_image: Union[Image.Image, torch.FloatTensor],
-        mask_image: Union[Image.Image, torch.FloatTensor],
-        strength: float,
-        mask_blur_radius: int = 8,
-        # Seam settings - when 0, doesn't fill seam
-        seam_size: int = 96,
-        seam_blur: int = 16,
-        seam_strength: float = 0.7,
-        seam_steps: int = 30,
-        tile_size: int = 32,
-        step_callback=None,
-        inpaint_replace=False,
-        enable_image_debugging=False,
-        infill_method=None,
-        inpaint_width=None,
-        inpaint_height=None,
-        inpaint_fill: Tuple[int, int, int, int] = (0x7F, 0x7F, 0x7F, 0xFF),
-        attention_maps_callback=None,
-        **kwargs,
-    ):
-        """
-        Returns a function returning an image derived from the prompt and
-        the initial image + mask.  Return value depends on the seed at
-        the time you call it.  kwargs are 'init_latent' and 'strength'
-        """
-
-        self.enable_image_debugging = enable_image_debugging
-        infill_method = infill_method or infill_methods()[0]
-        self.infill_method = infill_method
-
-        self.inpaint_width = inpaint_width
-        self.inpaint_height = inpaint_height
-
-        if isinstance(init_image, Image.Image):
-            self.pil_image = init_image.copy()
-
-            # Do infill
-            if infill_method == "patchmatch" and PatchMatch.patchmatch_available():
-                init_filled = self.infill_patchmatch(self.pil_image.copy())
-            elif infill_method == "tile":
-                init_filled = self.tile_fill_missing(self.pil_image.copy(), seed=self.seed, tile_size=tile_size)
-            elif infill_method == "solid":
-                solid_bg = Image.new("RGBA", init_image.size, inpaint_fill)
-                init_filled = Image.alpha_composite(solid_bg, init_image)
-            else:
-                raise ValueError(f"Non-supported infill type {infill_method}", infill_method)
-            init_filled.paste(init_image, (0, 0), init_image.split()[-1])
-
-            # Resize if requested for inpainting
-            if inpaint_width and inpaint_height:
-                init_filled = init_filled.resize((inpaint_width, inpaint_height))
-
-            debug_image(init_filled, "init_filled", debug_status=self.enable_image_debugging)
-
-            # Create init tensor
-            init_image = image_resized_to_grid_as_tensor(init_filled.convert("RGB"))
-
-        if isinstance(mask_image, Image.Image):
-            self.pil_mask = mask_image.copy()
-            debug_image(
-                mask_image,
-                "mask_image BEFORE multiply with pil_image",
-                debug_status=self.enable_image_debugging,
-            )
-
-            init_alpha = self.pil_image.getchannel("A")
-            if mask_image.mode != "L":
-                # FIXME: why do we get passed an RGB image here? We can only use single-channel.
-                mask_image = mask_image.convert("L")
-            mask_image = ImageChops.multiply(mask_image, init_alpha)
-            self.pil_mask = mask_image
-
-            # Resize if requested for inpainting
-            if inpaint_width and inpaint_height:
-                mask_image = mask_image.resize((inpaint_width, inpaint_height))
-
-            debug_image(
-                mask_image,
-                "mask_image AFTER multiply with pil_image",
-                debug_status=self.enable_image_debugging,
-            )
-            mask: torch.FloatTensor = image_resized_to_grid_as_tensor(mask_image, normalize=False)
-        else:
-            mask: torch.FloatTensor = mask_image
-
-        self.mask_blur_radius = mask_blur_radius
-
-        # noinspection PyTypeChecker
-        pipeline: StableDiffusionGeneratorPipeline = self.model
-
-        # todo: support cross-attention control
-        uc, c, _ = conditioning
-        conditioning_data = ConditioningData(uc, c, cfg_scale).add_scheduler_args_if_applicable(
-            pipeline.scheduler, eta=ddim_eta
-        )
-
-        def make_image(x_T: torch.Tensor, seed: int):
-            pipeline_output = pipeline.inpaint_from_embeddings(
-                init_image=init_image,
-                mask=1 - mask,  # expects white means "paint here."
-                strength=strength,
-                num_inference_steps=steps,
-                conditioning_data=conditioning_data,
-                noise_func=self.get_noise_like,
-                callback=step_callback,
-                seed=seed,
-            )
-
-            if pipeline_output.attention_map_saver is not None and attention_maps_callback is not None:
-                attention_maps_callback(pipeline_output.attention_map_saver)
-
-            result = self.postprocess_size_and_mask(pipeline.numpy_to_pil(pipeline_output.images)[0])
-
-            # Seam paint if this is our first pass (seam_size set to 0 during seam painting)
-            if seam_size > 0:
-                old_image = self.pil_image or init_image
-                old_mask = self.pil_mask or mask_image
-
-                result = self.seam_paint(
-                    result,
-                    seam_size,
-                    seam_blur,
-                    seed,
-                    seam_steps,
-                    cfg_scale,
-                    ddim_eta,
-                    conditioning,
-                    seam_strength,
-                    x_T,
-                    infill_method,
-                    step_callback,
-                )
-
-                # Restore original settings
-                self.get_make_image(
-                    steps,
-                    cfg_scale,
-                    ddim_eta,
-                    conditioning,
-                    old_image,
-                    old_mask,
-                    strength,
-                    mask_blur_radius,
-                    seam_size,
-                    seam_blur,
-                    seam_strength,
-                    seam_steps,
-                    tile_size,
-                    step_callback,
-                    inpaint_replace,
-                    enable_image_debugging,
-                    inpaint_width=inpaint_width,
-                    inpaint_height=inpaint_height,
-                    infill_method=infill_method,
-                    **kwargs,
-                )
-
-            return result
-
-        return make_image
-
-    def sample_to_image(self, samples) -> Image.Image:
-        gen_result = super().sample_to_image(samples).convert("RGB")
-        return self.postprocess_size_and_mask(gen_result)
-
-    def postprocess_size_and_mask(self, gen_result: Image.Image) -> Image.Image:
-        debug_image(gen_result, "gen_result", debug_status=self.enable_image_debugging)
-
-        # Resize if necessary
-        if self.inpaint_width and self.inpaint_height:
-            gen_result = gen_result.resize(self.pil_image.size)
-
-        if self.pil_image is None or self.pil_mask is None:
-            return gen_result
-
-        corrected_result = self.repaste_and_color_correct(
-            gen_result, self.pil_image, self.pil_mask, self.mask_blur_radius
-        )
-        debug_image(
-            corrected_result,
-            "corrected_result",
-            debug_status=self.enable_image_debugging,
-        )
-
-        return corrected_result

invokeai/backend/install/check_root.py

@@ -12,16 +12,17 @@ def check_invokeai_root(config: InvokeAIAppConfig):
         assert config.model_conf_path.exists(), f"{config.model_conf_path} not found"
         assert config.db_path.parent.exists(), f"{config.db_path.parent} not found"
         assert config.models_path.exists(), f"{config.models_path} not found"
-        for model in [
-            "CLIP-ViT-bigG-14-laion2B-39B-b160k",
-            "bert-base-uncased",
-            "clip-vit-large-patch14",
-            "sd-vae-ft-mse",
-            "stable-diffusion-2-clip",
-            "stable-diffusion-safety-checker",
-        ]:
-            path = config.models_path / f"core/convert/{model}"
-            assert path.exists(), f"{path} is missing"
+        if not config.ignore_missing_core_models:
+            for model in [
+                "CLIP-ViT-bigG-14-laion2B-39B-b160k",
+                "bert-base-uncased",
+                "clip-vit-large-patch14",
+                "sd-vae-ft-mse",
+                "stable-diffusion-2-clip",
+                "stable-diffusion-safety-checker",
+            ]:
+                path = config.models_path / f"core/convert/{model}"
+                assert path.exists(), f"{path} is missing"
     except Exception as e:
         print()
         print(f"An exception has occurred: {str(e)}")
@@ -32,5 +33,10 @@ def check_invokeai_root(config: InvokeAIAppConfig):
         print(
             '** From the command line, activate the virtual environment and run "invokeai-configure --yes --skip-sd-weights" **'
         )
+        print(
+            '** (To skip this check completely, add "--ignore_missing_core_models" to your CLI args. Not installing '
+            "these core models will prevent the loading of some or all .safetensors and .ckpt files. However, you can "
+            "always come back and install these core models in the future.)"
+        )
         input("Press any key to continue...")
         sys.exit(0)

invokeai/backend/install/invokeai_configure.py

@@ -10,15 +10,17 @@ import sys
 import argparse
 import io
 import os
+import psutil
 import shutil
 import textwrap
+import torch
 import traceback
 import yaml
 import warnings
 from argparse import Namespace
+from enum import Enum
 from pathlib import Path
 from shutil import get_terminal_size
-from typing import get_type_hints
 from urllib import request
 
 import npyscreen
@@ -44,6 +46,8 @@ from invokeai.app.services.config import (
 )
 from invokeai.backend.util.logging import InvokeAILogger
 from invokeai.frontend.install.model_install import addModelsForm, process_and_execute
+
+# TO DO - Move all the frontend code into invokeai.frontend.install
 from invokeai.frontend.install.widgets import (
     SingleSelectColumns,
     CenteredButtonPress,
@@ -53,6 +57,7 @@ from invokeai.frontend.install.widgets import (
     CyclingForm,
     MIN_COLS,
     MIN_LINES,
+    WindowTooSmallException,
 )
 from invokeai.backend.install.legacy_arg_parsing import legacy_parser
 from invokeai.backend.install.model_install_backend import (
@@ -61,6 +66,7 @@ from invokeai.backend.install.model_install_backend import (
     ModelInstall,
 )
 from invokeai.backend.model_management.model_probe import ModelType, BaseModelType
+from pydantic.error_wrappers import ValidationError
 
 warnings.filterwarnings("ignore")
 transformers.logging.set_verbosity_error()
@@ -76,6 +82,13 @@ Default_config_file = config.model_conf_path
 SD_Configs = config.legacy_conf_path
 
 PRECISION_CHOICES = ["auto", "float16", "float32"]
+GB = 1073741824  # GB in bytes
+HAS_CUDA = torch.cuda.is_available()
+_, MAX_VRAM = torch.cuda.mem_get_info() if HAS_CUDA else (0, 0)
+
+
+MAX_VRAM /= GB
+MAX_RAM = psutil.virtual_memory().total / GB
 
 INIT_FILE_PREAMBLE = """# InvokeAI initialization file
 # This is the InvokeAI initialization file, which contains command-line default values.
@@ -86,6 +99,12 @@ INIT_FILE_PREAMBLE = """# InvokeAI initialization file
 logger = InvokeAILogger.getLogger()
 
 
+class DummyWidgetValue(Enum):
+    zero = 0
+    true = True
+    false = False
+
+
 # --------------------------------------------
 def postscript(errors: None):
     if not any(errors):
@@ -181,7 +200,7 @@ def download_with_progress_bar(model_url: str, model_dest: str, label: str = "th
 
 
 def download_conversion_models():
-    target_dir = config.root_path / "models/core/convert"
+    target_dir = config.models_path / "core/convert"
     kwargs = dict()  # for future use
     try:
         logger.info("Downloading core tokenizers and text encoders")
@@ -376,15 +395,47 @@ Use cursor arrows to make a checkbox selection, and space to toggle.
             max_width=80,
             scroll_exit=True,
         )
-        self.max_cache_size = self.add_widget_intelligent(
-            IntTitleSlider,
-            name="Size of the RAM cache used for fast model switching (GB)",
-            value=old_opts.max_cache_size,
-            out_of=20,
-            lowest=3,
-            begin_entry_at=6,
+        self.nextrely += 1
+        self.add_widget_intelligent(
+            npyscreen.TitleFixedText,
+            name="RAM cache size (GB). Make this at least large enough to hold a single full model.",
+            begin_entry_at=0,
+            editable=False,
+            color="CONTROL",
             scroll_exit=True,
         )
+        self.nextrely -= 1
+        self.max_cache_size = self.add_widget_intelligent(
+            npyscreen.Slider,
+            value=clip(old_opts.max_cache_size, range=(3.0, MAX_RAM), step=0.5),
+            out_of=round(MAX_RAM),
+            lowest=0.0,
+            step=0.5,
+            relx=8,
+            scroll_exit=True,
+        )
+        if HAS_CUDA:
+            self.nextrely += 1
+            self.add_widget_intelligent(
+                npyscreen.TitleFixedText,
+                name="VRAM cache size (GB). Reserving a small amount of VRAM will modestly speed up the start of image generation.",
+                begin_entry_at=0,
+                editable=False,
+                color="CONTROL",
+                scroll_exit=True,
+            )
+            self.nextrely -= 1
+            self.max_vram_cache_size = self.add_widget_intelligent(
+                npyscreen.Slider,
+                value=clip(old_opts.max_vram_cache_size, range=(0, MAX_VRAM), step=0.25),
+                out_of=round(MAX_VRAM * 2) / 2,
+                lowest=0.0,
+                relx=8,
+                step=0.25,
+                scroll_exit=True,
+            )
+        else:
+            self.max_vram_cache_size = DummyWidgetValue.zero
         self.nextrely += 1
         self.outdir = self.add_widget_intelligent(
             FileBox,
@@ -401,7 +452,7 @@ Use cursor arrows to make a checkbox selection, and space to toggle.
         self.autoimport_dirs = {}
         self.autoimport_dirs["autoimport_dir"] = self.add_widget_intelligent(
             FileBox,
-            name=f"Folder to recursively scan for new checkpoints, ControlNets, LoRAs and TI models",
+            name="Folder to recursively scan for new checkpoints, ControlNets, LoRAs and TI models",
             value=str(config.root_path / config.autoimport_dir),
             select_dir=True,
             must_exist=False,
@@ -476,6 +527,7 @@ https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/LICENS
             "outdir",
             "free_gpu_mem",
             "max_cache_size",
+            "max_vram_cache_size",
             "xformers_enabled",
             "always_use_cpu",
         ]:
@@ -553,6 +605,16 @@ def default_user_selections(program_opts: Namespace) -> InstallSelections:
     )
 
 
+# -------------------------------------
+def clip(value: float, range: tuple[float, float], step: float) -> float:
+    minimum, maximum = range
+    if value < minimum:
+        value = minimum
+    if value > maximum:
+        value = maximum
+    return round(value / step) * step
+
+
 # -------------------------------------
 def initialize_rootdir(root: Path, yes_to_all: bool = False):
     logger.info("Initializing InvokeAI runtime directory")
@@ -592,13 +654,13 @@ def maybe_create_models_yaml(root: Path):
 
 # -------------------------------------
 def run_console_ui(program_opts: Namespace, initfile: Path = None) -> (Namespace, Namespace):
-    # parse_args() will read from init file if present
     invokeai_opts = default_startup_options(initfile)
     invokeai_opts.root = program_opts.root
 
-    # The third argument is needed in the Windows 11 environment to
-    # launch a console window running this program.
-    set_min_terminal_size(MIN_COLS, MIN_LINES)
+    if not set_min_terminal_size(MIN_COLS, MIN_LINES):
+        raise WindowTooSmallException(
+            "Could not increase terminal size. Try running again with a larger window or smaller font size."
+        )
 
     # the install-models application spawns a subprocess to install
     # models, and will crash unless this is set before running.
@@ -654,10 +716,13 @@ def migrate_init_file(legacy_format: Path):
     old = legacy_parser.parse_args([f"@{str(legacy_format)}"])
     new = InvokeAIAppConfig.get_config()
 
-    fields = list(get_type_hints(InvokeAIAppConfig).keys())
+    fields = [x for x, y in InvokeAIAppConfig.__fields__.items() if y.field_info.extra.get("category") != "DEPRECATED"]
     for attr in fields:
         if hasattr(old, attr):
-            setattr(new, attr, getattr(old, attr))
+            try:
+                setattr(new, attr, getattr(old, attr))
+            except ValidationError as e:
+                print(f"* Ignoring incompatible value for field {attr}:\n  {str(e)}")
 
     # a few places where the field names have changed and we have to
     # manually add in the new names/values
@@ -777,6 +842,7 @@ def main():
 
         models_to_download = default_user_selections(opt)
         new_init_file = config.root_path / "invokeai.yaml"
+
         if opt.yes_to_all:
             write_default_options(opt, new_init_file)
             init_options = Namespace(precision="float32" if opt.full_precision else "float16")
@@ -802,6 +868,8 @@ def main():
         postscript(errors=errors)
         if not opt.yes_to_all:
             input("Press any key to continue...")
+    except WindowTooSmallException as e:
+        logger.error(str(e))
     except KeyboardInterrupt:
         print("\nGoodbye! Come back soon.")
 

invokeai/backend/install/migrate_to_3.py

@@ -591,7 +591,6 @@ script, which will perform a full upgrade in place.""",
     # TODO: revisit - don't rely on invokeai.yaml to exist yet!
     dest_is_setup = (dest_root / "models/core").exists() and (dest_root / "databases").exists()
     if not dest_is_setup:
-        import invokeai.frontend.install.invokeai_configure
         from invokeai.backend.install.invokeai_configure import initialize_rootdir
 
         initialize_rootdir(dest_root, True)

invokeai/backend/install/model_install_backend.py

@@ -7,11 +7,13 @@ import warnings
 from dataclasses import dataclass, field
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import List, Dict, Callable, Union, Set
+from typing import Optional, List, Dict, Callable, Union, Set
 
 import requests
 from diffusers import DiffusionPipeline
 from diffusers import logging as dlogging
+import onnx
+import torch
 from huggingface_hub import hf_hub_url, HfFolder, HfApi
 from omegaconf import OmegaConf
 from tqdm import tqdm
@@ -22,6 +24,7 @@ from invokeai.app.services.config import InvokeAIAppConfig
 from invokeai.backend.model_management import ModelManager, ModelType, BaseModelType, ModelVariantType, AddModelResult
 from invokeai.backend.model_management.model_probe import ModelProbe, SchedulerPredictionType, ModelProbeInfo
 from invokeai.backend.util import download_with_resume
+from invokeai.backend.util.devices import torch_dtype, choose_torch_device
 from ..util.logging import InvokeAILogger
 
 warnings.filterwarnings("ignore")
@@ -86,8 +89,8 @@ class ModelLoadInfo:
     name: str
     model_type: ModelType
     base_type: BaseModelType
-    path: Path = None
-    repo_id: str = None
+    path: Optional[Path] = None
+    repo_id: Optional[str] = None
     description: str = ""
     installed: bool = False
     recommended: bool = False
@@ -98,9 +101,9 @@ class ModelInstall(object):
     def __init__(
         self,
         config: InvokeAIAppConfig,
-        prediction_type_helper: Callable[[Path], SchedulerPredictionType] = None,
-        model_manager: ModelManager = None,
-        access_token: str = None,
+        prediction_type_helper: Optional[Callable[[Path], SchedulerPredictionType]] = None,
+        model_manager: Optional[ModelManager] = None,
+        access_token: Optional[str] = None,
     ):
         self.config = config
         self.mgr = model_manager or ModelManager(config.model_conf_path)
@@ -128,7 +131,9 @@ class ModelInstall(object):
             model_dict[key] = ModelLoadInfo(**value)
 
         # supplement with entries in models.yaml
-        installed_models = self.mgr.list_models()
+        installed_models = [x for x in self.mgr.list_models()]
+        # suppresses autoloaded models
+        # installed_models = [x for x in self.mgr.list_models() if not self._is_autoloaded(x)]
 
         for md in installed_models:
             base = md["base_model"]
@@ -147,6 +152,17 @@ class ModelInstall(object):
                 )
         return {x: model_dict[x] for x in sorted(model_dict.keys(), key=lambda y: model_dict[y].name.lower())}
 
+    def _is_autoloaded(self, model_info: dict) -> bool:
+        path = model_info.get("path")
+        if not path:
+            return False
+        for autodir in ["autoimport_dir", "lora_dir", "embedding_dir", "controlnet_dir"]:
+            if autodir_path := getattr(self.config, autodir):
+                autodir_path = self.config.root_path / autodir_path
+                if Path(path).is_relative_to(autodir_path):
+                    return True
+        return False
+
     def list_models(self, model_type):
         installed = self.mgr.list_models(model_type=model_type)
         print(f"Installed models of type `{model_type}`:")
@@ -273,6 +289,7 @@ class ModelInstall(object):
                 logger.error(f"Unable to download {url}. Skipping.")
             info = ModelProbe().heuristic_probe(location)
             dest = self.config.models_path / info.base_type.value / info.model_type.value / location.name
+            dest.parent.mkdir(parents=True, exist_ok=True)
             models_path = shutil.move(location, dest)
 
         # staged version will be garbage-collected at this time
@@ -290,6 +307,8 @@ class ModelInstall(object):
             staging = Path(staging)
             if "model_index.json" in files:
                 location = self._download_hf_pipeline(repo_id, staging)  # pipeline
+            elif "unet/model.onnx" in files:
+                location = self._download_hf_model(repo_id, files, staging)
             else:
                 for suffix in ["safetensors", "bin"]:
                     if f"pytorch_lora_weights.{suffix}" in files:
@@ -346,7 +365,7 @@ class ModelInstall(object):
             if key in self.datasets:
                 description = self.datasets[key].get("description") or description
 
-        rel_path = self.relative_to_root(path)
+        rel_path = self.relative_to_root(path, self.config.models_path)
 
         attributes = dict(
             path=str(rel_path),
@@ -354,7 +373,7 @@ class ModelInstall(object):
             model_format=info.format,
         )
         legacy_conf = None
-        if info.model_type == ModelType.Main:
+        if info.model_type == ModelType.Main or info.model_type == ModelType.ONNX:
             attributes.update(
                 dict(
                     variant=info.variant_type,
@@ -386,8 +405,8 @@ class ModelInstall(object):
             attributes.update(dict(config=str(legacy_conf)))
         return attributes
 
-    def relative_to_root(self, path: Path) -> Path:
-        root = self.config.root_path
+    def relative_to_root(self, path: Path, root: Optional[Path] = None) -> Path:
+        root = root or self.config.root_path
         if path.is_relative_to(root):
             return path.relative_to(root)
         else:
@@ -399,15 +418,25 @@ class ModelInstall(object):
         does a save_pretrained() to the indicated staging area.
         """
         _, name = repo_id.split("/")
-        revisions = ["fp16", "main"] if self.config.precision == "float16" else ["main"]
+        precision = torch_dtype(choose_torch_device())
+        variants = ["fp16", None] if precision == torch.float16 else [None, "fp16"]
+
         model = None
-        for revision in revisions:
+        for variant in variants:
             try:
-                model = DiffusionPipeline.from_pretrained(repo_id, revision=revision, safety_checker=None)
-            except:  # most errors are due to fp16 not being present. Fix this to catch other errors
-                pass
+                model = DiffusionPipeline.from_pretrained(
+                    repo_id,
+                    variant=variant,
+                    torch_dtype=precision,
+                    safety_checker=None,
+                )
+            except Exception as e:  # most errors are due to fp16 not being present. Fix this to catch other errors
+                if "fp16" not in str(e):
+                    print(e)
+
             if model:
                 break
+
         if not model:
             logger.error(f"Diffusers model {repo_id} could not be downloaded. Skipping.")
             return None
@@ -419,8 +448,13 @@ class ModelInstall(object):
         location = staging / name
         paths = list()
         for filename in files:
+            filePath = Path(filename)
             p = hf_download_with_resume(
-                repo_id, model_dir=location, model_name=filename, access_token=self.access_token
+                repo_id,
+                model_dir=location / filePath.parent,
+                model_name=filePath.name,
+                access_token=self.access_token,
+                subfolder=filePath.parent,
             )
             if p:
                 paths.append(p)
@@ -468,11 +502,12 @@ def hf_download_with_resume(
     model_name: str,
     model_dest: Path = None,
     access_token: str = None,
+    subfolder: str = None,
 ) -> Path:
     model_dest = model_dest or Path(os.path.join(model_dir, model_name))
     os.makedirs(model_dir, exist_ok=True)
 
-    url = hf_hub_url(repo_id, model_name)
+    url = hf_hub_url(repo_id, model_name, subfolder=subfolder)
 
     header = {"Authorization": f"Bearer {access_token}"} if access_token else {}
     open_mode = "wb"

invokeai/backend/model_management/__init__.py

@@ -3,6 +3,7 @@ Initialization file for invokeai.backend.model_management
 """
 from .model_manager import ModelManager, ModelInfo, AddModelResult, SchedulerPredictionType
 from .model_cache import ModelCache
+from .lora import ModelPatcher, ONNXModelPatcher
 from .models import (
     BaseModelType,
     ModelType,
@@ -12,3 +13,4 @@ from .models import (
     DuplicateModelException,
 )
 from .model_merge import ModelMerger, MergeInterpolationMethod
+from .lora import ModelPatcher

invokeai/backend/model_management/convert_ckpt_to_diffusers.py

@@ -63,7 +63,7 @@ from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionS
 from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer
 
 from invokeai.backend.util.logging import InvokeAILogger
-from invokeai.app.services.config import InvokeAIAppConfig, MODEL_CORE
+from invokeai.app.services.config import InvokeAIAppConfig
 
 from picklescan.scanner import scan_file_path
 from .models import BaseModelType, ModelVariantType
@@ -81,7 +81,7 @@ if is_accelerate_available():
     from accelerate.utils import set_module_tensor_to_device
 
 logger = InvokeAILogger.getLogger(__name__)
-CONVERT_MODEL_ROOT = InvokeAIAppConfig.get_config().root_path / MODEL_CORE / "convert"
+CONVERT_MODEL_ROOT = InvokeAIAppConfig.get_config().models_path / "core/convert"
 
 
 def shave_segments(path, n_shave_prefix_segments=1):
@@ -1070,7 +1070,7 @@ def convert_controlnet_checkpoint(
     extract_ema,
     use_linear_projection=None,
     cross_attention_dim=None,
-    precision: torch.dtype = torch.float32,
+    precision: Optional[torch.dtype] = None,
 ):
     ctrlnet_config = create_unet_diffusers_config(original_config, image_size=image_size, controlnet=True)
     ctrlnet_config["upcast_attention"] = upcast_attention
@@ -1111,7 +1111,6 @@ def convert_controlnet_checkpoint(
     return controlnet.to(precision)
 
 
-# TO DO - PASS PRECISION
 def download_from_original_stable_diffusion_ckpt(
     checkpoint_path: str,
     model_version: BaseModelType,
@@ -1121,7 +1120,7 @@ def download_from_original_stable_diffusion_ckpt(
     prediction_type: str = None,
     model_type: str = None,
     extract_ema: bool = False,
-    precision: torch.dtype = torch.float32,
+    precision: Optional[torch.dtype] = None,
     scheduler_type: str = "pndm",
     num_in_channels: Optional[int] = None,
     upcast_attention: Optional[bool] = None,
@@ -1194,6 +1193,8 @@ def download_from_original_stable_diffusion_ckpt(
             [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer)
             to use. If this parameter is `None`, the function will load a new instance of [CLIPTokenizer] by itself, if
             needed.
+        precision (`torch.dtype`, *optional*, defauts to `None`):
+            If not provided the precision will be set to the precision of the original file.
         return: A StableDiffusionPipeline object representing the passed-in `.ckpt`/`.safetensors` file.
     """
 
@@ -1252,6 +1253,10 @@ def download_from_original_stable_diffusion_ckpt(
 
     logger.debug(f"model_type = {model_type}; original_config_file = {original_config_file}")
 
+    precision_probing_key = "model.diffusion_model.input_blocks.0.0.bias"
+    logger.debug(f"original checkpoint precision == {checkpoint[precision_probing_key].dtype}")
+    precision = precision or checkpoint[precision_probing_key].dtype
+
     if original_config_file is None:
         key_name_v2_1 = "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k.weight"
         key_name_sd_xl_base = "conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj.bias"
@@ -1279,9 +1284,12 @@ def download_from_original_stable_diffusion_ckpt(
         original_config_file = BytesIO(requests.get(config_url).content)
 
     original_config = OmegaConf.load(original_config_file)
+    if original_config["model"]["params"].get("use_ema") is not None:
+        extract_ema = original_config["model"]["params"]["use_ema"]
+
     if (
         model_version == BaseModelType.StableDiffusion2
-        and original_config["model"]["params"]["parameterization"] == "v"
+        and original_config["model"]["params"].get("parameterization") == "v"
     ):
         prediction_type = "v_prediction"
         upcast_attention = True
@@ -1447,7 +1455,7 @@ def download_from_original_stable_diffusion_ckpt(
             if controlnet:
                 pipe = pipeline_class(
                     vae=vae.to(precision),
-                    text_encoder=text_model,
+                    text_encoder=text_model.to(precision),
                     tokenizer=tokenizer,
                     unet=unet.to(precision),
                     scheduler=scheduler,
@@ -1459,7 +1467,7 @@ def download_from_original_stable_diffusion_ckpt(
             else:
                 pipe = pipeline_class(
                     vae=vae.to(precision),
-                    text_encoder=text_model,
+                    text_encoder=text_model.to(precision),
                     tokenizer=tokenizer,
                     unet=unet.to(precision),
                     scheduler=scheduler,
@@ -1484,8 +1492,8 @@ def download_from_original_stable_diffusion_ckpt(
                     image_noising_scheduler=image_noising_scheduler,
                     # regular denoising components
                     tokenizer=tokenizer,
-                    text_encoder=text_model,
-                    unet=unet,
+                    text_encoder=text_model.to(precision),
+                    unet=unet.to(precision),
                     scheduler=scheduler,
                     # vae
                     vae=vae,
@@ -1560,7 +1568,7 @@ def download_from_original_stable_diffusion_ckpt(
         if controlnet:
             pipe = pipeline_class(
                 vae=vae.to(precision),
-                text_encoder=text_model,
+                text_encoder=text_model.to(precision),
                 tokenizer=tokenizer,
                 unet=unet.to(precision),
                 controlnet=controlnet,
@@ -1571,7 +1579,7 @@ def download_from_original_stable_diffusion_ckpt(
         else:
             pipe = pipeline_class(
                 vae=vae.to(precision),
-                text_encoder=text_model,
+                text_encoder=text_model.to(precision),
                 tokenizer=tokenizer,
                 unet=unet.to(precision),
                 scheduler=scheduler,
@@ -1594,9 +1602,9 @@ def download_from_original_stable_diffusion_ckpt(
 
             pipe = StableDiffusionXLPipeline(
                 vae=vae.to(precision),
-                text_encoder=text_encoder,
+                text_encoder=text_encoder.to(precision),
                 tokenizer=tokenizer,
-                text_encoder_2=text_encoder_2,
+                text_encoder_2=text_encoder_2.to(precision),
                 tokenizer_2=tokenizer_2,
                 unet=unet.to(precision),
                 scheduler=scheduler,
@@ -1639,7 +1647,7 @@ def download_controlnet_from_original_ckpt(
     original_config_file: str,
     image_size: int = 512,
     extract_ema: bool = False,
-    precision: torch.dtype = torch.float32,
+    precision: Optional[torch.dtype] = None,
     num_in_channels: Optional[int] = None,
     upcast_attention: Optional[bool] = None,
     device: str = None,
@@ -1680,6 +1688,12 @@ def download_controlnet_from_original_ckpt(
     while "state_dict" in checkpoint:
         checkpoint = checkpoint["state_dict"]
 
+    # use original precision
+    precision_probing_key = "input_blocks.0.0.bias"
+    ckpt_precision = checkpoint[precision_probing_key].dtype
+    logger.debug(f"original controlnet precision = {ckpt_precision}")
+    precision = precision or ckpt_precision
+
     original_config = OmegaConf.load(original_config_file)
 
     if num_in_channels is not None:
@@ -1699,7 +1713,7 @@ def download_controlnet_from_original_ckpt(
         cross_attention_dim=cross_attention_dim,
     )
 
-    return controlnet
+    return controlnet.to(precision)
 
 
 def convert_ldm_vae_to_diffusers(checkpoint, vae_config: DictConfig, image_size: int) -> AutoencoderKL:

invokeai/backend/model_management/lora.py

@@ -6,427 +6,20 @@ from typing import Optional, Dict, Tuple, Any, Union, List
 from pathlib import Path
 
 import torch
+from safetensors.torch import load_file
+from torch.utils.hooks import RemovableHandle
+
+from diffusers.models import UNet2DConditionModel
+from transformers import CLIPTextModel
+from onnx import numpy_helper
+from onnxruntime import OrtValue
+import numpy as np
+
 from compel.embeddings_provider import BaseTextualInversionManager
 from diffusers.models import UNet2DConditionModel
 from safetensors.torch import load_file
 from transformers import CLIPTextModel, CLIPTokenizer
 
-
-class LoRALayerBase:
-    # rank: Optional[int]
-    # alpha: Optional[float]
-    # bias: Optional[torch.Tensor]
-    # layer_key: str
-
-    # @property
-    # def scale(self):
-    #    return self.alpha / self.rank if (self.alpha and self.rank) else 1.0
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: dict,
-    ):
-        if "alpha" in values:
-            self.alpha = values["alpha"].item()
-        else:
-            self.alpha = None
-
-        if "bias_indices" in values and "bias_values" in values and "bias_size" in values:
-            self.bias = torch.sparse_coo_tensor(
-                values["bias_indices"],
-                values["bias_values"],
-                tuple(values["bias_size"]),
-            )
-
-        else:
-            self.bias = None
-
-        self.rank = None  # set in layer implementation
-        self.layer_key = layer_key
-
-    def forward(
-        self,
-        module: torch.nn.Module,
-        input_h: Any,  # for real looks like Tuple[torch.nn.Tensor] but not sure
-        multiplier: float,
-    ):
-        if type(module) == torch.nn.Conv2d:
-            op = torch.nn.functional.conv2d
-            extra_args = dict(
-                stride=module.stride,
-                padding=module.padding,
-                dilation=module.dilation,
-                groups=module.groups,
-            )
-
-        else:
-            op = torch.nn.functional.linear
-            extra_args = {}
-
-        weight = self.get_weight()
-
-        bias = self.bias if self.bias is not None else 0
-        scale = self.alpha / self.rank if (self.alpha and self.rank) else 1.0
-        return (
-            op(
-                *input_h,
-                (weight + bias).view(module.weight.shape),
-                None,
-                **extra_args,
-            )
-            * multiplier
-            * scale
-        )
-
-    def get_weight(self):
-        raise NotImplementedError()
-
-    def calc_size(self) -> int:
-        model_size = 0
-        for val in [self.bias]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(
-        self,
-        device: Optional[torch.device] = None,
-        dtype: Optional[torch.dtype] = None,
-    ):
-        if self.bias is not None:
-            self.bias = self.bias.to(device=device, dtype=dtype)
-
-
-# TODO: find and debug lora/locon with bias
-class LoRALayer(LoRALayerBase):
-    # up: torch.Tensor
-    # mid: Optional[torch.Tensor]
-    # down: torch.Tensor
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: dict,
-    ):
-        super().__init__(layer_key, values)
-
-        self.up = values["lora_up.weight"]
-        self.down = values["lora_down.weight"]
-        if "lora_mid.weight" in values:
-            self.mid = values["lora_mid.weight"]
-        else:
-            self.mid = None
-
-        self.rank = self.down.shape[0]
-
-    def get_weight(self):
-        if self.mid is not None:
-            up = self.up.reshape(self.up.shape[0], self.up.shape[1])
-            down = self.down.reshape(self.down.shape[0], self.down.shape[1])
-            weight = torch.einsum("m n w h, i m, n j -> i j w h", self.mid, up, down)
-        else:
-            weight = self.up.reshape(self.up.shape[0], -1) @ self.down.reshape(self.down.shape[0], -1)
-
-        return weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        for val in [self.up, self.mid, self.down]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(
-        self,
-        device: Optional[torch.device] = None,
-        dtype: Optional[torch.dtype] = None,
-    ):
-        super().to(device=device, dtype=dtype)
-
-        self.up = self.up.to(device=device, dtype=dtype)
-        self.down = self.down.to(device=device, dtype=dtype)
-
-        if self.mid is not None:
-            self.mid = self.mid.to(device=device, dtype=dtype)
-
-
-class LoHALayer(LoRALayerBase):
-    # w1_a: torch.Tensor
-    # w1_b: torch.Tensor
-    # w2_a: torch.Tensor
-    # w2_b: torch.Tensor
-    # t1: Optional[torch.Tensor] = None
-    # t2: Optional[torch.Tensor] = None
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: dict,
-    ):
-        super().__init__(layer_key, values)
-
-        self.w1_a = values["hada_w1_a"]
-        self.w1_b = values["hada_w1_b"]
-        self.w2_a = values["hada_w2_a"]
-        self.w2_b = values["hada_w2_b"]
-
-        if "hada_t1" in values:
-            self.t1 = values["hada_t1"]
-        else:
-            self.t1 = None
-
-        if "hada_t2" in values:
-            self.t2 = values["hada_t2"]
-        else:
-            self.t2 = None
-
-        self.rank = self.w1_b.shape[0]
-
-    def get_weight(self):
-        if self.t1 is None:
-            weight = (self.w1_a @ self.w1_b) * (self.w2_a @ self.w2_b)
-
-        else:
-            rebuild1 = torch.einsum("i j k l, j r, i p -> p r k l", self.t1, self.w1_b, self.w1_a)
-            rebuild2 = torch.einsum("i j k l, j r, i p -> p r k l", self.t2, self.w2_b, self.w2_a)
-            weight = rebuild1 * rebuild2
-
-        return weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        for val in [self.w1_a, self.w1_b, self.w2_a, self.w2_b, self.t1, self.t2]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(
-        self,
-        device: Optional[torch.device] = None,
-        dtype: Optional[torch.dtype] = None,
-    ):
-        super().to(device=device, dtype=dtype)
-
-        self.w1_a = self.w1_a.to(device=device, dtype=dtype)
-        self.w1_b = self.w1_b.to(device=device, dtype=dtype)
-        if self.t1 is not None:
-            self.t1 = self.t1.to(device=device, dtype=dtype)
-
-        self.w2_a = self.w2_a.to(device=device, dtype=dtype)
-        self.w2_b = self.w2_b.to(device=device, dtype=dtype)
-        if self.t2 is not None:
-            self.t2 = self.t2.to(device=device, dtype=dtype)
-
-
-class LoKRLayer(LoRALayerBase):
-    # w1: Optional[torch.Tensor] = None
-    # w1_a: Optional[torch.Tensor] = None
-    # w1_b: Optional[torch.Tensor] = None
-    # w2: Optional[torch.Tensor] = None
-    # w2_a: Optional[torch.Tensor] = None
-    # w2_b: Optional[torch.Tensor] = None
-    # t2: Optional[torch.Tensor] = None
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: dict,
-    ):
-        super().__init__(layer_key, values)
-
-        if "lokr_w1" in values:
-            self.w1 = values["lokr_w1"]
-            self.w1_a = None
-            self.w1_b = None
-        else:
-            self.w1 = None
-            self.w1_a = values["lokr_w1_a"]
-            self.w1_b = values["lokr_w1_b"]
-
-        if "lokr_w2" in values:
-            self.w2 = values["lokr_w2"]
-            self.w2_a = None
-            self.w2_b = None
-        else:
-            self.w2 = None
-            self.w2_a = values["lokr_w2_a"]
-            self.w2_b = values["lokr_w2_b"]
-
-        if "lokr_t2" in values:
-            self.t2 = values["lokr_t2"]
-        else:
-            self.t2 = None
-
-        if "lokr_w1_b" in values:
-            self.rank = values["lokr_w1_b"].shape[0]
-        elif "lokr_w2_b" in values:
-            self.rank = values["lokr_w2_b"].shape[0]
-        else:
-            self.rank = None  # unscaled
-
-    def get_weight(self):
-        w1 = self.w1
-        if w1 is None:
-            w1 = self.w1_a @ self.w1_b
-
-        w2 = self.w2
-        if w2 is None:
-            if self.t2 is None:
-                w2 = self.w2_a @ self.w2_b
-            else:
-                w2 = torch.einsum("i j k l, i p, j r -> p r k l", self.t2, self.w2_a, self.w2_b)
-
-        if len(w2.shape) == 4:
-            w1 = w1.unsqueeze(2).unsqueeze(2)
-        w2 = w2.contiguous()
-        weight = torch.kron(w1, w2)
-
-        return weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        for val in [self.w1, self.w1_a, self.w1_b, self.w2, self.w2_a, self.w2_b, self.t2]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(
-        self,
-        device: Optional[torch.device] = None,
-        dtype: Optional[torch.dtype] = None,
-    ):
-        super().to(device=device, dtype=dtype)
-
-        if self.w1 is not None:
-            self.w1 = self.w1.to(device=device, dtype=dtype)
-        else:
-            self.w1_a = self.w1_a.to(device=device, dtype=dtype)
-            self.w1_b = self.w1_b.to(device=device, dtype=dtype)
-
-        if self.w2 is not None:
-            self.w2 = self.w2.to(device=device, dtype=dtype)
-        else:
-            self.w2_a = self.w2_a.to(device=device, dtype=dtype)
-            self.w2_b = self.w2_b.to(device=device, dtype=dtype)
-
-        if self.t2 is not None:
-            self.t2 = self.t2.to(device=device, dtype=dtype)
-
-
-class LoRAModel:  # (torch.nn.Module):
-    _name: str
-    layers: Dict[str, LoRALayer]
-    _device: torch.device
-    _dtype: torch.dtype
-
-    def __init__(
-        self,
-        name: str,
-        layers: Dict[str, LoRALayer],
-        device: torch.device,
-        dtype: torch.dtype,
-    ):
-        self._name = name
-        self._device = device or torch.cpu
-        self._dtype = dtype or torch.float32
-        self.layers = layers
-
-    @property
-    def name(self):
-        return self._name
-
-    @property
-    def device(self):
-        return self._device
-
-    @property
-    def dtype(self):
-        return self._dtype
-
-    def to(
-        self,
-        device: Optional[torch.device] = None,
-        dtype: Optional[torch.dtype] = None,
-    ) -> LoRAModel:
-        # TODO: try revert if exception?
-        for key, layer in self.layers.items():
-            layer.to(device=device, dtype=dtype)
-        self._device = device
-        self._dtype = dtype
-
-    def calc_size(self) -> int:
-        model_size = 0
-        for _, layer in self.layers.items():
-            model_size += layer.calc_size()
-        return model_size
-
-    @classmethod
-    def from_checkpoint(
-        cls,
-        file_path: Union[str, Path],
-        device: Optional[torch.device] = None,
-        dtype: Optional[torch.dtype] = None,
-    ):
-        device = device or torch.device("cpu")
-        dtype = dtype or torch.float32
-
-        if isinstance(file_path, str):
-            file_path = Path(file_path)
-
-        model = cls(
-            device=device,
-            dtype=dtype,
-            name=file_path.stem,  # TODO:
-            layers=dict(),
-        )
-
-        if file_path.suffix == ".safetensors":
-            state_dict = load_file(file_path.absolute().as_posix(), device="cpu")
-        else:
-            state_dict = torch.load(file_path, map_location="cpu")
-
-        state_dict = cls._group_state(state_dict)
-
-        for layer_key, values in state_dict.items():
-            # lora and locon
-            if "lora_down.weight" in values:
-                layer = LoRALayer(layer_key, values)
-
-            # loha
-            elif "hada_w1_b" in values:
-                layer = LoHALayer(layer_key, values)
-
-            # lokr
-            elif "lokr_w1_b" in values or "lokr_w1" in values:
-                layer = LoKRLayer(layer_key, values)
-
-            else:
-                # TODO: diff/ia3/... format
-                print(f">> Encountered unknown lora layer module in {model.name}: {layer_key}")
-                return
-
-            # lower memory consumption by removing already parsed layer values
-            state_dict[layer_key].clear()
-
-            layer.to(device=device, dtype=dtype)
-            model.layers[layer_key] = layer
-
-        return model
-
-    @staticmethod
-    def _group_state(state_dict: dict):
-        state_dict_groupped = dict()
-
-        for key, value in state_dict.items():
-            stem, leaf = key.split(".", 1)
-            if stem not in state_dict_groupped:
-                state_dict_groupped[stem] = dict()
-            state_dict_groupped[stem][leaf] = value
-
-        return state_dict_groupped
-
-
 """
 loras = [
     (lora_model1, 0.7),
@@ -505,6 +98,26 @@ class ModelPatcher:
         with cls.apply_lora(text_encoder, loras, "lora_te_"):
             yield
 
+    @classmethod
+    @contextmanager
+    def apply_sdxl_lora_text_encoder(
+        cls,
+        text_encoder: CLIPTextModel,
+        loras: List[Tuple[LoRAModel, float]],
+    ):
+        with cls.apply_lora(text_encoder, loras, "lora_te1_"):
+            yield
+
+    @classmethod
+    @contextmanager
+    def apply_sdxl_lora_text_encoder2(
+        cls,
+        text_encoder: CLIPTextModel,
+        loras: List[Tuple[LoRAModel, float]],
+    ):
+        with cls.apply_lora(text_encoder, loras, "lora_te2_"):
+            yield
+
     @classmethod
     @contextmanager
     def apply_lora(
@@ -530,7 +143,7 @@ class ModelPatcher:
                         # with torch.autocast(device_type="cpu"):
                         layer.to(dtype=torch.float32)
                         layer_scale = layer.alpha / layer.rank if (layer.alpha and layer.rank) else 1.0
-                        layer_weight = layer.get_weight() * lora_weight * layer_scale
+                        layer_weight = layer.get_weight(original_weights[module_key]) * lora_weight * layer_scale
 
                         if module.weight.shape != layer_weight.shape:
                             # TODO: debug on lycoris
@@ -551,7 +164,7 @@ class ModelPatcher:
         cls,
         tokenizer: CLIPTokenizer,
         text_encoder: CLIPTextModel,
-        ti_list: List[Any],
+        ti_list: List[Tuple[str, Any]],
     ) -> Tuple[CLIPTokenizer, TextualInversionManager]:
         init_tokens_count = None
         new_tokens_added = None
@@ -561,27 +174,27 @@ class ModelPatcher:
             ti_manager = TextualInversionManager(ti_tokenizer)
             init_tokens_count = text_encoder.resize_token_embeddings(None).num_embeddings
 
-            def _get_trigger(ti, index):
-                trigger = ti.name
+            def _get_trigger(ti_name, index):
+                trigger = ti_name
                 if index > 0:
                     trigger += f"-!pad-{i}"
                 return f"<{trigger}>"
 
             # modify tokenizer
             new_tokens_added = 0
-            for ti in ti_list:
+            for ti_name, ti in ti_list:
                 for i in range(ti.embedding.shape[0]):
-                    new_tokens_added += ti_tokenizer.add_tokens(_get_trigger(ti, i))
+                    new_tokens_added += ti_tokenizer.add_tokens(_get_trigger(ti_name, i))
 
             # modify text_encoder
             text_encoder.resize_token_embeddings(init_tokens_count + new_tokens_added)
             model_embeddings = text_encoder.get_input_embeddings()
 
-            for ti in ti_list:
+            for ti_name, ti in ti_list:
                 ti_tokens = []
                 for i in range(ti.embedding.shape[0]):
                     embedding = ti.embedding[i]
-                    trigger = _get_trigger(ti, i)
+                    trigger = _get_trigger(ti_name, i)
 
                     token_id = ti_tokenizer.convert_tokens_to_ids(trigger)
                     if token_id == ti_tokenizer.unk_token_id:
@@ -626,7 +239,6 @@ class ModelPatcher:
 
 
 class TextualInversionModel:
-    name: str
     embedding: torch.Tensor  # [n, 768]|[n, 1280]
 
     @classmethod
@@ -640,7 +252,6 @@ class TextualInversionModel:
             file_path = Path(file_path)
 
         result = cls()  # TODO:
-        result.name = file_path.stem  # TODO:
 
         if file_path.suffix == ".safetensors":
             state_dict = load_file(file_path.absolute().as_posix(), device="cpu")
@@ -698,3 +309,187 @@ class TextualInversionManager(BaseTextualInversionManager):
                 new_token_ids.extend(self.pad_tokens[token_id])
 
         return new_token_ids
+
+
+class ONNXModelPatcher:
+    from .models.base import IAIOnnxRuntimeModel, OnnxRuntimeModel
+
+    @classmethod
+    @contextmanager
+    def apply_lora_unet(
+        cls,
+        unet: OnnxRuntimeModel,
+        loras: List[Tuple[LoRAModel, float]],
+    ):
+        with cls.apply_lora(unet, loras, "lora_unet_"):
+            yield
+
+    @classmethod
+    @contextmanager
+    def apply_lora_text_encoder(
+        cls,
+        text_encoder: OnnxRuntimeModel,
+        loras: List[Tuple[LoRAModel, float]],
+    ):
+        with cls.apply_lora(text_encoder, loras, "lora_te_"):
+            yield
+
+    # based on
+    # https://github.com/ssube/onnx-web/blob/ca2e436f0623e18b4cfe8a0363fcfcf10508acf7/api/onnx_web/convert/diffusion/lora.py#L323
+    @classmethod
+    @contextmanager
+    def apply_lora(
+        cls,
+        model: IAIOnnxRuntimeModel,
+        loras: List[Tuple[LoraModel, float]],
+        prefix: str,
+    ):
+        from .models.base import IAIOnnxRuntimeModel
+
+        if not isinstance(model, IAIOnnxRuntimeModel):
+            raise Exception("Only IAIOnnxRuntimeModel models supported")
+
+        orig_weights = dict()
+
+        try:
+            blended_loras = dict()
+
+            for lora, lora_weight in loras:
+                for layer_key, layer in lora.layers.items():
+                    if not layer_key.startswith(prefix):
+                        continue
+
+                    layer.to(dtype=torch.float32)
+                    layer_key = layer_key.replace(prefix, "")
+                    # TODO: rewrite to pass original tensor weight(required by ia3)
+                    layer_weight = layer.get_weight(None).detach().cpu().numpy() * lora_weight
+                    if layer_key is blended_loras:
+                        blended_loras[layer_key] += layer_weight
+                    else:
+                        blended_loras[layer_key] = layer_weight
+
+            node_names = dict()
+            for node in model.nodes.values():
+                node_names[node.name.replace("/", "_").replace(".", "_").lstrip("_")] = node.name
+
+            for layer_key, lora_weight in blended_loras.items():
+                conv_key = layer_key + "_Conv"
+                gemm_key = layer_key + "_Gemm"
+                matmul_key = layer_key + "_MatMul"
+
+                if conv_key in node_names or gemm_key in node_names:
+                    if conv_key in node_names:
+                        conv_node = model.nodes[node_names[conv_key]]
+                    else:
+                        conv_node = model.nodes[node_names[gemm_key]]
+
+                    weight_name = [n for n in conv_node.input if ".weight" in n][0]
+                    orig_weight = model.tensors[weight_name]
+
+                    if orig_weight.shape[-2:] == (1, 1):
+                        if lora_weight.shape[-2:] == (1, 1):
+                            new_weight = orig_weight.squeeze((3, 2)) + lora_weight.squeeze((3, 2))
+                        else:
+                            new_weight = orig_weight.squeeze((3, 2)) + lora_weight
+
+                        new_weight = np.expand_dims(new_weight, (2, 3))
+                    else:
+                        if orig_weight.shape != lora_weight.shape:
+                            new_weight = orig_weight + lora_weight.reshape(orig_weight.shape)
+                        else:
+                            new_weight = orig_weight + lora_weight
+
+                    orig_weights[weight_name] = orig_weight
+                    model.tensors[weight_name] = new_weight.astype(orig_weight.dtype)
+
+                elif matmul_key in node_names:
+                    weight_node = model.nodes[node_names[matmul_key]]
+                    matmul_name = [n for n in weight_node.input if "MatMul" in n][0]
+
+                    orig_weight = model.tensors[matmul_name]
+                    new_weight = orig_weight + lora_weight.transpose()
+
+                    orig_weights[matmul_name] = orig_weight
+                    model.tensors[matmul_name] = new_weight.astype(orig_weight.dtype)
+
+                else:
+                    # warn? err?
+                    pass
+
+            yield
+
+        finally:
+            # restore original weights
+            for name, orig_weight in orig_weights.items():
+                model.tensors[name] = orig_weight
+
+    @classmethod
+    @contextmanager
+    def apply_ti(
+        cls,
+        tokenizer: CLIPTokenizer,
+        text_encoder: IAIOnnxRuntimeModel,
+        ti_list: List[Tuple[str, Any]],
+    ) -> Tuple[CLIPTokenizer, TextualInversionManager]:
+        from .models.base import IAIOnnxRuntimeModel
+
+        if not isinstance(text_encoder, IAIOnnxRuntimeModel):
+            raise Exception("Only IAIOnnxRuntimeModel models supported")
+
+        orig_embeddings = None
+
+        try:
+            ti_tokenizer = copy.deepcopy(tokenizer)
+            ti_manager = TextualInversionManager(ti_tokenizer)
+
+            def _get_trigger(ti_name, index):
+                trigger = ti_name
+                if index > 0:
+                    trigger += f"-!pad-{i}"
+                return f"<{trigger}>"
+
+            # modify tokenizer
+            new_tokens_added = 0
+            for ti_name, ti in ti_list:
+                for i in range(ti.embedding.shape[0]):
+                    new_tokens_added += ti_tokenizer.add_tokens(_get_trigger(ti_name, i))
+
+            # modify text_encoder
+            orig_embeddings = text_encoder.tensors["text_model.embeddings.token_embedding.weight"]
+
+            embeddings = np.concatenate(
+                (np.copy(orig_embeddings), np.zeros((new_tokens_added, orig_embeddings.shape[1]))),
+                axis=0,
+            )
+
+            for ti_name, ti in ti_list:
+                ti_tokens = []
+                for i in range(ti.embedding.shape[0]):
+                    embedding = ti.embedding[i].detach().numpy()
+                    trigger = _get_trigger(ti_name, i)
+
+                    token_id = ti_tokenizer.convert_tokens_to_ids(trigger)
+                    if token_id == ti_tokenizer.unk_token_id:
+                        raise RuntimeError(f"Unable to find token id for token '{trigger}'")
+
+                    if embeddings[token_id].shape != embedding.shape:
+                        raise ValueError(
+                            f"Cannot load embedding for {trigger}. It was trained on a model with token dimension {embedding.shape[0]}, but the current model has token dimension {embeddings[token_id].shape[0]}."
+                        )
+
+                    embeddings[token_id] = embedding
+                    ti_tokens.append(token_id)
+
+                if len(ti_tokens) > 1:
+                    ti_manager.pad_tokens[ti_tokens[0]] = ti_tokens[1:]
+
+            text_encoder.tensors["text_model.embeddings.token_embedding.weight"] = embeddings.astype(
+                orig_embeddings.dtype
+            )
+
+            yield ti_tokenizer, ti_manager
+
+        finally:
+            # restore
+            if orig_embeddings is not None:
+                text_encoder.tensors["text_model.embeddings.token_embedding.weight"] = orig_embeddings

invokeai/backend/model_management/model_cache.py

@@ -21,15 +21,13 @@ import os
 import sys
 import hashlib
 from contextlib import suppress
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Dict, Union, types, Optional, Type, Any
 
 import torch
 
-import logging
 import invokeai.backend.util.logging as logger
-from invokeai.app.services.config import get_invokeai_config
-from .lora import LoRAModel, TextualInversionModel
 from .models import BaseModelType, ModelType, SubModelType, ModelBase
 
 # Maximum size of the cache, in gigs
@@ -43,6 +41,18 @@ DEFAULT_MAX_VRAM_CACHE_SIZE = 2.75
 GIG = 1073741824
 
 
+@dataclass
+class CacheStats(object):
+    hits: int = 0  # cache hits
+    misses: int = 0  # cache misses
+    high_watermark: int = 0  # amount of cache used
+    in_cache: int = 0  # number of models in cache
+    cleared: int = 0  # number of models cleared to make space
+    cache_size: int = 0  # total size of cache
+    # {submodel_key => size}
+    loaded_model_sizes: Dict[str, int] = field(default_factory=dict)
+
+
 class ModelLocker(object):
     "Forward declaration"
     pass
@@ -117,6 +127,9 @@ class ModelCache(object):
         self.sha_chunksize = sha_chunksize
         self.logger = logger
 
+        # used for stats collection
+        self.stats = None
+
         self._cached_models = dict()
         self._cache_stack = list()
 
@@ -183,11 +196,14 @@ class ModelCache(object):
             model_type=model_type,
             submodel_type=submodel,
         )
-
         # TODO: lock for no copies on simultaneous calls?
         cache_entry = self._cached_models.get(key, None)
         if cache_entry is None:
-            self.logger.info(f"Loading model {model_path}, type {base_model}:{model_type}:{submodel}")
+            self.logger.info(
+                f"Loading model {model_path}, type {base_model.value}:{model_type.value}{':'+submodel.value if submodel else ''}"
+            )
+            if self.stats:
+                self.stats.misses += 1
 
             # this will remove older cached models until
             # there is sufficient room to load the requested model
@@ -201,6 +217,17 @@ class ModelCache(object):
 
             cache_entry = _CacheRecord(self, model, mem_used)
             self._cached_models[key] = cache_entry
+        else:
+            if self.stats:
+                self.stats.hits += 1
+
+        if self.stats:
+            self.stats.cache_size = self.max_cache_size * GIG
+            self.stats.high_watermark = max(self.stats.high_watermark, self._cache_size())
+            self.stats.in_cache = len(self._cached_models)
+            self.stats.loaded_model_sizes[key] = max(
+                self.stats.loaded_model_sizes.get(key, 0), model_info.get_size(submodel)
+            )
 
         with suppress(Exception):
             self._cache_stack.remove(key)
@@ -280,14 +307,14 @@ class ModelCache(object):
         """
         Given the HF repo id or path to a model on disk, returns a unique
         hash. Works for legacy checkpoint files, HF models on disk, and HF repo IDs
+
         :param model_path: Path to model file/directory on disk.
         """
         return self._local_model_hash(model_path)
 
     def cache_size(self) -> float:
-        "Return the current size of the cache, in GB"
-        current_cache_size = sum([m.size for m in self._cached_models.values()])
-        return current_cache_size / GIG
+        """Return the current size of the cache, in GB."""
+        return self._cache_size() / GIG
 
     def _has_cuda(self) -> bool:
         return self.execution_device.type == "cuda"
@@ -310,12 +337,15 @@ class ModelCache(object):
             f"Current VRAM/RAM usage: {vram}/{ram}; cached_models/loaded_models/locked_models/ = {cached_models}/{loaded_models}/{locked_models}"
         )
 
+    def _cache_size(self) -> int:
+        return sum([m.size for m in self._cached_models.values()])
+
     def _make_cache_room(self, model_size):
         # calculate how much memory this model will require
         # multiplier = 2 if self.precision==torch.float32 else 1
         bytes_needed = model_size
         maximum_size = self.max_cache_size * GIG  # stored in GB, convert to bytes
-        current_size = sum([m.size for m in self._cached_models.values()])
+        current_size = self._cache_size()
 
         if current_size + bytes_needed > maximum_size:
             self.logger.debug(
@@ -358,11 +388,14 @@ class ModelCache(object):
             # 2 refs:
             # 1 from cache_entry
             # 1 from getrefcount function
-            if not cache_entry.locked and refs <= 2:
+            # 1 from onnx runtime object
+            if not cache_entry.locked and refs <= 3 if "onnx" in model_key else 2:
                 self.logger.debug(
                     f"Unloading model {model_key} to free {(model_size/GIG):.2f} GB (-{(cache_entry.size/GIG):.2f} GB)"
                 )
                 current_size -= cache_entry.size
+                if self.stats:
+                    self.stats.cleared += 1
                 del self._cache_stack[pos]
                 del self._cached_models[model_key]
                 del cache_entry

invokeai/backend/model_management/model_manager.py

@@ -228,19 +228,19 @@ the root is the InvokeAI ROOTDIR.
 """
 from __future__ import annotations
 
-import os
 import hashlib
+import os
 import textwrap
-import yaml
+import types
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional, List, Tuple, Union, Dict, Set, Callable, types
 from shutil import rmtree, move
+from typing import Optional, List, Literal, Tuple, Union, Dict, Set, Callable
 
 import torch
+import yaml
 from omegaconf import OmegaConf
 from omegaconf.dictconfig import DictConfig
-
 from pydantic import BaseModel, Field
 
 import invokeai.backend.util.logging as logger
@@ -259,6 +259,7 @@ from .models import (
     ModelNotFoundException,
     InvalidModelException,
     DuplicateModelException,
+    ModelBase,
 )
 
 # We are only starting to number the config file with release 3.
@@ -276,7 +277,7 @@ class ModelInfo:
     hash: str
     location: Union[Path, str]
     precision: torch.dtype
-    _cache: ModelCache = None
+    _cache: Optional[ModelCache] = None
 
     def __enter__(self):
         return self.context.__enter__()
@@ -361,7 +362,7 @@ class ModelManager(object):
             if model_key.startswith("_"):
                 continue
             model_name, base_model, model_type = self.parse_key(model_key)
-            model_class = MODEL_CLASSES[base_model][model_type]
+            model_class = self._get_implementation(base_model, model_type)
             # alias for config file
             model_config["model_format"] = model_config.pop("format")
             self.models[model_key] = model_class.create_config(**model_config)
@@ -381,18 +382,24 @@ class ModelManager(object):
         # causing otherwise unreferenced models to be removed from memory
         self._read_models()
 
-    def model_exists(
-        self,
-        model_name: str,
-        base_model: BaseModelType,
-        model_type: ModelType,
-    ) -> bool:
+    def model_exists(self, model_name: str, base_model: BaseModelType, model_type: ModelType, *, rescan=False) -> bool:
         """
-        Given a model name, returns True if it is a valid
-        identifier.
+        Given a model name, returns True if it is a valid identifier.
+
+        :param model_name: symbolic name of the model in models.yaml
+        :param model_type: ModelType enum indicating the type of model to return
+        :param base_model: BaseModelType enum indicating the base model used by this model
+        :param rescan: if True, scan_models_directory
         """
         model_key = self.create_key(model_name, base_model, model_type)
-        return model_key in self.models
+        exists = model_key in self.models
+
+        # if model not found try to find it (maybe file just pasted)
+        if rescan and not exists:
+            self.scan_models_directory(base_model=base_model, model_type=model_type)
+            exists = self.model_exists(model_name, base_model, model_type, rescan=False)
+
+        return exists
 
     @classmethod
     def create_key(
@@ -423,7 +430,7 @@ class ModelManager(object):
         return (model_name, base_model, model_type)
 
     def _get_model_cache_path(self, model_path):
-        return self.app_config.models_path / ".cache" / hashlib.md5(str(model_path).encode()).hexdigest()
+        return self.resolve_model_path(Path(".cache") / hashlib.md5(str(model_path).encode()).hexdigest())
 
     @classmethod
     def initialize_model_config(cls, config_path: Path):
@@ -443,39 +450,32 @@ class ModelManager(object):
         :param model_name: symbolic name of the model in models.yaml
         :param model_type: ModelType enum indicating the type of model to return
         :param base_model: BaseModelType enum indicating the base model used by this model
-        :param submode_typel: an ModelType enum indicating the portion of
+        :param submodel_type: an ModelType enum indicating the portion of
                the model to retrieve (e.g. ModelType.Vae)
         """
-        model_class = MODEL_CLASSES[base_model][model_type]
         model_key = self.create_key(model_name, base_model, model_type)
 
-        # if model not found try to find it (maybe file just pasted)
-        if model_key not in self.models:
-            self.scan_models_directory(base_model=base_model, model_type=model_type)
-            if model_key not in self.models:
-                raise ModelNotFoundException(f"Model not found - {model_key}")
+        if not self.model_exists(model_name, base_model, model_type, rescan=True):
+            raise ModelNotFoundException(f"Model not found - {model_key}")
 
-        model_config = self.models[model_key]
-        model_path = self.app_config.root_path / model_config.path
+        model_config = self._get_model_config(base_model, model_name, model_type)
+
+        model_path, is_submodel_override = self._get_model_path(model_config, submodel_type)
+
+        if is_submodel_override:
+            model_type = submodel_type
+            submodel_type = None
+
+        model_class = self._get_implementation(base_model, model_type)
 
         if not model_path.exists():
             if model_class.save_to_config:
                 self.models[model_key].error = ModelError.NotFound
-                raise Exception(f'Files for model "{model_key}" not found')
+                raise Exception(f'Files for model "{model_key}" not found at {model_path}')
 
             else:
                 self.models.pop(model_key, None)
-                raise ModelNotFoundException(f"Model not found - {model_key}")
-
-        # vae/movq override
-        # TODO:
-        if submodel_type is not None and hasattr(model_config, submodel_type):
-            override_path = getattr(model_config, submodel_type)
-            if override_path:
-                model_path = self.app_config.root_path / override_path
-                model_type = submodel_type
-                submodel_type = None
-                model_class = MODEL_CLASSES[base_model][model_type]
+                raise ModelNotFoundException(f'Files for model "{model_key}" not found at {model_path}')
 
         # TODO: path
         # TODO: is it accurate to use path as id
@@ -513,12 +513,61 @@ class ModelManager(object):
             _cache=self.cache,
         )
 
+    def _get_model_path(
+        self, model_config: ModelConfigBase, submodel_type: Optional[SubModelType] = None
+    ) -> (Path, bool):
+        """Extract a model's filesystem path from its config.
+
+        :return: The fully qualified Path of the module (or submodule).
+        """
+        model_path = model_config.path
+        is_submodel_override = False
+
+        # Does the config explicitly override the submodel?
+        if submodel_type is not None and hasattr(model_config, submodel_type):
+            submodel_path = getattr(model_config, submodel_type)
+            if submodel_path is not None and len(submodel_path) > 0:
+                model_path = getattr(model_config, submodel_type)
+                is_submodel_override = True
+
+        model_path = self.resolve_model_path(model_path)
+        return model_path, is_submodel_override
+
+    def _get_model_config(self, base_model: BaseModelType, model_name: str, model_type: ModelType) -> ModelConfigBase:
+        """Get a model's config object."""
+        model_key = self.create_key(model_name, base_model, model_type)
+        try:
+            model_config = self.models[model_key]
+        except KeyError:
+            raise ModelNotFoundException(f"Model not found - {model_key}")
+        return model_config
+
+    def _get_implementation(self, base_model: BaseModelType, model_type: ModelType) -> type[ModelBase]:
+        """Get the concrete implementation class for a specific model type."""
+        model_class = MODEL_CLASSES[base_model][model_type]
+        return model_class
+
+    def _instantiate(
+        self,
+        model_name: str,
+        base_model: BaseModelType,
+        model_type: ModelType,
+        submodel_type: Optional[SubModelType] = None,
+    ) -> ModelBase:
+        """Make a new instance of this model, without loading it."""
+        model_config = self._get_model_config(base_model, model_name, model_type)
+        model_path, is_submodel_override = self._get_model_path(model_config, submodel_type)
+        # FIXME: do non-overriden submodels get the right class?
+        constructor = self._get_implementation(base_model, model_type)
+        instance = constructor(model_path, base_model, model_type)
+        return instance
+
     def model_info(
         self,
         model_name: str,
         base_model: BaseModelType,
         model_type: ModelType,
-    ) -> dict:
+    ) -> Union[dict, None]:
         """
         Given a model name returns the OmegaConf (dict-like) object describing it.
         """
@@ -540,13 +589,16 @@ class ModelManager(object):
         model_name: str,
         base_model: BaseModelType,
         model_type: ModelType,
-    ) -> dict:
+    ) -> Union[dict, None]:
         """
         Returns a dict describing one installed model, using
         the combined format of the list_models() method.
         """
         models = self.list_models(base_model, model_type, model_name)
-        return models[0] if models else None
+        if len(models) >= 1:
+            return models[0]
+        else:
+            return None
 
     def list_models(
         self,
@@ -560,7 +612,7 @@ class ModelManager(object):
 
         model_keys = (
             [self.create_key(model_name, base_model, model_type)]
-            if model_name
+            if model_name and base_model and model_type
             else sorted(self.models, key=str.casefold)
         )
         models = []
@@ -586,7 +638,7 @@ class ModelManager(object):
 
             # expose paths as absolute to help web UI
             if path := model_dict.get("path"):
-                model_dict["path"] = str(self.app_config.root_path / path)
+                model_dict["path"] = str(self.resolve_model_path(path))
             models.append(model_dict)
 
         return models
@@ -596,7 +648,7 @@ class ModelManager(object):
         Print a table of models and their descriptions. This needs to be redone
         """
         # TODO: redo
-        for model_type, model_dict in self.list_models().items():
+        for model_dict in self.list_models():
             for model_name, model_info in model_dict.items():
                 line = f'{model_info["name"]:25s} {model_info["type"]:10s} {model_info["description"]}'
                 print(line)
@@ -623,7 +675,7 @@ class ModelManager(object):
             self.cache.uncache_model(cache_id)
 
         # if model inside invoke models folder - delete files
-        model_path = self.app_config.root_path / model_cfg.path
+        model_path = self.resolve_model_path(model_cfg.path)
         cache_path = self._get_model_cache_path(model_path)
         if cache_path.exists():
             rmtree(str(cache_path))
@@ -654,12 +706,11 @@ class ModelManager(object):
         The returned dict has the same format as the dict returned by
         model_info().
         """
-        # relativize paths as they go in - this makes it easier to move the root directory around
+        # relativize paths as they go in - this makes it easier to move the models directory around
         if path := model_attributes.get("path"):
-            if Path(path).is_relative_to(self.app_config.root_path):
-                model_attributes["path"] = str(Path(path).relative_to(self.app_config.root_path))
+            model_attributes["path"] = str(self.relative_model_path(Path(path)))
 
-        model_class = MODEL_CLASSES[base_model][model_type]
+        model_class = self._get_implementation(base_model, model_type)
         model_config = model_class.create_config(**model_attributes)
         model_key = self.create_key(model_name, base_model, model_type)
 
@@ -671,7 +722,7 @@ class ModelManager(object):
             # TODO: if path changed and old_model.path inside models folder should we delete this too?
 
             # remove conversion cache as config changed
-            old_model_path = self.app_config.root_path / old_model.path
+            old_model_path = self.resolve_model_path(old_model.path)
             old_model_cache = self._get_model_cache_path(old_model_path)
             if old_model_cache.exists():
                 if old_model_cache.is_dir():
@@ -700,8 +751,8 @@ class ModelManager(object):
         model_name: str,
         base_model: BaseModelType,
         model_type: ModelType,
-        new_name: str = None,
-        new_base: BaseModelType = None,
+        new_name: Optional[str] = None,
+        new_base: Optional[BaseModelType] = None,
     ):
         """
         Rename or rebase a model.
@@ -715,7 +766,7 @@ class ModelManager(object):
         if not model_cfg:
             raise ModelNotFoundException(f"Unknown model: {model_key}")
 
-        old_path = self.app_config.root_path / model_cfg.path
+        old_path = self.resolve_model_path(model_cfg.path)
         new_name = new_name or model_name
         new_base = new_base or base_model
         new_key = self.create_key(new_name, new_base, model_type)
@@ -724,15 +775,15 @@ class ModelManager(object):
 
         # if this is a model file/directory that we manage ourselves, we need to move it
         if old_path.is_relative_to(self.app_config.models_path):
-            new_path = (
-                self.app_config.root_path
-                / "models"
-                / BaseModelType(new_base).value
-                / ModelType(model_type).value
-                / new_name
+            new_path = self.resolve_model_path(
+                Path(
+                    BaseModelType(new_base).value,
+                    ModelType(model_type).value,
+                    new_name,
+                )
             )
             move(old_path, new_path)
-            model_cfg.path = str(new_path.relative_to(self.app_config.root_path))
+            model_cfg.path = str(new_path.relative_to(self.app_config.models_path))
 
         # clean up caches
         old_model_cache = self._get_model_cache_path(old_path)
@@ -754,7 +805,7 @@ class ModelManager(object):
         self,
         model_name: str,
         base_model: BaseModelType,
-        model_type: Union[ModelType.Main, ModelType.Vae],
+        model_type: Literal[ModelType.Main, ModelType.Vae],
         dest_directory: Optional[Path] = None,
     ) -> AddModelResult:
         """
@@ -768,6 +819,10 @@ class ModelManager(object):
         This will raise a ValueError unless the model is a checkpoint.
         """
         info = self.model_info(model_name, base_model, model_type)
+
+        if info is None:
+            raise FileNotFoundError(f"model not found: {model_name}")
+
         if info["model_format"] != "checkpoint":
             raise ValueError(f"not a checkpoint format model: {model_name}")
 
@@ -781,8 +836,8 @@ class ModelManager(object):
             model_type,
             **submodel,
         )
-        checkpoint_path = self.app_config.root_path / info["path"]
-        old_diffusers_path = self.app_config.models_path / model.location
+        checkpoint_path = self.resolve_model_path(info["path"])
+        old_diffusers_path = self.resolve_model_path(model.location)
         new_diffusers_path = (
             dest_directory or self.app_config.models_path / base_model.value / model_type.value
         ) / model_name
@@ -795,7 +850,7 @@ class ModelManager(object):
             info["path"] = (
                 str(new_diffusers_path)
                 if dest_directory
-                else str(new_diffusers_path.relative_to(self.app_config.root_path))
+                else str(new_diffusers_path.relative_to(self.app_config.models_path))
             )
             info.pop("config")
 
@@ -810,6 +865,15 @@ class ModelManager(object):
 
         return result
 
+    def resolve_model_path(self, path: Union[Path, str]) -> Path:
+        """return relative paths based on configured models_path"""
+        return self.app_config.models_path / path
+
+    def relative_model_path(self, model_path: Path) -> Path:
+        if model_path.is_relative_to(self.app_config.models_path):
+            model_path = model_path.relative_to(self.app_config.models_path)
+        return model_path
+
     def search_models(self, search_folder):
         self.logger.info(f"Finding Models In: {search_folder}")
         models_folder_ckpt = Path(search_folder).glob("**/*.ckpt")
@@ -828,7 +892,7 @@ class ModelManager(object):
 
         return search_folder, found_models
 
-    def commit(self, conf_file: Path = None) -> None:
+    def commit(self, conf_file: Optional[Path] = None) -> None:
         """
         Write current configuration out to the indicated file.
         """
@@ -837,7 +901,7 @@ class ModelManager(object):
 
         for model_key, model_config in self.models.items():
             model_name, base_model, model_type = self.parse_key(model_key)
-            model_class = MODEL_CLASSES[base_model][model_type]
+            model_class = self._get_implementation(base_model, model_type)
             if model_class.save_to_config:
                 # TODO: or exclude_unset better fits here?
                 data_to_save[model_key] = model_config.dict(exclude_defaults=True, exclude={"error"})
@@ -883,12 +947,19 @@ class ModelManager(object):
         new_models_found = False
 
         self.logger.info(f"Scanning {self.app_config.models_path} for new models")
-        with Chdir(self.app_config.root_path):
+        with Chdir(self.app_config.models_path):
             for model_key, model_config in list(self.models.items()):
                 model_name, cur_base_model, cur_model_type = self.parse_key(model_key)
-                model_path = self.app_config.root_path.absolute() / model_config.path
+
+                # Patch for relative path bug in older models.yaml - paths should not
+                # be starting with a hard-coded 'models'. This will also fix up
+                # models.yaml when committed.
+                if model_config.path.startswith("models"):
+                    model_config.path = str(Path(*Path(model_config.path).parts[1:]))
+
+                model_path = self.resolve_model_path(model_config.path).absolute()
                 if not model_path.exists():
-                    model_class = MODEL_CLASSES[cur_base_model][cur_model_type]
+                    model_class = self._get_implementation(cur_base_model, cur_model_type)
                     if model_class.save_to_config:
                         model_config.error = ModelError.NotFound
                         self.models.pop(model_key, None)
@@ -904,8 +975,8 @@ class ModelManager(object):
                 for cur_model_type in ModelType:
                     if model_type is not None and cur_model_type != model_type:
                         continue
-                    model_class = MODEL_CLASSES[cur_base_model][cur_model_type]
-                    models_dir = self.app_config.models_path / cur_base_model.value / cur_model_type.value
+                    model_class = self._get_implementation(cur_base_model, cur_model_type)
+                    models_dir = self.resolve_model_path(Path(cur_base_model.value, cur_model_type.value))
 
                     if not models_dir.exists():
                         continue  # TODO: or create all folders?
@@ -919,10 +990,10 @@ class ModelManager(object):
                                 if model_key in self.models:
                                     raise DuplicateModelException(f"Model with key {model_key} added twice")
 
-                                if model_path.is_relative_to(self.app_config.root_path):
-                                    model_path = model_path.relative_to(self.app_config.root_path)
-
-                                model_config: ModelConfigBase = model_class.probe_config(str(model_path))
+                                model_path = self.relative_model_path(model_path)
+                                model_config: ModelConfigBase = model_class.probe_config(
+                                    str(model_path), model_base=cur_base_model
+                                )
                                 self.models[model_key] = model_config
                                 new_models_found = True
                             except DuplicateModelException as e:
@@ -932,12 +1003,11 @@ class ModelManager(object):
                             except NotImplementedError as e:
                                 self.logger.warning(e)
 
-        imported_models = self.autoimport()
-
+        imported_models = self.scan_autoimport_directory()
         if (new_models_found or imported_models) and self.config_path:
             self.commit()
 
-    def autoimport(self) -> Dict[str, AddModelResult]:
+    def scan_autoimport_directory(self) -> Dict[str, AddModelResult]:
         """
         Scan the autoimport directory (if defined) and import new models, delete defunct models.
         """
@@ -971,7 +1041,7 @@ class ModelManager(object):
         # LS: hacky
         # Patch in the SD VAE from core so that it is available for use by the UI
         try:
-            self.heuristic_import({config.root_path / "models/core/convert/sd-vae-ft-mse"})
+            self.heuristic_import({str(self.resolve_model_path("core/convert/sd-vae-ft-mse"))})
         except:
             pass
 
@@ -980,7 +1050,7 @@ class ModelManager(object):
             model_manager=self,
             prediction_type_helper=ask_user_for_prediction_type,
         )
-        known_paths = {config.root_path / x["path"] for x in self.list_models()}
+        known_paths = {self.resolve_model_path(x["path"]) for x in self.list_models()}
         directories = {
             config.root_path / x
             for x in [
@@ -999,7 +1069,7 @@ class ModelManager(object):
     def heuristic_import(
         self,
         items_to_import: Set[str],
-        prediction_type_helper: Callable[[Path], SchedulerPredictionType] = None,
+        prediction_type_helper: Optional[Callable[[Path], SchedulerPredictionType]] = None,
     ) -> Dict[str, AddModelResult]:
         """Import a list of paths, repo_ids or URLs. Returns the set of
         successfully imported items.

invokeai/backend/model_management/model_merge.py

@@ -33,7 +33,7 @@ class ModelMerger(object):
         self,
         model_paths: List[Path],
         alpha: float = 0.5,
-        interp: MergeInterpolationMethod = None,
+        interp: Optional[MergeInterpolationMethod] = None,
         force: bool = False,
         **kwargs,
     ) -> DiffusionPipeline:
@@ -73,7 +73,7 @@ class ModelMerger(object):
         base_model: Union[BaseModelType, str],
         merged_model_name: str,
         alpha: float = 0.5,
-        interp: MergeInterpolationMethod = None,
+        interp: Optional[MergeInterpolationMethod] = None,
         force: bool = False,
         merge_dest_directory: Optional[Path] = None,
         **kwargs,
@@ -109,7 +109,7 @@ class ModelMerger(object):
             # pick up the first model's vae
             if mod == model_names[0]:
                 vae = info.get("vae")
-            model_paths.extend([config.root_path / info["path"]])
+            model_paths.extend([(config.root_path / info["path"]).as_posix()])
 
         merge_method = None if interp == "weighted_sum" else MergeInterpolationMethod(interp)
         logger.debug(f"interp = {interp}, merge_method={merge_method}")
@@ -120,11 +120,11 @@ class ModelMerger(object):
             else config.models_path / base_model.value / ModelType.Main.value
         )
         dump_path.mkdir(parents=True, exist_ok=True)
-        dump_path = dump_path / merged_model_name
+        dump_path = (dump_path / merged_model_name).as_posix()
 
-        merged_pipe.save_pretrained(dump_path, safe_serialization=1)
+        merged_pipe.save_pretrained(dump_path, safe_serialization=True)
         attributes = dict(
-            path=str(dump_path),
+            path=dump_path,
             description=f"Merge of models {', '.join(model_names)}",
             model_format="diffusers",
             variant=ModelVariantType.Normal.value,

invokeai/backend/model_management/model_probe.py

@@ -17,6 +17,7 @@ from .models import (
     SilenceWarnings,
     InvalidModelException,
 )
+from .util import lora_token_vector_length
 from .models.base import read_checkpoint_meta
 
 
@@ -27,7 +28,7 @@ class ModelProbeInfo(object):
     variant_type: ModelVariantType
     prediction_type: SchedulerPredictionType
     upcast_attention: bool
-    format: Literal["diffusers", "checkpoint", "lycoris"]
+    format: Literal["diffusers", "checkpoint", "lycoris", "olive", "onnx"]
     image_size: int
 
 
@@ -41,6 +42,7 @@ class ModelProbe(object):
     PROBES = {
         "diffusers": {},
         "checkpoint": {},
+        "onnx": {},
     }
 
     CLASS2TYPE = {
@@ -53,7 +55,9 @@ class ModelProbe(object):
     }
 
     @classmethod
-    def register_probe(cls, format: Literal["diffusers", "checkpoint"], model_type: ModelType, probe_class: ProbeBase):
+    def register_probe(
+        cls, format: Literal["diffusers", "checkpoint", "onnx"], model_type: ModelType, probe_class: ProbeBase
+    ):
         cls.PROBES[format][model_type] = probe_class
 
     @classmethod
@@ -95,6 +99,7 @@ class ModelProbe(object):
                 if format_type == "diffusers"
                 else cls.get_model_type_from_checkpoint(model_path, model)
             )
+            format_type = "onnx" if model_type == ModelType.ONNX else format_type
             probe_class = cls.PROBES[format_type].get(model_type)
             if not probe_class:
                 return None
@@ -168,6 +173,8 @@ class ModelProbe(object):
         if model:
             class_name = model.__class__.__name__
         else:
+            if (folder_path / "unet/model.onnx").exists():
+                return ModelType.ONNX
             if (folder_path / "learned_embeds.bin").exists():
                 return ModelType.TextualInversion
 
@@ -309,21 +316,16 @@ class LoRACheckpointProbe(CheckpointProbeBase):
 
     def get_base_type(self) -> BaseModelType:
         checkpoint = self.checkpoint
-        key1 = "lora_te_text_model_encoder_layers_0_mlp_fc1.lora_down.weight"
-        key2 = "lora_te_text_model_encoder_layers_0_self_attn_k_proj.hada_w1_a"
-        lora_token_vector_length = (
-            checkpoint[key1].shape[1]
-            if key1 in checkpoint
-            else checkpoint[key2].shape[0]
-            if key2 in checkpoint
-            else 768
-        )
-        if lora_token_vector_length == 768:
+        token_vector_length = lora_token_vector_length(checkpoint)
+
+        if token_vector_length == 768:
             return BaseModelType.StableDiffusion1
-        elif lora_token_vector_length == 1024:
+        elif token_vector_length == 1024:
             return BaseModelType.StableDiffusion2
+        elif token_vector_length == 2048:
+            return BaseModelType.StableDiffusionXL
         else:
-            return None
+            raise InvalidModelException(f"Unknown LoRA type: {self.checkpoint_path}")
 
 
 class TextualInversionCheckpointProbe(CheckpointProbeBase):
@@ -460,6 +462,17 @@ class TextualInversionFolderProbe(FolderProbeBase):
         return TextualInversionCheckpointProbe(None, checkpoint=checkpoint).get_base_type()
 
 
+class ONNXFolderProbe(FolderProbeBase):
+    def get_format(self) -> str:
+        return "onnx"
+
+    def get_base_type(self) -> BaseModelType:
+        return BaseModelType.StableDiffusion1
+
+    def get_variant_type(self) -> ModelVariantType:
+        return ModelVariantType.Normal
+
+
 class ControlNetFolderProbe(FolderProbeBase):
     def get_base_type(self) -> BaseModelType:
         config_file = self.folder_path / "config.json"
@@ -468,9 +481,19 @@ class ControlNetFolderProbe(FolderProbeBase):
         with open(config_file, "r") as file:
             config = json.load(file)
         # no obvious way to distinguish between sd2-base and sd2-768
-        return (
-            BaseModelType.StableDiffusion1 if config["cross_attention_dim"] == 768 else BaseModelType.StableDiffusion2
+        dimension = config["cross_attention_dim"]
+        base_model = (
+            BaseModelType.StableDiffusion1
+            if dimension == 768
+            else BaseModelType.StableDiffusion2
+            if dimension == 1024
+            else BaseModelType.StableDiffusionXL
+            if dimension == 2048
+            else None
         )
+        if not base_model:
+            raise InvalidModelException(f"Unable to determine model base for {self.folder_path}")
+        return base_model
 
 
 class LoRAFolderProbe(FolderProbeBase):
@@ -497,3 +520,4 @@ ModelProbe.register_probe("checkpoint", ModelType.Vae, VaeCheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.Lora, LoRACheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.TextualInversion, TextualInversionCheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.ControlNet, ControlNetCheckpointProbe)
+ModelProbe.register_probe("onnx", ModelType.ONNX, ONNXFolderProbe)

invokeai/backend/model_management/models/__init__.py

@@ -23,8 +23,11 @@ from .lora import LoRAModel
 from .controlnet import ControlNetModel  # TODO:
 from .textual_inversion import TextualInversionModel
 
+from .stable_diffusion_onnx import ONNXStableDiffusion1Model, ONNXStableDiffusion2Model
+
 MODEL_CLASSES = {
     BaseModelType.StableDiffusion1: {
+        ModelType.ONNX: ONNXStableDiffusion1Model,
         ModelType.Main: StableDiffusion1Model,
         ModelType.Vae: VaeModel,
         ModelType.Lora: LoRAModel,
@@ -32,6 +35,7 @@ MODEL_CLASSES = {
         ModelType.TextualInversion: TextualInversionModel,
     },
     BaseModelType.StableDiffusion2: {
+        ModelType.ONNX: ONNXStableDiffusion2Model,
         ModelType.Main: StableDiffusion2Model,
         ModelType.Vae: VaeModel,
         ModelType.Lora: LoRAModel,
@@ -45,6 +49,7 @@ MODEL_CLASSES = {
         ModelType.Lora: LoRAModel,
         ModelType.ControlNet: ControlNetModel,
         ModelType.TextualInversion: TextualInversionModel,
+        ModelType.ONNX: ONNXStableDiffusion2Model,
     },
     BaseModelType.StableDiffusionXLRefiner: {
         ModelType.Main: StableDiffusionXLModel,
@@ -53,6 +58,7 @@ MODEL_CLASSES = {
         ModelType.Lora: LoRAModel,
         ModelType.ControlNet: ControlNetModel,
         ModelType.TextualInversion: TextualInversionModel,
+        ModelType.ONNX: ONNXStableDiffusion2Model,
     },
     # BaseModelType.Kandinsky2_1: {
     #    ModelType.Main: Kandinsky2_1Model,

invokeai/backend/model_management/models/base.py

@@ -8,13 +8,23 @@ from abc import ABCMeta, abstractmethod
 from pathlib import Path
 from picklescan.scanner import scan_file_path
 import torch
+import numpy as np
 import safetensors.torch
-from diffusers import DiffusionPipeline, ConfigMixin
+from pathlib import Path
+from diffusers import DiffusionPipeline, ConfigMixin, OnnxRuntimeModel
 
 from contextlib import suppress
 from pydantic import BaseModel, Field
 from typing import List, Dict, Optional, Type, Literal, TypeVar, Generic, Callable, Any, Union
 
+import onnx
+from onnx import numpy_helper
+from onnxruntime import (
+    InferenceSession,
+    SessionOptions,
+    get_available_providers,
+)
+
 
 class DuplicateModelException(Exception):
     pass
@@ -37,6 +47,7 @@ class BaseModelType(str, Enum):
 
 
 class ModelType(str, Enum):
+    ONNX = "onnx"
     Main = "main"
     Vae = "vae"
     Lora = "lora"
@@ -51,6 +62,8 @@ class SubModelType(str, Enum):
     Tokenizer = "tokenizer"
     Tokenizer2 = "tokenizer_2"
     Vae = "vae"
+    VaeDecoder = "vae_decoder"
+    VaeEncoder = "vae_encoder"
     Scheduler = "scheduler"
     SafetyChecker = "safety_checker"
     # MoVQ = "movq"
@@ -279,8 +292,9 @@ class DiffusersModel(ModelBase):
                 )
                 break
             except Exception as e:
-                # print("====ERR LOAD====")
-                # print(f"{variant}: {e}")
+                if not str(e).startswith("Error no file"):
+                    print("====ERR LOAD====")
+                    print(f"{variant}: {e}")
                 pass
         else:
             raise Exception(f"Failed to load {self.base_model}:{self.model_type}:{child_type} model")
@@ -362,6 +376,8 @@ def calc_model_size_by_data(model) -> int:
         return _calc_pipeline_by_data(model)
     elif isinstance(model, torch.nn.Module):
         return _calc_model_by_data(model)
+    elif isinstance(model, IAIOnnxRuntimeModel):
+        return _calc_onnx_model_by_data(model)
     else:
         return 0
 
@@ -382,6 +398,12 @@ def _calc_model_by_data(model) -> int:
     return mem
 
 
+def _calc_onnx_model_by_data(model) -> int:
+    tensor_size = model.tensors.size() * 2  # The session doubles this
+    mem = tensor_size  # in bytes
+    return mem
+
+
 def _fast_safetensors_reader(path: str):
     checkpoint = dict()
     device = torch.device("meta")
@@ -449,3 +471,208 @@ class SilenceWarnings(object):
         transformers_logging.set_verbosity(self.transformers_verbosity)
         diffusers_logging.set_verbosity(self.diffusers_verbosity)
         warnings.simplefilter("default")
+
+
+ONNX_WEIGHTS_NAME = "model.onnx"
+
+
+class IAIOnnxRuntimeModel:
+    class _tensor_access:
+        def __init__(self, model):
+            self.model = model
+            self.indexes = dict()
+            for idx, obj in enumerate(self.model.proto.graph.initializer):
+                self.indexes[obj.name] = idx
+
+        def __getitem__(self, key: str):
+            value = self.model.proto.graph.initializer[self.indexes[key]]
+            return numpy_helper.to_array(value)
+
+        def __setitem__(self, key: str, value: np.ndarray):
+            new_node = numpy_helper.from_array(value)
+            # set_external_data(new_node, location="in-memory-location")
+            new_node.name = key
+            # new_node.ClearField("raw_data")
+            del self.model.proto.graph.initializer[self.indexes[key]]
+            self.model.proto.graph.initializer.insert(self.indexes[key], new_node)
+            # self.model.data[key] = OrtValue.ortvalue_from_numpy(value)
+
+        # __delitem__
+
+        def __contains__(self, key: str):
+            return self.indexes[key] in self.model.proto.graph.initializer
+
+        def items(self):
+            raise NotImplementedError("tensor.items")
+            # return [(obj.name, obj) for obj in self.raw_proto]
+
+        def keys(self):
+            return self.indexes.keys()
+
+        def values(self):
+            raise NotImplementedError("tensor.values")
+            # return [obj for obj in self.raw_proto]
+
+        def size(self):
+            bytesSum = 0
+            for node in self.model.proto.graph.initializer:
+                bytesSum += sys.getsizeof(node.raw_data)
+            return bytesSum
+
+    class _access_helper:
+        def __init__(self, raw_proto):
+            self.indexes = dict()
+            self.raw_proto = raw_proto
+            for idx, obj in enumerate(raw_proto):
+                self.indexes[obj.name] = idx
+
+        def __getitem__(self, key: str):
+            return self.raw_proto[self.indexes[key]]
+
+        def __setitem__(self, key: str, value):
+            index = self.indexes[key]
+            del self.raw_proto[index]
+            self.raw_proto.insert(index, value)
+
+        # __delitem__
+
+        def __contains__(self, key: str):
+            return key in self.indexes
+
+        def items(self):
+            return [(obj.name, obj) for obj in self.raw_proto]
+
+        def keys(self):
+            return self.indexes.keys()
+
+        def values(self):
+            return [obj for obj in self.raw_proto]
+
+    def __init__(self, model_path: str, provider: Optional[str]):
+        self.path = model_path
+        self.session = None
+        self.provider = provider
+        """
+        self.data_path = self.path + "_data"
+        if not os.path.exists(self.data_path):
+            print(f"Moving model tensors to separate file: {self.data_path}")
+            tmp_proto = onnx.load(model_path, load_external_data=True)
+            onnx.save_model(tmp_proto, self.path, save_as_external_data=True, all_tensors_to_one_file=True, location=os.path.basename(self.data_path), size_threshold=1024, convert_attribute=False)
+            del tmp_proto
+            gc.collect()
+
+        self.proto = onnx.load(model_path, load_external_data=False)
+        """
+
+        self.proto = onnx.load(model_path, load_external_data=True)
+        # self.data = dict()
+        # for tensor in self.proto.graph.initializer:
+        #     name = tensor.name
+
+        #     if tensor.HasField("raw_data"):
+        #         npt = numpy_helper.to_array(tensor)
+        #         orv = OrtValue.ortvalue_from_numpy(npt)
+        #         # self.data[name] = orv
+        #         # set_external_data(tensor, location="in-memory-location")
+        #         tensor.name = name
+        #         # tensor.ClearField("raw_data")
+
+        self.nodes = self._access_helper(self.proto.graph.node)
+        # self.initializers = self._access_helper(self.proto.graph.initializer)
+        # print(self.proto.graph.input)
+        # print(self.proto.graph.initializer)
+
+        self.tensors = self._tensor_access(self)
+
+    # TODO: integrate with model manager/cache
+    def create_session(self, height=None, width=None):
+        if self.session is None or self.session_width != width or self.session_height != height:
+            # onnx.save(self.proto, "tmp.onnx")
+            # onnx.save_model(self.proto, "tmp.onnx", save_as_external_data=True, all_tensors_to_one_file=True, location="tmp.onnx_data", size_threshold=1024, convert_attribute=False)
+            # TODO: something to be able to get weight when they already moved outside of model proto
+            # (trimmed_model, external_data) = buffer_external_data_tensors(self.proto)
+            sess = SessionOptions()
+            # self._external_data.update(**external_data)
+            # sess.add_external_initializers(list(self.data.keys()), list(self.data.values()))
+            # sess.enable_profiling = True
+
+            # sess.intra_op_num_threads = 1
+            # sess.inter_op_num_threads = 1
+            # sess.execution_mode = ExecutionMode.ORT_SEQUENTIAL
+            # sess.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_ALL
+            # sess.enable_cpu_mem_arena = True
+            # sess.enable_mem_pattern = True
+            # sess.add_session_config_entry("session.intra_op.use_xnnpack_threadpool", "1") ########### It's the key code
+            self.session_height = height
+            self.session_width = width
+            if height and width:
+                sess.add_free_dimension_override_by_name("unet_sample_batch", 2)
+                sess.add_free_dimension_override_by_name("unet_sample_channels", 4)
+                sess.add_free_dimension_override_by_name("unet_hidden_batch", 2)
+                sess.add_free_dimension_override_by_name("unet_hidden_sequence", 77)
+                sess.add_free_dimension_override_by_name("unet_sample_height", self.session_height)
+                sess.add_free_dimension_override_by_name("unet_sample_width", self.session_width)
+                sess.add_free_dimension_override_by_name("unet_time_batch", 1)
+            providers = []
+            if self.provider:
+                providers.append(self.provider)
+            else:
+                providers = get_available_providers()
+            if "TensorrtExecutionProvider" in providers:
+                providers.remove("TensorrtExecutionProvider")
+            try:
+                self.session = InferenceSession(self.proto.SerializeToString(), providers=providers, sess_options=sess)
+            except Exception as e:
+                raise e
+            # self.session = InferenceSession("tmp.onnx", providers=[self.provider], sess_options=self.sess_options)
+            # self.io_binding = self.session.io_binding()
+
+    def release_session(self):
+        self.session = None
+        import gc
+
+        gc.collect()
+        return
+
+    def __call__(self, **kwargs):
+        if self.session is None:
+            raise Exception("You should call create_session before running model")
+
+        inputs = {k: np.array(v) for k, v in kwargs.items()}
+        output_names = self.session.get_outputs()
+        # for k in inputs:
+        #     self.io_binding.bind_cpu_input(k, inputs[k])
+        # for name in output_names:
+        #     self.io_binding.bind_output(name.name)
+        # self.session.run_with_iobinding(self.io_binding, None)
+        # return self.io_binding.copy_outputs_to_cpu()
+        return self.session.run(None, inputs)
+
+    # compatability with diffusers load code
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_id: Union[str, Path],
+        subfolder: Union[str, Path] = None,
+        file_name: Optional[str] = None,
+        provider: Optional[str] = None,
+        sess_options: Optional["SessionOptions"] = None,
+        **kwargs,
+    ):
+        file_name = file_name or ONNX_WEIGHTS_NAME
+
+        if os.path.isdir(model_id):
+            model_path = model_id
+            if subfolder is not None:
+                model_path = os.path.join(model_path, subfolder)
+            model_path = os.path.join(model_path, file_name)
+
+        else:
+            model_path = model_id
+
+        # load model from local directory
+        if not os.path.isfile(model_path):
+            raise Exception(f"Model not found: {model_path}")
+
+        # TODO: session options
+        return cls(model_path, provider=provider)

invokeai/backend/model_management/models/controlnet.py

@@ -17,6 +17,7 @@ from .base import (
     ModelNotFoundException,
 )
 from invokeai.app.services.config import InvokeAIAppConfig
+import invokeai.backend.util.logging as logger
 
 
 class ControlNetModelFormat(str, Enum):
@@ -66,7 +67,7 @@ class ControlNetModel(ModelBase):
         child_type: Optional[SubModelType] = None,
     ):
         if child_type is not None:
-            raise Exception("There is no child models in controlnet model")
+            raise Exception("There are no child models in controlnet model")
 
         model = None
         for variant in ["fp16", None]:
@@ -124,9 +125,7 @@ class ControlNetModel(ModelBase):
             return model_path
 
 
-@classmethod
 def _convert_controlnet_ckpt_and_cache(
-    cls,
     model_path: str,
     output_path: str,
     base_model: BaseModelType,
@@ -141,6 +140,7 @@ def _convert_controlnet_ckpt_and_cache(
     weights = app_config.root_path / model_path
     output_path = Path(output_path)
 
+    logger.info(f"Converting {weights} to diffusers format")
     # return cached version if it exists
     if output_path.exists():
         return output_path

invokeai/backend/model_management/models/lora.py

@@ -1,21 +1,23 @@
+import bisect
 import os
-import torch
 from enum import Enum
-from typing import Optional, Union, Literal
+from pathlib import Path
+from typing import Dict, Optional, Union
+
+import torch
+from safetensors.torch import load_file
+
 from .base import (
+    BaseModelType,
+    InvalidModelException,
     ModelBase,
     ModelConfigBase,
-    BaseModelType,
+    ModelNotFoundException,
     ModelType,
     SubModelType,
     classproperty,
-    InvalidModelException,
-    ModelNotFoundException,
 )
 
-# TODO: naming
-from ..lora import LoRAModel as LoRAModelRaw
-
 
 class LoRAModelFormat(str, Enum):
     LyCORIS = "lycoris"
@@ -50,6 +52,7 @@ class LoRAModel(ModelBase):
         model = LoRAModelRaw.from_checkpoint(
             file_path=self.model_path,
             dtype=torch_dtype,
+            base_model=self.base_model,
         )
 
         self.model_size = model.calc_size()
@@ -57,7 +60,7 @@ class LoRAModel(ModelBase):
 
     @classproperty
     def save_to_config(cls) -> bool:
-        return False
+        return True
 
     @classmethod
     def detect_format(cls, path: str):
@@ -87,3 +90,622 @@ class LoRAModel(ModelBase):
             raise NotImplementedError("Diffusers lora not supported")
         else:
             return model_path
+
+
+class LoRALayerBase:
+    # rank: Optional[int]
+    # alpha: Optional[float]
+    # bias: Optional[torch.Tensor]
+    # layer_key: str
+
+    # @property
+    # def scale(self):
+    #    return self.alpha / self.rank if (self.alpha and self.rank) else 1.0
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: dict,
+    ):
+        if "alpha" in values:
+            self.alpha = values["alpha"].item()
+        else:
+            self.alpha = None
+
+        if "bias_indices" in values and "bias_values" in values and "bias_size" in values:
+            self.bias = torch.sparse_coo_tensor(
+                values["bias_indices"],
+                values["bias_values"],
+                tuple(values["bias_size"]),
+            )
+
+        else:
+            self.bias = None
+
+        self.rank = None  # set in layer implementation
+        self.layer_key = layer_key
+
+    def get_weight(self, orig_weight: torch.Tensor):
+        raise NotImplementedError()
+
+    def calc_size(self) -> int:
+        model_size = 0
+        for val in [self.bias]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        if self.bias is not None:
+            self.bias = self.bias.to(device=device, dtype=dtype)
+
+
+# TODO: find and debug lora/locon with bias
+class LoRALayer(LoRALayerBase):
+    # up: torch.Tensor
+    # mid: Optional[torch.Tensor]
+    # down: torch.Tensor
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: dict,
+    ):
+        super().__init__(layer_key, values)
+
+        self.up = values["lora_up.weight"]
+        self.down = values["lora_down.weight"]
+        if "lora_mid.weight" in values:
+            self.mid = values["lora_mid.weight"]
+        else:
+            self.mid = None
+
+        self.rank = self.down.shape[0]
+
+    def get_weight(self, orig_weight: torch.Tensor):
+        if self.mid is not None:
+            up = self.up.reshape(self.up.shape[0], self.up.shape[1])
+            down = self.down.reshape(self.down.shape[0], self.down.shape[1])
+            weight = torch.einsum("m n w h, i m, n j -> i j w h", self.mid, up, down)
+        else:
+            weight = self.up.reshape(self.up.shape[0], -1) @ self.down.reshape(self.down.shape[0], -1)
+
+        return weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        for val in [self.up, self.mid, self.down]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        super().to(device=device, dtype=dtype)
+
+        self.up = self.up.to(device=device, dtype=dtype)
+        self.down = self.down.to(device=device, dtype=dtype)
+
+        if self.mid is not None:
+            self.mid = self.mid.to(device=device, dtype=dtype)
+
+
+class LoHALayer(LoRALayerBase):
+    # w1_a: torch.Tensor
+    # w1_b: torch.Tensor
+    # w2_a: torch.Tensor
+    # w2_b: torch.Tensor
+    # t1: Optional[torch.Tensor] = None
+    # t2: Optional[torch.Tensor] = None
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: dict,
+    ):
+        super().__init__(layer_key, values)
+
+        self.w1_a = values["hada_w1_a"]
+        self.w1_b = values["hada_w1_b"]
+        self.w2_a = values["hada_w2_a"]
+        self.w2_b = values["hada_w2_b"]
+
+        if "hada_t1" in values:
+            self.t1 = values["hada_t1"]
+        else:
+            self.t1 = None
+
+        if "hada_t2" in values:
+            self.t2 = values["hada_t2"]
+        else:
+            self.t2 = None
+
+        self.rank = self.w1_b.shape[0]
+
+    def get_weight(self, orig_weight: torch.Tensor):
+        if self.t1 is None:
+            weight = (self.w1_a @ self.w1_b) * (self.w2_a @ self.w2_b)
+
+        else:
+            rebuild1 = torch.einsum("i j k l, j r, i p -> p r k l", self.t1, self.w1_b, self.w1_a)
+            rebuild2 = torch.einsum("i j k l, j r, i p -> p r k l", self.t2, self.w2_b, self.w2_a)
+            weight = rebuild1 * rebuild2
+
+        return weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        for val in [self.w1_a, self.w1_b, self.w2_a, self.w2_b, self.t1, self.t2]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        super().to(device=device, dtype=dtype)
+
+        self.w1_a = self.w1_a.to(device=device, dtype=dtype)
+        self.w1_b = self.w1_b.to(device=device, dtype=dtype)
+        if self.t1 is not None:
+            self.t1 = self.t1.to(device=device, dtype=dtype)
+
+        self.w2_a = self.w2_a.to(device=device, dtype=dtype)
+        self.w2_b = self.w2_b.to(device=device, dtype=dtype)
+        if self.t2 is not None:
+            self.t2 = self.t2.to(device=device, dtype=dtype)
+
+
+class LoKRLayer(LoRALayerBase):
+    # w1: Optional[torch.Tensor] = None
+    # w1_a: Optional[torch.Tensor] = None
+    # w1_b: Optional[torch.Tensor] = None
+    # w2: Optional[torch.Tensor] = None
+    # w2_a: Optional[torch.Tensor] = None
+    # w2_b: Optional[torch.Tensor] = None
+    # t2: Optional[torch.Tensor] = None
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: dict,
+    ):
+        super().__init__(layer_key, values)
+
+        if "lokr_w1" in values:
+            self.w1 = values["lokr_w1"]
+            self.w1_a = None
+            self.w1_b = None
+        else:
+            self.w1 = None
+            self.w1_a = values["lokr_w1_a"]
+            self.w1_b = values["lokr_w1_b"]
+
+        if "lokr_w2" in values:
+            self.w2 = values["lokr_w2"]
+            self.w2_a = None
+            self.w2_b = None
+        else:
+            self.w2 = None
+            self.w2_a = values["lokr_w2_a"]
+            self.w2_b = values["lokr_w2_b"]
+
+        if "lokr_t2" in values:
+            self.t2 = values["lokr_t2"]
+        else:
+            self.t2 = None
+
+        if "lokr_w1_b" in values:
+            self.rank = values["lokr_w1_b"].shape[0]
+        elif "lokr_w2_b" in values:
+            self.rank = values["lokr_w2_b"].shape[0]
+        else:
+            self.rank = None  # unscaled
+
+    def get_weight(self, orig_weight: torch.Tensor):
+        w1 = self.w1
+        if w1 is None:
+            w1 = self.w1_a @ self.w1_b
+
+        w2 = self.w2
+        if w2 is None:
+            if self.t2 is None:
+                w2 = self.w2_a @ self.w2_b
+            else:
+                w2 = torch.einsum("i j k l, i p, j r -> p r k l", self.t2, self.w2_a, self.w2_b)
+
+        if len(w2.shape) == 4:
+            w1 = w1.unsqueeze(2).unsqueeze(2)
+        w2 = w2.contiguous()
+        weight = torch.kron(w1, w2)
+
+        return weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        for val in [self.w1, self.w1_a, self.w1_b, self.w2, self.w2_a, self.w2_b, self.t2]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        super().to(device=device, dtype=dtype)
+
+        if self.w1 is not None:
+            self.w1 = self.w1.to(device=device, dtype=dtype)
+        else:
+            self.w1_a = self.w1_a.to(device=device, dtype=dtype)
+            self.w1_b = self.w1_b.to(device=device, dtype=dtype)
+
+        if self.w2 is not None:
+            self.w2 = self.w2.to(device=device, dtype=dtype)
+        else:
+            self.w2_a = self.w2_a.to(device=device, dtype=dtype)
+            self.w2_b = self.w2_b.to(device=device, dtype=dtype)
+
+        if self.t2 is not None:
+            self.t2 = self.t2.to(device=device, dtype=dtype)
+
+
+class FullLayer(LoRALayerBase):
+    # weight: torch.Tensor
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: dict,
+    ):
+        super().__init__(layer_key, values)
+
+        self.weight = values["diff"]
+
+        if len(values.keys()) > 1:
+            _keys = list(values.keys())
+            _keys.remove("diff")
+            raise NotImplementedError(f"Unexpected keys in lora diff layer: {_keys}")
+
+        self.rank = None  # unscaled
+
+    def get_weight(self, orig_weight: torch.Tensor):
+        return self.weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        model_size += self.weight.nelement() * self.weight.element_size()
+        return model_size
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        super().to(device=device, dtype=dtype)
+
+        self.weight = self.weight.to(device=device, dtype=dtype)
+
+
+class IA3Layer(LoRALayerBase):
+    # weight: torch.Tensor
+    # on_input: torch.Tensor
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: dict,
+    ):
+        super().__init__(layer_key, values)
+
+        self.weight = values["weight"]
+        self.on_input = values["on_input"]
+
+        self.rank = None  # unscaled
+
+    def get_weight(self, orig_weight: torch.Tensor):
+        weight = self.weight
+        if not self.on_input:
+            weight = weight.reshape(-1, 1)
+        return orig_weight * weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        model_size += self.weight.nelement() * self.weight.element_size()
+        model_size += self.on_input.nelement() * self.on_input.element_size()
+        return model_size
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        super().to(device=device, dtype=dtype)
+
+        self.weight = self.weight.to(device=device, dtype=dtype)
+        self.on_input = self.on_input.to(device=device, dtype=dtype)
+
+
+# TODO: rename all methods used in model logic with Info postfix and remove here Raw postfix
+class LoRAModelRaw:  # (torch.nn.Module):
+    _name: str
+    layers: Dict[str, LoRALayer]
+    _device: torch.device
+    _dtype: torch.dtype
+
+    def __init__(
+        self,
+        name: str,
+        layers: Dict[str, LoRALayer],
+        device: torch.device,
+        dtype: torch.dtype,
+    ):
+        self._name = name
+        self._device = device or torch.cpu
+        self._dtype = dtype or torch.float32
+        self.layers = layers
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def dtype(self):
+        return self._dtype
+
+    def to(
+        self,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        # TODO: try revert if exception?
+        for key, layer in self.layers.items():
+            layer.to(device=device, dtype=dtype)
+        self._device = device
+        self._dtype = dtype
+
+    def calc_size(self) -> int:
+        model_size = 0
+        for _, layer in self.layers.items():
+            model_size += layer.calc_size()
+        return model_size
+
+    @classmethod
+    def _convert_sdxl_keys_to_diffusers_format(cls, state_dict):
+        """Convert the keys of an SDXL LoRA state_dict to diffusers format.
+
+        The input state_dict can be in either Stability AI format or diffusers format. If the state_dict is already in
+        diffusers format, then this function will have no effect.
+
+        This function is adapted from:
+        https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L385-L409
+
+        Args:
+            state_dict (Dict[str, Tensor]): The SDXL LoRA state_dict.
+
+        Raises:
+            ValueError: If state_dict contains an unrecognized key, or not all keys could be converted.
+
+        Returns:
+            Dict[str, Tensor]: The diffusers-format state_dict.
+        """
+        converted_count = 0  # The number of Stability AI keys converted to diffusers format.
+        not_converted_count = 0  # The number of keys that were not converted.
+
+        # Get a sorted list of Stability AI UNet keys so that we can efficiently search for keys with matching prefixes.
+        # For example, we want to efficiently find `input_blocks_4_1` in the list when searching for
+        # `input_blocks_4_1_proj_in`.
+        stability_unet_keys = list(SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP)
+        stability_unet_keys.sort()
+
+        new_state_dict = dict()
+        for full_key, value in state_dict.items():
+            if full_key.startswith("lora_unet_"):
+                search_key = full_key.replace("lora_unet_", "")
+                # Use bisect to find the key in stability_unet_keys that *may* match the search_key's prefix.
+                position = bisect.bisect_right(stability_unet_keys, search_key)
+                map_key = stability_unet_keys[position - 1]
+                # Now, check if the map_key *actually* matches the search_key.
+                if search_key.startswith(map_key):
+                    new_key = full_key.replace(map_key, SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP[map_key])
+                    new_state_dict[new_key] = value
+                    converted_count += 1
+                else:
+                    new_state_dict[full_key] = value
+                    not_converted_count += 1
+            elif full_key.startswith("lora_te1_") or full_key.startswith("lora_te2_"):
+                # The CLIP text encoders have the same keys in both Stability AI and diffusers formats.
+                new_state_dict[full_key] = value
+                continue
+            else:
+                raise ValueError(f"Unrecognized SDXL LoRA key prefix: '{full_key}'.")
+
+        if converted_count > 0 and not_converted_count > 0:
+            raise ValueError(
+                f"The SDXL LoRA could only be partially converted to diffusers format. converted={converted_count},"
+                f" not_converted={not_converted_count}"
+            )
+
+        return new_state_dict
+
+    @classmethod
+    def from_checkpoint(
+        cls,
+        file_path: Union[str, Path],
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+        base_model: Optional[BaseModelType] = None,
+    ):
+        device = device or torch.device("cpu")
+        dtype = dtype or torch.float32
+
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+
+        model = cls(
+            device=device,
+            dtype=dtype,
+            name=file_path.stem,  # TODO:
+            layers=dict(),
+        )
+
+        if file_path.suffix == ".safetensors":
+            state_dict = load_file(file_path.absolute().as_posix(), device="cpu")
+        else:
+            state_dict = torch.load(file_path, map_location="cpu")
+
+        state_dict = cls._group_state(state_dict)
+
+        if base_model == BaseModelType.StableDiffusionXL:
+            state_dict = cls._convert_sdxl_keys_to_diffusers_format(state_dict)
+
+        for layer_key, values in state_dict.items():
+            # lora and locon
+            if "lora_down.weight" in values:
+                layer = LoRALayer(layer_key, values)
+
+            # loha
+            elif "hada_w1_b" in values:
+                layer = LoHALayer(layer_key, values)
+
+            # lokr
+            elif "lokr_w1_b" in values or "lokr_w1" in values:
+                layer = LoKRLayer(layer_key, values)
+
+            # diff
+            elif "diff" in values:
+                layer = FullLayer(layer_key, values)
+
+            # ia3
+            elif "weight" in values and "on_input" in values:
+                layer = IA3Layer(layer_key, values)
+
+            else:
+                print(f">> Encountered unknown lora layer module in {model.name}: {layer_key} - {list(values.keys())}")
+                raise Exception("Unknown lora format!")
+
+            # lower memory consumption by removing already parsed layer values
+            state_dict[layer_key].clear()
+
+            layer.to(device=device, dtype=dtype)
+            model.layers[layer_key] = layer
+
+        return model
+
+    @staticmethod
+    def _group_state(state_dict: dict):
+        state_dict_groupped = dict()
+
+        for key, value in state_dict.items():
+            stem, leaf = key.split(".", 1)
+            if stem not in state_dict_groupped:
+                state_dict_groupped[stem] = dict()
+            state_dict_groupped[stem][leaf] = value
+
+        return state_dict_groupped
+
+
+# code from
+# https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L15C1-L97C32
+def make_sdxl_unet_conversion_map():
+    """Create a dict mapping state_dict keys from Stability AI SDXL format to diffusers SDXL format."""
+    unet_conversion_map_layer = []
+
+    for i in range(3):  # num_blocks is 3 in sdxl
+        # loop over downblocks/upblocks
+        for j in range(2):
+            # loop over resnets/attentions for downblocks
+            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
+            sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0."
+            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
+
+            if i < 3:
+                # no attention layers in down_blocks.3
+                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
+                sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1."
+                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
+
+        for j in range(3):
+            # loop over resnets/attentions for upblocks
+            hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}."
+            sd_up_res_prefix = f"output_blocks.{3*i + j}.0."
+            unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))
+
+            # if i > 0: commentout for sdxl
+            # no attention layers in up_blocks.0
+            hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}."
+            sd_up_atn_prefix = f"output_blocks.{3*i + j}.1."
+            unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))
+
+        if i < 3:
+            # no downsample in down_blocks.3
+            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
+            sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op."
+            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
+
+            # no upsample in up_blocks.3
+            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
+            sd_upsample_prefix = f"output_blocks.{3*i + 2}.{2}."  # change for sdxl
+            unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))
+
+    hf_mid_atn_prefix = "mid_block.attentions.0."
+    sd_mid_atn_prefix = "middle_block.1."
+    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
+
+    for j in range(2):
+        hf_mid_res_prefix = f"mid_block.resnets.{j}."
+        sd_mid_res_prefix = f"middle_block.{2*j}."
+        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
+
+    unet_conversion_map_resnet = [
+        # (stable-diffusion, HF Diffusers)
+        ("in_layers.0.", "norm1."),
+        ("in_layers.2.", "conv1."),
+        ("out_layers.0.", "norm2."),
+        ("out_layers.3.", "conv2."),
+        ("emb_layers.1.", "time_emb_proj."),
+        ("skip_connection.", "conv_shortcut."),
+    ]
+
+    unet_conversion_map = []
+    for sd, hf in unet_conversion_map_layer:
+        if "resnets" in hf:
+            for sd_res, hf_res in unet_conversion_map_resnet:
+                unet_conversion_map.append((sd + sd_res, hf + hf_res))
+        else:
+            unet_conversion_map.append((sd, hf))
+
+    for j in range(2):
+        hf_time_embed_prefix = f"time_embedding.linear_{j+1}."
+        sd_time_embed_prefix = f"time_embed.{j*2}."
+        unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix))
+
+    for j in range(2):
+        hf_label_embed_prefix = f"add_embedding.linear_{j+1}."
+        sd_label_embed_prefix = f"label_emb.0.{j*2}."
+        unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix))
+
+    unet_conversion_map.append(("input_blocks.0.0.", "conv_in."))
+    unet_conversion_map.append(("out.0.", "conv_norm_out."))
+    unet_conversion_map.append(("out.2.", "conv_out."))
+
+    return unet_conversion_map
+
+
+SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP = {
+    sd.rstrip(".").replace(".", "_"): hf.rstrip(".").replace(".", "_") for sd, hf in make_sdxl_unet_conversion_map()
+}

invokeai/backend/model_management/models/sdxl.py

@@ -1,6 +1,5 @@
 import os
 import json
-import invokeai.backend.util.logging as logger
 from enum import Enum
 from pydantic import Field
 from typing import Literal, Optional
@@ -12,6 +11,7 @@ from .base import (
     DiffusersModel,
     read_checkpoint_meta,
     classproperty,
+    InvalidModelException,
 )
 from omegaconf import OmegaConf
 
@@ -65,7 +65,7 @@ class StableDiffusionXLModel(DiffusersModel):
                 in_channels = unet_config["in_channels"]
 
             else:
-                raise Exception("Not supported stable diffusion diffusers format(possibly onnx?)")
+                raise InvalidModelException(f"{path} is not a recognized Stable Diffusion diffusers model")
 
         else:
             raise NotImplementedError(f"Unknown stable diffusion 2.* format: {model_format}")
@@ -80,8 +80,10 @@ class StableDiffusionXLModel(DiffusersModel):
             raise Exception("Unkown stable diffusion 2.* model format")
 
         if ckpt_config_path is None:
-            # TO DO: implement picking
-            pass
+            # avoid circular import
+            from .stable_diffusion import _select_ckpt_config
+
+            ckpt_config_path = _select_ckpt_config(kwargs.get("model_base", BaseModelType.StableDiffusionXL), variant)
 
         return cls.create_config(
             path=path,

invokeai/backend/model_management/models/stable_diffusion.py

@@ -4,6 +4,7 @@ from enum import Enum
 from pydantic import Field
 from pathlib import Path
 from typing import Literal, Optional, Union
+from diffusers import StableDiffusionInpaintPipeline, StableDiffusionPipeline
 from .base import (
     ModelConfigBase,
     BaseModelType,
@@ -123,6 +124,7 @@ class StableDiffusion1Model(DiffusersModel):
             return _convert_ckpt_and_cache(
                 version=BaseModelType.StableDiffusion1,
                 model_config=config,
+                load_safety_checker=False,
                 output_path=output_path,
             )
         else:
@@ -259,9 +261,11 @@ def _convert_ckpt_and_cache(
     """
     app_config = InvokeAIAppConfig.get_config()
 
-    weights = app_config.root_path / model_config.path
+    weights = app_config.models_path / model_config.path
     config_file = app_config.root_path / model_config.config
     output_path = Path(output_path)
+    variant = model_config.variant
+    pipeline_class = StableDiffusionInpaintPipeline if variant == "inpaint" else StableDiffusionPipeline
 
     # return cached version if it exists
     if output_path.exists():
@@ -288,6 +292,7 @@ def _convert_ckpt_and_cache(
             original_config_file=config_file,
             extract_ema=True,
             scan_needed=True,
+            pipeline_class=pipeline_class,
             from_safetensors=weights.suffix == ".safetensors",
             precision=torch_dtype(choose_torch_device()),
             **kwargs,

invokeai/backend/model_management/models/stable_diffusion_onnx.py

@@ -0,0 +1,157 @@
+import os
+import json
+from enum import Enum
+from pydantic import Field
+from pathlib import Path
+from typing import Literal, Optional, Union
+from .base import (
+    ModelBase,
+    ModelConfigBase,
+    BaseModelType,
+    ModelType,
+    SubModelType,
+    ModelVariantType,
+    DiffusersModel,
+    SchedulerPredictionType,
+    SilenceWarnings,
+    read_checkpoint_meta,
+    classproperty,
+    OnnxRuntimeModel,
+    IAIOnnxRuntimeModel,
+)
+from invokeai.app.services.config import InvokeAIAppConfig
+
+
+class StableDiffusionOnnxModelFormat(str, Enum):
+    Olive = "olive"
+    Onnx = "onnx"
+
+
+class ONNXStableDiffusion1Model(DiffusersModel):
+    class Config(ModelConfigBase):
+        model_format: Literal[StableDiffusionOnnxModelFormat.Onnx]
+        variant: ModelVariantType
+
+    def __init__(self, model_path: str, base_model: BaseModelType, model_type: ModelType):
+        assert base_model == BaseModelType.StableDiffusion1
+        assert model_type == ModelType.ONNX
+        super().__init__(
+            model_path=model_path,
+            base_model=BaseModelType.StableDiffusion1,
+            model_type=ModelType.ONNX,
+        )
+
+        for child_name, child_type in self.child_types.items():
+            if child_type is OnnxRuntimeModel:
+                self.child_types[child_name] = IAIOnnxRuntimeModel
+
+            # TODO: check that no optimum models provided
+
+    @classmethod
+    def probe_config(cls, path: str, **kwargs):
+        model_format = cls.detect_format(path)
+        in_channels = 4  # TODO:
+
+        if in_channels == 9:
+            variant = ModelVariantType.Inpaint
+        elif in_channels == 4:
+            variant = ModelVariantType.Normal
+        else:
+            raise Exception("Unkown stable diffusion 1.* model format")
+
+        return cls.create_config(
+            path=path,
+            model_format=model_format,
+            variant=variant,
+        )
+
+    @classproperty
+    def save_to_config(cls) -> bool:
+        return True
+
+    @classmethod
+    def detect_format(cls, model_path: str):
+        # TODO: Detect onnx vs olive
+        return StableDiffusionOnnxModelFormat.Onnx
+
+    @classmethod
+    def convert_if_required(
+        cls,
+        model_path: str,
+        output_path: str,
+        config: ModelConfigBase,
+        base_model: BaseModelType,
+    ) -> str:
+        return model_path
+
+
+class ONNXStableDiffusion2Model(DiffusersModel):
+    # TODO: check that configs overwriten properly
+    class Config(ModelConfigBase):
+        model_format: Literal[StableDiffusionOnnxModelFormat.Onnx]
+        variant: ModelVariantType
+        prediction_type: SchedulerPredictionType
+        upcast_attention: bool
+
+    def __init__(self, model_path: str, base_model: BaseModelType, model_type: ModelType):
+        assert base_model == BaseModelType.StableDiffusion2
+        assert model_type == ModelType.ONNX
+        super().__init__(
+            model_path=model_path,
+            base_model=BaseModelType.StableDiffusion2,
+            model_type=ModelType.ONNX,
+        )
+
+        for child_name, child_type in self.child_types.items():
+            if child_type is OnnxRuntimeModel:
+                self.child_types[child_name] = IAIOnnxRuntimeModel
+            # TODO: check that no optimum models provided
+
+    @classmethod
+    def probe_config(cls, path: str, **kwargs):
+        model_format = cls.detect_format(path)
+        in_channels = 4  # TODO:
+
+        if in_channels == 9:
+            variant = ModelVariantType.Inpaint
+        elif in_channels == 5:
+            variant = ModelVariantType.Depth
+        elif in_channels == 4:
+            variant = ModelVariantType.Normal
+        else:
+            raise Exception("Unkown stable diffusion 2.* model format")
+
+        if variant == ModelVariantType.Normal:
+            prediction_type = SchedulerPredictionType.VPrediction
+            upcast_attention = True
+
+        else:
+            prediction_type = SchedulerPredictionType.Epsilon
+            upcast_attention = False
+
+        return cls.create_config(
+            path=path,
+            model_format=model_format,
+            variant=variant,
+            prediction_type=prediction_type,
+            upcast_attention=upcast_attention,
+        )
+
+    @classproperty
+    def save_to_config(cls) -> bool:
+        return True
+
+    @classmethod
+    def detect_format(cls, model_path: str):
+        # TODO: Detect onnx vs olive
+        return StableDiffusionOnnxModelFormat.Onnx
+
+    @classmethod
+    def convert_if_required(
+        cls,
+        model_path: str,
+        output_path: str,
+        config: ModelConfigBase,
+        base_model: BaseModelType,
+    ) -> str:
+        return model_path