diff --git a/docs/help/FAQ.md b/docs/help/FAQ.md index 28401b5661..4c297f442a 100644 --- a/docs/help/FAQ.md +++ b/docs/help/FAQ.md @@ -18,12 +18,47 @@ Note that any releases marked as _pre-release_ are in a beta state. You may expe The Model Manager tab in the UI provides a few ways to install models, including using your already-downloaded models. You'll see a popup directing you there on first startup. For more information, see the [model install docs]. +## Missing models after updating to v4 + +If you find some models are missing after updating to v4, it's likely they weren't correctly registered before the update and didn't get picked up in the migration. + +You can use the `Scan Folder` tab in the Model Manager UI to fix this. The models will either be in the old, now-unused `autoimport` folder, or your `models` folder. + +- Find and copy your install's old `autoimport` folder path, install the main install folder. +- Go to the Model Manager and click `Scan Folder`. +- Paste the path and scan. +- IMPORTANT: Uncheck `Inplace install`. +- Click `Install All` to install all found models, or just install the models you want. + +Next, find and copy your install's `models` folder path (this could be your custom models folder path, or the `models` folder inside the main install folder). + +Follow the same steps to scan and import the missing models. + ## Slow generation - Check the [system requirements] to ensure that your system is capable of generating images. - Check the `ram` setting in `invokeai.yaml`. This setting tells Invoke how much of your system RAM can be used to cache models. Having this too high or too low can slow things down. That said, it's generally safest to not set this at all and instead let Invoke manage it. - Check the `vram` setting in `invokeai.yaml`. This setting tells Invoke how much of your GPU VRAM can be used to cache models. Counter-intuitively, if this setting is too high, Invoke will need to do a lot of shuffling of models as it juggles the VRAM cache and the currently-loaded model. The default value of 0.25 is generally works well for GPUs without 16GB or more VRAM. Even on a 24GB card, the default works well. - Check that your generations are happening on your GPU (if you have one). InvokeAI will log what is being used for generation upon startup. If your GPU isn't used, re-install to ensure the correct versions of torch get installed. +- If you are on Windows, you may have exceeded your GPU's VRAM capacity and are using slower [shared GPU memory](#shared-gpu-memory-windows). There's a guide to opt out of this behaviour in the linked FAQ entry. + +## Shared GPU Memory (Windows) + +!!! tip "Nvidia GPUs with driver 536.40" + + This only applies to current Nvidia cards with driver 536.40 or later, released in June 2023. + +When the GPU doesn't have enough VRAM for a task, Windows is able to allocate some of its CPU RAM to the GPU. This is much slower than VRAM, but it does allow the system to generate when it otherwise might no have enough VRAM. + +When shared GPU memory is used, generation slows down dramatically - but at least it doesn't crash. + +If you'd like to opt out of this behavior and instead get an error when you exceed your GPU's VRAM, follow [this guide from Nvidia](https://nvidia.custhelp.com/app/answers/detail/a_id/5490). + +Here's how to get the python path required in the linked guide: + +- Run `invoke.bat`. +- Select option 2 for developer console. +- At least one python path will be printed. Copy the path that includes your invoke installation directory (typically the first). ## Installer cannot find python (Windows) diff --git a/docs/installation/010_INSTALL_AUTOMATED.md b/docs/installation/010_INSTALL_AUTOMATED.md index 6b96ec0fe8..5e2db65d7b 100644 --- a/docs/installation/010_INSTALL_AUTOMATED.md +++ b/docs/installation/010_INSTALL_AUTOMATED.md @@ -44,7 +44,7 @@ The installation process is simple, with a few prompts: - Select the version to install. Unless you have a specific reason to install a specific version, select the default (the latest version). - Select location for the install. Be sure you have enough space in this folder for the base application, as described in the [installation requirements]. -- Select a GPU device. If you are unsure, you can let the installer figure it out. +- Select a GPU device. !!! info "Slow Installation" diff --git a/docs/installation/020_INSTALL_MANUAL.md b/docs/installation/020_INSTALL_MANUAL.md index 6370c7d5f2..36859a5795 100644 --- a/docs/installation/020_INSTALL_MANUAL.md +++ b/docs/installation/020_INSTALL_MANUAL.md @@ -6,11 +6,7 @@ ## Introduction -!!! tip "Conda" - - As of InvokeAI v2.3.0 installation using the `conda` package manager is no longer being supported. It will likely still work, but we are not testing this installation method. - -InvokeAI is distributed as a python package on PyPI, installable with `pip`. There are a few things that are handled by the installer that you'll need to manage manually, described in this guide. +InvokeAI is distributed as a python package on PyPI, installable with `pip`. There are a few things that are handled by the installer and launcher that you'll need to manage manually, described in this guide. ### Requirements @@ -40,11 +36,11 @@ Before you start, go through the [installation requirements]. 1. Enter the root (invokeai) directory and create a virtual Python environment within it named `.venv`. - !!! info "Virtual Environment Location" + !!! warning "Virtual Environment Location" While you may create the virtual environment anywhere in the file system, we recommend that you create it within the root directory as shown here. This allows the application to automatically detect its data directories. - If you choose a different location for the venv, then you must set the `INVOKEAI_ROOT` environment variable or pass the directory using the `--root` CLI arg. + If you choose a different location for the venv, then you _must_ set the `INVOKEAI_ROOT` environment variable or specify the root directory using the `--root` CLI arg. ```terminal cd $INVOKEAI_ROOT @@ -81,31 +77,23 @@ Before you start, go through the [installation requirements]. python3 -m pip install --upgrade pip ``` -1. Install the InvokeAI Package. The `--extra-index-url` option is used to select the correct `torch` backend: +1. Install the InvokeAI Package. The base command is `pip install InvokeAI --use-pep517`, but you may need to change this depending on your system and the desired features. - === "CUDA (NVidia)" + - You may need to provide an [extra index URL]. Select your platform configuration using [this tool on the PyTorch website]. Copy the `--extra-index-url` string from this and append it to your install command. - ```bash - pip install "InvokeAI[xformers]" --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu121 - ``` + !!! example "Install with an extra index URL" - === "ROCm (AMD)" + ```bash + pip install InvokeAI --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu121 + ``` - ```bash - pip install InvokeAI --use-pep517 --extra-index-url https://download.pytorch.org/whl/rocm5.6 - ``` + - If you have a CUDA GPU and want to install with `xformers`, you need to add an option to the package name. Note that `xformers` is not necessary. PyTorch includes an implementation of the SDP attention algorithm with the same performance. - === "CPU (Intel Macs & non-GPU systems)" + !!! example "Install with `xformers`" - ```bash - pip install InvokeAI --use-pep517 --extra-index-url https://download.pytorch.org/whl/cpu - ``` - - === "MPS (Apple Silicon)" - - ```bash - pip install InvokeAI --use-pep517 - ``` + ```bash + pip install "InvokeAI[xformers]" --use-pep517 + ``` 1. Deactivate and reactivate your runtime directory so that the invokeai-specific commands become available in the environment: @@ -126,37 +114,6 @@ Before you start, go through the [installation requirements]. Run `invokeai-web` to start the UI. You must activate the virtual environment before running the app. - If the virtual environment you selected is NOT inside `INVOKEAI_ROOT`, then you must specify the path to the root directory by adding - `--root_dir \path\to\invokeai`. + !!! warning - !!! tip - - You can permanently set the location of the runtime directory - by setting the environment variable `INVOKEAI_ROOT` to the - path of the directory. As mentioned previously, this is - recommended if your virtual environment is located outside of - your runtime directory. - -## Unsupported Conda Install - -Congratulations, you found the "secret" Conda installation instructions. If you really **really** want to use Conda with InvokeAI, you can do so using this unsupported recipe: - -```sh -mkdir ~/invokeai -conda create -n invokeai python=3.11 -conda activate invokeai -# Adjust this as described above for the appropriate torch backend -pip install InvokeAI[xformers] --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu121 -invokeai-web --root ~/invokeai -``` - -The `pip install` command shown in this recipe is for Linux/Windows -systems with an NVIDIA GPU. See step (6) above for the command to use -with other platforms/GPU combinations. If you don't wish to pass the -`--root` argument to `invokeai` with each launch, you may set the -environment variable `INVOKEAI_ROOT` to point to the installation directory. - -Note that if you run into problems with the Conda installation, the InvokeAI -staff will **not** be able to help you out. Caveat Emptor! - -[installation requirements]: INSTALL_REQUIREMENTS.md + If the virtual environment is _not_ inside the root directory, then you _must_ specify the path to the root directory with `--root_dir \path\to\invokeai` or the `INVOKEAI_ROOT` environment variable. diff --git a/installer/lib/installer.py b/installer/lib/installer.py index 45bc764023..11823b413e 100644 --- a/installer/lib/installer.py +++ b/installer/lib/installer.py @@ -3,6 +3,7 @@ InvokeAI installer script """ +import locale import os import platform import re @@ -316,7 +317,9 @@ def upgrade_pip(venv_path: Path) -> str | None: python = str(venv_path.expanduser().resolve() / python) try: - result = subprocess.check_output([python, "-m", "pip", "install", "--upgrade", "pip"]).decode() + result = subprocess.check_output([python, "-m", "pip", "install", "--upgrade", "pip"]).decode( + encoding=locale.getpreferredencoding() + ) except subprocess.CalledProcessError as e: print(e) result = None @@ -404,22 +407,29 @@ def get_torch_source() -> Tuple[str | None, str | None]: # device can be one of: "cuda", "rocm", "cpu", "cuda_and_dml, autodetect" device = select_gpu() + # The correct extra index URLs for torch are inconsistent, see https://pytorch.org/get-started/locally/#start-locally + url = None - optional_modules = "[onnx]" + optional_modules: str | None = None if OS == "Linux": if device.value == "rocm": url = "https://download.pytorch.org/whl/rocm5.6" elif device.value == "cpu": url = "https://download.pytorch.org/whl/cpu" - + elif device.value == "cuda": + # CUDA uses the default PyPi index + optional_modules = "[xformers,onnx-cuda]" elif OS == "Windows": if device.value == "cuda": url = "https://download.pytorch.org/whl/cu121" optional_modules = "[xformers,onnx-cuda]" - if device.value == "cuda_and_dml": - url = "https://download.pytorch.org/whl/cu121" - optional_modules = "[xformers,onnx-directml]" + elif device.value == "cpu": + # CPU uses the default PyPi index, no optional modules + pass + elif OS == "Darwin": + # macOS uses the default PyPi index, no optional modules + pass - # in all other cases, Torch wheels should be coming from PyPi as of Torch 1.13 + # Fall back to defaults return (url, optional_modules) diff --git a/installer/lib/messages.py b/installer/lib/messages.py index 257a587d9c..dcd65a9813 100644 --- a/installer/lib/messages.py +++ b/installer/lib/messages.py @@ -207,10 +207,8 @@ def dest_path(dest: Optional[str | Path] = None) -> Path | None: class GpuType(Enum): CUDA = "cuda" - CUDA_AND_DML = "cuda_and_dml" ROCM = "rocm" CPU = "cpu" - AUTODETECT = "autodetect" def select_gpu() -> GpuType: @@ -226,10 +224,6 @@ def select_gpu() -> GpuType: "an [gold1 b]NVIDIA[/] GPU (using CUDA™)", GpuType.CUDA, ) - nvidia_with_dml = ( - "an [gold1 b]NVIDIA[/] GPU (using CUDA™, and DirectML™ for ONNX) -- ALPHA", - GpuType.CUDA_AND_DML, - ) amd = ( "an [gold1 b]AMD[/] GPU (using ROCm™)", GpuType.ROCM, @@ -238,27 +232,19 @@ def select_gpu() -> GpuType: "Do not install any GPU support, use CPU for generation (slow)", GpuType.CPU, ) - autodetect = ( - "I'm not sure what to choose", - GpuType.AUTODETECT, - ) options = [] if OS == "Windows": - options = [nvidia, nvidia_with_dml, cpu] + options = [nvidia, cpu] if OS == "Linux": options = [nvidia, amd, cpu] elif OS == "Darwin": options = [cpu] - # future CoreML? if len(options) == 1: print(f'Your platform [gold1]{OS}-{ARCH}[/] only supports the "{options[0][1]}" driver. Proceeding with that.') return options[0][1] - # "I don't know" is always added the last option - options.append(autodetect) # type: ignore - options = {str(i): opt for i, opt in enumerate(options, 1)} console.rule(":space_invader: GPU (Graphics Card) selection :space_invader:") @@ -292,11 +278,6 @@ def select_gpu() -> GpuType: ), ) - if options[choice][1] is GpuType.AUTODETECT: - console.print( - "No problem. We will install CUDA support first :crossed_fingers: If Invoke does not detect a GPU, please re-run the installer and select one of the other GPU types." - ) - return options[choice][1] diff --git a/invokeai/app/api/routers/app_info.py b/invokeai/app/api/routers/app_info.py index 4cbdc81b28..21286ac2b0 100644 --- a/invokeai/app/api/routers/app_info.py +++ b/invokeai/app/api/routers/app_info.py @@ -12,7 +12,7 @@ from pydantic import BaseModel, Field from invokeai.app.invocations.upscale import ESRGAN_MODELS from invokeai.app.services.invocation_cache.invocation_cache_common import InvocationCacheStatus -from invokeai.backend.image_util.patchmatch import PatchMatch +from invokeai.backend.image_util.infill_methods.patchmatch import PatchMatch from invokeai.backend.image_util.safety_checker import SafetyChecker from invokeai.backend.util.logging import logging from invokeai.version import __version__ @@ -100,7 +100,7 @@ async def get_app_deps() -> AppDependencyVersions: @app_router.get("/config", operation_id="get_config", status_code=200, response_model=AppConfig) async def get_config() -> AppConfig: - infill_methods = ["tile", "lama", "cv2"] + infill_methods = ["tile", "lama", "cv2", "color"] # TODO: add mosaic back if PatchMatch.patchmatch_available(): infill_methods.append("patchmatch") diff --git a/invokeai/app/api/routers/model_manager.py b/invokeai/app/api/routers/model_manager.py index 015afa4678..7bb0f23dc8 100644 --- a/invokeai/app/api/routers/model_manager.py +++ b/invokeai/app/api/routers/model_manager.py @@ -219,28 +219,13 @@ async def scan_for_models( non_core_model_paths = [p for p in found_model_paths if not p.is_relative_to(core_models_path)] installed_models = ApiDependencies.invoker.services.model_manager.store.search_by_attr() - resolved_installed_model_paths: list[str] = [] - installed_model_sources: list[str] = [] - - # This call lists all installed models. - for model in installed_models: - path = pathlib.Path(model.path) - # If the model has a source, we need to add it to the list of installed sources. - if model.source: - installed_model_sources.append(model.source) - # If the path is not absolute, that means it is in the app models directory, and we need to join it with - # the models path before resolving. - if not path.is_absolute(): - resolved_installed_model_paths.append(str(pathlib.Path(models_path, path).resolve())) - continue - resolved_installed_model_paths.append(str(path.resolve())) scan_results: list[FoundModel] = [] - # Check if the model is installed by comparing the resolved paths, appending to the scan result. + # Check if the model is installed by comparing paths, appending to the scan result. for p in non_core_model_paths: path = str(p) - is_installed = path in resolved_installed_model_paths or path in installed_model_sources + is_installed = any(str(models_path / m.path) == path for m in installed_models) found_model = FoundModel(path=path, is_installed=is_installed) scan_results.append(found_model) except Exception as e: diff --git a/invokeai/app/invocations/infill.py b/invokeai/app/invocations/infill.py index 8d14c0a8fe..418bc62fdc 100644 --- a/invokeai/app/invocations/infill.py +++ b/invokeai/app/invocations/infill.py @@ -1,154 +1,91 @@ -# Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654) and the InvokeAI Team +from abc import abstractmethod +from typing import Literal, get_args -import math -from typing import Literal, Optional, get_args - -import numpy as np -from PIL import Image, ImageOps +from PIL import Image from invokeai.app.invocations.fields import ColorField, ImageField from invokeai.app.invocations.primitives import ImageOutput from invokeai.app.services.shared.invocation_context import InvocationContext -from invokeai.app.util.download_with_progress import download_with_progress_bar from invokeai.app.util.misc import SEED_MAX -from invokeai.backend.image_util.cv2_inpaint import cv2_inpaint -from invokeai.backend.image_util.lama import LaMA -from invokeai.backend.image_util.patchmatch import PatchMatch +from invokeai.backend.image_util.infill_methods.cv2_inpaint import cv2_inpaint +from invokeai.backend.image_util.infill_methods.lama import LaMA +from invokeai.backend.image_util.infill_methods.mosaic import infill_mosaic +from invokeai.backend.image_util.infill_methods.patchmatch import PatchMatch, infill_patchmatch +from invokeai.backend.image_util.infill_methods.tile import infill_tile +from invokeai.backend.util.logging import InvokeAILogger from .baseinvocation import BaseInvocation, invocation from .fields import InputField, WithBoard, WithMetadata from .image import PIL_RESAMPLING_MAP, PIL_RESAMPLING_MODES +logger = InvokeAILogger.get_logger() -def infill_methods() -> list[str]: - methods = ["tile", "solid", "lama", "cv2"] + +def get_infill_methods(): + methods = Literal["tile", "color", "lama", "cv2"] # TODO: add mosaic back if PatchMatch.patchmatch_available(): - methods.insert(0, "patchmatch") + methods = Literal["patchmatch", "tile", "color", "lama", "cv2"] # TODO: add mosaic back return methods -INFILL_METHODS = Literal[tuple(infill_methods())] +INFILL_METHODS = get_infill_methods() DEFAULT_INFILL_METHOD = "patchmatch" if "patchmatch" in get_args(INFILL_METHODS) else "tile" -def infill_lama(im: Image.Image) -> Image.Image: - lama = LaMA() - return lama(im) +class InfillImageProcessorInvocation(BaseInvocation, WithMetadata, WithBoard): + """Base class for invocations that preprocess images for Infilling""" + image: ImageField = InputField(description="The image to process") -def infill_patchmatch(im: Image.Image) -> Image.Image: - if im.mode != "RGBA": - return im + @abstractmethod + def infill(self, image: Image.Image) -> Image.Image: + """Infill the image with the specified method""" + pass - # Skip patchmatch if patchmatch isn't available - if not PatchMatch.patchmatch_available(): - return im + def load_image(self, context: InvocationContext) -> tuple[Image.Image, bool]: + """Process the image to have an alpha channel before being infilled""" + image = context.images.get_pil(self.image.image_name) + has_alpha = True if image.mode == "RGBA" else False + return image, has_alpha - # Patchmatch (note, we may want to expose patch_size? Increasing it significantly impacts performance though) - im_patched_np = PatchMatch.inpaint(im.convert("RGB"), ImageOps.invert(im.split()[-1]), patch_size=3) - im_patched = Image.fromarray(im_patched_np, mode="RGB") - return im_patched + def invoke(self, context: InvocationContext) -> ImageOutput: + # Retrieve and process image to be infilled + input_image, has_alpha = self.load_image(context) + # If the input image has no alpha channel, return it + if has_alpha is False: + return ImageOutput.build(context.images.get_dto(self.image.image_name)) -def infill_cv2(im: Image.Image) -> Image.Image: - return cv2_inpaint(im) + # Perform Infill action + infilled_image = self.infill(input_image) + # Create ImageDTO for Infilled Image + infilled_image_dto = context.images.save(image=infilled_image) -def get_tile_images(image: np.ndarray, width=8, height=8): - _nrows, _ncols, depth = image.shape - _strides = image.strides - - nrows, _m = divmod(_nrows, height) - ncols, _n = divmod(_ncols, width) - if _m != 0 or _n != 0: - return None - - return np.lib.stride_tricks.as_strided( - np.ravel(image), - shape=(nrows, ncols, height, width, depth), - strides=(height * _strides[0], width * _strides[1], *_strides), - writeable=False, - ) - - -def tile_fill_missing(im: Image.Image, tile_size: int = 16, seed: Optional[int] = None) -> Image.Image: - # Only fill if there's an alpha layer - if im.mode != "RGBA": - return im - - a = np.asarray(im, dtype=np.uint8) - - tile_size_tuple = (tile_size, tile_size) - - # Get the image as tiles of a specified size - tiles = get_tile_images(a, *tile_size_tuple).copy() - - # Get the mask as tiles - tiles_mask = tiles[:, :, :, :, 3] - - # Find any mask tiles with any fully transparent pixels (we will be replacing these later) - tmask_shape = tiles_mask.shape - tiles_mask = tiles_mask.reshape(math.prod(tiles_mask.shape)) - n, ny = (math.prod(tmask_shape[0:2])), math.prod(tmask_shape[2:]) - tiles_mask = tiles_mask > 0 - tiles_mask = tiles_mask.reshape((n, ny)).all(axis=1) - - # Get RGB tiles in single array and filter by the mask - tshape = tiles.shape - tiles_all = tiles.reshape((math.prod(tiles.shape[0:2]), *tiles.shape[2:])) - filtered_tiles = tiles_all[tiles_mask] - - if len(filtered_tiles) == 0: - return im - - # Find all invalid tiles and replace with a random valid tile - replace_count = (tiles_mask == False).sum() # noqa: E712 - rng = np.random.default_rng(seed=seed) - tiles_all[np.logical_not(tiles_mask)] = filtered_tiles[rng.choice(filtered_tiles.shape[0], replace_count), :, :, :] - - # Convert back to an image - tiles_all = tiles_all.reshape(tshape) - tiles_all = tiles_all.swapaxes(1, 2) - st = tiles_all.reshape( - ( - math.prod(tiles_all.shape[0:2]), - math.prod(tiles_all.shape[2:4]), - tiles_all.shape[4], - ) - ) - si = Image.fromarray(st, mode="RGBA") - - return si + # Return Infilled Image + return ImageOutput.build(infilled_image_dto) @invocation("infill_rgba", title="Solid Color Infill", tags=["image", "inpaint"], category="inpaint", version="1.2.2") -class InfillColorInvocation(BaseInvocation, WithMetadata, WithBoard): +class InfillColorInvocation(InfillImageProcessorInvocation): """Infills transparent areas of an image with a solid color""" - image: ImageField = InputField(description="The image to infill") color: ColorField = InputField( default=ColorField(r=127, g=127, b=127, a=255), description="The color to use to infill", ) - def invoke(self, context: InvocationContext) -> ImageOutput: - image = context.images.get_pil(self.image.image_name) - + def infill(self, image: Image.Image): solid_bg = Image.new("RGBA", image.size, self.color.tuple()) infilled = Image.alpha_composite(solid_bg, image.convert("RGBA")) - infilled.paste(image, (0, 0), image.split()[-1]) - - image_dto = context.images.save(image=infilled) - - return ImageOutput.build(image_dto) + return infilled @invocation("infill_tile", title="Tile Infill", tags=["image", "inpaint"], category="inpaint", version="1.2.3") -class InfillTileInvocation(BaseInvocation, WithMetadata, WithBoard): +class InfillTileInvocation(InfillImageProcessorInvocation): """Infills transparent areas of an image with tiles of the image""" - image: ImageField = InputField(description="The image to infill") tile_size: int = InputField(default=32, ge=1, description="The tile size (px)") seed: int = InputField( default=0, @@ -157,92 +94,74 @@ class InfillTileInvocation(BaseInvocation, WithMetadata, WithBoard): description="The seed to use for tile generation (omit for random)", ) - def invoke(self, context: InvocationContext) -> ImageOutput: - image = context.images.get_pil(self.image.image_name) - - infilled = tile_fill_missing(image.copy(), seed=self.seed, tile_size=self.tile_size) - infilled.paste(image, (0, 0), image.split()[-1]) - - image_dto = context.images.save(image=infilled) - - return ImageOutput.build(image_dto) + def infill(self, image: Image.Image): + output = infill_tile(image, seed=self.seed, tile_size=self.tile_size) + return output.infilled @invocation( "infill_patchmatch", title="PatchMatch Infill", tags=["image", "inpaint"], category="inpaint", version="1.2.2" ) -class InfillPatchMatchInvocation(BaseInvocation, WithMetadata, WithBoard): +class InfillPatchMatchInvocation(InfillImageProcessorInvocation): """Infills transparent areas of an image using the PatchMatch algorithm""" - image: ImageField = InputField(description="The image to infill") downscale: float = InputField(default=2.0, gt=0, description="Run patchmatch on downscaled image to speedup infill") resample_mode: PIL_RESAMPLING_MODES = InputField(default="bicubic", description="The resampling mode") - def invoke(self, context: InvocationContext) -> ImageOutput: - image = context.images.get_pil(self.image.image_name).convert("RGBA") - + def infill(self, image: Image.Image): resample_mode = PIL_RESAMPLING_MAP[self.resample_mode] - infill_image = image.copy() width = int(image.width / self.downscale) height = int(image.height / self.downscale) - infill_image = infill_image.resize( + + infilled = image.resize( (width, height), resample=resample_mode, ) - - if PatchMatch.patchmatch_available(): - infilled = infill_patchmatch(infill_image) - else: - raise ValueError("PatchMatch is not available on this system") - + infilled = infill_patchmatch(image) infilled = infilled.resize( (image.width, image.height), resample=resample_mode, ) - infilled.paste(image, (0, 0), mask=image.split()[-1]) - # image.paste(infilled, (0, 0), mask=image.split()[-1]) - image_dto = context.images.save(image=infilled) - - return ImageOutput.build(image_dto) + return infilled @invocation("infill_lama", title="LaMa Infill", tags=["image", "inpaint"], category="inpaint", version="1.2.2") -class LaMaInfillInvocation(BaseInvocation, WithMetadata, WithBoard): +class LaMaInfillInvocation(InfillImageProcessorInvocation): """Infills transparent areas of an image using the LaMa model""" - image: ImageField = InputField(description="The image to infill") - - def invoke(self, context: InvocationContext) -> ImageOutput: - image = context.images.get_pil(self.image.image_name) - - # Downloads the LaMa model if it doesn't already exist - download_with_progress_bar( - name="LaMa Inpainting Model", - url="https://github.com/Sanster/models/releases/download/add_big_lama/big-lama.pt", - dest_path=context.config.get().models_path / "core/misc/lama/lama.pt", - ) - - infilled = infill_lama(image.copy()) - - image_dto = context.images.save(image=infilled) - - return ImageOutput.build(image_dto) + def infill(self, image: Image.Image): + lama = LaMA() + return lama(image) @invocation("infill_cv2", title="CV2 Infill", tags=["image", "inpaint"], category="inpaint", version="1.2.2") -class CV2InfillInvocation(BaseInvocation, WithMetadata, WithBoard): +class CV2InfillInvocation(InfillImageProcessorInvocation): """Infills transparent areas of an image using OpenCV Inpainting""" + def infill(self, image: Image.Image): + return cv2_inpaint(image) + + +# @invocation( +# "infill_mosaic", title="Mosaic Infill", tags=["image", "inpaint", "outpaint"], category="inpaint", version="1.0.0" +# ) +class MosaicInfillInvocation(InfillImageProcessorInvocation): + """Infills transparent areas of an image with a mosaic pattern drawing colors from the rest of the image""" + image: ImageField = InputField(description="The image to infill") + tile_width: int = InputField(default=64, description="Width of the tile") + tile_height: int = InputField(default=64, description="Height of the tile") + min_color: ColorField = InputField( + default=ColorField(r=0, g=0, b=0, a=255), + description="The min threshold for color", + ) + max_color: ColorField = InputField( + default=ColorField(r=255, g=255, b=255, a=255), + description="The max threshold for color", + ) - def invoke(self, context: InvocationContext) -> ImageOutput: - image = context.images.get_pil(self.image.image_name) - - infilled = infill_cv2(image.copy()) - - image_dto = context.images.save(image=infilled) - - return ImageOutput.build(image_dto) + def infill(self, image: Image.Image): + return infill_mosaic(image, (self.tile_width, self.tile_height), self.min_color.tuple(), self.max_color.tuple()) diff --git a/invokeai/app/invocations/ip_adapter.py b/invokeai/app/invocations/ip_adapter.py index e302c2b97a..485414d263 100644 --- a/invokeai/app/invocations/ip_adapter.py +++ b/invokeai/app/invocations/ip_adapter.py @@ -1,21 +1,22 @@ from builtins import float -from typing import List, Union +from typing import List, Literal, Union from pydantic import BaseModel, Field, field_validator, model_validator from typing_extensions import Self -from invokeai.app.invocations.baseinvocation import ( - BaseInvocation, - BaseInvocationOutput, - invocation, - invocation_output, -) +from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, OutputField, UIType from invokeai.app.invocations.model import ModelIdentifierField from invokeai.app.invocations.primitives import ImageField from invokeai.app.invocations.util import validate_begin_end_step, validate_weights from invokeai.app.services.shared.invocation_context import InvocationContext -from invokeai.backend.model_manager.config import AnyModelConfig, BaseModelType, IPAdapterConfig, ModelType +from invokeai.backend.model_manager.config import ( + AnyModelConfig, + BaseModelType, + IPAdapterCheckpointConfig, + IPAdapterInvokeAIConfig, + ModelType, +) class IPAdapterField(BaseModel): @@ -48,12 +49,15 @@ class IPAdapterOutput(BaseInvocationOutput): ip_adapter: IPAdapterField = OutputField(description=FieldDescriptions.ip_adapter, title="IP-Adapter") +CLIP_VISION_MODEL_MAP = {"ViT-H": "ip_adapter_sd_image_encoder", "ViT-G": "ip_adapter_sdxl_image_encoder"} + + @invocation("ip_adapter", title="IP-Adapter", tags=["ip_adapter", "control"], category="ip_adapter", version="1.2.2") class IPAdapterInvocation(BaseInvocation): """Collects IP-Adapter info to pass to other nodes.""" # Inputs - image: Union[ImageField, List[ImageField]] = InputField(description="The IP-Adapter image prompt(s).") + image: Union[ImageField, List[ImageField]] = InputField(description="The IP-Adapter image prompt(s).", ui_order=1) ip_adapter_model: ModelIdentifierField = InputField( description="The IP-Adapter model.", title="IP-Adapter Model", @@ -61,7 +65,11 @@ class IPAdapterInvocation(BaseInvocation): ui_order=-1, ui_type=UIType.IPAdapterModel, ) - + clip_vision_model: Literal["ViT-H", "ViT-G"] = InputField( + description="CLIP Vision model to use. Overrides model settings. Mandatory for checkpoint models.", + default="ViT-H", + ui_order=2, + ) weight: Union[float, List[float]] = InputField( default=1, description="The weight given to the IP-Adapter", title="Weight" ) @@ -86,10 +94,16 @@ class IPAdapterInvocation(BaseInvocation): def invoke(self, context: InvocationContext) -> IPAdapterOutput: # Lookup the CLIP Vision encoder that is intended to be used with the IP-Adapter model. ip_adapter_info = context.models.get_config(self.ip_adapter_model.key) - assert isinstance(ip_adapter_info, IPAdapterConfig) - image_encoder_model_id = ip_adapter_info.image_encoder_model_id - image_encoder_model_name = image_encoder_model_id.split("/")[-1].strip() + assert isinstance(ip_adapter_info, (IPAdapterInvokeAIConfig, IPAdapterCheckpointConfig)) + + if isinstance(ip_adapter_info, IPAdapterInvokeAIConfig): + image_encoder_model_id = ip_adapter_info.image_encoder_model_id + image_encoder_model_name = image_encoder_model_id.split("/")[-1].strip() + else: + image_encoder_model_name = CLIP_VISION_MODEL_MAP[self.clip_vision_model] + image_encoder_model = self._get_image_encoder(context, image_encoder_model_name) + return IPAdapterOutput( ip_adapter=IPAdapterField( image=self.image, @@ -102,19 +116,25 @@ class IPAdapterInvocation(BaseInvocation): ) def _get_image_encoder(self, context: InvocationContext, image_encoder_model_name: str) -> AnyModelConfig: - found = False - while not found: + image_encoder_models = context.models.search_by_attrs( + name=image_encoder_model_name, base=BaseModelType.Any, type=ModelType.CLIPVision + ) + + if not len(image_encoder_models) > 0: + context.logger.warning( + f"The image encoder required by this IP Adapter ({image_encoder_model_name}) is not installed. \ + Downloading and installing now. This may take a while." + ) + + installer = context._services.model_manager.install + job = installer.heuristic_import(f"InvokeAI/{image_encoder_model_name}") + installer.wait_for_job(job, timeout=600) # Wait for up to 10 minutes image_encoder_models = context.models.search_by_attrs( name=image_encoder_model_name, base=BaseModelType.Any, type=ModelType.CLIPVision ) - found = len(image_encoder_models) > 0 - if not found: - context.logger.warning( - f"The image encoder required by this IP Adapter ({image_encoder_model_name}) is not installed." - ) - context.logger.warning("Downloading and installing now. This may take a while.") - installer = context._services.model_manager.install - job = installer.heuristic_import(f"InvokeAI/{image_encoder_model_name}") - installer.wait_for_job(job, timeout=600) # wait up to 10 minutes - then raise a TimeoutException - assert len(image_encoder_models) == 1 + + if len(image_encoder_models) == 0: + context.logger.error("Error while fetching CLIP Vision Image Encoder") + assert len(image_encoder_models) == 1 + return image_encoder_models[0] diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index f2e1822c30..764e744a2e 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -44,11 +44,7 @@ from invokeai.app.invocations.fields import ( WithMetadata, ) from invokeai.app.invocations.ip_adapter import IPAdapterField -from invokeai.app.invocations.primitives import ( - DenoiseMaskOutput, - ImageOutput, - LatentsOutput, -) +from invokeai.app.invocations.primitives import DenoiseMaskOutput, ImageOutput, LatentsOutput from invokeai.app.invocations.t2i_adapter import T2IAdapterField from invokeai.app.services.shared.invocation_context import InvocationContext from invokeai.app.util.controlnet_utils import prepare_control_image @@ -76,12 +72,7 @@ from ...backend.stable_diffusion.diffusers_pipeline import ( ) from ...backend.stable_diffusion.schedulers import SCHEDULER_MAP from ...backend.util.devices import choose_precision, choose_torch_device -from .baseinvocation import ( - BaseInvocation, - BaseInvocationOutput, - invocation, - invocation_output, -) +from .baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output from .controlnet_image_processors import ControlField from .model import ModelIdentifierField, UNetField, VAEField @@ -1423,7 +1414,7 @@ class IdealSizeInvocation(BaseInvocation): return tuple((x - x % multiple_of) for x in args) def invoke(self, context: InvocationContext) -> IdealSizeOutput: - unet_config = context.models.get_config(**self.unet.unet.model_dump()) + unet_config = context.models.get_config(self.unet.unet.key) aspect = self.width / self.height dimension: float = 512 if unet_config.base == BaseModelType.StableDiffusion2: diff --git a/invokeai/app/invocations/metadata.py b/invokeai/app/invocations/metadata.py index 6fc72a1c3f..2da482c833 100644 --- a/invokeai/app/invocations/metadata.py +++ b/invokeai/app/invocations/metadata.py @@ -2,16 +2,8 @@ from typing import Any, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field -from invokeai.app.invocations.baseinvocation import ( - BaseInvocation, - BaseInvocationOutput, - invocation, - invocation_output, -) -from invokeai.app.invocations.controlnet_image_processors import ( - CONTROLNET_MODE_VALUES, - CONTROLNET_RESIZE_VALUES, -) +from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output +from invokeai.app.invocations.controlnet_image_processors import CONTROLNET_MODE_VALUES, CONTROLNET_RESIZE_VALUES from invokeai.app.invocations.fields import ( FieldDescriptions, ImageField, @@ -43,6 +35,7 @@ class IPAdapterMetadataField(BaseModel): image: ImageField = Field(description="The IP-Adapter image prompt.") ip_adapter_model: ModelIdentifierField = Field(description="The IP-Adapter model.") + clip_vision_model: Literal["ViT-H", "ViT-G"] = Field(description="The CLIP Vision model") weight: Union[float, list[float]] = Field(description="The weight given to the IP-Adapter") begin_step_percent: float = Field(description="When the IP-Adapter is first applied (% of total steps)") end_step_percent: float = Field(description="When the IP-Adapter is last applied (% of total steps)") diff --git a/invokeai/app/services/config/config_default.py b/invokeai/app/services/config/config_default.py index 247835d533..f453a56584 100644 --- a/invokeai/app/services/config/config_default.py +++ b/invokeai/app/services/config/config_default.py @@ -3,6 +3,7 @@ from __future__ import annotations +import locale import os import re import shutil @@ -317,11 +318,10 @@ class InvokeAIAppConfig(BaseSettings): @staticmethod def find_root() -> Path: """Choose the runtime root directory when not specified on command line or init file.""" - venv = Path(os.environ.get("VIRTUAL_ENV") or ".") if os.environ.get("INVOKEAI_ROOT"): root = Path(os.environ["INVOKEAI_ROOT"]) - elif any((venv.parent / x).exists() for x in [INIT_FILE, LEGACY_INIT_FILE]): - root = (venv.parent).resolve() + elif venv := os.environ.get("VIRTUAL_ENV", None): + root = Path(venv).parent.resolve() else: root = Path("~/invokeai").expanduser().resolve() return root @@ -402,7 +402,7 @@ def load_and_migrate_config(config_path: Path) -> InvokeAIAppConfig: An instance of `InvokeAIAppConfig` with the loaded and migrated settings. """ assert config_path.suffix == ".yaml" - with open(config_path) as file: + with open(config_path, "rt", encoding=locale.getpreferredencoding()) as file: loaded_config_dict = yaml.safe_load(file) assert isinstance(loaded_config_dict, dict) diff --git a/invokeai/app/services/model_install/model_install_default.py b/invokeai/app/services/model_install/model_install_default.py index b557797a4b..20cfc1c4ff 100644 --- a/invokeai/app/services/model_install/model_install_default.py +++ b/invokeai/app/services/model_install/model_install_default.py @@ -1,5 +1,6 @@ """Model installation class.""" +import locale import os import re import signal @@ -323,7 +324,8 @@ class ModelInstallService(ModelInstallServiceBase): legacy_models_yaml_path = Path(self._app_config.root_path, legacy_models_yaml_path) if legacy_models_yaml_path.exists(): - legacy_models_yaml = yaml.safe_load(legacy_models_yaml_path.read_text()) + with open(legacy_models_yaml_path, "rt", encoding=locale.getpreferredencoding()) as file: + legacy_models_yaml = yaml.safe_load(file) yaml_metadata = legacy_models_yaml.pop("__metadata__") yaml_version = yaml_metadata.get("version") @@ -564,7 +566,7 @@ class ModelInstallService(ModelInstallServiceBase): # The model is not in the models directory - we don't need to move it. return model - new_path = (models_dir / model.base.value / model.type.value / model.name).with_suffix(old_path.suffix) + new_path = models_dir / model.base.value / model.type.value / old_path.name if old_path == new_path or new_path.exists() and old_path == new_path.resolve(): return model diff --git a/invokeai/app/services/model_manager/model_manager_default.py b/invokeai/app/services/model_manager/model_manager_default.py index b160ff6fed..de6e5f09d8 100644 --- a/invokeai/app/services/model_manager/model_manager_default.py +++ b/invokeai/app/services/model_manager/model_manager_default.py @@ -80,6 +80,7 @@ class ModelManagerService(ModelManagerServiceBase): ram_cache = ModelCache( max_cache_size=app_config.ram, max_vram_cache_size=app_config.vram, + lazy_offloading=app_config.lazy_offload, logger=logger, execution_device=execution_device, ) diff --git a/invokeai/backend/image_util/__init__.py b/invokeai/backend/image_util/__init__.py index 473ecc4c87..dec2a92150 100644 --- a/invokeai/backend/image_util/__init__.py +++ b/invokeai/backend/image_util/__init__.py @@ -2,7 +2,7 @@ Initialization file for invokeai.backend.image_util methods. """ -from .patchmatch import PatchMatch # noqa: F401 +from .infill_methods.patchmatch import PatchMatch # noqa: F401 from .pngwriter import PngWriter, PromptFormatter, retrieve_metadata, write_metadata # noqa: F401 from .seamless import configure_model_padding # noqa: F401 from .util import InitImageResizer, make_grid # noqa: F401 diff --git a/invokeai/backend/image_util/cv2_inpaint.py b/invokeai/backend/image_util/infill_methods/cv2_inpaint.py similarity index 100% rename from invokeai/backend/image_util/cv2_inpaint.py rename to invokeai/backend/image_util/infill_methods/cv2_inpaint.py diff --git a/invokeai/backend/image_util/lama.py b/invokeai/backend/image_util/infill_methods/lama.py similarity index 82% rename from invokeai/backend/image_util/lama.py rename to invokeai/backend/image_util/infill_methods/lama.py index 5b3fc3a9c7..fa354aeed1 100644 --- a/invokeai/backend/image_util/lama.py +++ b/invokeai/backend/image_util/infill_methods/lama.py @@ -7,6 +7,7 @@ from PIL import Image import invokeai.backend.util.logging as logger from invokeai.app.services.config.config_default import get_config +from invokeai.app.util.download_with_progress import download_with_progress_bar from invokeai.backend.util.devices import choose_torch_device @@ -30,6 +31,14 @@ class LaMA: def __call__(self, input_image: Image.Image, *args: Any, **kwds: Any) -> Any: device = choose_torch_device() model_location = get_config().models_path / "core/misc/lama/lama.pt" + + if not model_location.exists(): + download_with_progress_bar( + name="LaMa Inpainting Model", + url="https://github.com/Sanster/models/releases/download/add_big_lama/big-lama.pt", + dest_path=model_location, + ) + model = load_jit_model(model_location, device) image = np.asarray(input_image.convert("RGB")) diff --git a/invokeai/backend/image_util/infill_methods/mosaic.py b/invokeai/backend/image_util/infill_methods/mosaic.py new file mode 100644 index 0000000000..2715a100d2 --- /dev/null +++ b/invokeai/backend/image_util/infill_methods/mosaic.py @@ -0,0 +1,60 @@ +from typing import Tuple + +import numpy as np +from PIL import Image + + +def infill_mosaic( + image: Image.Image, + tile_shape: Tuple[int, int] = (64, 64), + min_color: Tuple[int, int, int, int] = (0, 0, 0, 0), + max_color: Tuple[int, int, int, int] = (255, 255, 255, 0), +) -> Image.Image: + """ + image:PIL - A PIL Image + tile_shape: Tuple[int,int] - Tile width & Tile Height + min_color: Tuple[int,int,int] - RGB values for the lowest color to clip to (0-255) + max_color: Tuple[int,int,int] - RGB values for the highest color to clip to (0-255) + """ + + np_image = np.array(image) # Convert image to np array + alpha = np_image[:, :, 3] # Get the mask from the alpha channel of the image + non_transparent_pixels = np_image[alpha != 0, :3] # List of non-transparent pixels + + # Create color tiles to paste in the empty areas of the image + tile_width, tile_height = tile_shape + + # Clip the range of colors in the image to a particular spectrum only + r_min, g_min, b_min, _ = min_color + r_max, g_max, b_max, _ = max_color + non_transparent_pixels[:, 0] = np.clip(non_transparent_pixels[:, 0], r_min, r_max) + non_transparent_pixels[:, 1] = np.clip(non_transparent_pixels[:, 1], g_min, g_max) + non_transparent_pixels[:, 2] = np.clip(non_transparent_pixels[:, 2], b_min, b_max) + + tiles = [] + for _ in range(256): + color = non_transparent_pixels[np.random.randint(len(non_transparent_pixels))] + tile = np.zeros((tile_height, tile_width, 3), dtype=np.uint8) + tile[:, :] = color + tiles.append(tile) + + # Fill the transparent area with tiles + filled_image = np.zeros((image.height, image.width, 3), dtype=np.uint8) + + for x in range(image.width): + for y in range(image.height): + tile = tiles[np.random.randint(len(tiles))] + try: + filled_image[ + y - (y % tile_height) : y - (y % tile_height) + tile_height, + x - (x % tile_width) : x - (x % tile_width) + tile_width, + ] = tile + except ValueError: + # Need to handle edge cases - literally + pass + + filled_image = Image.fromarray(filled_image) # Convert the filled tiles image to PIL + image = Image.composite( + image, filled_image, image.split()[-1] + ) # Composite the original image on top of the filled tiles + return image diff --git a/invokeai/backend/image_util/infill_methods/patchmatch.py b/invokeai/backend/image_util/infill_methods/patchmatch.py new file mode 100644 index 0000000000..7e9cdf8fa4 --- /dev/null +++ b/invokeai/backend/image_util/infill_methods/patchmatch.py @@ -0,0 +1,67 @@ +""" +This module defines a singleton object, "patchmatch" that +wraps the actual patchmatch object. It respects the global +"try_patchmatch" attribute, so that patchmatch loading can +be suppressed or deferred +""" + +import numpy as np +from PIL import Image + +import invokeai.backend.util.logging as logger +from invokeai.app.services.config.config_default import get_config + + +class PatchMatch: + """ + Thin class wrapper around the patchmatch function. + """ + + patch_match = None + tried_load: bool = False + + def __init__(self): + super().__init__() + + @classmethod + def _load_patch_match(cls): + if cls.tried_load: + return + if get_config().patchmatch: + from patchmatch import patch_match as pm + + if pm.patchmatch_available: + logger.info("Patchmatch initialized") + cls.patch_match = pm + else: + logger.info("Patchmatch not loaded (nonfatal)") + else: + logger.info("Patchmatch loading disabled") + cls.tried_load = True + + @classmethod + def patchmatch_available(cls) -> bool: + cls._load_patch_match() + if not cls.patch_match: + return False + return cls.patch_match.patchmatch_available + + @classmethod + def inpaint(cls, image: Image.Image) -> Image.Image: + if cls.patch_match is None or not cls.patchmatch_available(): + return image + + np_image = np.array(image) + mask = 255 - np_image[:, :, 3] + infilled = cls.patch_match.inpaint(np_image[:, :, :3], mask, patch_size=3) + return Image.fromarray(infilled, mode="RGB") + + +def infill_patchmatch(image: Image.Image) -> Image.Image: + IS_PATCHMATCH_AVAILABLE = PatchMatch.patchmatch_available() + + if not IS_PATCHMATCH_AVAILABLE: + logger.warning("PatchMatch is not available on this system") + return image + + return PatchMatch.inpaint(image) diff --git a/invokeai/backend/image_util/infill_methods/test_images/source1.webp b/invokeai/backend/image_util/infill_methods/test_images/source1.webp new file mode 100644 index 0000000000..7057eefa85 Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source1.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source10.webp b/invokeai/backend/image_util/infill_methods/test_images/source10.webp new file mode 100644 index 0000000000..f185d52a57 Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source10.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source2.webp b/invokeai/backend/image_util/infill_methods/test_images/source2.webp new file mode 100644 index 0000000000..b25060024a Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source2.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source3.webp b/invokeai/backend/image_util/infill_methods/test_images/source3.webp new file mode 100644 index 0000000000..64227084c7 Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source3.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source4.webp b/invokeai/backend/image_util/infill_methods/test_images/source4.webp new file mode 100644 index 0000000000..66a4260063 Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source4.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source5.webp b/invokeai/backend/image_util/infill_methods/test_images/source5.webp new file mode 100644 index 0000000000..49b87b268f Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source5.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source6.webp b/invokeai/backend/image_util/infill_methods/test_images/source6.webp new file mode 100644 index 0000000000..e16e132004 Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source6.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source7.webp b/invokeai/backend/image_util/infill_methods/test_images/source7.webp new file mode 100644 index 0000000000..723a5fddbd Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source7.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source8.webp b/invokeai/backend/image_util/infill_methods/test_images/source8.webp new file mode 100644 index 0000000000..32a0fea109 Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source8.webp differ diff --git a/invokeai/backend/image_util/infill_methods/test_images/source9.webp b/invokeai/backend/image_util/infill_methods/test_images/source9.webp new file mode 100644 index 0000000000..062f25ed8d Binary files /dev/null and b/invokeai/backend/image_util/infill_methods/test_images/source9.webp differ diff --git a/invokeai/backend/image_util/infill_methods/tile.ipynb b/invokeai/backend/image_util/infill_methods/tile.ipynb new file mode 100644 index 0000000000..eac7a43657 --- /dev/null +++ b/invokeai/backend/image_util/infill_methods/tile.ipynb @@ -0,0 +1,95 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Smoke test for the tile infill\"\"\"\n", + "\n", + "from pathlib import Path\n", + "from typing import Optional\n", + "from PIL import Image\n", + "from invokeai.backend.image_util.infill_methods.tile import infill_tile\n", + "\n", + "images: list[tuple[str, Image.Image]] = []\n", + "\n", + "for i in sorted(Path(\"./test_images/\").glob(\"*.webp\")):\n", + " images.append((i.name, Image.open(i)))\n", + " images.append((i.name, Image.open(i).transpose(Image.FLIP_LEFT_RIGHT)))\n", + " images.append((i.name, Image.open(i).transpose(Image.FLIP_TOP_BOTTOM)))\n", + " images.append((i.name, Image.open(i).resize((512, 512))))\n", + " images.append((i.name, Image.open(i).resize((1234, 461))))\n", + "\n", + "outputs: list[tuple[str, Image.Image, Image.Image, Optional[Image.Image]]] = []\n", + "\n", + "for name, image in images:\n", + " try:\n", + " output = infill_tile(image, seed=0, tile_size=32)\n", + " outputs.append((name, image, output.infilled, output.tile_image))\n", + " except ValueError as e:\n", + " print(f\"Skipping image {name}: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Display the images in jupyter notebook\n", + "import matplotlib.pyplot as plt\n", + "from PIL import ImageOps\n", + "\n", + "fig, axes = plt.subplots(len(outputs), 3, figsize=(10, 3 * len(outputs)))\n", + "plt.subplots_adjust(hspace=0)\n", + "\n", + "for i, (name, original, infilled, tile_image) in enumerate(outputs):\n", + " # Add a border to each image, helps to see the edges\n", + " size = original.size\n", + " original = ImageOps.expand(original, border=5, fill=\"red\")\n", + " filled = ImageOps.expand(infilled, border=5, fill=\"red\")\n", + " if tile_image:\n", + " tile_image = ImageOps.expand(tile_image, border=5, fill=\"red\")\n", + "\n", + " axes[i, 0].imshow(original)\n", + " axes[i, 0].axis(\"off\")\n", + " axes[i, 0].set_title(f\"Original ({name} - {size})\")\n", + "\n", + " if tile_image:\n", + " axes[i, 1].imshow(tile_image)\n", + " axes[i, 1].axis(\"off\")\n", + " axes[i, 1].set_title(\"Tile Image\")\n", + " else:\n", + " axes[i, 1].axis(\"off\")\n", + " axes[i, 1].set_title(\"NO TILES GENERATED (NO TRANSPARENCY)\")\n", + "\n", + " axes[i, 2].imshow(filled)\n", + " axes[i, 2].axis(\"off\")\n", + " axes[i, 2].set_title(\"Filled\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".invokeai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/invokeai/backend/image_util/infill_methods/tile.py b/invokeai/backend/image_util/infill_methods/tile.py new file mode 100644 index 0000000000..03cb6c1a8c --- /dev/null +++ b/invokeai/backend/image_util/infill_methods/tile.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +from typing import Optional + +import numpy as np +from PIL import Image + + +def create_tile_pool(img_array: np.ndarray, tile_size: tuple[int, int]) -> list[np.ndarray]: + """ + Create a pool of tiles from non-transparent areas of the image by systematically walking through the image. + + Args: + img_array: numpy array of the image. + tile_size: tuple (tile_width, tile_height) specifying the size of each tile. + + Returns: + A list of numpy arrays, each representing a tile. + """ + tiles: list[np.ndarray] = [] + rows, cols = img_array.shape[:2] + tile_width, tile_height = tile_size + + for y in range(0, rows - tile_height + 1, tile_height): + for x in range(0, cols - tile_width + 1, tile_width): + tile = img_array[y : y + tile_height, x : x + tile_width] + # Check if the image has an alpha channel and the tile is completely opaque + if img_array.shape[2] == 4 and np.all(tile[:, :, 3] == 255): + tiles.append(tile) + elif img_array.shape[2] == 3: # If no alpha channel, append the tile + tiles.append(tile) + + if not tiles: + raise ValueError( + "Not enough opaque pixels to generate any tiles. Use a smaller tile size or a different image." + ) + + return tiles + + +def create_filled_image( + img_array: np.ndarray, tile_pool: list[np.ndarray], tile_size: tuple[int, int], seed: int +) -> np.ndarray: + """ + Create an image of the same dimensions as the original, filled entirely with tiles from the pool. + + Args: + img_array: numpy array of the original image. + tile_pool: A list of numpy arrays, each representing a tile. + tile_size: tuple (tile_width, tile_height) specifying the size of each tile. + + Returns: + A numpy array representing the filled image. + """ + + rows, cols, _ = img_array.shape + tile_width, tile_height = tile_size + + # Prep an empty RGB image + filled_img_array = np.zeros((rows, cols, 3), dtype=img_array.dtype) + + # Make the random tile selection reproducible + rng = np.random.default_rng(seed) + + for y in range(0, rows, tile_height): + for x in range(0, cols, tile_width): + # Pick a random tile from the pool + tile = tile_pool[rng.integers(len(tile_pool))] + + # Calculate the space available (may be less than tile size near the edges) + space_y = min(tile_height, rows - y) + space_x = min(tile_width, cols - x) + + # Crop the tile if necessary to fit into the available space + cropped_tile = tile[:space_y, :space_x, :3] + + # Fill the available space with the (possibly cropped) tile + filled_img_array[y : y + space_y, x : x + space_x, :3] = cropped_tile + + return filled_img_array + + +@dataclass +class InfillTileOutput: + infilled: Image.Image + tile_image: Optional[Image.Image] = None + + +def infill_tile(image_to_infill: Image.Image, seed: int, tile_size: int) -> InfillTileOutput: + """Infills an image with random tiles from the image itself. + + If the image is not an RGBA image, it is returned untouched. + + Args: + image: The image to infill. + tile_size: The size of the tiles to use for infilling. + + Raises: + ValueError: If there are not enough opaque pixels to generate any tiles. + """ + + if image_to_infill.mode != "RGBA": + return InfillTileOutput(infilled=image_to_infill) + + # Internally, we want a tuple of (tile_width, tile_height). In the future, the tile size can be any rectangle. + _tile_size = (tile_size, tile_size) + np_image = np.array(image_to_infill, dtype=np.uint8) + + # Create the pool of tiles that we will use to infill + tile_pool = create_tile_pool(np_image, _tile_size) + + # Create an image from the tiles, same size as the original + tile_np_image = create_filled_image(np_image, tile_pool, _tile_size, seed) + + # Paste the OG image over the tile image, effectively infilling the area + tile_image = Image.fromarray(tile_np_image, "RGB") + infilled = tile_image.copy() + infilled.paste(image_to_infill, (0, 0), image_to_infill.split()[-1]) + + # I think we want this to be "RGBA"? + infilled.convert("RGBA") + + return InfillTileOutput(infilled=infilled, tile_image=tile_image) diff --git a/invokeai/backend/image_util/patchmatch.py b/invokeai/backend/image_util/patchmatch.py deleted file mode 100644 index 8b7b468397..0000000000 --- a/invokeai/backend/image_util/patchmatch.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -This module defines a singleton object, "patchmatch" that -wraps the actual patchmatch object. It respects the global -"try_patchmatch" attribute, so that patchmatch loading can -be suppressed or deferred -""" - -import numpy as np - -import invokeai.backend.util.logging as logger -from invokeai.app.services.config.config_default import get_config - - -class PatchMatch: - """ - Thin class wrapper around the patchmatch function. - """ - - patch_match = None - tried_load: bool = False - - def __init__(self): - super().__init__() - - @classmethod - def _load_patch_match(self): - if self.tried_load: - return - if get_config().patchmatch: - from patchmatch import patch_match as pm - - if pm.patchmatch_available: - logger.info("Patchmatch initialized") - else: - logger.info("Patchmatch not loaded (nonfatal)") - self.patch_match = pm - else: - logger.info("Patchmatch loading disabled") - self.tried_load = True - - @classmethod - def patchmatch_available(self) -> bool: - self._load_patch_match() - return self.patch_match and self.patch_match.patchmatch_available - - @classmethod - def inpaint(self, *args, **kwargs) -> np.ndarray: - if self.patchmatch_available(): - return self.patch_match.inpaint(*args, **kwargs) diff --git a/invokeai/backend/ip_adapter/ip_adapter.py b/invokeai/backend/ip_adapter/ip_adapter.py index e51966c779..f3be042146 100644 --- a/invokeai/backend/ip_adapter/ip_adapter.py +++ b/invokeai/backend/ip_adapter/ip_adapter.py @@ -1,8 +1,11 @@ # copied from https://github.com/tencent-ailab/IP-Adapter (Apache License 2.0) # and modified as needed -from typing import Optional, Union +import pathlib +from typing import List, Optional, TypedDict, Union +import safetensors +import safetensors.torch import torch from PIL import Image from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection @@ -13,10 +16,17 @@ from ..raw_model import RawModel from .resampler import Resampler +class IPAdapterStateDict(TypedDict): + ip_adapter: dict[str, torch.Tensor] + image_proj: dict[str, torch.Tensor] + + class ImageProjModel(torch.nn.Module): """Image Projection Model""" - def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4): + def __init__( + self, cross_attention_dim: int = 1024, clip_embeddings_dim: int = 1024, clip_extra_context_tokens: int = 4 + ): super().__init__() self.cross_attention_dim = cross_attention_dim @@ -25,7 +35,7 @@ class ImageProjModel(torch.nn.Module): self.norm = torch.nn.LayerNorm(cross_attention_dim) @classmethod - def from_state_dict(cls, state_dict: dict[torch.Tensor], clip_extra_context_tokens=4): + def from_state_dict(cls, state_dict: dict[str, torch.Tensor], clip_extra_context_tokens: int = 4): """Initialize an ImageProjModel from a state_dict. The cross_attention_dim and clip_embeddings_dim are inferred from the shape of the tensors in the state_dict. @@ -45,7 +55,7 @@ class ImageProjModel(torch.nn.Module): model.load_state_dict(state_dict) return model - def forward(self, image_embeds): + def forward(self, image_embeds: torch.Tensor): embeds = image_embeds clip_extra_context_tokens = self.proj(embeds).reshape( -1, self.clip_extra_context_tokens, self.cross_attention_dim @@ -57,7 +67,7 @@ class ImageProjModel(torch.nn.Module): class MLPProjModel(torch.nn.Module): """SD model with image prompt""" - def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024): + def __init__(self, cross_attention_dim: int = 1024, clip_embeddings_dim: int = 1024): super().__init__() self.proj = torch.nn.Sequential( @@ -68,7 +78,7 @@ class MLPProjModel(torch.nn.Module): ) @classmethod - def from_state_dict(cls, state_dict: dict[torch.Tensor]): + def from_state_dict(cls, state_dict: dict[str, torch.Tensor]): """Initialize an MLPProjModel from a state_dict. The cross_attention_dim and clip_embeddings_dim are inferred from the shape of the tensors in the state_dict. @@ -87,7 +97,7 @@ class MLPProjModel(torch.nn.Module): model.load_state_dict(state_dict) return model - def forward(self, image_embeds): + def forward(self, image_embeds: torch.Tensor): clip_extra_context_tokens = self.proj(image_embeds) return clip_extra_context_tokens @@ -97,7 +107,7 @@ class IPAdapter(RawModel): def __init__( self, - state_dict: dict[str, torch.Tensor], + state_dict: IPAdapterStateDict, device: torch.device, dtype: torch.dtype = torch.float16, num_tokens: int = 4, @@ -129,24 +139,27 @@ class IPAdapter(RawModel): return calc_model_size_by_data(self._image_proj_model) + calc_model_size_by_data(self.attn_weights) - def _init_image_proj_model(self, state_dict): + def _init_image_proj_model( + self, state_dict: dict[str, torch.Tensor] + ) -> Union[ImageProjModel, Resampler, MLPProjModel]: return ImageProjModel.from_state_dict(state_dict, self._num_tokens).to(self.device, dtype=self.dtype) @torch.inference_mode() - def get_image_embeds(self, pil_image, image_encoder: CLIPVisionModelWithProjection): - if isinstance(pil_image, Image.Image): - pil_image = [pil_image] + def get_image_embeds(self, pil_image: List[Image.Image], image_encoder: CLIPVisionModelWithProjection): clip_image = self._clip_image_processor(images=pil_image, return_tensors="pt").pixel_values clip_image_embeds = image_encoder(clip_image.to(self.device, dtype=self.dtype)).image_embeds - image_prompt_embeds = self._image_proj_model(clip_image_embeds) - uncond_image_prompt_embeds = self._image_proj_model(torch.zeros_like(clip_image_embeds)) - return image_prompt_embeds, uncond_image_prompt_embeds + try: + image_prompt_embeds = self._image_proj_model(clip_image_embeds) + uncond_image_prompt_embeds = self._image_proj_model(torch.zeros_like(clip_image_embeds)) + return image_prompt_embeds, uncond_image_prompt_embeds + except RuntimeError as e: + raise RuntimeError("Selected CLIP Vision Model is incompatible with the current IP Adapter") from e class IPAdapterPlus(IPAdapter): """IP-Adapter with fine-grained features""" - def _init_image_proj_model(self, state_dict): + def _init_image_proj_model(self, state_dict: dict[str, torch.Tensor]) -> Union[Resampler, MLPProjModel]: return Resampler.from_state_dict( state_dict=state_dict, depth=4, @@ -157,31 +170,32 @@ class IPAdapterPlus(IPAdapter): ).to(self.device, dtype=self.dtype) @torch.inference_mode() - def get_image_embeds(self, pil_image, image_encoder: CLIPVisionModelWithProjection): - if isinstance(pil_image, Image.Image): - pil_image = [pil_image] + def get_image_embeds(self, pil_image: List[Image.Image], image_encoder: CLIPVisionModelWithProjection): clip_image = self._clip_image_processor(images=pil_image, return_tensors="pt").pixel_values clip_image = clip_image.to(self.device, dtype=self.dtype) clip_image_embeds = image_encoder(clip_image, output_hidden_states=True).hidden_states[-2] - image_prompt_embeds = self._image_proj_model(clip_image_embeds) uncond_clip_image_embeds = image_encoder(torch.zeros_like(clip_image), output_hidden_states=True).hidden_states[ -2 ] - uncond_image_prompt_embeds = self._image_proj_model(uncond_clip_image_embeds) - return image_prompt_embeds, uncond_image_prompt_embeds + try: + image_prompt_embeds = self._image_proj_model(clip_image_embeds) + uncond_image_prompt_embeds = self._image_proj_model(uncond_clip_image_embeds) + return image_prompt_embeds, uncond_image_prompt_embeds + except RuntimeError as e: + raise RuntimeError("Selected CLIP Vision Model is incompatible with the current IP Adapter") from e class IPAdapterFull(IPAdapterPlus): """IP-Adapter Plus with full features.""" - def _init_image_proj_model(self, state_dict: dict[torch.Tensor]): + def _init_image_proj_model(self, state_dict: dict[str, torch.Tensor]): return MLPProjModel.from_state_dict(state_dict).to(self.device, dtype=self.dtype) class IPAdapterPlusXL(IPAdapterPlus): """IP-Adapter Plus for SDXL.""" - def _init_image_proj_model(self, state_dict): + def _init_image_proj_model(self, state_dict: dict[str, torch.Tensor]): return Resampler.from_state_dict( state_dict=state_dict, depth=4, @@ -192,24 +206,48 @@ class IPAdapterPlusXL(IPAdapterPlus): ).to(self.device, dtype=self.dtype) -def build_ip_adapter( - ip_adapter_ckpt_path: str, device: torch.device, dtype: torch.dtype = torch.float16 -) -> Union[IPAdapter, IPAdapterPlus]: - state_dict = torch.load(ip_adapter_ckpt_path, map_location="cpu") +def load_ip_adapter_tensors(ip_adapter_ckpt_path: pathlib.Path, device: str) -> IPAdapterStateDict: + state_dict: IPAdapterStateDict = {"ip_adapter": {}, "image_proj": {}} - if "proj.weight" in state_dict["image_proj"]: # IPAdapter (with ImageProjModel). + if ip_adapter_ckpt_path.suffix == ".safetensors": + model = safetensors.torch.load_file(ip_adapter_ckpt_path, device=device) + for key in model.keys(): + if key.startswith("image_proj."): + state_dict["image_proj"][key.replace("image_proj.", "")] = model[key] + elif key.startswith("ip_adapter."): + state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = model[key] + else: + raise RuntimeError(f"Encountered unexpected IP Adapter state dict key: '{key}'.") + else: + ip_adapter_diffusers_checkpoint_path = ip_adapter_ckpt_path / "ip_adapter.bin" + state_dict = torch.load(ip_adapter_diffusers_checkpoint_path, map_location="cpu") + + return state_dict + + +def build_ip_adapter( + ip_adapter_ckpt_path: pathlib.Path, device: torch.device, dtype: torch.dtype = torch.float16 +) -> Union[IPAdapter, IPAdapterPlus, IPAdapterPlusXL, IPAdapterPlus]: + state_dict = load_ip_adapter_tensors(ip_adapter_ckpt_path, device.type) + + # IPAdapter (with ImageProjModel) + if "proj.weight" in state_dict["image_proj"]: return IPAdapter(state_dict, device=device, dtype=dtype) - elif "proj_in.weight" in state_dict["image_proj"]: # IPAdaterPlus or IPAdapterPlusXL (with Resampler). + + # IPAdaterPlus or IPAdapterPlusXL (with Resampler) + elif "proj_in.weight" in state_dict["image_proj"]: cross_attention_dim = state_dict["ip_adapter"]["1.to_k_ip.weight"].shape[-1] if cross_attention_dim == 768: - # SD1 IP-Adapter Plus - return IPAdapterPlus(state_dict, device=device, dtype=dtype) + return IPAdapterPlus(state_dict, device=device, dtype=dtype) # SD1 IP-Adapter Plus elif cross_attention_dim == 2048: - # SDXL IP-Adapter Plus - return IPAdapterPlusXL(state_dict, device=device, dtype=dtype) + return IPAdapterPlusXL(state_dict, device=device, dtype=dtype) # SDXL IP-Adapter Plus else: raise Exception(f"Unsupported IP-Adapter Plus cross-attention dimension: {cross_attention_dim}.") - elif "proj.0.weight" in state_dict["image_proj"]: # IPAdapterFull (with MLPProjModel). + + # IPAdapterFull (with MLPProjModel) + elif "proj.0.weight" in state_dict["image_proj"]: return IPAdapterFull(state_dict, device=device, dtype=dtype) + + # Unrecognized IP Adapter Architectures else: raise ValueError(f"'{ip_adapter_ckpt_path}' has an unrecognized IP-Adapter model architecture.") diff --git a/invokeai/backend/ip_adapter/resampler.py b/invokeai/backend/ip_adapter/resampler.py index a8db22c0fd..a32eeacfdc 100644 --- a/invokeai/backend/ip_adapter/resampler.py +++ b/invokeai/backend/ip_adapter/resampler.py @@ -9,8 +9,8 @@ import torch.nn as nn # FFN -def FeedForward(dim, mult=4): - inner_dim = int(dim * mult) +def FeedForward(dim: int, mult: int = 4): + inner_dim = dim * mult return nn.Sequential( nn.LayerNorm(dim), nn.Linear(dim, inner_dim, bias=False), @@ -19,8 +19,8 @@ def FeedForward(dim, mult=4): ) -def reshape_tensor(x, heads): - bs, length, width = x.shape +def reshape_tensor(x: torch.Tensor, heads: int): + bs, length, _ = x.shape # (bs, length, width) --> (bs, length, n_heads, dim_per_head) x = x.view(bs, length, heads, -1) # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head) @@ -31,7 +31,7 @@ def reshape_tensor(x, heads): class PerceiverAttention(nn.Module): - def __init__(self, *, dim, dim_head=64, heads=8): + def __init__(self, *, dim: int, dim_head: int = 64, heads: int = 8): super().__init__() self.scale = dim_head**-0.5 self.dim_head = dim_head @@ -45,7 +45,7 @@ class PerceiverAttention(nn.Module): self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) self.to_out = nn.Linear(inner_dim, dim, bias=False) - def forward(self, x, latents): + def forward(self, x: torch.Tensor, latents: torch.Tensor): """ Args: x (torch.Tensor): image features @@ -80,14 +80,14 @@ class PerceiverAttention(nn.Module): class Resampler(nn.Module): def __init__( self, - dim=1024, - depth=8, - dim_head=64, - heads=16, - num_queries=8, - embedding_dim=768, - output_dim=1024, - ff_mult=4, + dim: int = 1024, + depth: int = 8, + dim_head: int = 64, + heads: int = 16, + num_queries: int = 8, + embedding_dim: int = 768, + output_dim: int = 1024, + ff_mult: int = 4, ): super().__init__() @@ -110,7 +110,15 @@ class Resampler(nn.Module): ) @classmethod - def from_state_dict(cls, state_dict: dict[torch.Tensor], depth=8, dim_head=64, heads=16, num_queries=8, ff_mult=4): + def from_state_dict( + cls, + state_dict: dict[str, torch.Tensor], + depth: int = 8, + dim_head: int = 64, + heads: int = 16, + num_queries: int = 8, + ff_mult: int = 4, + ): """A convenience function that initializes a Resampler from a state_dict. Some of the shape parameters are inferred from the state_dict (e.g. dim, embedding_dim, etc.). At the time of @@ -145,7 +153,7 @@ class Resampler(nn.Module): model.load_state_dict(state_dict) return model - def forward(self, x): + def forward(self, x: torch.Tensor): latents = self.latents.repeat(x.size(0), 1, 1) x = self.proj_in(x) diff --git a/invokeai/backend/model_manager/config.py b/invokeai/backend/model_manager/config.py index 524e39b2a1..82f88c0e81 100644 --- a/invokeai/backend/model_manager/config.py +++ b/invokeai/backend/model_manager/config.py @@ -323,10 +323,13 @@ class MainDiffusersConfig(DiffusersConfigBase, MainConfigBase): return Tag(f"{ModelType.Main.value}.{ModelFormat.Diffusers.value}") -class IPAdapterConfig(ModelConfigBase): - """Model config for IP Adaptor format models.""" - +class IPAdapterBaseConfig(ModelConfigBase): type: Literal[ModelType.IPAdapter] = ModelType.IPAdapter + + +class IPAdapterInvokeAIConfig(IPAdapterBaseConfig): + """Model config for IP Adapter diffusers format models.""" + image_encoder_model_id: str format: Literal[ModelFormat.InvokeAI] @@ -335,6 +338,16 @@ class IPAdapterConfig(ModelConfigBase): return Tag(f"{ModelType.IPAdapter.value}.{ModelFormat.InvokeAI.value}") +class IPAdapterCheckpointConfig(IPAdapterBaseConfig): + """Model config for IP Adapter checkpoint format models.""" + + format: Literal[ModelFormat.Checkpoint] + + @staticmethod + def get_tag() -> Tag: + return Tag(f"{ModelType.IPAdapter.value}.{ModelFormat.Checkpoint.value}") + + class CLIPVisionDiffusersConfig(DiffusersConfigBase): """Model config for CLIPVision.""" @@ -390,7 +403,8 @@ AnyModelConfig = Annotated[ Annotated[LoRADiffusersConfig, LoRADiffusersConfig.get_tag()], Annotated[TextualInversionFileConfig, TextualInversionFileConfig.get_tag()], Annotated[TextualInversionFolderConfig, TextualInversionFolderConfig.get_tag()], - Annotated[IPAdapterConfig, IPAdapterConfig.get_tag()], + Annotated[IPAdapterInvokeAIConfig, IPAdapterInvokeAIConfig.get_tag()], + Annotated[IPAdapterCheckpointConfig, IPAdapterCheckpointConfig.get_tag()], Annotated[T2IAdapterConfig, T2IAdapterConfig.get_tag()], Annotated[CLIPVisionDiffusersConfig, CLIPVisionDiffusersConfig.get_tag()], ], diff --git a/invokeai/backend/model_manager/load/load_default.py b/invokeai/backend/model_manager/load/load_default.py index 60cc1f5e6c..6774fc2989 100644 --- a/invokeai/backend/model_manager/load/load_default.py +++ b/invokeai/backend/model_manager/load/load_default.py @@ -37,7 +37,7 @@ class ModelLoader(ModelLoaderBase): self._logger = logger self._ram_cache = ram_cache self._convert_cache = convert_cache - self._torch_dtype = torch_dtype(choose_torch_device(), app_config) + self._torch_dtype = torch_dtype(choose_torch_device()) def load_model(self, model_config: AnyModelConfig, submodel_type: Optional[SubModelType] = None) -> LoadedModel: """ diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache_base.py b/invokeai/backend/model_manager/load/model_cache/model_cache_base.py index eb82f87cb2..a8c2dd3e92 100644 --- a/invokeai/backend/model_manager/load/model_cache/model_cache_base.py +++ b/invokeai/backend/model_manager/load/model_cache/model_cache_base.py @@ -117,7 +117,7 @@ class ModelCacheBase(ABC, Generic[T]): @property @abstractmethod - def stats(self) -> CacheStats: + def stats(self) -> Optional[CacheStats]: """Return collected CacheStats object.""" pass diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache_default.py b/invokeai/backend/model_manager/load/model_cache/model_cache_default.py index 6173d48abe..f2e0c01a94 100644 --- a/invokeai/backend/model_manager/load/model_cache/model_cache_default.py +++ b/invokeai/backend/model_manager/load/model_cache/model_cache_default.py @@ -269,9 +269,6 @@ class ModelCache(ModelCacheBase[AnyModel]): if torch.device(source_device).type == torch.device(target_device).type: return - # may raise an exception here if insufficient GPU VRAM - self._check_free_vram(target_device, cache_entry.size) - start_model_to_time = time.time() snapshot_before = self._capture_memory_snapshot() cache_entry.model.to(target_device) @@ -329,11 +326,11 @@ class ModelCache(ModelCacheBase[AnyModel]): f" {in_ram_models}/{in_vram_models}({locked_in_vram_models})" ) - def make_room(self, model_size: int) -> None: + def make_room(self, size: int) -> None: """Make enough room in the cache to accommodate a new model of indicated size.""" # calculate how much memory this model will require # multiplier = 2 if self.precision==torch.float32 else 1 - bytes_needed = model_size + bytes_needed = size maximum_size = self.max_cache_size * GIG # stored in GB, convert to bytes current_size = self.cache_size() @@ -388,7 +385,7 @@ class ModelCache(ModelCacheBase[AnyModel]): # 1 from onnx runtime object if not cache_entry.locked and refs <= (3 if "onnx" in model_key else 2): self.logger.debug( - f"Removing {model_key} from RAM cache to free at least {(model_size/GIG):.2f} GB (-{(cache_entry.size/GIG):.2f} GB)" + f"Removing {model_key} from RAM cache to free at least {(size/GIG):.2f} GB (-{(cache_entry.size/GIG):.2f} GB)" ) current_size -= cache_entry.size models_cleared += 1 @@ -420,13 +417,3 @@ class ModelCache(ModelCacheBase[AnyModel]): mps.empty_cache() self.logger.debug(f"After making room: cached_models={len(self._cached_models)}") - - def _check_free_vram(self, target_device: torch.device, needed_size: int) -> None: - if target_device.type != "cuda": - return - vram_device = ( # mem_get_info() needs an indexed device - target_device if target_device.index is not None else torch.device(str(target_device), index=0) - ) - free_mem, _ = torch.cuda.mem_get_info(torch.device(vram_device)) - if needed_size > free_mem: - raise torch.cuda.OutOfMemoryError diff --git a/invokeai/backend/model_manager/load/model_cache/model_locker.py b/invokeai/backend/model_manager/load/model_cache/model_locker.py index 81dca346e5..a275987773 100644 --- a/invokeai/backend/model_manager/load/model_cache/model_locker.py +++ b/invokeai/backend/model_manager/load/model_cache/model_locker.py @@ -34,7 +34,6 @@ class ModelLocker(ModelLockerBase): # NOTE that the model has to have the to() method in order for this code to move it into GPU! self._cache_entry.lock() - try: if self._cache.lazy_offloading: self._cache.offload_unlocked_models(self._cache_entry.size) @@ -51,6 +50,7 @@ class ModelLocker(ModelLockerBase): except Exception: self._cache_entry.unlock() raise + return self.model def unlock(self) -> None: diff --git a/invokeai/backend/model_manager/load/model_loaders/ip_adapter.py b/invokeai/backend/model_manager/load/model_loaders/ip_adapter.py index 89dd46e929..55eed81fcd 100644 --- a/invokeai/backend/model_manager/load/model_loaders/ip_adapter.py +++ b/invokeai/backend/model_manager/load/model_loaders/ip_adapter.py @@ -7,19 +7,13 @@ from typing import Optional import torch from invokeai.backend.ip_adapter.ip_adapter import build_ip_adapter -from invokeai.backend.model_manager import ( - AnyModel, - AnyModelConfig, - BaseModelType, - ModelFormat, - ModelType, - SubModelType, -) +from invokeai.backend.model_manager import AnyModel, AnyModelConfig, BaseModelType, ModelFormat, ModelType, SubModelType from invokeai.backend.model_manager.load import ModelLoader, ModelLoaderRegistry from invokeai.backend.raw_model import RawModel @ModelLoaderRegistry.register(base=BaseModelType.Any, type=ModelType.IPAdapter, format=ModelFormat.InvokeAI) +@ModelLoaderRegistry.register(base=BaseModelType.Any, type=ModelType.IPAdapter, format=ModelFormat.Checkpoint) class IPAdapterInvokeAILoader(ModelLoader): """Class to load IP Adapter diffusers models.""" @@ -32,7 +26,7 @@ class IPAdapterInvokeAILoader(ModelLoader): raise ValueError("There are no submodels in an IP-Adapter model.") model_path = Path(config.path) model: RawModel = build_ip_adapter( - ip_adapter_ckpt_path=str(model_path / "ip_adapter.bin"), + ip_adapter_ckpt_path=model_path, device=torch.device("cpu"), dtype=self._torch_dtype, ) diff --git a/invokeai/backend/model_manager/probe.py b/invokeai/backend/model_manager/probe.py index ddd9e99eda..bf21a7fe7b 100644 --- a/invokeai/backend/model_manager/probe.py +++ b/invokeai/backend/model_manager/probe.py @@ -230,9 +230,10 @@ class ModelProbe(object): return ModelType.LoRA elif any(key.startswith(v) for v in {"controlnet", "control_model", "input_blocks"}): return ModelType.ControlNet + elif any(key.startswith(v) for v in {"image_proj.", "ip_adapter."}): + return ModelType.IPAdapter elif key in {"emb_params", "string_to_param"}: return ModelType.TextualInversion - else: # diffusers-ti if len(ckpt) < 10 and all(isinstance(v, torch.Tensor) for v in ckpt.values()): @@ -323,7 +324,7 @@ class ModelProbe(object): with SilenceWarnings(): if model_path.suffix.endswith((".ckpt", ".pt", ".pth", ".bin")): cls._scan_model(model_path.name, model_path) - model = torch.load(model_path) + model = torch.load(model_path, map_location="cpu") assert isinstance(model, dict) return model else: @@ -527,8 +528,25 @@ class ControlNetCheckpointProbe(CheckpointProbeBase): class IPAdapterCheckpointProbe(CheckpointProbeBase): + """Class for probing IP Adapters""" + def get_base_type(self) -> BaseModelType: - raise NotImplementedError() + checkpoint = self.checkpoint + for key in checkpoint.keys(): + if not key.startswith(("image_proj.", "ip_adapter.")): + continue + cross_attention_dim = checkpoint["ip_adapter.1.to_k_ip.weight"].shape[-1] + if cross_attention_dim == 768: + return BaseModelType.StableDiffusion1 + elif cross_attention_dim == 1024: + return BaseModelType.StableDiffusion2 + elif cross_attention_dim == 2048: + return BaseModelType.StableDiffusionXL + else: + raise InvalidModelConfigException( + f"IP-Adapter had unexpected cross-attention dimension: {cross_attention_dim}." + ) + raise InvalidModelConfigException(f"{self.model_path}: Unable to determine base type") class CLIPVisionCheckpointProbe(CheckpointProbeBase): @@ -768,7 +786,7 @@ class T2IAdapterFolderProbe(FolderProbeBase): ) -############## register probe classes ###### +# Register probe classes ModelProbe.register_probe("diffusers", ModelType.Main, PipelineFolderProbe) ModelProbe.register_probe("diffusers", ModelType.VAE, VaeFolderProbe) ModelProbe.register_probe("diffusers", ModelType.LoRA, LoRAFolderProbe) diff --git a/invokeai/backend/util/devices.py b/invokeai/backend/util/devices.py index 0be53c842a..cb6b93eaac 100644 --- a/invokeai/backend/util/devices.py +++ b/invokeai/backend/util/devices.py @@ -6,8 +6,7 @@ from typing import Literal, Optional, Union import torch from torch import autocast -from invokeai.app.services.config import InvokeAIAppConfig -from invokeai.app.services.config.config_default import get_config +from invokeai.app.services.config.config_default import PRECISION, get_config CPU_DEVICE = torch.device("cpu") CUDA_DEVICE = torch.device("cuda") @@ -33,35 +32,34 @@ def get_torch_device_name() -> str: return torch.cuda.get_device_name(device) if device.type == "cuda" else device.type.upper() -# We are in transition here from using a single global AppConfig to allowing multiple -# configurations. It is strongly recommended to pass the app_config to this function. -def choose_precision( - device: torch.device, app_config: Optional[InvokeAIAppConfig] = None -) -> Literal["float32", "float16", "bfloat16"]: +def choose_precision(device: torch.device) -> Literal["float32", "float16", "bfloat16"]: """Return an appropriate precision for the given torch device.""" - app_config = app_config or get_config() + app_config = get_config() if device.type == "cuda": device_name = torch.cuda.get_device_name(device) - if not ("GeForce GTX 1660" in device_name or "GeForce GTX 1650" in device_name): - if app_config.precision == "float32": - return "float32" - elif app_config.precision == "bfloat16": - return "bfloat16" - else: - return "float16" + if "GeForce GTX 1660" in device_name or "GeForce GTX 1650" in device_name: + # These GPUs have limited support for float16 + return "float32" + elif app_config.precision == "auto" or app_config.precision == "autocast": + # Default to float16 for CUDA devices + return "float16" + else: + # Use the user-defined precision + return app_config.precision elif device.type == "mps": - return "float16" + if app_config.precision == "auto" or app_config.precision == "autocast": + # Default to float16 for MPS devices + return "float16" + else: + # Use the user-defined precision + return app_config.precision + # CPU / safe fallback return "float32" -# We are in transition here from using a single global AppConfig to allowing multiple -# configurations. It is strongly recommended to pass the app_config to this function. -def torch_dtype( - device: Optional[torch.device] = None, - app_config: Optional[InvokeAIAppConfig] = None, -) -> torch.dtype: +def torch_dtype(device: Optional[torch.device] = None) -> torch.dtype: device = device or choose_torch_device() - precision = choose_precision(device, app_config) + precision = choose_precision(device) if precision == "float16": return torch.float16 if precision == "bfloat16": @@ -71,7 +69,7 @@ def torch_dtype( return torch.float32 -def choose_autocast(precision): +def choose_autocast(precision: PRECISION): """Returns an autocast context or nullcontext for the given precision string""" # float16 currently requires autocast to avoid errors like: # 'expected scalar type Half but found Float' diff --git a/invokeai/frontend/web/public/locales/ar.json b/invokeai/frontend/web/public/locales/ar.json index d5be1b1fce..ee370d1e42 100644 --- a/invokeai/frontend/web/public/locales/ar.json +++ b/invokeai/frontend/web/public/locales/ar.json @@ -291,7 +291,6 @@ "canvasMerged": "تم دمج الخط", "sentToImageToImage": "تم إرسال إلى صورة إلى صورة", "sentToUnifiedCanvas": "تم إرسال إلى لوحة موحدة", - "parametersSet": "تم تعيين المعلمات", "parametersNotSet": "لم يتم تعيين المعلمات", "metadataLoadFailed": "فشل تحميل البيانات الوصفية" }, diff --git a/invokeai/frontend/web/public/locales/de.json b/invokeai/frontend/web/public/locales/de.json index 48a8c5127e..033dffdc44 100644 --- a/invokeai/frontend/web/public/locales/de.json +++ b/invokeai/frontend/web/public/locales/de.json @@ -75,7 +75,8 @@ "copy": "Kopieren", "aboutHeading": "Nutzen Sie Ihre kreative Energie", "toResolve": "Lösen", - "add": "Hinzufügen" + "add": "Hinzufügen", + "loglevel": "Protokoll Stufe" }, "gallery": { "galleryImageSize": "Bildgröße", @@ -388,7 +389,14 @@ "vaePrecision": "VAE-Präzision", "variant": "Variante", "modelDeleteFailed": "Modell konnte nicht gelöscht werden", - "noModelSelected": "Kein Modell ausgewählt" + "noModelSelected": "Kein Modell ausgewählt", + "huggingFace": "HuggingFace", + "defaultSettings": "Standardeinstellungen", + "edit": "Bearbeiten", + "cancel": "Stornieren", + "defaultSettingsSaved": "Standardeinstellungen gespeichert", + "addModels": "Model hinzufügen", + "deleteModelImage": "Lösche Model Bild" }, "parameters": { "images": "Bilder", @@ -472,7 +480,6 @@ "canvasMerged": "Leinwand zusammengeführt", "sentToImageToImage": "Gesendet an Bild zu Bild", "sentToUnifiedCanvas": "Gesendet an Leinwand", - "parametersSet": "Parameter festlegen", "parametersNotSet": "Parameter nicht festgelegt", "metadataLoadFailed": "Metadaten konnten nicht geladen werden", "setCanvasInitialImage": "Ausgangsbild setzen", @@ -677,7 +684,8 @@ "body": "Körper", "hands": "Hände", "dwOpenpose": "DW Openpose", - "dwOpenposeDescription": "Posenschätzung mit DW Openpose" + "dwOpenposeDescription": "Posenschätzung mit DW Openpose", + "selectCLIPVisionModel": "Wähle ein CLIP Vision Model aus" }, "queue": { "status": "Status", @@ -765,7 +773,10 @@ "recallParameters": "Parameter wiederherstellen", "cfgRescaleMultiplier": "$t(parameters.cfgRescaleMultiplier)", "allPrompts": "Alle Prompts", - "imageDimensions": "Bilder Auslösungen" + "imageDimensions": "Bilder Auslösungen", + "parameterSet": "Parameter {{parameter}} setzen", + "recallParameter": "{{label}} Abrufen", + "parsingFailed": "Parsing Fehlgeschlagen" }, "popovers": { "noiseUseCPU": { @@ -1030,7 +1041,8 @@ "title": "Bild" }, "advanced": { - "title": "Erweitert" + "title": "Erweitert", + "options": "$t(accordions.advanced.title) Optionen" }, "control": { "title": "Kontrolle" diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index 1601169e03..9686f8a02a 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -217,6 +217,7 @@ "saveControlImage": "Save Control Image", "scribble": "scribble", "selectModel": "Select a model", + "selectCLIPVisionModel": "Select a CLIP Vision model", "setControlImageDimensions": "Set Control Image Dimensions To W/H", "showAdvanced": "Show Advanced", "small": "Small", @@ -655,6 +656,7 @@ "install": "Install", "installAll": "Install All", "installRepo": "Install Repo", + "ipAdapters": "IP Adapters", "load": "Load", "localOnly": "local only", "manual": "Manual", @@ -682,6 +684,7 @@ "noModelsInstalled": "No Models Installed", "noModelsInstalledDesc1": "Install models with the", "noModelSelected": "No Model Selected", + "noMatchingModels": "No matching Models", "none": "none", "path": "Path", "pathToConfig": "Path To Config", @@ -885,6 +888,11 @@ "imageFit": "Fit Initial Image To Output Size", "images": "Images", "infillMethod": "Infill Method", + "infillMosaicTileWidth": "Tile Width", + "infillMosaicTileHeight": "Tile Height", + "infillMosaicMinColor": "Min Color", + "infillMosaicMaxColor": "Max Color", + "infillColorValue": "Fill Color", "info": "Info", "invoke": { "addingImagesTo": "Adding images to", @@ -1033,10 +1041,10 @@ "metadataLoadFailed": "Failed to load metadata", "modelAddedSimple": "Model Added to Queue", "modelImportCanceled": "Model Import Canceled", + "parameters": "Parameters", "parameterNotSet": "{{parameter}} not set", "parameterSet": "{{parameter}} set", "parametersNotSet": "Parameters Not Set", - "parametersSet": "Parameters Set", "problemCopyingCanvas": "Problem Copying Canvas", "problemCopyingCanvasDesc": "Unable to export base layer", "problemCopyingImage": "Unable to Copy Image", @@ -1415,6 +1423,7 @@ "eraseBoundingBox": "Erase Bounding Box", "eraser": "Eraser", "fillBoundingBox": "Fill Bounding Box", + "initialFitImageSize": "Fit Image Size on Drop", "invertBrushSizeScrollDirection": "Invert Scroll for Brush Size", "layer": "Layer", "limitStrokesToBox": "Limit Strokes to Box", diff --git a/invokeai/frontend/web/public/locales/es.json b/invokeai/frontend/web/public/locales/es.json index c7af596556..3037045db5 100644 --- a/invokeai/frontend/web/public/locales/es.json +++ b/invokeai/frontend/web/public/locales/es.json @@ -363,7 +363,6 @@ "canvasMerged": "Lienzo consolidado", "sentToImageToImage": "Enviar hacia Imagen a Imagen", "sentToUnifiedCanvas": "Enviar hacia Lienzo Consolidado", - "parametersSet": "Parámetros establecidos", "parametersNotSet": "Parámetros no establecidos", "metadataLoadFailed": "Error al cargar metadatos", "serverError": "Error en el servidor", diff --git a/invokeai/frontend/web/public/locales/fr.json b/invokeai/frontend/web/public/locales/fr.json index 095ee5d0d5..b8f560e265 100644 --- a/invokeai/frontend/web/public/locales/fr.json +++ b/invokeai/frontend/web/public/locales/fr.json @@ -298,7 +298,6 @@ "canvasMerged": "Canvas fusionné", "sentToImageToImage": "Envoyé à Image à Image", "sentToUnifiedCanvas": "Envoyé à Canvas unifié", - "parametersSet": "Paramètres définis", "parametersNotSet": "Paramètres non définis", "metadataLoadFailed": "Échec du chargement des métadonnées" }, diff --git a/invokeai/frontend/web/public/locales/he.json b/invokeai/frontend/web/public/locales/he.json index efb90f61c7..dbbb3cbec4 100644 --- a/invokeai/frontend/web/public/locales/he.json +++ b/invokeai/frontend/web/public/locales/he.json @@ -306,7 +306,6 @@ "canvasMerged": "קנבס מוזג", "sentToImageToImage": "נשלח לתמונה לתמונה", "sentToUnifiedCanvas": "נשלח אל קנבס מאוחד", - "parametersSet": "הגדרת פרמטרים", "parametersNotSet": "פרמטרים לא הוגדרו", "metadataLoadFailed": "טעינת מטא-נתונים נכשלה" }, diff --git a/invokeai/frontend/web/public/locales/it.json b/invokeai/frontend/web/public/locales/it.json index a9eb75c0f5..ff4e44c487 100644 --- a/invokeai/frontend/web/public/locales/it.json +++ b/invokeai/frontend/web/public/locales/it.json @@ -366,7 +366,7 @@ "modelConverted": "Modello convertito", "alpha": "Alpha", "convertToDiffusersHelpText1": "Questo modello verrà convertito nel formato 🧨 Diffusori.", - "convertToDiffusersHelpText3": "Il file Checkpoint su disco verrà eliminato se si trova nella cartella principale di InvokeAI. Se si trova invece in una posizione personalizzata, NON verrà eliminato.", + "convertToDiffusersHelpText3": "Il file del modello su disco verrà eliminato se si trova nella cartella principale di InvokeAI. Se si trova invece in una posizione personalizzata, NON verrà eliminato.", "v2_base": "v2 (512px)", "v2_768": "v2 (768px)", "none": "nessuno", @@ -443,7 +443,8 @@ "noModelsInstalled": "Nessun modello installato", "hfTokenInvalidErrorMessage2": "Aggiornalo in ", "main": "Principali", - "noModelsInstalledDesc1": "Installa i modelli con" + "noModelsInstalledDesc1": "Installa i modelli con", + "ipAdapters": "Adattatori IP" }, "parameters": { "images": "Immagini", @@ -568,7 +569,6 @@ "canvasMerged": "Tela unita", "sentToImageToImage": "Inviato a Immagine a Immagine", "sentToUnifiedCanvas": "Inviato a Tela Unificata", - "parametersSet": "Parametri impostati", "parametersNotSet": "Parametri non impostati", "metadataLoadFailed": "Impossibile caricare i metadati", "serverError": "Errore del Server", @@ -937,7 +937,8 @@ "controlnet": "$t(controlnet.controlAdapter_one) #{{number}} ($t(common.controlNet))", "mediapipeFace": "Mediapipe Volto", "ip_adapter": "$t(controlnet.controlAdapter_one) #{{number}} ($t(common.ipAdapter))", - "t2i_adapter": "$t(controlnet.controlAdapter_one) #{{number}} ($t(common.t2iAdapter))" + "t2i_adapter": "$t(controlnet.controlAdapter_one) #{{number}} ($t(common.t2iAdapter))", + "selectCLIPVisionModel": "Seleziona un modello CLIP Vision" }, "queue": { "queueFront": "Aggiungi all'inizio della coda", diff --git a/invokeai/frontend/web/public/locales/nl.json b/invokeai/frontend/web/public/locales/nl.json index 8fd8c96ee4..70adbb371d 100644 --- a/invokeai/frontend/web/public/locales/nl.json +++ b/invokeai/frontend/web/public/locales/nl.json @@ -420,7 +420,6 @@ "canvasMerged": "Canvas samengevoegd", "sentToImageToImage": "Gestuurd naar Afbeelding naar afbeelding", "sentToUnifiedCanvas": "Gestuurd naar Centraal canvas", - "parametersSet": "Parameters ingesteld", "parametersNotSet": "Parameters niet ingesteld", "metadataLoadFailed": "Fout bij laden metagegevens", "serverError": "Serverfout", diff --git a/invokeai/frontend/web/public/locales/pl.json b/invokeai/frontend/web/public/locales/pl.json index 399417db58..b7592c3fae 100644 --- a/invokeai/frontend/web/public/locales/pl.json +++ b/invokeai/frontend/web/public/locales/pl.json @@ -267,7 +267,6 @@ "canvasMerged": "Scalono widoczne warstwy", "sentToImageToImage": "Wysłano do Obraz na obraz", "sentToUnifiedCanvas": "Wysłano do trybu uniwersalnego", - "parametersSet": "Ustawiono parametry", "parametersNotSet": "Nie ustawiono parametrów", "metadataLoadFailed": "Błąd wczytywania metadanych" }, diff --git a/invokeai/frontend/web/public/locales/pt.json b/invokeai/frontend/web/public/locales/pt.json index 34f99b7075..3003a1732b 100644 --- a/invokeai/frontend/web/public/locales/pt.json +++ b/invokeai/frontend/web/public/locales/pt.json @@ -310,7 +310,6 @@ "canvasMerged": "Tela Fundida", "sentToImageToImage": "Mandar Para Imagem Para Imagem", "sentToUnifiedCanvas": "Enviada para a Tela Unificada", - "parametersSet": "Parâmetros Definidos", "parametersNotSet": "Parâmetros Não Definidos", "metadataLoadFailed": "Falha ao tentar carregar metadados" }, diff --git a/invokeai/frontend/web/public/locales/pt_BR.json b/invokeai/frontend/web/public/locales/pt_BR.json index 2859eb31db..c966c6db50 100644 --- a/invokeai/frontend/web/public/locales/pt_BR.json +++ b/invokeai/frontend/web/public/locales/pt_BR.json @@ -307,7 +307,6 @@ "canvasMerged": "Tela Fundida", "sentToImageToImage": "Mandar Para Imagem Para Imagem", "sentToUnifiedCanvas": "Enviada para a Tela Unificada", - "parametersSet": "Parâmetros Definidos", "parametersNotSet": "Parâmetros Não Definidos", "metadataLoadFailed": "Falha ao tentar carregar metadados" }, diff --git a/invokeai/frontend/web/public/locales/ru.json b/invokeai/frontend/web/public/locales/ru.json index 258bceeb05..4dd2ad895a 100644 --- a/invokeai/frontend/web/public/locales/ru.json +++ b/invokeai/frontend/web/public/locales/ru.json @@ -575,7 +575,6 @@ "canvasMerged": "Холст объединен", "sentToImageToImage": "Отправить в img2img", "sentToUnifiedCanvas": "Отправлено на Единый холст", - "parametersSet": "Параметры заданы", "parametersNotSet": "Параметры не заданы", "metadataLoadFailed": "Не удалось загрузить метаданные", "serverError": "Ошибка сервера", diff --git a/invokeai/frontend/web/public/locales/uk.json b/invokeai/frontend/web/public/locales/uk.json index f97909525c..9bb38c21b3 100644 --- a/invokeai/frontend/web/public/locales/uk.json +++ b/invokeai/frontend/web/public/locales/uk.json @@ -315,7 +315,6 @@ "canvasMerged": "Полотно об'єднане", "sentToImageToImage": "Надіслати до img2img", "sentToUnifiedCanvas": "Надіслати на полотно", - "parametersSet": "Параметри задані", "parametersNotSet": "Параметри не задані", "metadataLoadFailed": "Не вдалося завантажити метадані", "serverError": "Помилка сервера", diff --git a/invokeai/frontend/web/public/locales/zh_CN.json b/invokeai/frontend/web/public/locales/zh_CN.json index 77a06ea77b..a88f540990 100644 --- a/invokeai/frontend/web/public/locales/zh_CN.json +++ b/invokeai/frontend/web/public/locales/zh_CN.json @@ -487,7 +487,6 @@ "canvasMerged": "画布已合并", "sentToImageToImage": "已发送到图生图", "sentToUnifiedCanvas": "已发送到统一画布", - "parametersSet": "参数已设定", "parametersNotSet": "参数未设定", "metadataLoadFailed": "加载元数据失败", "uploadFailedInvalidUploadDesc": "必须是单张的 PNG 或 JPEG 图片", diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketio/socketModelInstall.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketio/socketModelInstall.ts index 4d04ef92be..f474c2736b 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketio/socketModelInstall.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketio/socketModelInstall.ts @@ -43,6 +43,7 @@ export const addModelInstallEventListener = (startAppListening: AppStartListenin }) ); dispatch(api.util.invalidateTags([{ type: 'ModelConfig', id: LIST_TAG }])); + dispatch(api.util.invalidateTags([{ type: 'ModelScanFolderResults', id: LIST_TAG }])); }, }); diff --git a/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasSettingsButtonPopover.tsx b/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasSettingsButtonPopover.tsx index 0228b158dd..83ee900a43 100644 --- a/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasSettingsButtonPopover.tsx +++ b/invokeai/frontend/web/src/features/canvas/components/IAICanvasToolbar/IAICanvasSettingsButtonPopover.tsx @@ -18,6 +18,7 @@ import { setShouldAutoSave, setShouldCropToBoundingBoxOnSave, setShouldDarkenOutsideBoundingBox, + setShouldFitImageSize, setShouldInvertBrushSizeScrollDirection, setShouldRestrictStrokesToBox, setShouldShowCanvasDebugInfo, @@ -48,6 +49,7 @@ const IAICanvasSettingsButtonPopover = () => { const shouldSnapToGrid = useAppSelector((s) => s.canvas.shouldSnapToGrid); const shouldRestrictStrokesToBox = useAppSelector((s) => s.canvas.shouldRestrictStrokesToBox); const shouldAntialias = useAppSelector((s) => s.canvas.shouldAntialias); + const shouldFitImageSize = useAppSelector((s) => s.canvas.shouldFitImageSize); useHotkeys( ['n'], @@ -102,6 +104,10 @@ const IAICanvasSettingsButtonPopover = () => { (e: ChangeEvent) => dispatch(setShouldAntialias(e.target.checked)), [dispatch] ); + const handleChangeShouldFitImageSize = useCallback( + (e: ChangeEvent) => dispatch(setShouldFitImageSize(e.target.checked)), + [dispatch] + ); return ( @@ -165,6 +171,10 @@ const IAICanvasSettingsButtonPopover = () => { {t('unifiedCanvas.antialiasing')} + + {t('unifiedCanvas.initialFitImageSize')} + + diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts b/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts index c5a60bdd26..bb469c67f0 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasSlice.ts @@ -66,6 +66,7 @@ const initialCanvasState: CanvasState = { shouldAutoSave: false, shouldCropToBoundingBoxOnSave: false, shouldDarkenOutsideBoundingBox: false, + shouldFitImageSize: true, shouldInvertBrushSizeScrollDirection: false, shouldLockBoundingBox: false, shouldPreserveMaskedArea: false, @@ -144,12 +145,20 @@ export const canvasSlice = createSlice({ reducer: (state, action: PayloadActionWithOptimalDimension) => { const { width, height, image_name } = action.payload; const { optimalDimension } = action.meta; - const { stageDimensions } = state; + const { stageDimensions, shouldFitImageSize } = state; - const newBoundingBoxDimensions = { - width: roundDownToMultiple(clamp(width, CANVAS_GRID_SIZE_FINE, optimalDimension), CANVAS_GRID_SIZE_FINE), - height: roundDownToMultiple(clamp(height, CANVAS_GRID_SIZE_FINE, optimalDimension), CANVAS_GRID_SIZE_FINE), - }; + const newBoundingBoxDimensions = shouldFitImageSize + ? { + width: roundDownToMultiple(width, CANVAS_GRID_SIZE_FINE), + height: roundDownToMultiple(height, CANVAS_GRID_SIZE_FINE), + } + : { + width: roundDownToMultiple(clamp(width, CANVAS_GRID_SIZE_FINE, optimalDimension), CANVAS_GRID_SIZE_FINE), + height: roundDownToMultiple( + clamp(height, CANVAS_GRID_SIZE_FINE, optimalDimension), + CANVAS_GRID_SIZE_FINE + ), + }; const newBoundingBoxCoordinates = { x: roundToMultiple(width / 2 - newBoundingBoxDimensions.width / 2, CANVAS_GRID_SIZE_FINE), @@ -289,12 +298,19 @@ export const canvasSlice = createSlice({ const { images, selectedImageIndex } = state.layerState.stagingArea; pushToPrevLayerStates(state); - if (!images.length) { - return; - } - images.splice(selectedImageIndex, 1); + if (images.length === 0) { + pushToPrevLayerStates(state); + + state.layerState.stagingArea = deepClone(initialLayerState.stagingArea); + + state.futureLayerStates = []; + state.shouldShowStagingOutline = true; + state.shouldShowStagingImage = true; + state.batchIds = []; + } + if (selectedImageIndex >= images.length) { state.layerState.stagingArea.selectedImageIndex = images.length - 1; } @@ -575,6 +591,9 @@ export const canvasSlice = createSlice({ setShouldAntialias: (state, action: PayloadAction) => { state.shouldAntialias = action.payload; }, + setShouldFitImageSize: (state, action: PayloadAction) => { + state.shouldFitImageSize = action.payload; + }, setShouldCropToBoundingBoxOnSave: (state, action: PayloadAction) => { state.shouldCropToBoundingBoxOnSave = action.payload; }, @@ -685,6 +704,7 @@ export const { setShouldRestrictStrokesToBox, stagingAreaInitialized, setShouldAntialias, + setShouldFitImageSize, canvasResized, canvasBatchIdAdded, canvasBatchIdsReset, diff --git a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts index 7fc39fde1f..2d30e18760 100644 --- a/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts +++ b/invokeai/frontend/web/src/features/canvas/store/canvasTypes.ts @@ -120,6 +120,7 @@ export interface CanvasState { shouldAutoSave: boolean; shouldCropToBoundingBoxOnSave: boolean; shouldDarkenOutsideBoundingBox: boolean; + shouldFitImageSize: boolean; shouldInvertBrushSizeScrollDirection: boolean; shouldLockBoundingBox: boolean; shouldPreserveMaskedArea: boolean; diff --git a/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterModel.tsx b/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterModel.tsx index 25d327e54e..91f8822352 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterModel.tsx +++ b/invokeai/frontend/web/src/features/controlAdapters/components/parameters/ParamControlAdapterModel.tsx @@ -1,12 +1,18 @@ -import { Combobox, FormControl, Tooltip } from '@invoke-ai/ui-library'; +import type { ComboboxOnChange, ComboboxOption } from '@invoke-ai/ui-library'; +import { Combobox, Flex, FormControl, Tooltip } from '@invoke-ai/ui-library'; import { createMemoizedSelector } from 'app/store/createMemoizedSelector'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import { useGroupedModelCombobox } from 'common/hooks/useGroupedModelCombobox'; +import { useControlAdapterCLIPVisionModel } from 'features/controlAdapters/hooks/useControlAdapterCLIPVisionModel'; import { useControlAdapterIsEnabled } from 'features/controlAdapters/hooks/useControlAdapterIsEnabled'; import { useControlAdapterModel } from 'features/controlAdapters/hooks/useControlAdapterModel'; import { useControlAdapterModels } from 'features/controlAdapters/hooks/useControlAdapterModels'; import { useControlAdapterType } from 'features/controlAdapters/hooks/useControlAdapterType'; -import { controlAdapterModelChanged } from 'features/controlAdapters/store/controlAdaptersSlice'; +import { + controlAdapterCLIPVisionModelChanged, + controlAdapterModelChanged, +} from 'features/controlAdapters/store/controlAdaptersSlice'; +import type { CLIPVisionModel } from 'features/controlAdapters/store/types'; import { selectGenerationSlice } from 'features/parameters/store/generationSlice'; import { memo, useCallback, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; @@ -29,6 +35,7 @@ const ParamControlAdapterModel = ({ id }: ParamControlAdapterModelProps) => { const { modelConfig } = useControlAdapterModel(id); const dispatch = useAppDispatch(); const currentBaseModel = useAppSelector((s) => s.generation.model?.base); + const currentCLIPVisionModel = useControlAdapterCLIPVisionModel(id); const mainModel = useAppSelector(selectMainModel); const { t } = useTranslation(); @@ -49,6 +56,16 @@ const ParamControlAdapterModel = ({ id }: ParamControlAdapterModelProps) => { [dispatch, id] ); + const onCLIPVisionModelChange = useCallback( + (v) => { + if (!v?.value) { + return; + } + dispatch(controlAdapterCLIPVisionModelChanged({ id, clipVisionModel: v.value as CLIPVisionModel })); + }, + [dispatch, id] + ); + const selectedModel = useMemo( () => (modelConfig && controlAdapterType ? { ...modelConfig, model_type: controlAdapterType } : null), [controlAdapterType, modelConfig] @@ -71,18 +88,51 @@ const ParamControlAdapterModel = ({ id }: ParamControlAdapterModelProps) => { isLoading, }); + const clipVisionOptions = useMemo( + () => [ + { label: 'ViT-H', value: 'ViT-H' }, + { label: 'ViT-G', value: 'ViT-G' }, + ], + [] + ); + + const clipVisionModel = useMemo( + () => clipVisionOptions.find((o) => o.value === currentCLIPVisionModel), + [clipVisionOptions, currentCLIPVisionModel] + ); + return ( - - - - - + + + + + + + {modelConfig?.type === 'ip_adapter' && modelConfig.format === 'checkpoint' && ( + + + + )} + ); }; diff --git a/invokeai/frontend/web/src/features/controlAdapters/hooks/useControlAdapterCLIPVisionModel.ts b/invokeai/frontend/web/src/features/controlAdapters/hooks/useControlAdapterCLIPVisionModel.ts new file mode 100644 index 0000000000..249d2022fe --- /dev/null +++ b/invokeai/frontend/web/src/features/controlAdapters/hooks/useControlAdapterCLIPVisionModel.ts @@ -0,0 +1,24 @@ +import { createMemoizedSelector } from 'app/store/createMemoizedSelector'; +import { useAppSelector } from 'app/store/storeHooks'; +import { + selectControlAdapterById, + selectControlAdaptersSlice, +} from 'features/controlAdapters/store/controlAdaptersSlice'; +import { useMemo } from 'react'; + +export const useControlAdapterCLIPVisionModel = (id: string) => { + const selector = useMemo( + () => + createMemoizedSelector(selectControlAdaptersSlice, (controlAdapters) => { + const cn = selectControlAdapterById(controlAdapters, id); + if (cn && cn?.type === 'ip_adapter') { + return cn.clipVisionModel; + } + }), + [id] + ); + + const clipVisionModel = useAppSelector(selector); + + return clipVisionModel; +}; diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts b/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts index f4edca41bb..100bb3f6ad 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts @@ -14,6 +14,7 @@ import { v4 as uuidv4 } from 'uuid'; import { controlAdapterImageProcessed } from './actions'; import { CONTROLNET_PROCESSORS } from './constants'; import type { + CLIPVisionModel, ControlAdapterConfig, ControlAdapterProcessorType, ControlAdaptersState, @@ -244,6 +245,13 @@ export const controlAdaptersSlice = createSlice({ } caAdapter.updateOne(state, { id, changes: { controlMode } }); }, + controlAdapterCLIPVisionModelChanged: ( + state, + action: PayloadAction<{ id: string; clipVisionModel: CLIPVisionModel }> + ) => { + const { id, clipVisionModel } = action.payload; + caAdapter.updateOne(state, { id, changes: { clipVisionModel } }); + }, controlAdapterResizeModeChanged: ( state, action: PayloadAction<{ @@ -381,6 +389,7 @@ export const { controlAdapterProcessedImageChanged, controlAdapterIsEnabledChanged, controlAdapterModelChanged, + controlAdapterCLIPVisionModelChanged, controlAdapterWeightChanged, controlAdapterBeginStepPctChanged, controlAdapterEndStepPctChanged, diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/types.ts b/invokeai/frontend/web/src/features/controlAdapters/store/types.ts index 93d4915cdf..329c318759 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/types.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/types.ts @@ -243,12 +243,15 @@ export type T2IAdapterConfig = { shouldAutoConfig: boolean; }; +export type CLIPVisionModel = 'ViT-H' | 'ViT-G'; + export type IPAdapterConfig = { type: 'ip_adapter'; id: string; isEnabled: boolean; controlImage: string | null; model: ParameterIPAdapterModel | null; + clipVisionModel: CLIPVisionModel; weight: number; beginStepPct: number; endStepPct: number; diff --git a/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts b/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts index d4796572d4..dc893ceb1c 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/util/buildControlAdapter.ts @@ -46,6 +46,7 @@ export const initialIPAdapter: Omit = { isEnabled: true, controlImage: null, model: null, + clipVisionModel: 'ViT-H', weight: 1, beginStepPct: 0, endStepPct: 1, diff --git a/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx b/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx index 5b9f15c21a..ce75ea62e0 100644 --- a/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx +++ b/invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx @@ -33,6 +33,7 @@ const ImageMetadataActions = (props: Props) => { + diff --git a/invokeai/frontend/web/src/features/metadata/util/handlers.ts b/invokeai/frontend/web/src/features/metadata/util/handlers.ts index b64426b422..2fb840afcb 100644 --- a/invokeai/frontend/web/src/features/metadata/util/handlers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/handlers.ts @@ -189,6 +189,12 @@ export const handlers = { recaller: recallers.cfgScale, }), height: buildHandlers({ getLabel: () => t('metadata.height'), parser: parsers.height, recaller: recallers.height }), + initialImage: buildHandlers({ + getLabel: () => t('metadata.initImage'), + parser: parsers.initialImage, + recaller: recallers.initialImage, + renderValue: async (imageDTO) => imageDTO.image_name, + }), negativePrompt: buildHandlers({ getLabel: () => t('metadata.negativePrompt'), parser: parsers.negativePrompt, @@ -405,6 +411,6 @@ export const parseAndRecallAllMetadata = async (metadata: unknown, skip: (keyof }) ); if (results.some((result) => result.status === 'fulfilled')) { - parameterSetToast(t('toast.parametersSet')); + parameterSetToast(t('toast.parameters')); } }; diff --git a/invokeai/frontend/web/src/features/metadata/util/parsers.ts b/invokeai/frontend/web/src/features/metadata/util/parsers.ts index c7c9616bd0..9f5c14d94e 100644 --- a/invokeai/frontend/web/src/features/metadata/util/parsers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/parsers.ts @@ -1,3 +1,4 @@ +import { getStore } from 'app/store/nanostores/store'; import { initialControlNet, initialIPAdapter, @@ -57,6 +58,8 @@ import { isParameterWidth, } from 'features/parameters/types/parameterSchemas'; import { get, isArray, isString } from 'lodash-es'; +import { imagesApi } from 'services/api/endpoints/images'; +import type { ImageDTO } from 'services/api/types'; import { isControlNetModelConfig, isIPAdapterModelConfig, @@ -135,6 +138,14 @@ const parseCFGRescaleMultiplier: MetadataParseFunc = (metadata) => getProperty(metadata, 'scheduler', isParameterScheduler); +const parseInitialImage: MetadataParseFunc = async (metadata) => { + const imageName = await getProperty(metadata, 'init_image', isString); + const imageDTORequest = getStore().dispatch(imagesApi.endpoints.getImageDTO.initiate(imageName)); + const imageDTO = await imageDTORequest.unwrap(); + imageDTORequest.unsubscribe(); + return imageDTO; +}; + const parseWidth: MetadataParseFunc = (metadata) => getProperty(metadata, 'width', isParameterWidth); const parseHeight: MetadataParseFunc = (metadata) => @@ -372,6 +383,7 @@ const parseIPAdapter: MetadataParseFunc = async (metada type: 'ip_adapter', isEnabled: true, model: zModelIdentifierField.parse(ipAdapterModel), + clipVisionModel: 'ViT-H', controlImage: image?.image_name ?? null, weight: weight ?? initialIPAdapter.weight, beginStepPct: begin_step_percent ?? initialIPAdapter.beginStepPct, @@ -401,6 +413,7 @@ export const parsers = { cfgScale: parseCFGScale, cfgRescaleMultiplier: parseCFGRescaleMultiplier, scheduler: parseScheduler, + initialImage: parseInitialImage, width: parseWidth, height: parseHeight, steps: parseSteps, diff --git a/invokeai/frontend/web/src/features/metadata/util/recallers.ts b/invokeai/frontend/web/src/features/metadata/util/recallers.ts index f35399c139..88af390a20 100644 --- a/invokeai/frontend/web/src/features/metadata/util/recallers.ts +++ b/invokeai/frontend/web/src/features/metadata/util/recallers.ts @@ -17,6 +17,7 @@ import type { import { modelSelected } from 'features/parameters/store/actions'; import { heightRecalled, + initialImageChanged, setCfgRescaleMultiplier, setCfgScale, setImg2imgStrength, @@ -61,6 +62,7 @@ import { setRefinerStart, setRefinerSteps, } from 'features/sdxl/store/sdxlSlice'; +import type { ImageDTO } from 'services/api/types'; const recallPositivePrompt: MetadataRecallFunc = (positivePrompt) => { getStore().dispatch(setPositivePrompt(positivePrompt)); @@ -94,6 +96,10 @@ const recallScheduler: MetadataRecallFunc = (scheduler) => { getStore().dispatch(setScheduler(scheduler)); }; +const recallInitialImage: MetadataRecallFunc = async (imageDTO) => { + getStore().dispatch(initialImageChanged(imageDTO)); +}; + const recallWidth: MetadataRecallFunc = (width) => { getStore().dispatch(widthRecalled(width)); }; @@ -235,6 +241,7 @@ export const recallers = { cfgScale: recallCFGScale, cfgRescaleMultiplier: recallCFGRescaleMultiplier, scheduler: recallScheduler, + initialImage: recallInitialImage, width: recallWidth, height: recallHeight, steps: recallSteps, diff --git a/invokeai/frontend/web/src/features/modelManagerV2/store/modelManagerV2Slice.ts b/invokeai/frontend/web/src/features/modelManagerV2/store/modelManagerV2Slice.ts index 6bdd829bb1..c637d30fd8 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/store/modelManagerV2Slice.ts +++ b/invokeai/frontend/web/src/features/modelManagerV2/store/modelManagerV2Slice.ts @@ -3,7 +3,7 @@ import { createSlice } from '@reduxjs/toolkit'; import type { PersistConfig } from 'app/store/store'; import type { ModelType } from 'services/api/types'; -export type FilterableModelType = Exclude; +export type FilterableModelType = Exclude | 'refiner'; type ModelManagerState = { _version: 1; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ModelInstallQueue/ModelInstallQueueItem.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ModelInstallQueue/ModelInstallQueueItem.tsx index 6811b90907..d1fc600510 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ModelInstallQueue/ModelInstallQueueItem.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ModelInstallQueue/ModelInstallQueueItem.tsx @@ -87,6 +87,10 @@ export const ModelInstallQueueItem = (props: ModelListItemProps) => { }, [installJob.source]); const progressValue = useMemo(() => { + if (installJob.status === 'completed' || installJob.status === 'error' || installJob.status === 'cancelled') { + return 100; + } + if (isNil(installJob.bytes) || isNil(installJob.total_bytes)) { return null; } @@ -96,7 +100,7 @@ export const ModelInstallQueueItem = (props: ModelListItemProps) => { } return (installJob.bytes / installJob.total_bytes) * 100; - }, [installJob.bytes, installJob.total_bytes]); + }, [installJob.bytes, installJob.status, installJob.total_bytes]); return ( diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResultItem.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResultItem.tsx index 34b207505a..4f2f77470d 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResultItem.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResultItem.tsx @@ -1,48 +1,19 @@ import { Badge, Box, Flex, IconButton, Text } from '@invoke-ai/ui-library'; -import { useAppDispatch } from 'app/store/storeHooks'; -import { addToast } from 'features/system/store/systemSlice'; -import { makeToast } from 'features/system/util/makeToast'; import { useCallback } from 'react'; import { useTranslation } from 'react-i18next'; import { PiPlusBold } from 'react-icons/pi'; import type { ScanFolderResponse } from 'services/api/endpoints/models'; -import { useInstallModelMutation } from 'services/api/endpoints/models'; type Props = { result: ScanFolderResponse[number]; + installModel: (source: string) => void; }; -export const ScanModelResultItem = ({ result }: Props) => { +export const ScanModelResultItem = ({ result, installModel }: Props) => { const { t } = useTranslation(); - const dispatch = useAppDispatch(); - const [installModel] = useInstallModelMutation(); - - const handleQuickAdd = useCallback(() => { - installModel({ source: result.path }) - .unwrap() - .then((_) => { - dispatch( - addToast( - makeToast({ - title: t('toast.modelAddedSimple'), - status: 'success', - }) - ) - ); - }) - .catch((error) => { - if (error) { - dispatch( - addToast( - makeToast({ - title: `${error.data.detail} `, - status: 'error', - }) - ) - ); - } - }); - }, [installModel, result, dispatch, t]); + const handleInstall = useCallback(() => { + installModel(result.path); + }, [installModel, result]); return ( @@ -54,7 +25,7 @@ export const ScanModelResultItem = ({ result }: Props) => { {result.is_installed ? ( {t('common.installed')} ) : ( - } onClick={handleQuickAdd} size="sm" /> + } onClick={handleInstall} size="sm" /> )} diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResults.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResults.tsx index 3033c7715c..360d6c1403 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResults.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/AddModelPanel/ScanFolder/ScanFolderResults.tsx @@ -1,7 +1,10 @@ import { Button, + Checkbox, Divider, Flex, + FormControl, + FormLabel, Heading, IconButton, Input, @@ -12,7 +15,7 @@ import { useAppDispatch } from 'app/store/storeHooks'; import ScrollableContent from 'common/components/OverlayScrollbars/ScrollableContent'; import { addToast } from 'features/system/store/systemSlice'; import { makeToast } from 'features/system/util/makeToast'; -import type { ChangeEventHandler } from 'react'; +import type { ChangeEvent, ChangeEventHandler } from 'react'; import { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { PiXBold } from 'react-icons/pi'; @@ -28,7 +31,7 @@ export const ScanModelsResults = ({ results }: ScanModelResultsProps) => { const { t } = useTranslation(); const [searchTerm, setSearchTerm] = useState(''); const dispatch = useAppDispatch(); - + const [inplace, setInplace] = useState(true); const [installModel] = useInstallModelMutation(); const filteredResults = useMemo(() => { @@ -42,6 +45,10 @@ export const ScanModelsResults = ({ results }: ScanModelResultsProps) => { setSearchTerm(e.target.value.trim()); }, []); + const onChangeInplace = useCallback((e: ChangeEvent) => { + setInplace(e.target.checked); + }, []); + const clearSearch = useCallback(() => { setSearchTerm(''); }, []); @@ -51,7 +58,7 @@ export const ScanModelsResults = ({ results }: ScanModelResultsProps) => { if (result.is_installed) { continue; } - installModel({ source: result.path }) + installModel({ source: result.path, inplace }) .unwrap() .then((_) => { dispatch( @@ -76,7 +83,37 @@ export const ScanModelsResults = ({ results }: ScanModelResultsProps) => { } }); } - }, [installModel, filteredResults, dispatch, t]); + }, [filteredResults, installModel, inplace, dispatch, t]); + + const handleInstallOne = useCallback( + (source: string) => { + installModel({ source, inplace }) + .unwrap() + .then((_) => { + dispatch( + addToast( + makeToast({ + title: t('toast.modelAddedSimple'), + status: 'success', + }) + ) + ); + }) + .catch((error) => { + if (error) { + dispatch( + addToast( + makeToast({ + title: `${error.data.detail} `, + status: 'error', + }) + ) + ); + } + }); + }, + [installModel, inplace, dispatch, t] + ); return ( <> @@ -85,6 +122,10 @@ export const ScanModelsResults = ({ results }: ScanModelResultsProps) => { {t('modelManager.scanResults')} + + {t('modelManager.inplaceInstall')} + + @@ -116,7 +157,7 @@ export const ScanModelsResults = ({ results }: ScanModelResultsProps) => { {filteredResults.map((result) => ( - + ))} diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelList.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelList.tsx index 033841ec79..67e65dbfb6 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelList.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelList.tsx @@ -1,6 +1,7 @@ -import { Flex } from '@invoke-ai/ui-library'; +import { Flex, Text } from '@invoke-ai/ui-library'; import { useAppSelector } from 'app/store/storeHooks'; import ScrollableContent from 'common/components/OverlayScrollbars/ScrollableContent'; +import type { FilterableModelType } from 'features/modelManagerV2/store/modelManagerV2Slice'; import { memo, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { @@ -9,10 +10,11 @@ import { useIPAdapterModels, useLoRAModels, useMainModels, + useRefinerModels, useT2IAdapterModels, useVAEModels, } from 'services/api/hooks/modelsByType'; -import type { AnyModelConfig, ModelType } from 'services/api/types'; +import type { AnyModelConfig } from 'services/api/types'; import { FetchingModelsLoader } from './FetchingModelsLoader'; import { ModelListWrapper } from './ModelListWrapper'; @@ -27,6 +29,12 @@ const ModelList = () => { [mainModels, searchTerm, filteredModelType] ); + const [refinerModels, { isLoading: isLoadingRefinerModels }] = useRefinerModels(); + const filteredRefinerModels = useMemo( + () => modelsFilter(refinerModels, searchTerm, filteredModelType), + [refinerModels, searchTerm, filteredModelType] + ); + const [loraModels, { isLoading: isLoadingLoRAModels }] = useLoRAModels(); const filteredLoRAModels = useMemo( () => modelsFilter(loraModels, searchTerm, filteredModelType), @@ -63,6 +71,28 @@ const ModelList = () => { [vaeModels, searchTerm, filteredModelType] ); + const totalFilteredModels = useMemo(() => { + return ( + filteredMainModels.length + + filteredRefinerModels.length + + filteredLoRAModels.length + + filteredEmbeddingModels.length + + filteredControlNetModels.length + + filteredT2IAdapterModels.length + + filteredIPAdapterModels.length + + filteredVAEModels.length + ); + }, [ + filteredControlNetModels.length, + filteredEmbeddingModels.length, + filteredIPAdapterModels.length, + filteredLoRAModels.length, + filteredMainModels.length, + filteredRefinerModels.length, + filteredT2IAdapterModels.length, + filteredVAEModels.length, + ]); + return ( @@ -71,6 +101,11 @@ const ModelList = () => { {!isLoadingMainModels && filteredMainModels.length > 0 && ( )} + {/* Refiner Model List */} + {isLoadingRefinerModels && } + {!isLoadingRefinerModels && filteredRefinerModels.length > 0 && ( + + )} {/* LoRAs List */} {isLoadingLoRAModels && } {!isLoadingLoRAModels && filteredLoRAModels.length > 0 && ( @@ -108,6 +143,11 @@ const ModelList = () => { {!isLoadingT2IAdapterModels && filteredT2IAdapterModels.length > 0 && ( )} + {totalFilteredModels === 0 && ( + + {t('modelManager.noMatchingModels')} + + )} ); @@ -118,12 +158,24 @@ export default memo(ModelList); const modelsFilter = ( data: T[], nameFilter: string, - filteredModelType: ModelType | null + filteredModelType: FilterableModelType | null ): T[] => { return data.filter((model) => { const matchesFilter = model.name.toLowerCase().includes(nameFilter.toLowerCase()); - const matchesType = filteredModelType ? model.type === filteredModelType : true; + const matchesType = getMatchesType(model, filteredModelType); return matchesFilter && matchesType; }); }; + +const getMatchesType = (modelConfig: AnyModelConfig, filteredModelType: FilterableModelType | null): boolean => { + if (filteredModelType === 'refiner') { + return modelConfig.base === 'sdxl-refiner'; + } + + if (filteredModelType === 'main' && modelConfig.base === 'sdxl-refiner') { + return false; + } + + return filteredModelType ? modelConfig.type === filteredModelType : true; +}; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelListItem.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelListItem.tsx index 995a599048..c9d0c03ed8 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelListItem.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelListItem.tsx @@ -90,11 +90,13 @@ const ModelListItem = (props: ModelListItemProps) => { cursor="pointer" onClick={handleSelectModel} > - + - + - {model.name} + + {model.name} + diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelTypeFilter.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelTypeFilter.tsx index 0b8ad3f600..76802b36e7 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelTypeFilter.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelTypeFilter.tsx @@ -13,6 +13,7 @@ export const ModelTypeFilter = () => { const MODEL_TYPE_LABELS: Record = useMemo( () => ({ main: t('modelManager.main'), + refiner: t('sdxl.refiner'), lora: 'LoRA', embedding: t('modelManager.textualInversions'), controlnet: 'ControlNet', diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/Model.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/Model.tsx index 6c5583ade9..d95eed8d24 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/Model.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/Model.tsx @@ -87,9 +87,9 @@ export const Model = () => { - + - + {data.name} @@ -114,7 +114,7 @@ export const Model = () => { )} {data.source && ( - + {t('modelManager.source')}: {data?.source} )} diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelAttrView.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelAttrView.tsx index f45bfca993..ebdedffebf 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelAttrView.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelAttrView.tsx @@ -9,7 +9,9 @@ export const ModelAttrView = ({ label, value }: Props) => { return ( {label} - {value || '-'} + + {value || '-'} + ); }; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx index adb123f24d..0618af5dd0 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx @@ -53,7 +53,7 @@ export const ModelView = () => { )} - {data.type === 'ip_adapter' && ( + {data.type === 'ip_adapter' && data.format === 'invokeai' && ( diff --git a/invokeai/frontend/web/src/features/nodes/components/flow/nodes/Invocation/fields/inputs/NumberFieldInputComponent.tsx b/invokeai/frontend/web/src/features/nodes/components/flow/nodes/Invocation/fields/inputs/NumberFieldInputComponent.tsx index 0cb250bb22..e3f33d8a45 100644 --- a/invokeai/frontend/web/src/features/nodes/components/flow/nodes/Invocation/fields/inputs/NumberFieldInputComponent.tsx +++ b/invokeai/frontend/web/src/features/nodes/components/flow/nodes/Invocation/fields/inputs/NumberFieldInputComponent.tsx @@ -37,34 +37,50 @@ const NumberFieldInputComponent = ( ); const min = useMemo(() => { + let min = -NUMPY_RAND_MAX; if (!isNil(fieldTemplate.minimum)) { - return fieldTemplate.minimum; + min = fieldTemplate.minimum; } if (!isNil(fieldTemplate.exclusiveMinimum)) { - return fieldTemplate.exclusiveMinimum + 0.01; + min = fieldTemplate.exclusiveMinimum + 0.01; } - return; + return min; }, [fieldTemplate.exclusiveMinimum, fieldTemplate.minimum]); const max = useMemo(() => { + let max = NUMPY_RAND_MAX; if (!isNil(fieldTemplate.maximum)) { - return fieldTemplate.maximum; + max = fieldTemplate.maximum; } if (!isNil(fieldTemplate.exclusiveMaximum)) { - return fieldTemplate.exclusiveMaximum - 0.01; + max = fieldTemplate.exclusiveMaximum - 0.01; } - return; + return max; }, [fieldTemplate.exclusiveMaximum, fieldTemplate.maximum]); + const step = useMemo(() => { + if (isNil(fieldTemplate.multipleOf)) { + return isIntegerField ? 1 : 0.1; + } + return fieldTemplate.multipleOf; + }, [fieldTemplate.multipleOf, isIntegerField]); + + const fineStep = useMemo(() => { + if (isNil(fieldTemplate.multipleOf)) { + return isIntegerField ? 1 : 0.01; + } + return fieldTemplate.multipleOf; + }, [fieldTemplate.multipleOf, isIntegerField]); + return ( ); diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts index 2298e84d43..ad563de468 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts @@ -48,7 +48,7 @@ export const addIPAdapterToLinearGraph = async ( if (!ipAdapter.model) { return; } - const { id, weight, model, beginStepPct, endStepPct, controlImage } = ipAdapter; + const { id, weight, model, clipVisionModel, beginStepPct, endStepPct, controlImage } = ipAdapter; assert(controlImage, 'IP Adapter image is required'); @@ -58,6 +58,7 @@ export const addIPAdapterToLinearGraph = async ( is_intermediate: true, weight: weight, ip_adapter_model: model, + clip_vision_model: clipVisionModel, begin_step_percent: beginStepPct, end_step_percent: endStepPct, image: { @@ -83,7 +84,7 @@ export const addIPAdapterToLinearGraph = async ( }; const buildIPAdapterMetadata = (ipAdapter: IPAdapterConfig): S['IPAdapterMetadataField'] => { - const { controlImage, beginStepPct, endStepPct, model, weight } = ipAdapter; + const { controlImage, beginStepPct, endStepPct, model, clipVisionModel, weight } = ipAdapter; assert(model, 'IP Adapter model is required'); @@ -99,6 +100,7 @@ const buildIPAdapterMetadata = (ipAdapter: IPAdapterConfig): S['IPAdapterMetadat return { ip_adapter_model: model, + clip_vision_model: clipVisionModel, weight, begin_step_percent: beginStepPct, end_step_percent: endStepPct, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts index d847ccbfb5..6a59c51872 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts @@ -65,6 +65,11 @@ export const buildCanvasOutpaintGraph = async ( infillTileSize, infillPatchmatchDownscaleSize, infillMethod, + // infillMosaicTileWidth, + // infillMosaicTileHeight, + // infillMosaicMinColor, + // infillMosaicMaxColor, + infillColorValue, clipSkip, seamlessXAxis, seamlessYAxis, @@ -356,6 +361,28 @@ export const buildCanvasOutpaintGraph = async ( }; } + // TODO: add mosaic back + // if (infillMethod === 'mosaic') { + // graph.nodes[INPAINT_INFILL] = { + // type: 'infill_mosaic', + // id: INPAINT_INFILL, + // is_intermediate, + // tile_width: infillMosaicTileWidth, + // tile_height: infillMosaicTileHeight, + // min_color: infillMosaicMinColor, + // max_color: infillMosaicMaxColor, + // }; + // } + + if (infillMethod === 'color') { + graph.nodes[INPAINT_INFILL] = { + type: 'infill_rgba', + id: INPAINT_INFILL, + color: infillColorValue, + is_intermediate, + }; + } + // Handle Scale Before Processing if (isUsingScaledDimensions) { const scaledWidth: number = scaledBoundingBoxDimensions.width; diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts index 39a54fd9d1..b932b26660 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts @@ -66,6 +66,11 @@ export const buildCanvasSDXLOutpaintGraph = async ( infillTileSize, infillPatchmatchDownscaleSize, infillMethod, + // infillMosaicTileWidth, + // infillMosaicTileHeight, + // infillMosaicMinColor, + // infillMosaicMaxColor, + infillColorValue, seamlessXAxis, seamlessYAxis, canvasCoherenceMode, @@ -365,6 +370,28 @@ export const buildCanvasSDXLOutpaintGraph = async ( }; } + // TODO: add mosaic back + // if (infillMethod === 'mosaic') { + // graph.nodes[INPAINT_INFILL] = { + // type: 'infill_mosaic', + // id: INPAINT_INFILL, + // is_intermediate, + // tile_width: infillMosaicTileWidth, + // tile_height: infillMosaicTileHeight, + // min_color: infillMosaicMinColor, + // max_color: infillMosaicMaxColor, + // }; + // } + + if (infillMethod === 'color') { + graph.nodes[INPAINT_INFILL] = { + type: 'infill_rgba', + id: INPAINT_INFILL, + is_intermediate, + color: infillColorValue, + }; + } + // Handle Scale Before Processing if (isUsingScaledDimensions) { const scaledWidth: number = scaledBoundingBoxDimensions.width; diff --git a/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillColorOptions.tsx b/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillColorOptions.tsx new file mode 100644 index 0000000000..1cafe4310e --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillColorOptions.tsx @@ -0,0 +1,46 @@ +import { Box, Flex, FormControl, FormLabel } from '@invoke-ai/ui-library'; +import { createSelector } from '@reduxjs/toolkit'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAIColorPicker from 'common/components/IAIColorPicker'; +import { selectGenerationSlice, setInfillColorValue } from 'features/parameters/store/generationSlice'; +import { memo, useCallback, useMemo } from 'react'; +import type { RgbaColor } from 'react-colorful'; +import { useTranslation } from 'react-i18next'; + +const ParamInfillColorOptions = () => { + const dispatch = useAppDispatch(); + + const selector = useMemo( + () => + createSelector(selectGenerationSlice, (generation) => ({ + infillColor: generation.infillColorValue, + })), + [] + ); + + const { infillColor } = useAppSelector(selector); + + const infillMethod = useAppSelector((s) => s.generation.infillMethod); + + const { t } = useTranslation(); + + const handleInfillColor = useCallback( + (v: RgbaColor) => { + dispatch(setInfillColorValue(v)); + }, + [dispatch] + ); + + return ( + + + {t('parameters.infillColorValue')} + + + + + + ); +}; + +export default memo(ParamInfillColorOptions); diff --git a/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillMosaicOptions.tsx b/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillMosaicOptions.tsx new file mode 100644 index 0000000000..f164bb903e --- /dev/null +++ b/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillMosaicOptions.tsx @@ -0,0 +1,127 @@ +import { Box, CompositeNumberInput, CompositeSlider, Flex, FormControl, FormLabel } from '@invoke-ai/ui-library'; +import { createSelector } from '@reduxjs/toolkit'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import IAIColorPicker from 'common/components/IAIColorPicker'; +import { + selectGenerationSlice, + setInfillMosaicMaxColor, + setInfillMosaicMinColor, + setInfillMosaicTileHeight, + setInfillMosaicTileWidth, +} from 'features/parameters/store/generationSlice'; +import { memo, useCallback, useMemo } from 'react'; +import type { RgbaColor } from 'react-colorful'; +import { useTranslation } from 'react-i18next'; + +const ParamInfillMosaicTileSize = () => { + const dispatch = useAppDispatch(); + + const selector = useMemo( + () => + createSelector(selectGenerationSlice, (generation) => ({ + infillMosaicTileWidth: generation.infillMosaicTileWidth, + infillMosaicTileHeight: generation.infillMosaicTileHeight, + infillMosaicMinColor: generation.infillMosaicMinColor, + infillMosaicMaxColor: generation.infillMosaicMaxColor, + })), + [] + ); + + const { infillMosaicTileWidth, infillMosaicTileHeight, infillMosaicMinColor, infillMosaicMaxColor } = + useAppSelector(selector); + + const infillMethod = useAppSelector((s) => s.generation.infillMethod); + + const { t } = useTranslation(); + + const handleInfillMosaicTileWidthChange = useCallback( + (v: number) => { + dispatch(setInfillMosaicTileWidth(v)); + }, + [dispatch] + ); + + const handleInfillMosaicTileHeightChange = useCallback( + (v: number) => { + dispatch(setInfillMosaicTileHeight(v)); + }, + [dispatch] + ); + + const handleInfillMosaicMinColor = useCallback( + (v: RgbaColor) => { + dispatch(setInfillMosaicMinColor(v)); + }, + [dispatch] + ); + + const handleInfillMosaicMaxColor = useCallback( + (v: RgbaColor) => { + dispatch(setInfillMosaicMaxColor(v)); + }, + [dispatch] + ); + + return ( + + + {t('parameters.infillMosaicTileWidth')} + + + + + {t('parameters.infillMosaicTileHeight')} + + + + + {t('parameters.infillMosaicMinColor')} + + + + + + {t('parameters.infillMosaicMaxColor')} + + + + + + ); +}; + +export default memo(ParamInfillMosaicTileSize); diff --git a/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillOptions.tsx b/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillOptions.tsx index 16cbffe56a..04e4727885 100644 --- a/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillOptions.tsx +++ b/invokeai/frontend/web/src/features/parameters/components/Canvas/InfillAndScaling/ParamInfillOptions.tsx @@ -1,6 +1,8 @@ import { useAppSelector } from 'app/store/storeHooks'; import { memo } from 'react'; +import ParamInfillColorOptions from './ParamInfillColorOptions'; +import ParamInfillMosaicOptions from './ParamInfillMosaicOptions'; import ParamInfillPatchmatchDownscaleSize from './ParamInfillPatchmatchDownscaleSize'; import ParamInfillTilesize from './ParamInfillTilesize'; @@ -14,6 +16,14 @@ const ParamInfillOptions = () => { return ; } + if (infillMethod === 'mosaic') { + return ; + } + + if (infillMethod === 'color') { + return ; + } + return null; }; diff --git a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts index e272cd278e..0da6e21d9f 100644 --- a/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts +++ b/invokeai/frontend/web/src/features/parameters/store/generationSlice.ts @@ -19,6 +19,7 @@ import type { import { getIsSizeOptimal, getOptimalDimension } from 'features/parameters/util/optimalDimension'; import { configChanged } from 'features/system/store/configSlice'; import { clamp } from 'lodash-es'; +import type { RgbaColor } from 'react-colorful'; import type { ImageDTO } from 'services/api/types'; import type { GenerationState } from './types'; @@ -43,8 +44,6 @@ const initialGenerationState: GenerationState = { shouldFitToWidthHeight: true, shouldRandomizeSeed: true, steps: 50, - infillTileSize: 32, - infillPatchmatchDownscaleSize: 1, width: 512, model: null, vae: null, @@ -55,6 +54,13 @@ const initialGenerationState: GenerationState = { shouldUseCpuNoise: true, shouldShowAdvancedOptions: false, aspectRatio: { ...initialAspectRatioState }, + infillTileSize: 32, + infillPatchmatchDownscaleSize: 1, + infillMosaicTileWidth: 64, + infillMosaicTileHeight: 64, + infillMosaicMinColor: { r: 0, g: 0, b: 0, a: 1 }, + infillMosaicMaxColor: { r: 255, g: 255, b: 255, a: 1 }, + infillColorValue: { r: 0, g: 0, b: 0, a: 1 }, }; export const generationSlice = createSlice({ @@ -116,15 +122,6 @@ export const generationSlice = createSlice({ setCanvasCoherenceMinDenoise: (state, action: PayloadAction) => { state.canvasCoherenceMinDenoise = action.payload; }, - setInfillMethod: (state, action: PayloadAction) => { - state.infillMethod = action.payload; - }, - setInfillTileSize: (state, action: PayloadAction) => { - state.infillTileSize = action.payload; - }, - setInfillPatchmatchDownscaleSize: (state, action: PayloadAction) => { - state.infillPatchmatchDownscaleSize = action.payload; - }, initialImageChanged: (state, action: PayloadAction) => { const { image_name, width, height } = action.payload; state.initialImage = { imageName: image_name, width, height }; @@ -206,6 +203,30 @@ export const generationSlice = createSlice({ aspectRatioChanged: (state, action: PayloadAction) => { state.aspectRatio = action.payload; }, + setInfillMethod: (state, action: PayloadAction) => { + state.infillMethod = action.payload; + }, + setInfillTileSize: (state, action: PayloadAction) => { + state.infillTileSize = action.payload; + }, + setInfillPatchmatchDownscaleSize: (state, action: PayloadAction) => { + state.infillPatchmatchDownscaleSize = action.payload; + }, + setInfillMosaicTileWidth: (state, action: PayloadAction) => { + state.infillMosaicTileWidth = action.payload; + }, + setInfillMosaicTileHeight: (state, action: PayloadAction) => { + state.infillMosaicTileHeight = action.payload; + }, + setInfillMosaicMinColor: (state, action: PayloadAction) => { + state.infillMosaicMinColor = action.payload; + }, + setInfillMosaicMaxColor: (state, action: PayloadAction) => { + state.infillMosaicMaxColor = action.payload; + }, + setInfillColorValue: (state, action: PayloadAction) => { + state.infillColorValue = action.payload; + }, }, extraReducers: (builder) => { builder.addCase(configChanged, (state, action) => { @@ -249,8 +270,6 @@ export const { setShouldFitToWidthHeight, setShouldRandomizeSeed, setSteps, - setInfillTileSize, - setInfillPatchmatchDownscaleSize, initialImageChanged, modelChanged, vaeSelected, @@ -264,6 +283,13 @@ export const { heightChanged, widthRecalled, heightRecalled, + setInfillTileSize, + setInfillPatchmatchDownscaleSize, + setInfillMosaicTileWidth, + setInfillMosaicTileHeight, + setInfillMosaicMinColor, + setInfillMosaicMaxColor, + setInfillColorValue, } = generationSlice.actions; export const { selectOptimalDimension } = generationSlice.selectors; diff --git a/invokeai/frontend/web/src/features/parameters/store/types.ts b/invokeai/frontend/web/src/features/parameters/store/types.ts index 73185754ee..773cfbf925 100644 --- a/invokeai/frontend/web/src/features/parameters/store/types.ts +++ b/invokeai/frontend/web/src/features/parameters/store/types.ts @@ -17,6 +17,7 @@ import type { ParameterVAEModel, ParameterWidth, } from 'features/parameters/types/parameterSchemas'; +import type { RgbaColor } from 'react-colorful'; export interface GenerationState { _version: 2; @@ -39,8 +40,6 @@ export interface GenerationState { shouldFitToWidthHeight: boolean; shouldRandomizeSeed: boolean; steps: ParameterSteps; - infillTileSize: number; - infillPatchmatchDownscaleSize: number; width: ParameterWidth; model: ParameterModel | null; vae: ParameterVAEModel | null; @@ -51,6 +50,13 @@ export interface GenerationState { shouldUseCpuNoise: boolean; shouldShowAdvancedOptions: boolean; aspectRatio: AspectRatioState; + infillTileSize: number; + infillPatchmatchDownscaleSize: number; + infillMosaicTileWidth: number; + infillMosaicTileHeight: number; + infillMosaicMinColor: RgbaColor; + infillMosaicMaxColor: RgbaColor; + infillColorValue: RgbaColor; } export type PayloadActionWithOptimalDimension = PayloadAction; diff --git a/invokeai/frontend/web/src/features/settingsAccordions/components/AdvancedSettingsAccordion/AdvancedSettingsAccordion.tsx b/invokeai/frontend/web/src/features/settingsAccordions/components/AdvancedSettingsAccordion/AdvancedSettingsAccordion.tsx index 05ca73927c..087efba616 100644 --- a/invokeai/frontend/web/src/features/settingsAccordions/components/AdvancedSettingsAccordion/AdvancedSettingsAccordion.tsx +++ b/invokeai/frontend/web/src/features/settingsAccordions/components/AdvancedSettingsAccordion/AdvancedSettingsAccordion.tsx @@ -61,7 +61,7 @@ export const AdvancedSettingsAccordion = memo(() => { return ( - + diff --git a/invokeai/frontend/web/src/features/settingsAccordions/components/ControlSettingsAccordion/ControlSettingsAccordion.tsx b/invokeai/frontend/web/src/features/settingsAccordions/components/ControlSettingsAccordion/ControlSettingsAccordion.tsx index a02eadef36..3cddc927e9 100644 --- a/invokeai/frontend/web/src/features/settingsAccordions/components/ControlSettingsAccordion/ControlSettingsAccordion.tsx +++ b/invokeai/frontend/web/src/features/settingsAccordions/components/ControlSettingsAccordion/ControlSettingsAccordion.tsx @@ -77,7 +77,7 @@ export const ControlSettingsAccordion: React.FC = memo(() => { return ( - +