From 556c6a1d84e094a50ea24f3458ae0e0a324167ea Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Wed, 31 Jul 2024 00:51:55 +0530 Subject: [PATCH] fix: Update DepthAnything to use the transformers implementation --- .../controlnet_image_processors.py | 25 +- .../image_util/depth_anything/__init__.py | 65 --- .../image_util/depth_anything/utils/blocks.py | 147 ------- .../depth_anything/utils/transform.py | 159 ------- .../image_util/depth_anything/v2/dinov2.py | 405 ------------------ .../v2/dinov2_layers/__init__.py | 12 - .../v2/dinov2_layers/attention.py | 76 ---- .../depth_anything/v2/dinov2_layers/block.py | 248 ----------- .../v2/dinov2_layers/drop_path.py | 35 -- .../v2/dinov2_layers/layer_scale.py | 27 -- .../depth_anything/v2/dinov2_layers/mlp.py | 41 -- .../v2/dinov2_layers/patch_embed.py | 89 ---- .../v2/dinov2_layers/swiglu_ffn.py | 63 --- .../image_util/depth_anything/v2/dpt.py | 231 ---------- 14 files changed, 10 insertions(+), 1613 deletions(-) delete mode 100644 invokeai/backend/image_util/depth_anything/__init__.py delete mode 100644 invokeai/backend/image_util/depth_anything/utils/blocks.py delete mode 100644 invokeai/backend/image_util/depth_anything/utils/transform.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/__init__.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/attention.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/block.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/drop_path.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/layer_scale.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/mlp.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/patch_embed.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dinov2_layers/swiglu_ffn.py delete mode 100644 invokeai/backend/image_util/depth_anything/v2/dpt.py diff --git a/invokeai/app/invocations/controlnet_image_processors.py b/invokeai/app/invocations/controlnet_image_processors.py index 497d07e7cf..1b022a071d 100644 --- a/invokeai/app/invocations/controlnet_image_processors.py +++ b/invokeai/app/invocations/controlnet_image_processors.py @@ -2,7 +2,6 @@ # initial implementation by Gregg Helt, 2023 # heavily leverages controlnet_aux package: https://github.com/patrickvonplaten/controlnet_aux from builtins import bool, float -from pathlib import Path from typing import Dict, List, Literal, Union import cv2 @@ -21,6 +20,7 @@ from controlnet_aux import ( from controlnet_aux.util import HWC3, ade_palette from PIL import Image from pydantic import BaseModel, Field, field_validator, model_validator +from transformers import pipeline from invokeai.app.invocations.baseinvocation import ( BaseInvocation, @@ -44,13 +44,11 @@ from invokeai.app.invocations.util import validate_begin_end_step, validate_weig from invokeai.app.services.shared.invocation_context import InvocationContext from invokeai.app.util.controlnet_utils import CONTROLNET_MODE_VALUES, CONTROLNET_RESIZE_VALUES, heuristic_resize from invokeai.backend.image_util.canny import get_canny_edges -from invokeai.backend.image_util.depth_anything import DEPTH_ANYTHING_MODELS, DepthAnythingDetector from invokeai.backend.image_util.dw_openpose import DWPOSE_MODELS, DWOpenposeDetector from invokeai.backend.image_util.hed import HEDProcessor from invokeai.backend.image_util.lineart import LineartProcessor from invokeai.backend.image_util.lineart_anime import LineartAnimeProcessor from invokeai.backend.image_util.util import np_to_pil, pil_to_np -from invokeai.backend.util.devices import TorchDevice class ControlField(BaseModel): @@ -593,6 +591,11 @@ class ColorMapImageProcessorInvocation(ImageProcessorInvocation): DEPTH_ANYTHING_MODEL_SIZES = Literal["large", "base", "small"] +DEPTH_ANYTHING_MODELS = { + "large": "LiheYoung/depth-anything-large-hf", + "base": "LiheYoung/depth-anything-base-hf", + "small": "depth-anything/Depth-Anything-V2-Small-hf", +} @invocation( @@ -600,7 +603,7 @@ DEPTH_ANYTHING_MODEL_SIZES = Literal["large", "base", "small"] title="Depth Anything Processor", tags=["controlnet", "depth", "depth anything"], category="controlnet", - version="1.1.2", + version="1.1.3", ) class DepthAnythingImageProcessorInvocation(ImageProcessorInvocation): """Generates a depth map based on the Depth Anything algorithm""" @@ -611,17 +614,9 @@ class DepthAnythingImageProcessorInvocation(ImageProcessorInvocation): resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res) def run_processor(self, image: Image.Image) -> Image.Image: - def loader(model_path: Path): - return DepthAnythingDetector.load_model( - model_path, model_size=self.model_size, device=TorchDevice.choose_torch_device() - ) - - with self._context.models.load_remote_model( - source=DEPTH_ANYTHING_MODELS[self.model_size], loader=loader - ) as model: - depth_anything_detector = DepthAnythingDetector(model, TorchDevice.choose_torch_device()) - processed_image = depth_anything_detector(image=image, resolution=self.resolution) - return processed_image + depth_anything_pipeline = pipeline(task="depth-estimation", model=DEPTH_ANYTHING_MODELS[self.model_size]) + depth_map = depth_anything_pipeline(image)["depth"] + return depth_map @invocation( diff --git a/invokeai/backend/image_util/depth_anything/__init__.py b/invokeai/backend/image_util/depth_anything/__init__.py deleted file mode 100644 index 5ed9b2e20b..0000000000 --- a/invokeai/backend/image_util/depth_anything/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -from pathlib import Path -from typing import Literal - -import numpy as np -import torch -from einops import repeat -from PIL import Image - -from invokeai.app.services.config.config_default import get_config -from invokeai.backend.image_util.depth_anything.v2.dpt import DepthAnythingV2 -from invokeai.backend.util.logging import InvokeAILogger - -config = get_config() -logger = InvokeAILogger.get_logger(config=config) - -DEPTH_ANYTHING_MODELS = { - "large": "https://huggingface.co/spaces/LiheYoung/Depth-Anything/resolve/main/checkpoints/depth_anything_vitl14.pth?download=true", - "base": "https://huggingface.co/spaces/LiheYoung/Depth-Anything/resolve/main/checkpoints/depth_anything_vitb14.pth?download=true", - "small": "https://huggingface.co/depth-anything/Depth-Anything-V2-Small/resolve/main/depth_anything_v2_vits.pth?download=true", -} - - -class DepthAnythingDetector: - def __init__(self, model: DepthAnythingV2, device: torch.device) -> None: - self.model = model - self.device = device - - @staticmethod - def load_model( - model_path: Path, device: torch.device, model_size: Literal["large", "base", "small", "giant"] = "small" - ) -> DepthAnythingV2: - match model_size: - case "small": - model = DepthAnythingV2(encoder="vits", features=64, out_channels=[48, 96, 192, 384]) - case "base": - model = DepthAnythingV2(encoder="vitb", features=128, out_channels=[96, 192, 384, 768]) - case "large": - model = DepthAnythingV2(encoder="vitl", features=256, out_channels=[256, 512, 1024, 1024]) - case "giant": - model = DepthAnythingV2(encoder="vitg", features=384, out_channels=[1536, 1536, 1536, 1536]) - - model.load_state_dict(torch.load(model_path.as_posix(), map_location="cpu")) - model.eval() - model.to(device) - return model - - def __call__(self, image: Image.Image, resolution: int = 512) -> Image.Image: - if not self.model: - logger.warn("DepthAnything model was not loaded. Returning original image") - return image - - np_image = np.array(image, dtype=np.uint8) - image_height, image_width = np_image.shape[:2] - - with torch.no_grad(): - depth = self.model.infer_image(np_image) - depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0 - - depth_map = repeat(depth, "h w -> h w 3").astype(np.uint8) - depth_map = Image.fromarray(depth_map) - - new_height = int(image_height * (resolution / image_width)) - depth_map = depth_map.resize((resolution, new_height)) - - return depth_map diff --git a/invokeai/backend/image_util/depth_anything/utils/blocks.py b/invokeai/backend/image_util/depth_anything/utils/blocks.py deleted file mode 100644 index cb1161c62f..0000000000 --- a/invokeai/backend/image_util/depth_anything/utils/blocks.py +++ /dev/null @@ -1,147 +0,0 @@ -# Referenced from: https://github.com/DepthAnything/Depth-Anything-V2/blob/main/depth_anything_v2/util/blocks.py - -import torch.nn as nn - - -def _make_scratch(in_shape, out_shape, groups=1, expand=False): - scratch = nn.Module() - - out_shape1 = out_shape - out_shape2 = out_shape - out_shape3 = out_shape - if len(in_shape) >= 4: - out_shape4 = out_shape - - if expand: - out_shape1 = out_shape - out_shape2 = out_shape * 2 - out_shape3 = out_shape * 4 - if len(in_shape) >= 4: - out_shape4 = out_shape * 8 - - scratch.layer1_rn = nn.Conv2d( - in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups - ) - scratch.layer2_rn = nn.Conv2d( - in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups - ) - scratch.layer3_rn = nn.Conv2d( - in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups - ) - if len(in_shape) >= 4: - scratch.layer4_rn = nn.Conv2d( - in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups - ) - - return scratch - - -class ResidualConvUnit(nn.Module): - """Residual convolution module.""" - - def __init__(self, features, activation, bn): - """Init. - - Args: - features (int): number of features - """ - super().__init__() - - self.bn = bn - - self.groups = 1 - - self.conv1 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups) - - self.conv2 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups) - - if self.bn: - self.bn1 = nn.BatchNorm2d(features) - self.bn2 = nn.BatchNorm2d(features) - - self.activation = activation - - self.skip_add = nn.quantized.FloatFunctional() - - def forward(self, x): - """Forward pass. - - Args: - x (tensor): input - - Returns: - tensor: output - """ - - out = self.activation(x) - out = self.conv1(out) - if self.bn: - out = self.bn1(out) - - out = self.activation(out) - out = self.conv2(out) - if self.bn: - out = self.bn2(out) - - if self.groups > 1: - out = self.conv_merge(out) - - return self.skip_add.add(out, x) - - -class FeatureFusionBlock(nn.Module): - """Feature fusion block.""" - - def __init__(self, features, activation, deconv=False, bn=False, expand=False, align_corners=True, size=None): - """Init. - - Args: - features (int): number of features - """ - super(FeatureFusionBlock, self).__init__() - - self.deconv = deconv - self.align_corners = align_corners - - self.groups = 1 - - self.expand = expand - out_features = features - if self.expand: - out_features = features // 2 - - self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1) - - self.resConfUnit1 = ResidualConvUnit(features, activation, bn) - self.resConfUnit2 = ResidualConvUnit(features, activation, bn) - - self.skip_add = nn.quantized.FloatFunctional() - - self.size = size - - def forward(self, *xs, size=None): - """Forward pass. - - Returns: - tensor: output - """ - output = xs[0] - - if len(xs) == 2: - res = self.resConfUnit1(xs[1]) - output = self.skip_add.add(output, res) - - output = self.resConfUnit2(output) - - if (size is None) and (self.size is None): - modifier = {"scale_factor": 2} - elif size is None: - modifier = {"size": self.size} - else: - modifier = {"size": size} - - output = nn.functional.interpolate(output, **modifier, mode="bilinear", align_corners=self.align_corners) - - output = self.out_conv(output) - - return output diff --git a/invokeai/backend/image_util/depth_anything/utils/transform.py b/invokeai/backend/image_util/depth_anything/utils/transform.py deleted file mode 100644 index 933ce4c913..0000000000 --- a/invokeai/backend/image_util/depth_anything/utils/transform.py +++ /dev/null @@ -1,159 +0,0 @@ -# Referenced from: https://github.com/DepthAnything/Depth-Anything-V2/blob/main/depth_anything_v2/util/transform.py - -import cv2 -import numpy as np - - -class Resize(object): - """Resize sample to given size (width, height).""" - - def __init__( - self, - width, - height, - resize_target=True, - keep_aspect_ratio=False, - ensure_multiple_of=1, - resize_method="lower_bound", - image_interpolation_method=cv2.INTER_AREA, - ): - """Init. - - Args: - width (int): desired output width - height (int): desired output height - resize_target (bool, optional): - True: Resize the full sample (image, mask, target). - False: Resize image only. - Defaults to True. - keep_aspect_ratio (bool, optional): - True: Keep the aspect ratio of the input sample. - Output sample might not have the given width and height, and - resize behaviour depends on the parameter 'resize_method'. - Defaults to False. - ensure_multiple_of (int, optional): - Output width and height is constrained to be multiple of this parameter. - Defaults to 1. - resize_method (str, optional): - "lower_bound": Output will be at least as large as the given size. - "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) - "minimal": Scale as least as possible. (Output size might be smaller than given size.) - Defaults to "lower_bound". - """ - self.__width = width - self.__height = height - - self.__resize_target = resize_target - self.__keep_aspect_ratio = keep_aspect_ratio - self.__multiple_of = ensure_multiple_of - self.__resize_method = resize_method - self.__image_interpolation_method = image_interpolation_method - - def constrain_to_multiple_of(self, x, min_val=0, max_val=None): - y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) - - if max_val is not None and y > max_val: - y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) - - if y < min_val: - y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) - - return y - - def get_size(self, width, height): - # determine new height and width - scale_height = self.__height / height - scale_width = self.__width / width - - if self.__keep_aspect_ratio: - if self.__resize_method == "lower_bound": - # scale such that output size is lower bound - if scale_width > scale_height: - # fit width - scale_height = scale_width - else: - # fit height - scale_width = scale_height - elif self.__resize_method == "upper_bound": - # scale such that output size is upper bound - if scale_width < scale_height: - # fit width - scale_height = scale_width - else: - # fit height - scale_width = scale_height - elif self.__resize_method == "minimal": - # scale as least as possbile - if abs(1 - scale_width) < abs(1 - scale_height): - # fit width - scale_height = scale_width - else: - # fit height - scale_width = scale_height - else: - raise ValueError(f"resize_method {self.__resize_method} not implemented") - - if self.__resize_method == "lower_bound": - new_height = self.constrain_to_multiple_of(scale_height * height, min_val=self.__height) - new_width = self.constrain_to_multiple_of(scale_width * width, min_val=self.__width) - elif self.__resize_method == "upper_bound": - new_height = self.constrain_to_multiple_of(scale_height * height, max_val=self.__height) - new_width = self.constrain_to_multiple_of(scale_width * width, max_val=self.__width) - elif self.__resize_method == "minimal": - new_height = self.constrain_to_multiple_of(scale_height * height) - new_width = self.constrain_to_multiple_of(scale_width * width) - else: - raise ValueError(f"resize_method {self.__resize_method} not implemented") - - return (new_width, new_height) - - def __call__(self, sample): - width, height = self.get_size(sample["image"].shape[1], sample["image"].shape[0]) - - # resize sample - sample["image"] = cv2.resize(sample["image"], (width, height), interpolation=self.__image_interpolation_method) - - if self.__resize_target: - if "depth" in sample: - sample["depth"] = cv2.resize(sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST) - - if "mask" in sample: - sample["mask"] = cv2.resize( - sample["mask"].astype(np.float32), (width, height), interpolation=cv2.INTER_NEAREST - ) - - return sample - - -class NormalizeImage(object): - """Normlize image by given mean and std.""" - - def __init__(self, mean, std): - self.__mean = mean - self.__std = std - - def __call__(self, sample): - sample["image"] = (sample["image"] - self.__mean) / self.__std - - return sample - - -class PrepareForNet(object): - """Prepare sample for usage as network input.""" - - def __init__(self): - pass - - def __call__(self, sample): - image = np.transpose(sample["image"], (2, 0, 1)) - sample["image"] = np.ascontiguousarray(image).astype(np.float32) - - if "depth" in sample: - depth = sample["depth"].astype(np.float32) - sample["depth"] = np.ascontiguousarray(depth) - - if "mask" in sample: - sample["mask"] = sample["mask"].astype(np.float32) - sample["mask"] = np.ascontiguousarray(sample["mask"]) - - return sample diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2.py b/invokeai/backend/image_util/depth_anything/v2/dinov2.py deleted file mode 100644 index da460924f6..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the Apache License, Version 2.0 -# found in the LICENSE file in the root directory of this source tree. - -# References: -# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py -# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py - - -import math -from functools import partial -from typing import Callable, Sequence, Tuple, Union - -import torch -import torch.nn as nn -import torch.utils.checkpoint -from torch.nn.init import trunc_normal_ - -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers import MemEffAttention, Mlp, PatchEmbed, SwiGLUFFNFused -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers import NestedTensorBlock as Block - - -def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module: - if not depth_first and include_root: - fn(module=module, name=name) - for child_name, child_module in module.named_children(): - child_name = ".".join((name, child_name)) if name else child_name - named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True) - if depth_first and include_root: - fn(module=module, name=name) - return module - - -class BlockChunk(nn.ModuleList): - def forward(self, x): - for b in self: - x = b(x) - return x - - -class DinoVisionTransformer(nn.Module): - def __init__( - self, - img_size=224, - patch_size=16, - in_chans=3, - embed_dim=768, - depth=12, - num_heads=12, - mlp_ratio=4.0, - qkv_bias=True, - ffn_bias=True, - proj_bias=True, - drop_path_rate=0.0, - drop_path_uniform=False, - init_values=None, # for layerscale: None or 0 => no layerscale - embed_layer=PatchEmbed, - act_layer=nn.GELU, - block_fn=Block, - ffn_layer="mlp", - block_chunks=1, - num_register_tokens=0, - interpolate_antialias=False, - interpolate_offset=0.1, - ): - """ - Args: - img_size (int, tuple): input image size - patch_size (int, tuple): patch size - in_chans (int): number of input channels - embed_dim (int): embedding dimension - depth (int): depth of transformer - num_heads (int): number of attention heads - mlp_ratio (int): ratio of mlp hidden dim to embedding dim - qkv_bias (bool): enable bias for qkv if True - proj_bias (bool): enable bias for proj in attn if True - ffn_bias (bool): enable bias for ffn if True - drop_path_rate (float): stochastic depth rate - drop_path_uniform (bool): apply uniform drop rate across blocks - weight_init (str): weight init scheme - init_values (float): layer-scale init values - embed_layer (nn.Module): patch embedding layer - act_layer (nn.Module): MLP activation layer - block_fn (nn.Module): transformer block class - ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity" - block_chunks: (int) split block sequence into block_chunks units for FSDP wrap - num_register_tokens: (int) number of extra cls tokens (so-called "registers") - interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings - interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings - """ - super().__init__() - norm_layer = partial(nn.LayerNorm, eps=1e-6) - - self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models - self.num_tokens = 1 - self.n_blocks = depth - self.num_heads = num_heads - self.patch_size = patch_size - self.num_register_tokens = num_register_tokens - self.interpolate_antialias = interpolate_antialias - self.interpolate_offset = interpolate_offset - - self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) - num_patches = self.patch_embed.num_patches - - self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) - self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) - assert num_register_tokens >= 0 - self.register_tokens = ( - nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None - ) - - if drop_path_uniform is True: - dpr = [drop_path_rate] * depth - else: - dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule - - if ffn_layer == "mlp": - ffn_layer = Mlp - elif ffn_layer == "swiglufused" or ffn_layer == "swiglu": - ffn_layer = SwiGLUFFNFused - elif ffn_layer == "identity": - - def f(*args, **kwargs): - return nn.Identity() - - ffn_layer = f - else: - raise NotImplementedError - - blocks_list = [ - block_fn( - dim=embed_dim, - num_heads=num_heads, - mlp_ratio=mlp_ratio, - qkv_bias=qkv_bias, - proj_bias=proj_bias, - ffn_bias=ffn_bias, - drop_path=dpr[i], - norm_layer=norm_layer, - act_layer=act_layer, - ffn_layer=ffn_layer, - init_values=init_values, - ) - for i in range(depth) - ] - if block_chunks > 0: - self.chunked_blocks = True - chunked_blocks = [] - chunksize = depth // block_chunks - for i in range(0, depth, chunksize): - # this is to keep the block index consistent if we chunk the block list - chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize]) - self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks]) - else: - self.chunked_blocks = False - self.blocks = nn.ModuleList(blocks_list) - - self.norm = norm_layer(embed_dim) - self.head = nn.Identity() - - self.mask_token = nn.Parameter(torch.zeros(1, embed_dim)) - - self.init_weights() - - def init_weights(self): - trunc_normal_(self.pos_embed, std=0.02) - nn.init.normal_(self.cls_token, std=1e-6) - if self.register_tokens is not None: - nn.init.normal_(self.register_tokens, std=1e-6) - named_apply(init_weights_vit_timm, self) - - def interpolate_pos_encoding(self, x, w, h): - previous_dtype = x.dtype - npatch = x.shape[1] - 1 - N = self.pos_embed.shape[1] - 1 - if npatch == N and w == h: - return self.pos_embed - pos_embed = self.pos_embed.float() - class_pos_embed = pos_embed[:, 0] - patch_pos_embed = pos_embed[:, 1:] - dim = x.shape[-1] - w0 = w // self.patch_size - h0 = h // self.patch_size - # we add a small number to avoid floating point error in the interpolation - # see discussion at https://github.com/facebookresearch/dino/issues/8 - # DINOv2 with register modify the interpolate_offset from 0.1 to 0.0 - w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset - # w0, h0 = w0 + 0.1, h0 + 0.1 - - sqrt_N = math.sqrt(N) - sx, sy = float(w0) / sqrt_N, float(h0) / sqrt_N - patch_pos_embed = nn.functional.interpolate( - patch_pos_embed.reshape(1, int(sqrt_N), int(sqrt_N), dim).permute(0, 3, 1, 2), - scale_factor=(sx, sy), - # (int(w0), int(h0)), # to solve the upsampling shape issue - mode="bicubic", - antialias=self.interpolate_antialias, - ) - - assert int(w0) == patch_pos_embed.shape[-2] - assert int(h0) == patch_pos_embed.shape[-1] - patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) - return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype) - - def prepare_tokens_with_masks(self, x, masks=None): - B, nc, w, h = x.shape - x = self.patch_embed(x) - if masks is not None: - x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x) - - x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) - x = x + self.interpolate_pos_encoding(x, w, h) - - if self.register_tokens is not None: - x = torch.cat( - ( - x[:, :1], - self.register_tokens.expand(x.shape[0], -1, -1), - x[:, 1:], - ), - dim=1, - ) - - return x - - def forward_features_list(self, x_list, masks_list): - x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list, strict=False)] - for blk in self.blocks: - x = blk(x) - - all_x = x - output = [] - for x, masks in zip(all_x, masks_list, strict=False): - x_norm = self.norm(x) - output.append( - { - "x_norm_clstoken": x_norm[:, 0], - "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1], - "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :], - "x_prenorm": x, - "masks": masks, - } - ) - return output - - def forward_features(self, x, masks=None): - if isinstance(x, list): - return self.forward_features_list(x, masks) - - x = self.prepare_tokens_with_masks(x, masks) - - for blk in self.blocks: - x = blk(x) - - x_norm = self.norm(x) - return { - "x_norm_clstoken": x_norm[:, 0], - "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1], - "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :], - "x_prenorm": x, - "masks": masks, - } - - def _get_intermediate_layers_not_chunked(self, x, n=1): - x = self.prepare_tokens_with_masks(x) - # If n is an int, take the n last blocks. If it's a list, take them - output, total_block_len = [], len(self.blocks) - blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n - for i, blk in enumerate(self.blocks): - x = blk(x) - if i in blocks_to_take: - output.append(x) - assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" - return output - - def _get_intermediate_layers_chunked(self, x, n=1): - x = self.prepare_tokens_with_masks(x) - output, i, total_block_len = [], 0, len(self.blocks[-1]) - # If n is an int, take the n last blocks. If it's a list, take them - blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n - for block_chunk in self.blocks: - for blk in block_chunk[i:]: # Passing the nn.Identity() - x = blk(x) - if i in blocks_to_take: - output.append(x) - i += 1 - assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" - return output - - def get_intermediate_layers( - self, - x: torch.Tensor, - n: Union[int, Sequence] = 1, # Layers or n last layers to take - reshape: bool = False, - return_class_token: bool = False, - norm: bool = True, - ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: - if self.chunked_blocks: - outputs = self._get_intermediate_layers_chunked(x, n) - else: - outputs = self._get_intermediate_layers_not_chunked(x, n) - if norm: - outputs = [self.norm(out) for out in outputs] - class_tokens = [out[:, 0] for out in outputs] - outputs = [out[:, 1 + self.num_register_tokens :] for out in outputs] - if reshape: - B, _, w, h = x.shape - outputs = [ - out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous() - for out in outputs - ] - if return_class_token: - return tuple(zip(outputs, class_tokens, strict=False)) - return tuple(outputs) - - def forward(self, *args, is_training=False, **kwargs): - ret = self.forward_features(*args, **kwargs) - if is_training: - return ret - else: - return self.head(ret["x_norm_clstoken"]) - - -def init_weights_vit_timm(module: nn.Module, name: str = ""): - """ViT weight initialization, original timm impl (for reproducibility)""" - if isinstance(module, nn.Linear): - trunc_normal_(module.weight, std=0.02) - if module.bias is not None: - nn.init.zeros_(module.bias) - - -def vit_small(patch_size=16, num_register_tokens=0, **kwargs): - model = DinoVisionTransformer( - patch_size=patch_size, - embed_dim=384, - depth=12, - num_heads=6, - mlp_ratio=4, - block_fn=partial(Block, attn_class=MemEffAttention), - num_register_tokens=num_register_tokens, - **kwargs, - ) - return model - - -def vit_base(patch_size=16, num_register_tokens=0, **kwargs): - model = DinoVisionTransformer( - patch_size=patch_size, - embed_dim=768, - depth=12, - num_heads=12, - mlp_ratio=4, - block_fn=partial(Block, attn_class=MemEffAttention), - num_register_tokens=num_register_tokens, - **kwargs, - ) - return model - - -def vit_large(patch_size=16, num_register_tokens=0, **kwargs): - model = DinoVisionTransformer( - patch_size=patch_size, - embed_dim=1024, - depth=24, - num_heads=16, - mlp_ratio=4, - block_fn=partial(Block, attn_class=MemEffAttention), - num_register_tokens=num_register_tokens, - **kwargs, - ) - return model - - -def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs): - """ - Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64 - """ - model = DinoVisionTransformer( - patch_size=patch_size, - embed_dim=1536, - depth=40, - num_heads=24, - mlp_ratio=4, - block_fn=partial(Block, attn_class=MemEffAttention), - num_register_tokens=num_register_tokens, - **kwargs, - ) - return model - - -def DINOv2(model_name): - model_zoo = {"vits": vit_small, "vitb": vit_base, "vitl": vit_large, "vitg": vit_giant2} - - return model_zoo[model_name]( - img_size=518, - patch_size=14, - init_values=1.0, - ffn_layer="mlp" if model_name != "vitg" else "swiglufused", - block_chunks=0, - num_register_tokens=0, - interpolate_antialias=False, - interpolate_offset=0.1, - ) diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/__init__.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/__init__.py deleted file mode 100644 index 3980163fea..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - - -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.attention import MemEffAttention # noqa -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.block import NestedTensorBlock # noqa -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.mlp import Mlp # noqa -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.patch_embed import PatchEmbed # noqa -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused # noqa diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/attention.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/attention.py deleted file mode 100644 index d2a5e5b96a..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/attention.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -# References: -# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py -# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py - -# Referenced from: https://github.com/DepthAnything/Depth-Anything-V2 - - -from torch import Tensor, nn - -try: - from xformers.ops import memory_efficient_attention, unbind - - XFORMERS_AVAILABLE = True -except ImportError: - XFORMERS_AVAILABLE = False - - -class Attention(nn.Module): - def __init__( - self, - dim: int, - num_heads: int = 8, - qkv_bias: bool = False, - proj_bias: bool = True, - attn_drop: float = 0.0, - proj_drop: float = 0.0, - ) -> None: - super().__init__() - self.num_heads = num_heads - head_dim = dim // num_heads - self.scale = head_dim**-0.5 - - self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) - self.attn_drop = nn.Dropout(attn_drop) - self.proj = nn.Linear(dim, dim, bias=proj_bias) - self.proj_drop = nn.Dropout(proj_drop) - - def forward(self, x: Tensor) -> Tensor: - B, N, C = x.shape - qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) - - q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] - attn = q @ k.transpose(-2, -1) - - attn = attn.softmax(dim=-1) - attn = self.attn_drop(attn) - - x = (attn @ v).transpose(1, 2).reshape(B, N, C) - x = self.proj(x) - x = self.proj_drop(x) - return x - - -class MemEffAttention(Attention): - def forward(self, x: Tensor, attn_bias=None) -> Tensor: - if not XFORMERS_AVAILABLE: - assert attn_bias is None, "xFormers is required for nested tensors usage" - return super().forward(x) - - B, N, C = x.shape - qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) - - q, k, v = unbind(qkv, 2) - - x = memory_efficient_attention(q, k, v, attn_bias=attn_bias) - x = x.reshape([B, N, C]) - - x = self.proj(x) - x = self.proj_drop(x) - return x diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/block.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/block.py deleted file mode 100644 index 825dbfb86f..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/block.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -# References: -# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py -# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py - - -from typing import Any, Callable, Dict, List, Tuple - -import torch -from torch import Tensor, nn - -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.attention import Attention, MemEffAttention -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.drop_path import DropPath -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.layer_scale import LayerScale -from invokeai.backend.image_util.depth_anything.v2.dinov2_layers.mlp import Mlp - -try: - from xformers.ops import fmha, index_select_cat, scaled_index_add - - XFORMERS_AVAILABLE = True -except ImportError: - XFORMERS_AVAILABLE = False - - -class Block(nn.Module): - def __init__( - self, - dim: int, - num_heads: int, - mlp_ratio: float = 4.0, - qkv_bias: bool = False, - proj_bias: bool = True, - ffn_bias: bool = True, - drop: float = 0.0, - attn_drop: float = 0.0, - init_values=None, - drop_path: float = 0.0, - act_layer: Callable[..., nn.Module] = nn.GELU, - norm_layer: Callable[..., nn.Module] = nn.LayerNorm, - attn_class: Callable[..., nn.Module] = Attention, - ffn_layer: Callable[..., nn.Module] = Mlp, - ) -> None: - super().__init__() - # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}") - self.norm1 = norm_layer(dim) - self.attn = attn_class( - dim, - num_heads=num_heads, - qkv_bias=qkv_bias, - proj_bias=proj_bias, - attn_drop=attn_drop, - proj_drop=drop, - ) - self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() - self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - - self.norm2 = norm_layer(dim) - mlp_hidden_dim = int(dim * mlp_ratio) - self.mlp = ffn_layer( - in_features=dim, - hidden_features=mlp_hidden_dim, - act_layer=act_layer, - drop=drop, - bias=ffn_bias, - ) - self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() - self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - - self.sample_drop_ratio = drop_path - - def forward(self, x: Tensor) -> Tensor: - def attn_residual_func(x: Tensor) -> Tensor: - return self.ls1(self.attn(self.norm1(x))) - - def ffn_residual_func(x: Tensor) -> Tensor: - return self.ls2(self.mlp(self.norm2(x))) - - if self.training and self.sample_drop_ratio > 0.1: - # the overhead is compensated only for a drop path rate larger than 0.1 - x = drop_add_residual_stochastic_depth( - x, - residual_func=attn_residual_func, - sample_drop_ratio=self.sample_drop_ratio, - ) - x = drop_add_residual_stochastic_depth( - x, - residual_func=ffn_residual_func, - sample_drop_ratio=self.sample_drop_ratio, - ) - elif self.training and self.sample_drop_ratio > 0.0: - x = x + self.drop_path1(attn_residual_func(x)) - x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2 - else: - x = x + attn_residual_func(x) - x = x + ffn_residual_func(x) - return x - - -def drop_add_residual_stochastic_depth( - x: Tensor, - residual_func: Callable[[Tensor], Tensor], - sample_drop_ratio: float = 0.0, -) -> Tensor: - # 1) extract subset using permutation - b, n, d = x.shape - sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) - brange = (torch.randperm(b, device=x.device))[:sample_subset_size] - x_subset = x[brange] - - # 2) apply residual_func to get residual - residual = residual_func(x_subset) - - x_flat = x.flatten(1) - residual = residual.flatten(1) - - residual_scale_factor = b / sample_subset_size - - # 3) add the residual - x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) - return x_plus_residual.view_as(x) - - -def get_branges_scales(x, sample_drop_ratio=0.0): - b, n, d = x.shape - sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) - brange = (torch.randperm(b, device=x.device))[:sample_subset_size] - residual_scale_factor = b / sample_subset_size - return brange, residual_scale_factor - - -def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None): - if scaling_vector is None: - x_flat = x.flatten(1) - residual = residual.flatten(1) - x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) - else: - x_plus_residual = scaled_index_add( - x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor - ) - return x_plus_residual - - -attn_bias_cache: Dict[Tuple, Any] = {} - - -def get_attn_bias_and_cat(x_list, branges=None): - """ - this will perform the index select, cat the tensors, and provide the attn_bias from cache - """ - batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list] - all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list, strict=False)) - if all_shapes not in attn_bias_cache.keys(): - seqlens = [] - for b, x in zip(batch_sizes, x_list, strict=False): - for _ in range(b): - seqlens.append(x.shape[1]) - attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens) - attn_bias._batch_sizes = batch_sizes - attn_bias_cache[all_shapes] = attn_bias - - if branges is not None: - cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1]) - else: - tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list) - cat_tensors = torch.cat(tensors_bs1, dim=1) - - return attn_bias_cache[all_shapes], cat_tensors - - -def drop_add_residual_stochastic_depth_list( - x_list: List[Tensor], - residual_func: Callable[[Tensor, Any], Tensor], - sample_drop_ratio: float = 0.0, - scaling_vector=None, -) -> Tensor: - # 1) generate random set of indices for dropping samples in the batch - branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list] - branges = [s[0] for s in branges_scales] - residual_scale_factors = [s[1] for s in branges_scales] - - # 2) get attention bias and index+concat the tensors - attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges) - - # 3) apply residual_func to get residual, and split the result - residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore - - outputs = [] - for x, brange, residual, residual_scale_factor in zip( - x_list, branges, residual_list, residual_scale_factors, strict=False - ): - outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x)) - return outputs - - -class NestedTensorBlock(Block): - def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]: - """ - x_list contains a list of tensors to nest together and run - """ - assert isinstance(self.attn, MemEffAttention) - - if self.training and self.sample_drop_ratio > 0.0: - - def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: - return self.attn(self.norm1(x), attn_bias=attn_bias) - - def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: - return self.mlp(self.norm2(x)) - - x_list = drop_add_residual_stochastic_depth_list( - x_list, - residual_func=attn_residual_func, - sample_drop_ratio=self.sample_drop_ratio, - scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None, - ) - x_list = drop_add_residual_stochastic_depth_list( - x_list, - residual_func=ffn_residual_func, - sample_drop_ratio=self.sample_drop_ratio, - scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None, - ) - return x_list - else: - - def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: - return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias)) - - def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: - return self.ls2(self.mlp(self.norm2(x))) - - attn_bias, x = get_attn_bias_and_cat(x_list) - x = x + attn_residual_func(x, attn_bias=attn_bias) - x = x + ffn_residual_func(x) - return attn_bias.split(x) - - def forward(self, x_or_x_list): - if isinstance(x_or_x_list, Tensor): - return super().forward(x_or_x_list) - elif isinstance(x_or_x_list, list): - assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage" - return self.forward_nested(x_or_x_list) - else: - raise AssertionError diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/drop_path.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/drop_path.py deleted file mode 100644 index af05625984..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/drop_path.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -# References: -# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py -# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py - - -from torch import nn - - -def drop_path(x, drop_prob: float = 0.0, training: bool = False): - if drop_prob == 0.0 or not training: - return x - keep_prob = 1 - drop_prob - shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets - random_tensor = x.new_empty(shape).bernoulli_(keep_prob) - if keep_prob > 0.0: - random_tensor.div_(keep_prob) - output = x * random_tensor - return output - - -class DropPath(nn.Module): - """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" - - def __init__(self, drop_prob=None): - super(DropPath, self).__init__() - self.drop_prob = drop_prob - - def forward(self, x): - return drop_path(x, self.drop_prob, self.training) diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/layer_scale.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/layer_scale.py deleted file mode 100644 index 7035d1e9c6..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/layer_scale.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 - -from typing import Union - -import torch -from torch import Tensor, nn - - -class LayerScale(nn.Module): - def __init__( - self, - dim: int, - init_values: Union[float, Tensor] = 1e-5, - inplace: bool = False, - ) -> None: - super().__init__() - self.inplace = inplace - self.gamma = nn.Parameter(init_values * torch.ones(dim)) - - def forward(self, x: Tensor) -> Tensor: - return x.mul_(self.gamma) if self.inplace else x * self.gamma diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/mlp.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/mlp.py deleted file mode 100644 index 5e4b315f97..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/mlp.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -# References: -# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py -# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py - - -from typing import Callable, Optional - -from torch import Tensor, nn - - -class Mlp(nn.Module): - def __init__( - self, - in_features: int, - hidden_features: Optional[int] = None, - out_features: Optional[int] = None, - act_layer: Callable[..., nn.Module] = nn.GELU, - drop: float = 0.0, - bias: bool = True, - ) -> None: - super().__init__() - out_features = out_features or in_features - hidden_features = hidden_features or in_features - self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) - self.act = act_layer() - self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) - self.drop = nn.Dropout(drop) - - def forward(self, x: Tensor) -> Tensor: - x = self.fc1(x) - x = self.act(x) - x = self.drop(x) - x = self.fc2(x) - x = self.drop(x) - return x diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/patch_embed.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/patch_embed.py deleted file mode 100644 index 79a50fc64e..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/patch_embed.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -# References: -# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py -# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py - -from typing import Callable, Optional, Tuple, Union - -import torch.nn as nn -from torch import Tensor - - -def make_2tuple(x): - if isinstance(x, tuple): - assert len(x) == 2 - return x - - assert isinstance(x, int) - return (x, x) - - -class PatchEmbed(nn.Module): - """ - 2D image to patch embedding: (B,C,H,W) -> (B,N,D) - - Args: - img_size: Image size. - patch_size: Patch token size. - in_chans: Number of input image channels. - embed_dim: Number of linear projection output channels. - norm_layer: Normalization layer. - """ - - def __init__( - self, - img_size: Union[int, Tuple[int, int]] = 224, - patch_size: Union[int, Tuple[int, int]] = 16, - in_chans: int = 3, - embed_dim: int = 768, - norm_layer: Optional[Callable] = None, - flatten_embedding: bool = True, - ) -> None: - super().__init__() - - image_HW = make_2tuple(img_size) - patch_HW = make_2tuple(patch_size) - patch_grid_size = ( - image_HW[0] // patch_HW[0], - image_HW[1] // patch_HW[1], - ) - - self.img_size = image_HW - self.patch_size = patch_HW - self.patches_resolution = patch_grid_size - self.num_patches = patch_grid_size[0] * patch_grid_size[1] - - self.in_chans = in_chans - self.embed_dim = embed_dim - - self.flatten_embedding = flatten_embedding - - self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW) - self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() - - def forward(self, x: Tensor) -> Tensor: - _, _, H, W = x.shape - patch_H, patch_W = self.patch_size - - assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" - assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" - - x = self.proj(x) # B C H W - H, W = x.size(2), x.size(3) - x = x.flatten(2).transpose(1, 2) # B HW C - x = self.norm(x) - if not self.flatten_embedding: - x = x.reshape(-1, H, W, self.embed_dim) # B H W C - return x - - def flops(self) -> float: - Ho, Wo = self.patches_resolution - flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) - if self.norm is not None: - flops += Ho * Wo * self.embed_dim - return flops diff --git a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/swiglu_ffn.py b/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/swiglu_ffn.py deleted file mode 100644 index e82999e9b0..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dinov2_layers/swiglu_ffn.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Callable, Optional - -import torch.nn.functional as F -from torch import Tensor, nn - - -class SwiGLUFFN(nn.Module): - def __init__( - self, - in_features: int, - hidden_features: Optional[int] = None, - out_features: Optional[int] = None, - act_layer: Callable[..., nn.Module] = None, - drop: float = 0.0, - bias: bool = True, - ) -> None: - super().__init__() - out_features = out_features or in_features - hidden_features = hidden_features or in_features - self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias) - self.w3 = nn.Linear(hidden_features, out_features, bias=bias) - - def forward(self, x: Tensor) -> Tensor: - x12 = self.w12(x) - x1, x2 = x12.chunk(2, dim=-1) - hidden = F.silu(x1) * x2 - return self.w3(hidden) - - -try: - from xformers.ops import SwiGLU - - XFORMERS_AVAILABLE = True -except ImportError: - SwiGLU = SwiGLUFFN - XFORMERS_AVAILABLE = False - - -class SwiGLUFFNFused(SwiGLU): - def __init__( - self, - in_features: int, - hidden_features: Optional[int] = None, - out_features: Optional[int] = None, - act_layer: Callable[..., nn.Module] = None, - drop: float = 0.0, - bias: bool = True, - ) -> None: - out_features = out_features or in_features - hidden_features = hidden_features or in_features - hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8 - super().__init__( - in_features=in_features, - hidden_features=hidden_features, - out_features=out_features, - bias=bias, - ) diff --git a/invokeai/backend/image_util/depth_anything/v2/dpt.py b/invokeai/backend/image_util/depth_anything/v2/dpt.py deleted file mode 100644 index d9146ee672..0000000000 --- a/invokeai/backend/image_util/depth_anything/v2/dpt.py +++ /dev/null @@ -1,231 +0,0 @@ -# Referenced from https://github.com/DepthAnything/Depth-Anything-V2/blob/main/depth_anything_v2/dpt.py - -from typing import List, Literal, Optional - -import cv2 -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from torchvision.transforms import Compose - -from invokeai.backend.image_util.depth_anything.utils.blocks import FeatureFusionBlock, _make_scratch -from invokeai.backend.image_util.depth_anything.utils.transform import NormalizeImage, PrepareForNet, Resize -from invokeai.backend.image_util.depth_anything.v2.dinov2 import DINOv2 - - -def _make_fusion_block(features, use_bn, size=None): - return FeatureFusionBlock( - features, - nn.ReLU(False), - deconv=False, - bn=use_bn, - expand=False, - align_corners=True, - size=size, - ) - - -class ConvBlock(nn.Module): - def __init__(self, in_feature, out_feature): - super().__init__() - - self.conv_block = nn.Sequential( - nn.Conv2d(in_feature, out_feature, kernel_size=3, stride=1, padding=1), - nn.BatchNorm2d(out_feature), - nn.ReLU(True), - ) - - def forward(self, x): - return self.conv_block(x) - - -class DPTHead(nn.Module): - def __init__( - self, - in_channels: int, - features: int = 256, - use_bn: bool = False, - out_channels: Optional[List[int]] = None, - use_clstoken: bool = False, - ): - super(DPTHead, self).__init__() - - if out_channels is None: - out_channels = [256, 512, 1024, 1024] - - self.use_clstoken = use_clstoken - - self.projects = nn.ModuleList( - [ - nn.Conv2d( - in_channels=in_channels, - out_channels=out_channel, - kernel_size=1, - stride=1, - padding=0, - ) - for out_channel in out_channels - ] - ) - - self.resize_layers = nn.ModuleList( - [ - nn.ConvTranspose2d( - in_channels=out_channels[0], out_channels=out_channels[0], kernel_size=4, stride=4, padding=0 - ), - nn.ConvTranspose2d( - in_channels=out_channels[1], out_channels=out_channels[1], kernel_size=2, stride=2, padding=0 - ), - nn.Identity(), - nn.Conv2d( - in_channels=out_channels[3], out_channels=out_channels[3], kernel_size=3, stride=2, padding=1 - ), - ] - ) - - if use_clstoken: - self.readout_projects = nn.ModuleList() - for _ in range(len(self.projects)): - self.readout_projects.append(nn.Sequential(nn.Linear(2 * in_channels, in_channels), nn.GELU())) - - self.scratch = _make_scratch( - out_channels, - features, - groups=1, - expand=False, - ) - - self.scratch.stem_transpose = None - - self.scratch.refinenet1 = _make_fusion_block(features, use_bn) - self.scratch.refinenet2 = _make_fusion_block(features, use_bn) - self.scratch.refinenet3 = _make_fusion_block(features, use_bn) - self.scratch.refinenet4 = _make_fusion_block(features, use_bn) - - head_features_1 = features - head_features_2 = 32 - - self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1) - self.scratch.output_conv2 = nn.Sequential( - nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1), - nn.ReLU(True), - nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0), - nn.ReLU(True), - nn.Identity(), - ) - - def forward(self, out_features, patch_h, patch_w): - out = [] - for i, x in enumerate(out_features): - if self.use_clstoken: - x, cls_token = x[0], x[1] - readout = cls_token.unsqueeze(1).expand_as(x) - x = self.readout_projects[i](torch.cat((x, readout), -1)) - else: - x = x[0] - - x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)) - - x = self.projects[i](x) - x = self.resize_layers[i](x) - - out.append(x) - - layer_1, layer_2, layer_3, layer_4 = out - - layer_1_rn = self.scratch.layer1_rn(layer_1) - layer_2_rn = self.scratch.layer2_rn(layer_2) - layer_3_rn = self.scratch.layer3_rn(layer_3) - layer_4_rn = self.scratch.layer4_rn(layer_4) - - path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) - path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:]) - path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:]) - path_1 = self.scratch.refinenet1(path_2, layer_1_rn) - - out = self.scratch.output_conv1(path_1) - out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) - out = self.scratch.output_conv2(out) - - return out - - -class DepthAnythingV2(nn.Module): - def __init__( - self, - encoder: Literal["vits", "vitb", "vitl", "vitg"] = "vitl", - features: int = 256, - out_channels: Optional[List[int]] = None, - use_bn: bool = False, - use_clstoken: bool = False, - ): - super(DepthAnythingV2, self).__init__() - - if out_channels is None: - out_channels = [256, 512, 1024, 1024] - - self.intermediate_layer_idx = { - "vits": [2, 5, 8, 11], - "vitb": [2, 5, 8, 11], - "vitl": [4, 11, 17, 23], - "vitg": [9, 19, 29, 39], - } - - self.encoder = encoder - self.pretrained = DINOv2(model_name=encoder) - - self.depth_head = DPTHead( - self.pretrained.embed_dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken - ) - - def forward(self, x: torch.Tensor): - patch_h, patch_w = x.shape[-2] // 14, x.shape[-1] // 14 - - features = self.pretrained.get_intermediate_layers( - x, self.intermediate_layer_idx[self.encoder], return_class_token=True - ) - - depth = self.depth_head(features, patch_h, patch_w) - depth = F.relu(depth) - - return depth.squeeze(1) - - @torch.no_grad() - def infer_image(self, raw_image: np.ndarray, input_size: int = 518): - image, (h, w) = self.image2tensor(raw_image, input_size) - - depth = self.forward(image) - - depth = F.interpolate(depth[:, None], (h, w), mode="bilinear", align_corners=True)[0, 0] - - return depth.cpu().numpy() - - def image2tensor(self, raw_image, input_size=518): - transform = Compose( - [ - Resize( - width=input_size, - height=input_size, - resize_target=False, - keep_aspect_ratio=True, - ensure_multiple_of=14, - resize_method="lower_bound", - image_interpolation_method=cv2.INTER_CUBIC, - ), - NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - PrepareForNet(), - ] - ) - - h, w = raw_image.shape[:2] - - image = cv2.cvtColor(raw_image, cv2.COLOR_BGR2RGB) / 255.0 - - image = transform({"image": image})["image"] - image = torch.from_numpy(image).unsqueeze(0) - - DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" - image = image.to(DEVICE) - - return image, (h, w)