From d742479810f692a0f0dd4015b5efc01abfa3f670 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Fri, 17 Nov 2023 18:36:28 -0500 Subject: [PATCH 01/33] Add nodes for tile splitting and merging. The main motivation for these nodes is for use in tiled upscaling workflows. --- invokeai/app/invocations/tiles.py | 162 +++++++++++++++++++++++++++++ invokeai/backend/tiles/__init__.py | 0 invokeai/backend/tiles/tiles.py | 155 +++++++++++++++++++++++++++ invokeai/backend/tiles/utils.py | 36 +++++++ 4 files changed, 353 insertions(+) create mode 100644 invokeai/app/invocations/tiles.py create mode 100644 invokeai/backend/tiles/__init__.py create mode 100644 invokeai/backend/tiles/tiles.py create mode 100644 invokeai/backend/tiles/utils.py diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py new file mode 100644 index 0000000000..acc87a7864 --- /dev/null +++ b/invokeai/app/invocations/tiles.py @@ -0,0 +1,162 @@ +import numpy as np +from PIL import Image +from pydantic import BaseModel + +from invokeai.app.invocations.baseinvocation import ( + BaseInvocation, + BaseInvocationOutput, + InputField, + InvocationContext, + OutputField, + WithMetadata, + WithWorkflow, + invocation, + invocation_output, +) +from invokeai.app.invocations.primitives import ImageField, ImageOutput +from invokeai.app.services.image_records.image_records_common import ImageCategory, ResourceOrigin +from invokeai.backend.tiles.tiles import calc_tiles, merge_tiles_with_linear_blending +from invokeai.backend.tiles.utils import Tile + +# TODO(ryand): Is this important? +_DIMENSION_MULTIPLE_OF = 8 + + +class TileWithImage(BaseModel): + tile: Tile + image: ImageField + + +@invocation_output("calc_tiles_output") +class CalcTilesOutput(BaseInvocationOutput): + # TODO(ryand): Add description from FieldDescriptions. + tiles: list[Tile] = OutputField(description="") + + +@invocation("calculate_tiles", title="Calculate Tiles", tags=["tiles"], category="tiles", version="1.0.0") +class CalcTiles(BaseInvocation): + """TODO(ryand)""" + + # Inputs + image_height: int = InputField(ge=1) + image_width: int = InputField(ge=1) + tile_height: int = InputField(ge=1, multiple_of=_DIMENSION_MULTIPLE_OF, default=576) + tile_width: int = InputField(ge=1, multiple_of=_DIMENSION_MULTIPLE_OF, default=576) + overlap: int = InputField(ge=0, multiple_of=_DIMENSION_MULTIPLE_OF, default=64) + + def invoke(self, context: InvocationContext) -> CalcTilesOutput: + tiles = calc_tiles( + image_height=self.image_height, + image_width=self.image_width, + tile_height=self.tile_height, + tile_width=self.tile_width, + overlap=self.overlap, + ) + return CalcTilesOutput(tiles=tiles) + + +@invocation_output("tile_to_properties_output") +class TileToPropertiesOutput(BaseInvocationOutput): + # TODO(ryand): Add descriptions. + coords_top: int = OutputField(description="") + coords_bottom: int = OutputField(description="") + coords_left: int = OutputField(description="") + coords_right: int = OutputField(description="") + + overlap_top: int = OutputField(description="") + overlap_bottom: int = OutputField(description="") + overlap_left: int = OutputField(description="") + overlap_right: int = OutputField(description="") + + +@invocation("tile_to_properties") +class TileToProperties(BaseInvocation): + """Split a Tile into its individual properties.""" + + tile: Tile = InputField() + + def invoke(self, context: InvocationContext) -> TileToPropertiesOutput: + return TileToPropertiesOutput( + coords_top=self.tile.coords.top, + coords_bottom=self.tile.coords.bottom, + coords_left=self.tile.coords.left, + coords_right=self.tile.coords.right, + overlap_top=self.tile.overlap.top, + overlap_bottom=self.tile.overlap.bottom, + overlap_left=self.tile.overlap.left, + overlap_right=self.tile.overlap.right, + ) + + +# HACK(ryand): The only reason that PairTileImage is needed is because the iterate/collect nodes don't preserve order. +# Can this be fixed? + + +@invocation_output("pair_tile_image_output") +class PairTileImageOutput(BaseInvocationOutput): + tile_with_image: TileWithImage = OutputField(description="") + + +@invocation("pair_tile_image", title="Pair Tile with Image", tags=["tiles"], category="tiles", version="1.0.0") +class PairTileImage(BaseInvocation): + image: ImageField = InputField() + tile: Tile = InputField() + + def invoke(self, context: InvocationContext) -> PairTileImageOutput: + return PairTileImageOutput( + tile_with_image=TileWithImage( + tile=self.tile, + image=self.image, + ) + ) + + +@invocation("merge_tiles_to_image", title="Merge Tiles To Image", tags=["tiles"], category="tiles", version="1.0.0") +class MergeTilesToImage(BaseInvocation, WithMetadata, WithWorkflow): + """TODO(ryand)""" + + # Inputs + image_height: int = InputField(ge=1) + image_width: int = InputField(ge=1) + tiles_with_images: list[TileWithImage] = InputField() + blend_amount: int = InputField(ge=0) + + def invoke(self, context: InvocationContext) -> ImageOutput: + images = [twi.image for twi in self.tiles_with_images] + tiles = [twi.tile for twi in self.tiles_with_images] + + # Get all tile images for processing. + # TODO(ryand): It pains me that we spend time PNG decoding each tile from disk when they almost certainly + # existed in memory at an earlier point in the graph. + tile_np_images: list[np.ndarray] = [] + for image in images: + pil_image = context.services.images.get_pil_image(image.image_name) + pil_image = pil_image.convert("RGB") + tile_np_images.append(np.array(pil_image)) + + # Prepare the output image buffer. + # Check the first tile to determine how many image channels are expected in the output. + channels = tile_np_images[0].shape[-1] + dtype = tile_np_images[0].dtype + np_image = np.zeros(shape=(self.image_height, self.image_width, channels), dtype=dtype) + + merge_tiles_with_linear_blending( + dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount + ) + pil_image = Image.fromarray(np_image) + + image_dto = context.services.images.create( + image=pil_image, + image_origin=ResourceOrigin.INTERNAL, + image_category=ImageCategory.GENERAL, + node_id=self.id, + session_id=context.graph_execution_state_id, + is_intermediate=self.is_intermediate, + metadata=self.metadata, + workflow=self.workflow, + ) + return ImageOutput( + image=ImageField(image_name=image_dto.image_name), + width=image_dto.width, + height=image_dto.height, + ) diff --git a/invokeai/backend/tiles/__init__.py b/invokeai/backend/tiles/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py new file mode 100644 index 0000000000..566381d1ff --- /dev/null +++ b/invokeai/backend/tiles/tiles.py @@ -0,0 +1,155 @@ +import math + +import numpy as np + +from invokeai.backend.tiles.utils import TBLR, Tile, paste + +# TODO(ryand) +# Test the following: +# - Tile too big in x, y +# - Overlap too big in x, y +# - Single tile fits +# - Multiple tiles fit perfectly +# - Not evenly divisible by tile size(with overlap) + + +def calc_tiles_with_overlap( + image_height: int, image_width: int, tile_height: int, tile_width: int, overlap: int = 0 +) -> list[Tile]: + """Calculate the tile coordinates for a given image shape under a simple tiling scheme with overlaps. + + Args: + image_height (int): The image height in px. + image_width (int): The image width in px. + tile_height (int): The tile height in px. All tiles will have this height. + tile_width (int): The tile width in px. All tiles will have this width. + overlap (int, optional): The target overlap between adjacent tiles. If the tiles do not evenly cover the image + shape, then the last row/column of tiles will overlap more than this. Defaults to 0. + + Returns: + list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. + """ + assert image_height >= tile_height + assert image_width >= tile_width + assert overlap < tile_height + assert overlap < tile_width + + non_overlap_per_tile_height = tile_height - overlap + non_overlap_per_tile_width = tile_width - overlap + + num_tiles_y = math.ceil((image_height - overlap) / non_overlap_per_tile_height) + num_tiles_x = math.ceil((image_width - overlap) / non_overlap_per_tile_width) + + # Calculate tile coordinates and overlaps. + tiles: list[Tile] = [] + for tile_idx_y in range(num_tiles_y): + for tile_idx_x in range(num_tiles_x): + tile = Tile( + coords=TBLR( + top=tile_idx_y * non_overlap_per_tile_height, + bottom=tile_idx_y * non_overlap_per_tile_height + tile_height, + left=tile_idx_x * non_overlap_per_tile_width, + right=tile_idx_x * non_overlap_per_tile_width + tile_width, + ), + overlap=TBLR( + top=0 if tile_idx_y == 0 else overlap, + bottom=overlap, + left=0 if tile_idx_x == 0 else overlap, + right=overlap, + ), + ) + + if tile.coords.bottom > image_height: + # If this tile would go off the bottom of the image, shift it so that it is aligned with the bottom + # of the image. + tile.coords.bottom = image_height + tile.coords.top = image_height - tile_height + tile.overlap.bottom = 0 + # Note that this could result in a large overlap between this tile and the one above it. + top_neighbor_bottom = (tile_idx_y - 1) * non_overlap_per_tile_height + tile_height + tile.overlap.top = top_neighbor_bottom - tile.coords.top + + if tile.coords.right > image_width: + # If this tile would go off the right edge of the image, shift it so that it is aligned with the + # right edge of the image. + tile.coords.right = image_width + tile.coords.left = image_width - tile_width + tile.overlap.right = 0 + # Note that this could result in a large overlap between this tile and the one to its left. + left_neighbor_right = (tile_idx_x - 1) * non_overlap_per_tile_width + tile_width + tile.overlap.left = left_neighbor_right - tile.coords.left + + tiles.append(tile) + + return tiles + + +# TODO(ryand): +# - Test with blend_amount=0 +# - Test tiles that go off of the dst_image. +# - Test mismatched tiles and tile_images lengths. +# - Test mismatched + + +def merge_tiles_with_linear_blending( + dst_image: np.ndarray, tiles: list[Tile], tile_images: list[np.ndarray], blend_amount: int +): + """Merge a set of image tiles into `dst_image` with linear blending between the tiles. + + We expect every tile edge to either: + 1) have an overlap of 0, because it is aligned with the image edge, or + 2) have an overlap >= blend_amount. + If neither of these conditions are satisfied, we raise an exception. + + The linear blending is centered at the halfway point of the overlap between adjacent tiles. + + Args: + dst_image (np.ndarray): The destination image. Shape: (H, W, C). + tiles (list[Tile]): The list of tiles describing the locations of the respective `tile_images`. + tile_images (list[np.ndarray]): The tile images to merge into `dst_image`. + blend_amount (int): The amount of blending (in px) between adjacent overlapping tiles. + """ + # Sort tiles and images first by left x coordinate, then by top y coordinate. During tile processing, we want to + # iterate over tiles left-to-right, top-to-bottom. + tiles_and_images = list(zip(tiles, tile_images, strict=True)) + tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.left) + tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.top) + + # Prepare 1D linear gradients for blending. + gradient_left_x = np.linspace(start=0.0, stop=1.0, num=blend_amount) + gradient_top_y = np.linspace(start=0.0, stop=1.0, num=blend_amount) + # Convert shape: (blend_amount, ) -> (blend_amount, 1). The extra dimension enables the gradient to be applied + # to a 2D image via broadcasting. Note that no additional dimension is needed on gradient_left_x for + # broadcasting to work correctly. + gradient_top_y = np.expand_dims(gradient_top_y, axis=1) + + for tile, tile_image in tiles_and_images: + # We expect tiles to be written left-to-right, top-to-bottom. We construct a mask that applies linear blending + # to the top and to the left of the current tile. The inverse linear blending is automatically applied to the + # bottom/right of the tiles that have already been pasted by the paste(...) operation. + tile_height, tile_width, _ = tile_image.shape + mask = np.ones(shape=(tile_height, tile_width), dtype=np.float64) + # Top blending: + if tile.overlap.top > 0: + assert tile.overlap.top >= blend_amount + # Center the blending gradient in the middle of the overlap. + blend_start_top = tile.overlap.top // 2 - blend_amount // 2 + # The region above the blending region is masked completely. + mask[:blend_start_top, :] = 0.0 + # Apply the blend gradient to the mask. Note that we use `*=` rather than `=` to achieve more natural + # behavior on the corners where vertical and horizontal blending gradients overlap. + mask[blend_start_top : blend_start_top + blend_amount, :] *= gradient_top_y + # HACK(ryand): For debugging + # tile_image[blend_start_top : blend_start_top + blend_amount, :] = 0 + + # Left blending: + # (See comments under 'top blending' for an explanation of the logic.) + if tile.overlap.left > 0: + assert tile.overlap.left >= blend_amount + blend_start_left = tile.overlap.left // 2 - blend_amount // 2 + mask[:, :blend_start_left] = 0.0 + mask[:, blend_start_left : blend_start_left + blend_amount] *= gradient_left_x + # HACK(ryand): For debugging + # tile_image[:, blend_start_left : blend_start_left + blend_amount] = 0 + + paste(dst_image=dst_image, src_image=tile_image, box=tile.coords, mask=mask) diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py new file mode 100644 index 0000000000..cf8e926aa5 --- /dev/null +++ b/invokeai/backend/tiles/utils.py @@ -0,0 +1,36 @@ +from typing import Optional + +import numpy as np +from pydantic import BaseModel, Field + + +class TBLR(BaseModel): + top: int + bottom: int + left: int + right: int + + +class Tile(BaseModel): + coords: TBLR = Field(description="The coordinates of this tile relative to its parent image.") + overlap: TBLR = Field(description="The amount of overlap with adjacent tiles on each side of this tile.") + + +def paste(dst_image: np.ndarray, src_image: np.ndarray, box: TBLR, mask: Optional[np.ndarray] = None): + """Paste a source image into a destination image. + + Args: + dst_image (torch.Tensor): The destination image to paste into. Shape: (H, W, C). + src_image (torch.Tensor): The source image to paste. Shape: (H, W, C). H and W must be compatible with 'box'. + box (TBLR): Box defining the region in the 'dst_image' where 'src_image' will be pasted. + mask (Optional[torch.Tensor]): A mask that defines the blending between 'src_image' and 'dst_image'. + Range: [0.0, 1.0], Shape: (H, W). The output is calculate per-pixel according to + `src * mask + dst * (1 - mask)`. + """ + + if mask is None: + dst_image[box.top : box.bottom, box.left : box.right] = src_image + else: + mask = np.expand_dims(mask, -1) + dst_image_box = dst_image[box.top : box.bottom, box.left : box.right] + dst_image[box.top : box.bottom, box.left : box.right] = src_image * mask + dst_image_box * (1.0 - mask) From caf47dee099903e70330e2083f27d2a0f5568c45 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 20 Nov 2023 11:53:40 -0500 Subject: [PATCH 02/33] Add unit tests for tile paste(...) util function. --- tests/backend/tiles/test_utils.py | 101 ++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/backend/tiles/test_utils.py diff --git a/tests/backend/tiles/test_utils.py b/tests/backend/tiles/test_utils.py new file mode 100644 index 0000000000..bbef233ca5 --- /dev/null +++ b/tests/backend/tiles/test_utils.py @@ -0,0 +1,101 @@ +import numpy as np +import pytest + +from invokeai.backend.tiles.utils import TBLR, paste + + +def test_paste_no_mask_success(): + """Test successful paste with mask=None.""" + dst_image = np.zeros((5, 5, 3), dtype=np.uint8) + + # Create src_image with a pattern that can be used to validate that it was pasted correctly. + src_image = np.zeros((3, 3, 3), dtype=np.uint8) + src_image[0, :, 0] = 1 # Row of 1s in channel 0. + src_image[:, 0, 1] = 2 # Column of 2s in channel 1. + + # Paste in bottom-center of dst_image. + box = TBLR(top=2, bottom=5, left=1, right=4) + + # Construct expected output image. + expected_output = np.zeros((5, 5, 3), dtype=np.uint8) + expected_output[2, 1:4, 0] = 1 + expected_output[2:5, 1, 1] = 2 + + paste(dst_image=dst_image, src_image=src_image, box=box) + + np.testing.assert_array_equal(dst_image, expected_output, strict=True) + + +def test_paste_with_mask_success(): + """Test successful paste with a mask.""" + dst_image = np.zeros((5, 5, 3), dtype=np.uint8) + + # Create src_image with a pattern that can be used to validate that it was pasted correctly. + src_image = np.zeros((3, 3, 3), dtype=np.uint8) + src_image[0, :, 0] = 64 # Row of 64s in channel 0. + src_image[:, 0, 1] = 128 # Column of 128s in channel 1. + + # Paste in bottom-center of dst_image. + box = TBLR(top=2, bottom=5, left=1, right=4) + + # Create a mask that blends the top-left corner of 'src_image' at 50%, and ignores the rest of src_image. + mask = np.zeros((3, 3)) + mask[0, 0] = 0.5 + + # Construct expected output image. + expected_output = np.zeros((5, 5, 3), dtype=np.uint8) + expected_output[2, 1, 0] = 32 + expected_output[2, 1, 1] = 64 + + paste(dst_image=dst_image, src_image=src_image, box=box, mask=mask) + + np.testing.assert_array_equal(dst_image, expected_output, strict=True) + + +@pytest.mark.parametrize("use_mask", [True, False]) +def test_paste_box_overflows_dst_image(use_mask: bool): + """Test that an exception is raised if 'box' overflows the 'dst_image'.""" + dst_image = np.zeros((5, 5, 3), dtype=np.uint8) + src_image = np.zeros((3, 3, 3), dtype=np.uint8) + mask = None + if use_mask: + mask = np.zeros((3, 3)) + + # Construct box that overflows bottom of dst_image. + top = 3 + left = 0 + box = TBLR(top=top, bottom=top + src_image.shape[0], left=left, right=left + src_image.shape[1]) + + with pytest.raises(ValueError): + paste(dst_image=dst_image, src_image=src_image, box=box, mask=mask) + + +@pytest.mark.parametrize("use_mask", [True, False]) +def test_paste_src_image_does_not_match_box(use_mask: bool): + """Test that an exception is raised if the 'src_image' shape does not match the 'box' dimensions.""" + dst_image = np.zeros((5, 5, 3), dtype=np.uint8) + src_image = np.zeros((3, 3, 3), dtype=np.uint8) + mask = None + if use_mask: + mask = np.zeros((3, 3)) + + # Construct box that is smaller than src_image. + box = TBLR(top=0, bottom=src_image.shape[0] - 1, left=0, right=src_image.shape[1]) + + with pytest.raises(ValueError): + paste(dst_image=dst_image, src_image=src_image, box=box, mask=mask) + + +def test_paste_mask_does_not_match_src_image(): + """Test that an exception is raised if the 'mask' shape is different than the 'src_image' shape.""" + dst_image = np.zeros((5, 5, 3), dtype=np.uint8) + src_image = np.zeros((3, 3, 3), dtype=np.uint8) + + # Construct mask that is smaller than the src_image. + mask = np.zeros((src_image.shape[0] - 1, src_image.shape[1])) + + # Construct box that matches src_image shape. + box = TBLR(top=0, bottom=src_image.shape[0], left=0, right=src_image.shape[1]) + + with pytest.raises(ValueError): + paste(dst_image=dst_image, src_image=src_image, box=box, mask=mask) From 1f63fa8236daab3f3792f3145bae5a12bc7cef12 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 20 Nov 2023 14:23:49 -0500 Subject: [PATCH 03/33] Add unit tests for calc_tiles_with_overlap(...) and fix a bug in its implementation. --- invokeai/backend/tiles/tiles.py | 52 ++++++++++--------- invokeai/backend/tiles/utils.py | 11 ++++ tests/backend/tiles/test_tiles.py | 84 +++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 23 deletions(-) create mode 100644 tests/backend/tiles/test_tiles.py diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 566381d1ff..5e5c4b7050 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -1,17 +1,10 @@ import math +from typing import Union import numpy as np from invokeai.backend.tiles.utils import TBLR, Tile, paste -# TODO(ryand) -# Test the following: -# - Tile too big in x, y -# - Overlap too big in x, y -# - Single tile fits -# - Multiple tiles fit perfectly -# - Not evenly divisible by tile size(with overlap) - def calc_tiles_with_overlap( image_height: int, image_width: int, tile_height: int, tile_width: int, overlap: int = 0 @@ -40,8 +33,10 @@ def calc_tiles_with_overlap( num_tiles_y = math.ceil((image_height - overlap) / non_overlap_per_tile_height) num_tiles_x = math.ceil((image_width - overlap) / non_overlap_per_tile_width) - # Calculate tile coordinates and overlaps. + # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. tiles: list[Tile] = [] + + # Calculate tile coordinates. (Ignore overlap values for now.) for tile_idx_y in range(num_tiles_y): for tile_idx_x in range(num_tiles_x): tile = Tile( @@ -51,12 +46,7 @@ def calc_tiles_with_overlap( left=tile_idx_x * non_overlap_per_tile_width, right=tile_idx_x * non_overlap_per_tile_width + tile_width, ), - overlap=TBLR( - top=0 if tile_idx_y == 0 else overlap, - bottom=overlap, - left=0 if tile_idx_x == 0 else overlap, - right=overlap, - ), + overlap=TBLR(top=0, bottom=0, left=0, right=0), ) if tile.coords.bottom > image_height: @@ -64,23 +54,39 @@ def calc_tiles_with_overlap( # of the image. tile.coords.bottom = image_height tile.coords.top = image_height - tile_height - tile.overlap.bottom = 0 - # Note that this could result in a large overlap between this tile and the one above it. - top_neighbor_bottom = (tile_idx_y - 1) * non_overlap_per_tile_height + tile_height - tile.overlap.top = top_neighbor_bottom - tile.coords.top if tile.coords.right > image_width: # If this tile would go off the right edge of the image, shift it so that it is aligned with the # right edge of the image. tile.coords.right = image_width tile.coords.left = image_width - tile_width - tile.overlap.right = 0 - # Note that this could result in a large overlap between this tile and the one to its left. - left_neighbor_right = (tile_idx_x - 1) * non_overlap_per_tile_width + tile_width - tile.overlap.left = left_neighbor_right - tile.coords.left tiles.append(tile) + def get_tile_or_none(idx_y: int, idx_x: int) -> Union[Tile, None]: + if idx_y < 0 or idx_y > num_tiles_y or idx_x < 0 or idx_x > num_tiles_x: + return None + return tiles[idx_y * num_tiles_x + idx_x] + + # Iterate over tiles again and calculate overlaps. + for tile_idx_y in range(num_tiles_y): + for tile_idx_x in range(num_tiles_x): + cur_tile = get_tile_or_none(tile_idx_y, tile_idx_x) + top_neighbor_tile = get_tile_or_none(tile_idx_y - 1, tile_idx_x) + left_neighbor_tile = get_tile_or_none(tile_idx_y, tile_idx_x - 1) + + assert cur_tile is not None + + # Update cur_tile top-overlap and corresponding top-neighbor bottom-overlap. + if top_neighbor_tile is not None: + cur_tile.overlap.top = max(0, top_neighbor_tile.coords.bottom - cur_tile.coords.top) + top_neighbor_tile.overlap.bottom = cur_tile.overlap.top + + # Update cur_tile left-overlap and corresponding left-neighbor right-overlap. + if left_neighbor_tile is not None: + cur_tile.overlap.left = max(0, left_neighbor_tile.coords.right - cur_tile.coords.left) + left_neighbor_tile.overlap.right = cur_tile.overlap.left + return tiles diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py index cf8e926aa5..4ad40ffa35 100644 --- a/invokeai/backend/tiles/utils.py +++ b/invokeai/backend/tiles/utils.py @@ -10,11 +10,22 @@ class TBLR(BaseModel): left: int right: int + def __eq__(self, other): + return ( + self.top == other.top + and self.bottom == other.bottom + and self.left == other.left + and self.right == other.right + ) + class Tile(BaseModel): coords: TBLR = Field(description="The coordinates of this tile relative to its parent image.") overlap: TBLR = Field(description="The amount of overlap with adjacent tiles on each side of this tile.") + def __eq__(self, other): + return self.coords == other.coords and self.overlap == other.overlap + def paste(dst_image: np.ndarray, src_image: np.ndarray, box: TBLR, mask: Optional[np.ndarray] = None): """Paste a source image into a destination image. diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py new file mode 100644 index 0000000000..332ab15005 --- /dev/null +++ b/tests/backend/tiles/test_tiles.py @@ -0,0 +1,84 @@ +import pytest + +from invokeai.backend.tiles.tiles import calc_tiles_with_overlap +from invokeai.backend.tiles.utils import TBLR, Tile + +#################################### +# Test calc_tiles_with_overlap(...) +#################################### + + +def test_calc_tiles_with_overlap_single_tile(): + """Test calc_tiles_with_overlap() behavior when a single tile covers the image.""" + tiles = calc_tiles_with_overlap(image_height=512, image_width=1024, tile_height=512, tile_width=1024, overlap=64) + + expected_tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=1024), overlap=TBLR(top=0, bottom=0, left=0, right=0)) + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_with_overlap_evenly_divisible(): + """Test calc_tiles_with_overlap() behavior when the image is evenly covered by multiple tiles.""" + # Parameters chosen so that image is evenly covered by 2 rows, 3 columns of tiles. + tiles = calc_tiles_with_overlap(image_height=576, image_width=1600, tile_height=320, tile_width=576, overlap=64) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=320, left=0, right=576), overlap=TBLR(top=0, bottom=64, left=0, right=64)), + Tile(coords=TBLR(top=0, bottom=320, left=512, right=1088), overlap=TBLR(top=0, bottom=64, left=64, right=64)), + Tile(coords=TBLR(top=0, bottom=320, left=1024, right=1600), overlap=TBLR(top=0, bottom=64, left=64, right=0)), + # Row 1 + Tile(coords=TBLR(top=256, bottom=576, left=0, right=576), overlap=TBLR(top=64, bottom=0, left=0, right=64)), + Tile(coords=TBLR(top=256, bottom=576, left=512, right=1088), overlap=TBLR(top=64, bottom=0, left=64, right=64)), + Tile(coords=TBLR(top=256, bottom=576, left=1024, right=1600), overlap=TBLR(top=64, bottom=0, left=64, right=0)), + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_with_overlap_not_evenly_divisible(): + """Test calc_tiles_with_overlap() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" + # Parameters chosen so that image is covered by 2 rows and 3 columns of tiles, with uneven overlaps. + tiles = calc_tiles_with_overlap(image_height=400, image_width=1200, tile_height=256, tile_width=512, overlap=64) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=256, left=0, right=512), overlap=TBLR(top=0, bottom=112, left=0, right=64)), + Tile(coords=TBLR(top=0, bottom=256, left=448, right=960), overlap=TBLR(top=0, bottom=112, left=64, right=272)), + Tile(coords=TBLR(top=0, bottom=256, left=688, right=1200), overlap=TBLR(top=0, bottom=112, left=272, right=0)), + # Row 1 + Tile(coords=TBLR(top=144, bottom=400, left=0, right=512), overlap=TBLR(top=112, bottom=0, left=0, right=64)), + Tile( + coords=TBLR(top=144, bottom=400, left=448, right=960), overlap=TBLR(top=112, bottom=0, left=64, right=272) + ), + Tile( + coords=TBLR(top=144, bottom=400, left=688, right=1200), overlap=TBLR(top=112, bottom=0, left=272, right=0) + ), + ] + + assert tiles == expected_tiles + + +@pytest.mark.parametrize( + ["image_height", "image_width", "tile_height", "tile_width", "overlap", "raises"], + [ + (128, 128, 128, 128, 127, False), # OK + (128, 128, 128, 128, 0, False), # OK + (128, 128, 64, 64, 0, False), # OK + (128, 128, 129, 128, 0, True), # tile_height exceeds image_height. + (128, 128, 128, 129, 0, True), # tile_width exceeds image_width. + (128, 128, 64, 128, 64, True), # overlap equals tile_height. + (128, 128, 128, 64, 64, True), # overlap equals tile_width. + ], +) +def test_calc_tiles_with_overlap_input_validation( + image_height: int, image_width: int, tile_height: int, tile_width: int, overlap: int, raises: bool +): + """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" + if raises: + with pytest.raises(AssertionError): + calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) + else: + calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) From 1d0dc7eeabfe58bc6190f03d38c4dd27ed63e603 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 20 Nov 2023 15:42:23 -0500 Subject: [PATCH 04/33] Add unit tests for merge_tiles_with_linear_blending(...). --- invokeai/backend/tiles/tiles.py | 11 +-- tests/backend/tiles/test_tiles.py | 142 +++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 10 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 5e5c4b7050..3d64e3e145 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -90,13 +90,6 @@ def calc_tiles_with_overlap( return tiles -# TODO(ryand): -# - Test with blend_amount=0 -# - Test tiles that go off of the dst_image. -# - Test mismatched tiles and tile_images lengths. -# - Test mismatched - - def merge_tiles_with_linear_blending( dst_image: np.ndarray, tiles: list[Tile], tile_images: list[np.ndarray], blend_amount: int ): @@ -145,7 +138,7 @@ def merge_tiles_with_linear_blending( # Apply the blend gradient to the mask. Note that we use `*=` rather than `=` to achieve more natural # behavior on the corners where vertical and horizontal blending gradients overlap. mask[blend_start_top : blend_start_top + blend_amount, :] *= gradient_top_y - # HACK(ryand): For debugging + # For visual debugging: # tile_image[blend_start_top : blend_start_top + blend_amount, :] = 0 # Left blending: @@ -155,7 +148,7 @@ def merge_tiles_with_linear_blending( blend_start_left = tile.overlap.left // 2 - blend_amount // 2 mask[:, :blend_start_left] = 0.0 mask[:, blend_start_left : blend_start_left + blend_amount] *= gradient_left_x - # HACK(ryand): For debugging + # For visual debugging: # tile_image[:, blend_start_left : blend_start_left + blend_amount] = 0 paste(dst_image=dst_image, src_image=tile_image, box=tile.coords, mask=mask) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 332ab15005..353e65d336 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -1,6 +1,7 @@ +import numpy as np import pytest -from invokeai.backend.tiles.tiles import calc_tiles_with_overlap +from invokeai.backend.tiles.tiles import calc_tiles_with_overlap, merge_tiles_with_linear_blending from invokeai.backend.tiles.utils import TBLR, Tile #################################### @@ -82,3 +83,142 @@ def test_calc_tiles_with_overlap_input_validation( calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) else: calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) + + +############################################# +# Test merge_tiles_with_linear_blending(...) +############################################# + + +@pytest.mark.parametrize("blend_amount", [0, 32]) +def test_merge_tiles_with_linear_blending_horizontal(blend_amount: int): + """Test merge_tiles_with_linear_blending(...) behavior when merging horizontally.""" + # Initialize 2 tiles side-by-side. + tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=64)), + Tile(coords=TBLR(top=0, bottom=512, left=448, right=960), overlap=TBLR(top=0, bottom=0, left=64, right=0)), + ] + + dst_image = np.zeros((512, 960, 3), dtype=np.uint8) + + # Prepare tile_images that match tiles. Pixel values are set based on the tile index. + tile_images = [ + np.zeros((512, 512, 3)) + 64, + np.zeros((512, 512, 3)) + 128, + ] + + # Calculate expected output. + expected_output = np.zeros((512, 960, 3), dtype=np.uint8) + expected_output[:, : 480 - (blend_amount // 2), :] = 64 + if blend_amount > 0: + gradient = np.linspace(start=64, stop=128, num=blend_amount, dtype=np.uint8).reshape((1, blend_amount, 1)) + expected_output[:, 480 - (blend_amount // 2) : 480 + (blend_amount // 2), :] = gradient + expected_output[:, 480 + (blend_amount // 2) :, :] = 128 + + merge_tiles_with_linear_blending( + dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=blend_amount + ) + + np.testing.assert_array_equal(dst_image, expected_output, strict=True) + + +@pytest.mark.parametrize("blend_amount", [0, 32]) +def test_merge_tiles_with_linear_blending_vertical(blend_amount: int): + """Test merge_tiles_with_linear_blending(...) behavior when merging vertically.""" + # Initialize 2 tiles stacked vertically. + tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=64, left=0, right=0)), + Tile(coords=TBLR(top=448, bottom=960, left=0, right=512), overlap=TBLR(top=64, bottom=0, left=0, right=0)), + ] + + dst_image = np.zeros((960, 512, 3), dtype=np.uint8) + + # Prepare tile_images that match tiles. Pixel values are set based on the tile index. + tile_images = [ + np.zeros((512, 512, 3)) + 64, + np.zeros((512, 512, 3)) + 128, + ] + + # Calculate expected output. + expected_output = np.zeros((960, 512, 3), dtype=np.uint8) + expected_output[: 480 - (blend_amount // 2), :, :] = 64 + if blend_amount > 0: + gradient = np.linspace(start=64, stop=128, num=blend_amount, dtype=np.uint8).reshape((blend_amount, 1, 1)) + expected_output[480 - (blend_amount // 2) : 480 + (blend_amount // 2), :, :] = gradient + expected_output[480 + (blend_amount // 2) :, :, :] = 128 + + merge_tiles_with_linear_blending( + dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=blend_amount + ) + + np.testing.assert_array_equal(dst_image, expected_output, strict=True) + + +def test_merge_tiles_with_linear_blending_blend_amount_exceeds_vertical_overlap(): + """Test that merge_tiles_with_linear_blending(...) raises an exception if 'blend_amount' exceeds the overlap between + any vertically adjacent tiles. + """ + # Initialize 2 tiles stacked vertically. + tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=64, left=0, right=0)), + Tile(coords=TBLR(top=448, bottom=960, left=0, right=512), overlap=TBLR(top=64, bottom=0, left=0, right=0)), + ] + + dst_image = np.zeros((960, 512, 3), dtype=np.uint8) + + # Prepare tile_images that match tiles. + tile_images = [np.zeros((512, 512, 3)), np.zeros((512, 512, 3))] + + # blend_amount=128 exceeds overlap of 64, so should raise exception. + with pytest.raises(AssertionError): + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128) + + +def test_merge_tiles_with_linear_blending_blend_amount_exceeds_horizontal_overlap(): + """Test that merge_tiles_with_linear_blending(...) raises an exception if 'blend_amount' exceeds the overlap between + any horizontally adjacent tiles. + """ + # Initialize 2 tiles side-by-side. + tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=64)), + Tile(coords=TBLR(top=0, bottom=512, left=448, right=960), overlap=TBLR(top=0, bottom=0, left=64, right=0)), + ] + + dst_image = np.zeros((512, 960, 3), dtype=np.uint8) + + # Prepare tile_images that match tiles. + tile_images = [np.zeros((512, 512, 3)), np.zeros((512, 512, 3))] + + # blend_amount=128 exceeds overlap of 64, so should raise exception. + with pytest.raises(AssertionError): + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128) + + +def test_merge_tiles_with_linear_blending_tiles_overflow_dst_image(): + """Test that merge_tiles_with_linear_blending(...) raises an exception if any of the tiles overflows the + dst_image. + """ + tiles = [Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=0))] + + dst_image = np.zeros((256, 512, 3), dtype=np.uint8) + + # Prepare tile_images that match tiles. + tile_images = [np.zeros((512, 512, 3))] + + with pytest.raises(ValueError): + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0) + + +def test_merge_tiles_with_linear_blending_mismatched_list_lengths(): + """Test that merge_tiles_with_linear_blending(...) raises an exception if the lengths of 'tiles' and 'tile_images' + do not match. + """ + tiles = [Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=0))] + + dst_image = np.zeros((256, 512, 3), dtype=np.uint8) + + # tile_images is longer than tiles, so should cause an exception. + tile_images = [np.zeros((512, 512, 3)), np.zeros((512, 512, 3))] + + with pytest.raises(ValueError): + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0) From 3980f79ed539c8250120a2224915861a26debfa2 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 20 Nov 2023 17:18:13 -0500 Subject: [PATCH 05/33] Tidy up tiles invocations, add documentation. --- invokeai/app/invocations/tiles.py | 92 ++++++++++++++++--------------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index acc87a7864..c6499c45d6 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -15,65 +15,65 @@ from invokeai.app.invocations.baseinvocation import ( ) from invokeai.app.invocations.primitives import ImageField, ImageOutput from invokeai.app.services.image_records.image_records_common import ImageCategory, ResourceOrigin -from invokeai.backend.tiles.tiles import calc_tiles, merge_tiles_with_linear_blending +from invokeai.backend.tiles.tiles import calc_tiles_with_overlap, merge_tiles_with_linear_blending from invokeai.backend.tiles.utils import Tile -# TODO(ryand): Is this important? -_DIMENSION_MULTIPLE_OF = 8 - class TileWithImage(BaseModel): tile: Tile image: ImageField -@invocation_output("calc_tiles_output") -class CalcTilesOutput(BaseInvocationOutput): - # TODO(ryand): Add description from FieldDescriptions. - tiles: list[Tile] = OutputField(description="") +@invocation_output("calculate_image_tiles_output") +class CalculateImageTilesOutput(BaseInvocationOutput): + tiles: list[Tile] = OutputField(description="The tiles coordinates that cover a particular image shape.") -@invocation("calculate_tiles", title="Calculate Tiles", tags=["tiles"], category="tiles", version="1.0.0") -class CalcTiles(BaseInvocation): - """TODO(ryand)""" +@invocation("calculate_image_tiles", title="Calculate Image Tiles", tags=["tiles"], category="tiles", version="1.0.0") +class CalculateImageTiles(BaseInvocation): + """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" - # Inputs - image_height: int = InputField(ge=1) - image_width: int = InputField(ge=1) - tile_height: int = InputField(ge=1, multiple_of=_DIMENSION_MULTIPLE_OF, default=576) - tile_width: int = InputField(ge=1, multiple_of=_DIMENSION_MULTIPLE_OF, default=576) - overlap: int = InputField(ge=0, multiple_of=_DIMENSION_MULTIPLE_OF, default=64) + image_height: int = InputField( + ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." + ) + image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") + tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") + tile_width: int = InputField(ge=1, default=576, description="The tile width, in pixels.") + overlap: int = InputField( + ge=0, + default=128, + description="The target overlap, in pixels, between adjacent tiles. Adjacent tiles will overlap by at least this amount", + ) - def invoke(self, context: InvocationContext) -> CalcTilesOutput: - tiles = calc_tiles( + def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: + tiles = calc_tiles_with_overlap( image_height=self.image_height, image_width=self.image_width, tile_height=self.tile_height, tile_width=self.tile_width, overlap=self.overlap, ) - return CalcTilesOutput(tiles=tiles) + return CalculateImageTilesOutput(tiles=tiles) @invocation_output("tile_to_properties_output") class TileToPropertiesOutput(BaseInvocationOutput): - # TODO(ryand): Add descriptions. - coords_top: int = OutputField(description="") - coords_bottom: int = OutputField(description="") - coords_left: int = OutputField(description="") - coords_right: int = OutputField(description="") + coords_top: int = OutputField(description="Top coordinate of the tile relative to its parent image.") + coords_bottom: int = OutputField(description="Bottom coordinate of the tile relative to its parent image.") + coords_left: int = OutputField(description="Left coordinate of the tile relative to its parent image.") + coords_right: int = OutputField(description="Right coordinate of the tile relative to its parent image.") - overlap_top: int = OutputField(description="") - overlap_bottom: int = OutputField(description="") - overlap_left: int = OutputField(description="") - overlap_right: int = OutputField(description="") + overlap_top: int = OutputField(description="Overlap between this tile and its top neighbor.") + overlap_bottom: int = OutputField(description="Overlap between this tile and its bottom neighbor.") + overlap_left: int = OutputField(description="Overlap between this tile and its left neighbor.") + overlap_right: int = OutputField(description="Overlap between this tile and its right neighbor.") -@invocation("tile_to_properties") +@invocation("tile_to_properties", title="Tile to Properties", tags=["tiles"], category="tiles", version="1.0.0") class TileToProperties(BaseInvocation): """Split a Tile into its individual properties.""" - tile: Tile = InputField() + tile: Tile = InputField(description="The tile to split into properties.") def invoke(self, context: InvocationContext) -> TileToPropertiesOutput: return TileToPropertiesOutput( @@ -88,19 +88,20 @@ class TileToProperties(BaseInvocation): ) -# HACK(ryand): The only reason that PairTileImage is needed is because the iterate/collect nodes don't preserve order. -# Can this be fixed? - - @invocation_output("pair_tile_image_output") class PairTileImageOutput(BaseInvocationOutput): - tile_with_image: TileWithImage = OutputField(description="") + tile_with_image: TileWithImage = OutputField(description="A tile description with its corresponding image.") @invocation("pair_tile_image", title="Pair Tile with Image", tags=["tiles"], category="tiles", version="1.0.0") class PairTileImage(BaseInvocation): - image: ImageField = InputField() - tile: Tile = InputField() + """Pair an image with its tile properties.""" + + # TODO(ryand): The only reason that PairTileImage is needed is because the iterate/collect nodes don't preserve + # order. Can this be fixed? + + image: ImageField = InputField(description="The tile image.") + tile: Tile = InputField(description="The tile properties.") def invoke(self, context: InvocationContext) -> PairTileImageOutput: return PairTileImageOutput( @@ -111,15 +112,18 @@ class PairTileImage(BaseInvocation): ) -@invocation("merge_tiles_to_image", title="Merge Tiles To Image", tags=["tiles"], category="tiles", version="1.0.0") +@invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.0.0") class MergeTilesToImage(BaseInvocation, WithMetadata, WithWorkflow): - """TODO(ryand)""" + """Merge multiple tile images into a single image.""" # Inputs - image_height: int = InputField(ge=1) - image_width: int = InputField(ge=1) - tiles_with_images: list[TileWithImage] = InputField() - blend_amount: int = InputField(ge=0) + image_height: int = InputField(ge=1, description="The height of the output image, in pixels.") + image_width: int = InputField(ge=1, description="The width of the output image, in pixels.") + tiles_with_images: list[TileWithImage] = InputField(description="A list of tile images with tile properties.") + blend_amount: int = InputField( + ge=0, + description="The amount to blend adjacent tiles in pixels. Must be <= the amount of overlap between adjacent tiles.", + ) def invoke(self, context: InvocationContext) -> ImageOutput: images = [twi.image for twi in self.tiles_with_images] From 436560da3901e507a588800c2e6ba2cace66be94 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Thu, 23 Nov 2023 10:52:03 -0500 Subject: [PATCH 06/33] (minor) Add 'Invocation' suffix to all tiling node classes. --- invokeai/app/invocations/tiles.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index c6499c45d6..927e99be64 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -30,7 +30,7 @@ class CalculateImageTilesOutput(BaseInvocationOutput): @invocation("calculate_image_tiles", title="Calculate Image Tiles", tags=["tiles"], category="tiles", version="1.0.0") -class CalculateImageTiles(BaseInvocation): +class CalculateImageTilesInvocation(BaseInvocation): """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" image_height: int = InputField( @@ -70,7 +70,7 @@ class TileToPropertiesOutput(BaseInvocationOutput): @invocation("tile_to_properties", title="Tile to Properties", tags=["tiles"], category="tiles", version="1.0.0") -class TileToProperties(BaseInvocation): +class TileToPropertiesInvocation(BaseInvocation): """Split a Tile into its individual properties.""" tile: Tile = InputField(description="The tile to split into properties.") @@ -94,7 +94,7 @@ class PairTileImageOutput(BaseInvocationOutput): @invocation("pair_tile_image", title="Pair Tile with Image", tags=["tiles"], category="tiles", version="1.0.0") -class PairTileImage(BaseInvocation): +class PairTileImageInvocation(BaseInvocation): """Pair an image with its tile properties.""" # TODO(ryand): The only reason that PairTileImage is needed is because the iterate/collect nodes don't preserve @@ -113,7 +113,7 @@ class PairTileImage(BaseInvocation): @invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.0.0") -class MergeTilesToImage(BaseInvocation, WithMetadata, WithWorkflow): +class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): """Merge multiple tile images into a single image.""" # Inputs From 121b930abfdc9f7e365b0c4f3fd024e05f35c631 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 11:02:10 -0500 Subject: [PATCH 07/33] Copy CropLatentsInvocation from https://github.com/skunkworxdark/XYGrid_nodes/blob/74647fa9c1fa57d317a94bd43ca689af7f0aae5e/images_to_grids.py#L1117C1-L1167C80. --- invokeai/app/invocations/latent.py | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index d438bcae02..c143eb891c 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -1162,3 +1162,56 @@ class BlendLatentsInvocation(BaseInvocation): # context.services.latents.set(name, resized_latents) context.services.latents.save(name, blended_latents) return build_latents_output(latents_name=name, latents=blended_latents) + + +@invocation( + "lcrop", + title="Crop Latents", + tags=["latents", "crop"], + category="latents", + version="1.0.0", +) +class CropLatentsInvocation(BaseInvocation): + """Crops latents""" + + latents: LatentsField = InputField( + description=FieldDescriptions.latents, + input=Input.Connection, + ) + width: int = InputField( + ge=64, + multiple_of=_downsampling_factor, + description=FieldDescriptions.width, + ) + height: int = InputField( + ge=64, + multiple_of=_downsampling_factor, + description=FieldDescriptions.width, + ) + x_offset: int = InputField( + ge=0, + multiple_of=_downsampling_factor, + description="x-coordinate", + ) + y_offset: int = InputField( + ge=0, + multiple_of=_downsampling_factor, + description="y-coordinate", + ) + + def invoke(self, context: InvocationContext) -> LatentsOutput: + latents = context.services.latents.get(self.latents.latents_name) + + x1 = self.x_offset // _downsampling_factor + y1 = self.y_offset // _downsampling_factor + x2 = x1 + (self.width // _downsampling_factor) + y2 = y1 + (self.height // _downsampling_factor) + + cropped_latents = latents[:, :, y1:y2, x1:x2] + + # resized_latents = resized_latents.to("cpu") + + name = f"{context.graph_execution_state_id}__{self.id}" + context.services.latents.save(name, cropped_latents) + + return build_latents_output(latents_name=name, latents=cropped_latents) From e1c53a2465583869126833704b5b5a0b1d42f062 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 11:12:15 -0500 Subject: [PATCH 08/33] Use LATENT_SCALE_FACTOR = 8 constant in CropLatentsInvocation. --- invokeai/app/invocations/latent.py | 34 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index c143eb891c..b5c9c876c8 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -79,6 +79,12 @@ DEFAULT_PRECISION = choose_precision(choose_torch_device()) SAMPLER_NAME_VALUES = Literal[tuple(SCHEDULER_MAP.keys())] +# HACK: Many nodes are currently hard-coded to use a fixed latent scale factor of 8. This is fragile, and will need to +# be addressed if future models use a different latent scale factor. Also, note that there may be places where the scale +# factor is hard-coded to a literal '8' rather than using this constant. +# The ratio of image:latent dimensions is LATENT_SCALE_FACTOR:1, or 8:1. +LATENT_SCALE_FACTOR = 8 + @invocation_output("scheduler_output") class SchedulerOutput(BaseInvocationOutput): @@ -390,9 +396,9 @@ class DenoiseLatentsInvocation(BaseInvocation): exit_stack: ExitStack, do_classifier_free_guidance: bool = True, ) -> List[ControlNetData]: - # assuming fixed dimensional scaling of 8:1 for image:latents - control_height_resize = latents_shape[2] * 8 - control_width_resize = latents_shape[3] * 8 + # Assuming fixed dimensional scaling of LATENT_SCALE_FACTOR. + control_height_resize = latents_shape[2] * LATENT_SCALE_FACTOR + control_width_resize = latents_shape[3] * LATENT_SCALE_FACTOR if control_input is None: control_list = None elif isinstance(control_input, list) and len(control_input) == 0: @@ -905,12 +911,12 @@ class ResizeLatentsInvocation(BaseInvocation): ) width: int = InputField( ge=64, - multiple_of=8, + multiple_of=LATENT_SCALE_FACTOR, description=FieldDescriptions.width, ) height: int = InputField( ge=64, - multiple_of=8, + multiple_of=LATENT_SCALE_FACTOR, description=FieldDescriptions.width, ) mode: LATENTS_INTERPOLATION_MODE = InputField(default="bilinear", description=FieldDescriptions.interp_mode) @@ -924,7 +930,7 @@ class ResizeLatentsInvocation(BaseInvocation): resized_latents = torch.nn.functional.interpolate( latents.to(device), - size=(self.height // 8, self.width // 8), + size=(self.height // LATENT_SCALE_FACTOR, self.width // LATENT_SCALE_FACTOR), mode=self.mode, antialias=self.antialias if self.mode in ["bilinear", "bicubic"] else False, ) @@ -1180,32 +1186,32 @@ class CropLatentsInvocation(BaseInvocation): ) width: int = InputField( ge=64, - multiple_of=_downsampling_factor, + multiple_of=LATENT_SCALE_FACTOR, description=FieldDescriptions.width, ) height: int = InputField( ge=64, - multiple_of=_downsampling_factor, + multiple_of=LATENT_SCALE_FACTOR, description=FieldDescriptions.width, ) x_offset: int = InputField( ge=0, - multiple_of=_downsampling_factor, + multiple_of=LATENT_SCALE_FACTOR, description="x-coordinate", ) y_offset: int = InputField( ge=0, - multiple_of=_downsampling_factor, + multiple_of=LATENT_SCALE_FACTOR, description="y-coordinate", ) def invoke(self, context: InvocationContext) -> LatentsOutput: latents = context.services.latents.get(self.latents.latents_name) - x1 = self.x_offset // _downsampling_factor - y1 = self.y_offset // _downsampling_factor - x2 = x1 + (self.width // _downsampling_factor) - y2 = y1 + (self.height // _downsampling_factor) + x1 = self.x_offset // LATENT_SCALE_FACTOR + y1 = self.y_offset // LATENT_SCALE_FACTOR + x2 = x1 + (self.width // LATENT_SCALE_FACTOR) + y2 = y1 + (self.height // LATENT_SCALE_FACTOR) cropped_latents = latents[:, :, y1:y2, x1:x2] From 9b4e6da22618ca927affee83c9870422d7c8d38a Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 11:30:00 -0500 Subject: [PATCH 09/33] Improve documentation of CropLatentsInvocation. --- invokeai/app/invocations/latent.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index b5c9c876c8..ae0497eea7 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -1178,31 +1178,33 @@ class BlendLatentsInvocation(BaseInvocation): version="1.0.0", ) class CropLatentsInvocation(BaseInvocation): - """Crops latents""" + """Crops a latent-space tensor to a box specified in image-space. The box dimensions and coordinates must be + divisible by the latent scale factor of 8. + """ latents: LatentsField = InputField( description=FieldDescriptions.latents, input=Input.Connection, ) width: int = InputField( - ge=64, + ge=1, multiple_of=LATENT_SCALE_FACTOR, - description=FieldDescriptions.width, + description="The width (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", ) height: int = InputField( - ge=64, + ge=1, multiple_of=LATENT_SCALE_FACTOR, - description=FieldDescriptions.width, + description="The height (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", ) x_offset: int = InputField( ge=0, multiple_of=LATENT_SCALE_FACTOR, - description="x-coordinate", + description="The left x coordinate (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", ) y_offset: int = InputField( ge=0, multiple_of=LATENT_SCALE_FACTOR, - description="y-coordinate", + description="The top y coordinate (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", ) def invoke(self, context: InvocationContext) -> LatentsOutput: From 04e0fefdeeff60f639afcd652bd4921f4c77d2f0 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 12:05:55 -0500 Subject: [PATCH 10/33] Rename CropLatentsInvocation -> CropLatentsCoreInvocation to prevent conflict with custom node. And other minor tidying. --- invokeai/app/invocations/latent.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index ae0497eea7..ff9cc1dcf6 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -1170,14 +1170,18 @@ class BlendLatentsInvocation(BaseInvocation): return build_latents_output(latents_name=name, latents=blended_latents) +# The Crop Latents node was copied from @skunkworxdark's implementation here: +# https://github.com/skunkworxdark/XYGrid_nodes/blob/74647fa9c1fa57d317a94bd43ca689af7f0aae5e/images_to_grids.py#L1117C1-L1167C80 @invocation( - "lcrop", + "crop_latents", title="Crop Latents", tags=["latents", "crop"], category="latents", version="1.0.0", ) -class CropLatentsInvocation(BaseInvocation): +# TODO(ryand): Named `CropLatentsCoreInvocation` to prevent a conflict with custom node `CropLatentsInvocation`. +# Currently, if the class names conflict then 'GET /openapi.json' fails. +class CropLatentsCoreInvocation(BaseInvocation): """Crops a latent-space tensor to a box specified in image-space. The box dimensions and coordinates must be divisible by the latent scale factor of 8. """ @@ -1215,9 +1219,7 @@ class CropLatentsInvocation(BaseInvocation): x2 = x1 + (self.width // LATENT_SCALE_FACTOR) y2 = y1 + (self.height // LATENT_SCALE_FACTOR) - cropped_latents = latents[:, :, y1:y2, x1:x2] - - # resized_latents = resized_latents.to("cpu") + cropped_latents = latents[..., y1:y2, x1:x2] name = f"{context.graph_execution_state_id}__{self.id}" context.services.latents.save(name, cropped_latents) From 7e4a6893707d96a63c81cab60ed8357de5f0fd87 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 12:30:10 -0500 Subject: [PATCH 11/33] Update tiling nodes to use width-before-height field ordering convention. --- invokeai/app/invocations/tiles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 927e99be64..350141a2f3 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -33,12 +33,12 @@ class CalculateImageTilesOutput(BaseInvocationOutput): class CalculateImageTilesInvocation(BaseInvocation): """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" + image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") image_height: int = InputField( ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." ) - image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") - tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") tile_width: int = InputField(ge=1, default=576, description="The tile width, in pixels.") + tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") overlap: int = InputField( ge=0, default=128, @@ -117,8 +117,8 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): """Merge multiple tile images into a single image.""" # Inputs - image_height: int = InputField(ge=1, description="The height of the output image, in pixels.") image_width: int = InputField(ge=1, description="The width of the output image, in pixels.") + image_height: int = InputField(ge=1, description="The height of the output image, in pixels.") tiles_with_images: list[TileWithImage] = InputField(description="A list of tile images with tile properties.") blend_amount: int = InputField( ge=0, From 303791d5c6e43370d24a6b2f90ac3be2d9edb0ad Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 13:49:33 -0500 Subject: [PATCH 12/33] Add width and height fields to TileToPropertiesInvocation output to avoid having to calculate them with math nodes. --- invokeai/app/invocations/tiles.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 350141a2f3..934861f008 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -63,6 +63,14 @@ class TileToPropertiesOutput(BaseInvocationOutput): coords_left: int = OutputField(description="Left coordinate of the tile relative to its parent image.") coords_right: int = OutputField(description="Right coordinate of the tile relative to its parent image.") + # HACK: The width and height fields are 'meta' fields that can easily be calculated from the other fields on this + # object. Including redundant fields that can cheaply/easily be re-calculated goes against conventional API design + # principles. These fields are included, because 1) they are often useful in tiled workflows, and 2) they are + # difficult to calculate in a workflow (even though it's just a couple of subtraction nodes the graph gets + # surprisingly complicated). + width: int = OutputField(description="The width of the tile. Equal to coords_right - coords_left.") + height: int = OutputField(description="The height of the tile. Equal to coords_bottom - coords_top.") + overlap_top: int = OutputField(description="Overlap between this tile and its top neighbor.") overlap_bottom: int = OutputField(description="Overlap between this tile and its bottom neighbor.") overlap_left: int = OutputField(description="Overlap between this tile and its left neighbor.") @@ -81,6 +89,8 @@ class TileToPropertiesInvocation(BaseInvocation): coords_bottom=self.tile.coords.bottom, coords_left=self.tile.coords.left, coords_right=self.tile.coords.right, + width=self.tile.coords.right - self.tile.coords.left, + height=self.tile.coords.bottom - self.tile.coords.top, overlap_top=self.tile.overlap.top, overlap_bottom=self.tile.overlap.bottom, overlap_left=self.tile.overlap.left, From 932de08fc0d6acf694d918ffbe085d1746754d34 Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 14:07:38 -0500 Subject: [PATCH 13/33] Infer a tight-fitting output image size from the passed tiles in MergeTilesToImageInvocation. --- invokeai/app/invocations/tiles.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 934861f008..d1b51a43f0 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -127,8 +127,6 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): """Merge multiple tile images into a single image.""" # Inputs - image_width: int = InputField(ge=1, description="The width of the output image, in pixels.") - image_height: int = InputField(ge=1, description="The height of the output image, in pixels.") tiles_with_images: list[TileWithImage] = InputField(description="A list of tile images with tile properties.") blend_amount: int = InputField( ge=0, @@ -139,6 +137,13 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): images = [twi.image for twi in self.tiles_with_images] tiles = [twi.tile for twi in self.tiles_with_images] + # Infer the output image dimensions from the max/min tile limits. + height = 0 + width = 0 + for tile in tiles: + height = max(height, tile.coords.bottom) + width = max(width, tile.coords.right) + # Get all tile images for processing. # TODO(ryand): It pains me that we spend time PNG decoding each tile from disk when they almost certainly # existed in memory at an earlier point in the graph. @@ -152,7 +157,7 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): # Check the first tile to determine how many image channels are expected in the output. channels = tile_np_images[0].shape[-1] dtype = tile_np_images[0].dtype - np_image = np.zeros(shape=(self.image_height, self.image_width, channels), dtype=dtype) + np_image = np.zeros(shape=(height, width, channels), dtype=dtype) merge_tiles_with_linear_blending( dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount From 049b0239da4353086580793752e8f8321e19e9fb Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Mon, 27 Nov 2023 23:34:45 -0500 Subject: [PATCH 14/33] Re-organize merge_tiles_with_linear_blending(...) to merge rows horizontally first and then vertically. This change achieves slightly more natural blending on the corners where 4 tiles overlap. --- invokeai/backend/tiles/tiles.py | 101 +++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 3d64e3e145..3a678d825e 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -114,6 +114,24 @@ def merge_tiles_with_linear_blending( tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.left) tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.top) + # Organize tiles into rows. + tile_and_image_rows: list[list[tuple[Tile, np.ndarray]]] = [] + cur_tile_and_image_row: list[tuple[Tile, np.ndarray]] = [] + first_tile_in_cur_row, _ = tiles_and_images[0] + for tile_and_image in tiles_and_images: + tile, _ = tile_and_image + if not ( + tile.coords.top == first_tile_in_cur_row.coords.top + and tile.coords.bottom == first_tile_in_cur_row.coords.bottom + ): + # Store the previous row, and start a new one. + tile_and_image_rows.append(cur_tile_and_image_row) + cur_tile_and_image_row = [] + first_tile_in_cur_row, _ = tile_and_image + + cur_tile_and_image_row.append(tile_and_image) + tile_and_image_rows.append(cur_tile_and_image_row) + # Prepare 1D linear gradients for blending. gradient_left_x = np.linspace(start=0.0, stop=1.0, num=blend_amount) gradient_top_y = np.linspace(start=0.0, stop=1.0, num=blend_amount) @@ -122,33 +140,62 @@ def merge_tiles_with_linear_blending( # broadcasting to work correctly. gradient_top_y = np.expand_dims(gradient_top_y, axis=1) - for tile, tile_image in tiles_and_images: - # We expect tiles to be written left-to-right, top-to-bottom. We construct a mask that applies linear blending - # to the top and to the left of the current tile. The inverse linear blending is automatically applied to the - # bottom/right of the tiles that have already been pasted by the paste(...) operation. - tile_height, tile_width, _ = tile_image.shape - mask = np.ones(shape=(tile_height, tile_width), dtype=np.float64) + for tile_and_image_row in tile_and_image_rows: + first_tile_in_row, _ = tile_and_image_row[0] + row_height = first_tile_in_row.coords.bottom - first_tile_in_row.coords.top + row_image = np.zeros((row_height, dst_image.shape[1], dst_image.shape[2]), dtype=dst_image.dtype) + + # Blend the tiles in the row horizontally. + for tile, tile_image in tile_and_image_row: + # We expect the tiles to be ordered left-to-right. For each tile, we construct a mask that applies linear + # blending to the left of the current tile. The inverse linear blending is automatically applied to the + # right of the tiles that have already been pasted by the paste(...) operation. + tile_height, tile_width, _ = tile_image.shape + mask = np.ones(shape=(tile_height, tile_width), dtype=np.float64) + + # Left blending: + if tile.overlap.left > 0: + assert tile.overlap.left >= blend_amount + # Center the blending gradient in the middle of the overlap. + blend_start_left = tile.overlap.left // 2 - blend_amount // 2 + # The region left of the blending region is masked completely. + mask[:, :blend_start_left] = 0.0 + # Apply the blend gradient to the mask. + mask[:, blend_start_left : blend_start_left + blend_amount] = gradient_left_x + # For visual debugging: + # tile_image[:, blend_start_left : blend_start_left + blend_amount] = 0 + + paste( + dst_image=row_image, + src_image=tile_image, + box=TBLR( + top=0, bottom=tile.coords.bottom - tile.coords.top, left=tile.coords.left, right=tile.coords.right + ), + mask=mask, + ) + + # Blend the row into the dst_image vertically. + # We construct a mask that applies linear blending to the top of the current row. The inverse linear blending is + # automatically applied to the bottom of the tiles that have already been pasted by the paste(...) operation. + mask = np.ones(shape=(row_image.shape[0], row_image.shape[1]), dtype=np.float64) # Top blending: - if tile.overlap.top > 0: - assert tile.overlap.top >= blend_amount - # Center the blending gradient in the middle of the overlap. - blend_start_top = tile.overlap.top // 2 - blend_amount // 2 - # The region above the blending region is masked completely. + # (See comments under 'Left blending' for an explanation of the logic.) + # We assume that the entire row has the same vertical overlaps as the first_tile_in_row. + if first_tile_in_row.overlap.top > 0: + assert first_tile_in_row.overlap.top >= blend_amount + blend_start_top = first_tile_in_row.overlap.top // 2 - blend_amount // 2 mask[:blend_start_top, :] = 0.0 - # Apply the blend gradient to the mask. Note that we use `*=` rather than `=` to achieve more natural - # behavior on the corners where vertical and horizontal blending gradients overlap. - mask[blend_start_top : blend_start_top + blend_amount, :] *= gradient_top_y + mask[blend_start_top : blend_start_top + blend_amount, :] = gradient_top_y # For visual debugging: - # tile_image[blend_start_top : blend_start_top + blend_amount, :] = 0 - - # Left blending: - # (See comments under 'top blending' for an explanation of the logic.) - if tile.overlap.left > 0: - assert tile.overlap.left >= blend_amount - blend_start_left = tile.overlap.left // 2 - blend_amount // 2 - mask[:, :blend_start_left] = 0.0 - mask[:, blend_start_left : blend_start_left + blend_amount] *= gradient_left_x - # For visual debugging: - # tile_image[:, blend_start_left : blend_start_left + blend_amount] = 0 - - paste(dst_image=dst_image, src_image=tile_image, box=tile.coords, mask=mask) + # row_image[blend_start_top : blend_start_top + blend_amount, :] = 0 + paste( + dst_image=dst_image, + src_image=row_image, + box=TBLR( + top=first_tile_in_row.coords.top, + bottom=first_tile_in_row.coords.bottom, + left=0, + right=row_image.shape[1], + ), + mask=mask, + ) From bb87c988cbc97dd4dac9c785d5fd19d43cadd75d Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Wed, 29 Nov 2023 10:23:55 -0500 Subject: [PATCH 15/33] Change input field ordering of CropLatentsCoreInvocation to match ImageCropInvocation. --- invokeai/app/invocations/latent.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index ff9cc1dcf6..46a9c6a270 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -1190,16 +1190,6 @@ class CropLatentsCoreInvocation(BaseInvocation): description=FieldDescriptions.latents, input=Input.Connection, ) - width: int = InputField( - ge=1, - multiple_of=LATENT_SCALE_FACTOR, - description="The width (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", - ) - height: int = InputField( - ge=1, - multiple_of=LATENT_SCALE_FACTOR, - description="The height (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", - ) x_offset: int = InputField( ge=0, multiple_of=LATENT_SCALE_FACTOR, @@ -1210,6 +1200,16 @@ class CropLatentsCoreInvocation(BaseInvocation): multiple_of=LATENT_SCALE_FACTOR, description="The top y coordinate (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", ) + width: int = InputField( + ge=1, + multiple_of=LATENT_SCALE_FACTOR, + description="The width (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", + ) + height: int = InputField( + ge=1, + multiple_of=LATENT_SCALE_FACTOR, + description="The height (in px) of the crop rectangle in image space. This value will be converted to a dimension in latent space.", + ) def invoke(self, context: InvocationContext) -> LatentsOutput: latents = context.services.latents.get(self.latents.latents_name) From 14254e8be8056ad7ff5a615cb2145cbb607fdc68 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Tue, 5 Dec 2023 12:29:55 +0000 Subject: [PATCH 16/33] First check-in of new tile nodes - calc_tiles_even_split - calc_tiles_min_overlap - merge_tiles_with_seam_blending Update MergeTilesToImageInvocation with seam blending --- invokeai/app/invocations/tiles.py | 114 +++++++++++++++- invokeai/backend/tiles/tiles.py | 213 +++++++++++++++++++++++++++--- invokeai/backend/tiles/utils.py | 132 +++++++++++++++++- 3 files changed, 433 insertions(+), 26 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index d1b51a43f0..f21ee1bf5c 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -1,3 +1,5 @@ +from typing import Literal + import numpy as np from PIL import Image from pydantic import BaseModel @@ -5,6 +7,7 @@ from pydantic import BaseModel from invokeai.app.invocations.baseinvocation import ( BaseInvocation, BaseInvocationOutput, + Input, InputField, InvocationContext, OutputField, @@ -15,7 +18,13 @@ from invokeai.app.invocations.baseinvocation import ( ) from invokeai.app.invocations.primitives import ImageField, ImageOutput from invokeai.app.services.image_records.image_records_common import ImageCategory, ResourceOrigin -from invokeai.backend.tiles.tiles import calc_tiles_with_overlap, merge_tiles_with_linear_blending +from invokeai.backend.tiles.tiles import ( + calc_tiles_even_split, + calc_tiles_min_overlap, + calc_tiles_with_overlap, + merge_tiles_with_linear_blending, + merge_tiles_with_seam_blending, +) from invokeai.backend.tiles.utils import Tile @@ -56,6 +65,86 @@ class CalculateImageTilesInvocation(BaseInvocation): return CalculateImageTilesOutput(tiles=tiles) +@invocation( + "calculate_image_tiles_Even_Split", + title="Calculate Image Tiles Even Split", + tags=["tiles"], + category="tiles", + version="1.0.0", +) +class CalculateImageTilesEvenSplitInvocation(BaseInvocation): + """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" + + image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") + image_height: int = InputField( + ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." + ) + num_tiles_x: int = InputField( + default=2, + ge=1, + description="Number of tiles to divide image into on the x axis", + ) + num_tiles_y: int = InputField( + default=2, + ge=1, + description="Number of tiles to divide image into on the y axis", + ) + overlap: float = InputField( + default=0.25, + ge=0, + lt=1, + description="Overlap amount of tile size (0-1)", + ) + + def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: + tiles = calc_tiles_even_split( + image_height=self.image_height, + image_width=self.image_width, + num_tiles_x=self.num_tiles_x, + num_tiles_y=self.num_tiles_y, + overlap=self.overlap, + ) + return CalculateImageTilesOutput(tiles=tiles) + + +@invocation( + "calculate_image_tiles_min_overlap", + title="Calculate Image Tiles Minimum Overlap", + tags=["tiles"], + category="tiles", + version="1.0.0", +) +class CalculateImageTilesMinimumOverlapInvocation(BaseInvocation): + """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" + + image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") + image_height: int = InputField( + ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." + ) + tile_width: int = InputField(ge=1, default=576, description="The tile width, in pixels.") + tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") + min_overlap: int = InputField( + default=128, + ge=0, + description="minimum tile overlap size (must be a multiple of 8)", + ) + round_to_8: bool = InputField( + default=False, + description="Round outputs down to the nearest 8 (for pulling from a large noise field)", + ) + + def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: + tiles = calc_tiles_min_overlap( + image_height=self.image_height, + image_width=self.image_width, + tile_height=self.tile_height, + tile_width=self.tile_width, + min_overlap=self.min_overlap, + round_to_8=self.round_to_8, + ) + return CalculateImageTilesOutput(tiles=tiles) + + @invocation_output("tile_to_properties_output") class TileToPropertiesOutput(BaseInvocationOutput): coords_top: int = OutputField(description="Top coordinate of the tile relative to its parent image.") @@ -122,13 +211,22 @@ class PairTileImageInvocation(BaseInvocation): ) -@invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.0.0") +BLEND_MODES = Literal["Linear", "Seam"] + + +@invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.1.0") class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): """Merge multiple tile images into a single image.""" # Inputs tiles_with_images: list[TileWithImage] = InputField(description="A list of tile images with tile properties.") + blend_mode: BLEND_MODES = InputField( + default="Seam", + description="blending type Linear or Seam", + input=Input.Direct, + ) blend_amount: int = InputField( + default=32, ge=0, description="The amount to blend adjacent tiles in pixels. Must be <= the amount of overlap between adjacent tiles.", ) @@ -158,10 +256,16 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): channels = tile_np_images[0].shape[-1] dtype = tile_np_images[0].dtype np_image = np.zeros(shape=(height, width, channels), dtype=dtype) + if self.blend_mode == "Linear": + merge_tiles_with_linear_blending( + dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount + ) + else: + merge_tiles_with_seam_blending( + dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount + ) - merge_tiles_with_linear_blending( - dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount - ) + # Convert into a PIL image and save pil_image = Image.fromarray(np_image) image_dto = context.services.images.create( diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 3a678d825e..18584350a5 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -1,9 +1,8 @@ import math -from typing import Union import numpy as np -from invokeai.backend.tiles.utils import TBLR, Tile, paste +from invokeai.backend.tiles.utils import TBLR, Tile, calc_overlap, paste, seam_blend def calc_tiles_with_overlap( @@ -63,31 +62,117 @@ def calc_tiles_with_overlap( tiles.append(tile) - def get_tile_or_none(idx_y: int, idx_x: int) -> Union[Tile, None]: - if idx_y < 0 or idx_y > num_tiles_y or idx_x < 0 or idx_x > num_tiles_x: - return None - return tiles[idx_y * num_tiles_x + idx_x] + return calc_overlap(tiles, num_tiles_x, num_tiles_y) - # Iterate over tiles again and calculate overlaps. + +def calc_tiles_even_split( + image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: float = 0 +) -> list[Tile]: + """Calculate the tile coordinates for a given image shape with the number of tiles requested. + + Args: + image_height (int): The image height in px. + image_width (int): The image width in px. + num_x_tiles (int): The number of tile to split the image into on the X-axis. + num_y_tiles (int): The number of tile to split the image into on the Y-axis. + overlap (int, optional): The target overlap amount of the tiles size. Defaults to 0. + + Returns: + list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. + """ + + # Ensure tile size is divisible by 8 + if image_width % 8 != 0 or image_height % 8 != 0: + raise ValueError(f"image size (({image_width}, {image_height})) must be divisible by 8") + + # Calculate the overlap size based on the percentage and adjust it to be divisible by 8 (rounding up) + overlap_x = 8 * math.ceil(int((image_width / num_tiles_x) * overlap) / 8) + overlap_y = 8 * math.ceil(int((image_height / num_tiles_y) * overlap) / 8) + + # Calculate the tile size based on the number of tiles and overlap, and ensure it's divisible by 8 (rounding down) + tile_size_x = 8 * math.floor(((image_width + overlap_x * (num_tiles_x - 1)) // num_tiles_x) / 8) + tile_size_y = 8 * math.floor(((image_height + overlap_y * (num_tiles_y - 1)) // num_tiles_y) / 8) + + # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. + tiles: list[Tile] = [] + + # Calculate tile coordinates. (Ignore overlap values for now.) for tile_idx_y in range(num_tiles_y): + # Calculate the top and bottom of the row + top = tile_idx_y * (tile_size_y - overlap_y) + bottom = min(top + tile_size_y, image_height) + # For the last row adjust bottom to be the height of the image + if tile_idx_y == num_tiles_y - 1: + bottom = image_height + for tile_idx_x in range(num_tiles_x): - cur_tile = get_tile_or_none(tile_idx_y, tile_idx_x) - top_neighbor_tile = get_tile_or_none(tile_idx_y - 1, tile_idx_x) - left_neighbor_tile = get_tile_or_none(tile_idx_y, tile_idx_x - 1) + # Calculate the left & right coordinate of each tile + left = tile_idx_x * (tile_size_x - overlap_x) + right = min(left + tile_size_x, image_width) + # For the last tile in the row adjust right to be the width of the image + if tile_idx_x == num_tiles_x - 1: + right = image_width - assert cur_tile is not None + tile = Tile( + coords=TBLR(top=top, bottom=bottom, left=left, right=right), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) - # Update cur_tile top-overlap and corresponding top-neighbor bottom-overlap. - if top_neighbor_tile is not None: - cur_tile.overlap.top = max(0, top_neighbor_tile.coords.bottom - cur_tile.coords.top) - top_neighbor_tile.overlap.bottom = cur_tile.overlap.top + tiles.append(tile) - # Update cur_tile left-overlap and corresponding left-neighbor right-overlap. - if left_neighbor_tile is not None: - cur_tile.overlap.left = max(0, left_neighbor_tile.coords.right - cur_tile.coords.left) - left_neighbor_tile.overlap.right = cur_tile.overlap.left + return calc_overlap(tiles, num_tiles_x, num_tiles_y) - return tiles + +def calc_tiles_min_overlap( + image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int, round_to_8: bool +) -> list[Tile]: + """Calculate the tile coordinates for a given image shape under a simple tiling scheme with overlaps. + + Args: + image_height (int): The image height in px. + image_width (int): The image width in px. + tile_height (int): The tile height in px. All tiles will have this height. + tile_width (int): The tile width in px. All tiles will have this width. + min_overlap (int): The target minimum overlap between adjacent tiles. If the tiles do not evenly cover the image + shape, then the overlap will be spread between the tiles. + + Returns: + list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. + """ + assert image_height >= tile_height + assert image_width >= tile_width + assert min_overlap < tile_height + assert min_overlap < tile_width + + num_tiles_x = math.ceil((image_width - min_overlap) / (tile_width - min_overlap)) if tile_width < image_width else 1 + num_tiles_y = ( + math.ceil((image_height - min_overlap) / (tile_height - min_overlap)) if tile_height < image_height else 1 + ) + + # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. + tiles: list[Tile] = [] + + # Calculate tile coordinates. (Ignore overlap values for now.) + for tile_idx_y in range(num_tiles_y): + top = (tile_idx_y * (image_height - tile_height)) // (num_tiles_y - 1) if num_tiles_y > 1 else 0 + if round_to_8: + top = 8 * (top // 8) + bottom = top + tile_height + + for tile_idx_x in range(num_tiles_x): + left = (tile_idx_x * (image_width - tile_width)) // (num_tiles_x - 1) if num_tiles_x > 1 else 0 + if round_to_8: + left = 8 * (left // 8) + right = left + tile_width + + tile = Tile( + coords=TBLR(top=top, bottom=bottom, left=left, right=right), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) + + tiles.append(tile) + + return calc_overlap(tiles, num_tiles_x, num_tiles_y) def merge_tiles_with_linear_blending( @@ -199,3 +284,91 @@ def merge_tiles_with_linear_blending( ), mask=mask, ) + + +def merge_tiles_with_seam_blending( + dst_image: np.ndarray, tiles: list[Tile], tile_images: list[np.ndarray], blend_amount: int +): + """Merge a set of image tiles into `dst_image` with seam blending between the tiles. + + We expect every tile edge to either: + 1) have an overlap of 0, because it is aligned with the image edge, or + 2) have an overlap >= blend_amount. + If neither of these conditions are satisfied, we raise an exception. + + The seam blending is centered on a seam of least energy of the overlap between adjacent tiles. + + Args: + dst_image (np.ndarray): The destination image. Shape: (H, W, C). + tiles (list[Tile]): The list of tiles describing the locations of the respective `tile_images`. + tile_images (list[np.ndarray]): The tile images to merge into `dst_image`. + blend_amount (int): The amount of blending (in px) between adjacent overlapping tiles. + """ + # Sort tiles and images first by left x coordinate, then by top y coordinate. During tile processing, we want to + # iterate over tiles left-to-right, top-to-bottom. + tiles_and_images = list(zip(tiles, tile_images, strict=True)) + tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.left) + tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.top) + + # Organize tiles into rows. + tile_and_image_rows: list[list[tuple[Tile, np.ndarray]]] = [] + cur_tile_and_image_row: list[tuple[Tile, np.ndarray]] = [] + first_tile_in_cur_row, _ = tiles_and_images[0] + for tile_and_image in tiles_and_images: + tile, _ = tile_and_image + if not ( + tile.coords.top == first_tile_in_cur_row.coords.top + and tile.coords.bottom == first_tile_in_cur_row.coords.bottom + ): + # Store the previous row, and start a new one. + tile_and_image_rows.append(cur_tile_and_image_row) + cur_tile_and_image_row = [] + first_tile_in_cur_row, _ = tile_and_image + + cur_tile_and_image_row.append(tile_and_image) + tile_and_image_rows.append(cur_tile_and_image_row) + + for tile_and_image_row in tile_and_image_rows: + first_tile_in_row, _ = tile_and_image_row[0] + row_height = first_tile_in_row.coords.bottom - first_tile_in_row.coords.top + row_image = np.zeros((row_height, dst_image.shape[1], dst_image.shape[2]), dtype=dst_image.dtype) + + # Blend the tiles in the row horizontally. + for tile, tile_image in tile_and_image_row: + # We expect the tiles to be ordered left-to-right. + # For each tile: + # - extract the overlap regions and pass to seam_blend() + # - apply blended region to the row_image + # - apply the un-blended region to the row_image + tile_height, tile_width, _ = tile_image.shape + overlap_size = tile.overlap.left + # Left blending: + if overlap_size > 0: + assert overlap_size >= blend_amount + + overlap_coord_right = tile.coords.left + overlap_size + src_overlap = row_image[:, tile.coords.left : overlap_coord_right] + dst_overlap = tile_image[:, :overlap_size] + blended_overlap = seam_blend(src_overlap, dst_overlap, blend_amount, x_seam=False) + row_image[:, tile.coords.left : overlap_coord_right] = blended_overlap + row_image[:, overlap_coord_right : tile.coords.right] = tile_image[:, overlap_size:] + else: + # no overlap just paste the tile + row_image[:, tile.coords.left : tile.coords.right] = tile_image + + # Blend the row into the dst_image + # We assume that the entire row has the same vertical overlaps as the first_tile_in_row. + # Rows are processed in the same way as tiles (extract overlap, blend, apply) + row_overlap_size = first_tile_in_row.overlap.top + if row_overlap_size > 0: + assert row_overlap_size >= blend_amount + + overlap_coords_bottom = first_tile_in_row.coords.top + row_overlap_size + src_overlap = dst_image[first_tile_in_row.coords.top : overlap_coords_bottom, :] + dst_overlap = row_image[:row_overlap_size, :] + blended_overlap = seam_blend(src_overlap, dst_overlap, blend_amount, x_seam=True) + dst_image[first_tile_in_row.coords.top : overlap_coords_bottom, :] = blended_overlap + dst_image[overlap_coords_bottom : first_tile_in_row.coords.bottom, :] = row_image[row_overlap_size:, :] + else: + # no overlap just paste the row + row_image[first_tile_in_row.coords.top:first_tile_in_row.coords.bottom, :] = row_image diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py index 4ad40ffa35..34bb28aebb 100644 --- a/invokeai/backend/tiles/utils.py +++ b/invokeai/backend/tiles/utils.py @@ -1,6 +1,9 @@ -from typing import Optional +import math +from typing import Optional, Union +import cv2 import numpy as np +#from PIL import Image from pydantic import BaseModel, Field @@ -45,3 +48,130 @@ def paste(dst_image: np.ndarray, src_image: np.ndarray, box: TBLR, mask: Optiona mask = np.expand_dims(mask, -1) dst_image_box = dst_image[box.top : box.bottom, box.left : box.right] dst_image[box.top : box.bottom, box.left : box.right] = src_image * mask + dst_image_box * (1.0 - mask) + + +def calc_overlap(tiles: list[Tile], num_tiles_x, num_tiles_y) -> list[Tile]: + """Calculate and update the overlap of a list of tiles. + + Args: + tiles (list[Tile]): The list of tiles describing the locations of the respective `tile_images`. + num_tiles_x: the number of tiles on the x axis. + num_tiles_y: the number of tiles on the y axis. + """ + def get_tile_or_none(idx_y: int, idx_x: int) -> Union[Tile, None]: + if idx_y < 0 or idx_y > num_tiles_y or idx_x < 0 or idx_x > num_tiles_x: + return None + return tiles[idx_y * num_tiles_x + idx_x] + + for tile_idx_y in range(num_tiles_y): + for tile_idx_x in range(num_tiles_x): + cur_tile = get_tile_or_none(tile_idx_y, tile_idx_x) + top_neighbor_tile = get_tile_or_none(tile_idx_y - 1, tile_idx_x) + left_neighbor_tile = get_tile_or_none(tile_idx_y, tile_idx_x - 1) + + assert cur_tile is not None + + # Update cur_tile top-overlap and corresponding top-neighbor bottom-overlap. + if top_neighbor_tile is not None: + cur_tile.overlap.top = max(0, top_neighbor_tile.coords.bottom - cur_tile.coords.top) + top_neighbor_tile.overlap.bottom = cur_tile.overlap.top + + # Update cur_tile left-overlap and corresponding left-neighbor right-overlap. + if left_neighbor_tile is not None: + cur_tile.overlap.left = max(0, left_neighbor_tile.coords.right - cur_tile.coords.left) + left_neighbor_tile.overlap.right = cur_tile.overlap.left + return tiles + + +def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool,) -> np.ndarray: + """Blend two overlapping tile sections using a seams to find a path. + + It is assumed that input images will be RGB np arrays and are the same size. + + Args: + ia1 (torch.Tensor): Image array 1 Shape: (H, W, C). + ia2 (torch.Tensor): Image array 2 Shape: (H, W, C). + x_seam (bool): If the images should be blended on the x axis or not. + blend_amount (int): The size of the blur to use on the seam. Half of this value will be used to avoid the edges of the image. + """ + + def shift(arr, num, fill_value=255.0): + result = np.full_like(arr, fill_value) + if num > 0: + result[num:] = arr[:-num] + elif num < 0: + result[:num] = arr[-num:] + else: + result[:] = arr + return result + + # Assume RGB and convert to grey + iag1 = np.dot(ia1, [0.2989, 0.5870, 0.1140]) + iag2 = np.dot(ia2, [0.2989, 0.5870, 0.1140]) + + # Calc Difference between the images + ia = iag2 - iag1 + + # If the seam is on the X-axis rotate the array so we can treat it like a vertical seam + if x_seam: + ia = np.rot90(ia, 1) + + # Calc max and min X & Y limits + # gutter is used to avoid the blur hitting the edge of the image + gutter = math.ceil(blend_amount / 2) if blend_amount > 0 else 0 + max_y, max_x = ia.shape + max_x -= gutter + min_x = gutter + + # Calc the energy in the difference + energy = np.abs(np.gradient(ia, axis=0)) + np.abs(np.gradient(ia, axis=1)) + + #Find the starting position of the seam + res = np.copy(energy) + for y in range(1, max_y): + row = res[y, :] + rowl = shift(row, -1) + rowr = shift(row, 1) + res[y, :] = res[y - 1, :] + np.min([row, rowl, rowr], axis=0) + + # create an array max_y long + lowest_energy_line = np.empty([max_y], dtype="uint16") + lowest_energy_line[max_y - 1] = np.argmin(res[max_y - 1, min_x : max_x - 1]) + + #Calc the path of the seam + for ypos in range(max_y - 2, -1, -1): + lowest_pos = lowest_energy_line[ypos + 1] + lpos = lowest_pos - 1 + rpos = lowest_pos + 1 + lpos = np.clip(lpos, min_x, max_x - 1) + rpos = np.clip(rpos, min_x, max_x - 1) + lowest_energy_line[ypos] = np.argmin(energy[ypos, lpos : rpos + 1]) + lpos + + # Draw the mask + mask = np.zeros_like(ia) + for ypos in range(0, max_y): + to_fill = lowest_energy_line[ypos] + mask[ypos, :to_fill] = 1 + + # If the seam is on the X-axis rotate the array back + if x_seam: + mask = np.rot90(mask, 3) + + # blur the seam mask if required + if blend_amount > 0: + mask = cv2.blur(mask, (blend_amount, blend_amount)) + + # copy ia2 over ia1 while applying the seam mask + mask = np.expand_dims(mask, -1) + blended_image = ia1 * mask + ia2 * (1.0 - mask) + + # for debugging to see the final blended overlap image + #image = Image.fromarray((mask * 255.0).astype("uint8")) + #i1 = Image.fromarray(ia1.astype("uint8")) + #i2 = Image.fromarray(ia2.astype("uint8")) + #bimage = Image.fromarray(blended_image.astype("uint8")) + + #print(f"{ia1.shape}, {ia2.shape}, {mask.shape}, {blended_image.shape}") + #print(f"{i1.size}, {i2.size}, {image.size}, {bimage.size}") + + return blended_image From 674d9796d079f981a34569756c97d68ba6507381 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Tue, 5 Dec 2023 21:03:16 +0000 Subject: [PATCH 17/33] First check-in of new tile nodes - calc_tiles_even_split - calc_tiles_min_overlap - merge_tiles_with_seam_blending Update MergeTilesToImageInvocation with seam blending --- invokeai/app/invocations/tiles.py | 114 +++++++++++++- invokeai/backend/tiles/tiles.py | 245 +++++++++++++++++++++++++++--- invokeai/backend/tiles/utils.py | 100 ++++++++++++ 3 files changed, 435 insertions(+), 24 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 3055c1baae..609b539610 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -1,3 +1,5 @@ +from typing import Literal + import numpy as np from PIL import Image from pydantic import BaseModel @@ -5,6 +7,7 @@ from pydantic import BaseModel from invokeai.app.invocations.baseinvocation import ( BaseInvocation, BaseInvocationOutput, + Input, InputField, InvocationContext, OutputField, @@ -15,7 +18,13 @@ from invokeai.app.invocations.baseinvocation import ( ) from invokeai.app.invocations.primitives import ImageField, ImageOutput from invokeai.app.services.image_records.image_records_common import ImageCategory, ResourceOrigin -from invokeai.backend.tiles.tiles import calc_tiles_with_overlap, merge_tiles_with_linear_blending +from invokeai.backend.tiles.tiles import ( + calc_tiles_even_split, + calc_tiles_min_overlap, + calc_tiles_with_overlap, + merge_tiles_with_linear_blending, + merge_tiles_with_seam_blending, +) from invokeai.backend.tiles.utils import Tile @@ -56,6 +65,86 @@ class CalculateImageTilesInvocation(BaseInvocation): return CalculateImageTilesOutput(tiles=tiles) +@invocation( + "calculate_image_tiles_Even_Split", + title="Calculate Image Tiles Even Split", + tags=["tiles"], + category="tiles", + version="1.0.0", +) +class CalculateImageTilesEvenSplitInvocation(BaseInvocation): + """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" + + image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") + image_height: int = InputField( + ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." + ) + num_tiles_x: int = InputField( + default=2, + ge=1, + description="Number of tiles to divide image into on the x axis", + ) + num_tiles_y: int = InputField( + default=2, + ge=1, + description="Number of tiles to divide image into on the y axis", + ) + overlap: float = InputField( + default=0.25, + ge=0, + lt=1, + description="Overlap amount of tile size (0-1)", + ) + + def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: + tiles = calc_tiles_even_split( + image_height=self.image_height, + image_width=self.image_width, + num_tiles_x=self.num_tiles_x, + num_tiles_y=self.num_tiles_y, + overlap=self.overlap, + ) + return CalculateImageTilesOutput(tiles=tiles) + + +@invocation( + "calculate_image_tiles_min_overlap", + title="Calculate Image Tiles Minimum Overlap", + tags=["tiles"], + category="tiles", + version="1.0.0", +) +class CalculateImageTilesMinimumOverlapInvocation(BaseInvocation): + """Calculate the coordinates and overlaps of tiles that cover a target image shape.""" + + image_width: int = InputField(ge=1, default=1024, description="The image width, in pixels, to calculate tiles for.") + image_height: int = InputField( + ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." + ) + tile_width: int = InputField(ge=1, default=576, description="The tile width, in pixels.") + tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") + min_overlap: int = InputField( + default=128, + ge=0, + description="minimum tile overlap size (must be a multiple of 8)", + ) + round_to_8: bool = InputField( + default=False, + description="Round outputs down to the nearest 8 (for pulling from a large noise field)", + ) + + def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: + tiles = calc_tiles_min_overlap( + image_height=self.image_height, + image_width=self.image_width, + tile_height=self.tile_height, + tile_width=self.tile_width, + min_overlap=self.min_overlap, + round_to_8=self.round_to_8, + ) + return CalculateImageTilesOutput(tiles=tiles) + + @invocation_output("tile_to_properties_output") class TileToPropertiesOutput(BaseInvocationOutput): coords_left: int = OutputField(description="Left coordinate of the tile relative to its parent image.") @@ -122,13 +211,22 @@ class PairTileImageInvocation(BaseInvocation): ) -@invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.0.0") +BLEND_MODES = Literal["Linear", "Seam"] + + +@invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.1.0") class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): """Merge multiple tile images into a single image.""" # Inputs tiles_with_images: list[TileWithImage] = InputField(description="A list of tile images with tile properties.") + blend_mode: BLEND_MODES = InputField( + default="Seam", + description="blending type Linear or Seam", + input=Input.Direct, + ) blend_amount: int = InputField( + default=32, ge=0, description="The amount to blend adjacent tiles in pixels. Must be <= the amount of overlap between adjacent tiles.", ) @@ -158,10 +256,16 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): channels = tile_np_images[0].shape[-1] dtype = tile_np_images[0].dtype np_image = np.zeros(shape=(height, width, channels), dtype=dtype) + if self.blend_mode == "Linear": + merge_tiles_with_linear_blending( + dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount + ) + else: + merge_tiles_with_seam_blending( + dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount + ) - merge_tiles_with_linear_blending( - dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount - ) + # Convert into a PIL image and save pil_image = Image.fromarray(np_image) image_dto = context.services.images.create( diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 3a678d825e..33c5f5c445 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -3,7 +3,40 @@ from typing import Union import numpy as np -from invokeai.backend.tiles.utils import TBLR, Tile, paste +from invokeai.backend.tiles.utils import TBLR, Tile, paste, seam_blend + + +def calc_overlap(tiles: list[Tile], num_tiles_x, num_tiles_y) -> list[Tile]: + """Calculate and update the overlap of a list of tiles. + + Args: + tiles (list[Tile]): The list of tiles describing the locations of the respective `tile_images`. + num_tiles_x: the number of tiles on the x axis. + num_tiles_y: the number of tiles on the y axis. + """ + def get_tile_or_none(idx_y: int, idx_x: int) -> Union[Tile, None]: + if idx_y < 0 or idx_y > num_tiles_y or idx_x < 0 or idx_x > num_tiles_x: + return None + return tiles[idx_y * num_tiles_x + idx_x] + + for tile_idx_y in range(num_tiles_y): + for tile_idx_x in range(num_tiles_x): + cur_tile = get_tile_or_none(tile_idx_y, tile_idx_x) + top_neighbor_tile = get_tile_or_none(tile_idx_y - 1, tile_idx_x) + left_neighbor_tile = get_tile_or_none(tile_idx_y, tile_idx_x - 1) + + assert cur_tile is not None + + # Update cur_tile top-overlap and corresponding top-neighbor bottom-overlap. + if top_neighbor_tile is not None: + cur_tile.overlap.top = max(0, top_neighbor_tile.coords.bottom - cur_tile.coords.top) + top_neighbor_tile.overlap.bottom = cur_tile.overlap.top + + # Update cur_tile left-overlap and corresponding left-neighbor right-overlap. + if left_neighbor_tile is not None: + cur_tile.overlap.left = max(0, left_neighbor_tile.coords.right - cur_tile.coords.left) + left_neighbor_tile.overlap.right = cur_tile.overlap.left + return tiles def calc_tiles_with_overlap( @@ -63,31 +96,117 @@ def calc_tiles_with_overlap( tiles.append(tile) - def get_tile_or_none(idx_y: int, idx_x: int) -> Union[Tile, None]: - if idx_y < 0 or idx_y > num_tiles_y or idx_x < 0 or idx_x > num_tiles_x: - return None - return tiles[idx_y * num_tiles_x + idx_x] + return calc_overlap(tiles, num_tiles_x, num_tiles_y) - # Iterate over tiles again and calculate overlaps. + +def calc_tiles_even_split( + image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: float = 0 +) -> list[Tile]: + """Calculate the tile coordinates for a given image shape with the number of tiles requested. + + Args: + image_height (int): The image height in px. + image_width (int): The image width in px. + num_x_tiles (int): The number of tile to split the image into on the X-axis. + num_y_tiles (int): The number of tile to split the image into on the Y-axis. + overlap (int, optional): The target overlap amount of the tiles size. Defaults to 0. + + Returns: + list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. + """ + + # Ensure tile size is divisible by 8 + if image_width % 8 != 0 or image_height % 8 != 0: + raise ValueError(f"image size (({image_width}, {image_height})) must be divisible by 8") + + # Calculate the overlap size based on the percentage and adjust it to be divisible by 8 (rounding up) + overlap_x = 8 * math.ceil(int((image_width / num_tiles_x) * overlap) / 8) + overlap_y = 8 * math.ceil(int((image_height / num_tiles_y) * overlap) / 8) + + # Calculate the tile size based on the number of tiles and overlap, and ensure it's divisible by 8 (rounding down) + tile_size_x = 8 * math.floor(((image_width + overlap_x * (num_tiles_x - 1)) // num_tiles_x) / 8) + tile_size_y = 8 * math.floor(((image_height + overlap_y * (num_tiles_y - 1)) // num_tiles_y) / 8) + + # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. + tiles: list[Tile] = [] + + # Calculate tile coordinates. (Ignore overlap values for now.) for tile_idx_y in range(num_tiles_y): + # Calculate the top and bottom of the row + top = tile_idx_y * (tile_size_y - overlap_y) + bottom = min(top + tile_size_y, image_height) + # For the last row adjust bottom to be the height of the image + if tile_idx_y == num_tiles_y - 1: + bottom = image_height + for tile_idx_x in range(num_tiles_x): - cur_tile = get_tile_or_none(tile_idx_y, tile_idx_x) - top_neighbor_tile = get_tile_or_none(tile_idx_y - 1, tile_idx_x) - left_neighbor_tile = get_tile_or_none(tile_idx_y, tile_idx_x - 1) + # Calculate the left & right coordinate of each tile + left = tile_idx_x * (tile_size_x - overlap_x) + right = min(left + tile_size_x, image_width) + # For the last tile in the row adjust right to be the width of the image + if tile_idx_x == num_tiles_x - 1: + right = image_width - assert cur_tile is not None + tile = Tile( + coords=TBLR(top=top, bottom=bottom, left=left, right=right), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) - # Update cur_tile top-overlap and corresponding top-neighbor bottom-overlap. - if top_neighbor_tile is not None: - cur_tile.overlap.top = max(0, top_neighbor_tile.coords.bottom - cur_tile.coords.top) - top_neighbor_tile.overlap.bottom = cur_tile.overlap.top + tiles.append(tile) - # Update cur_tile left-overlap and corresponding left-neighbor right-overlap. - if left_neighbor_tile is not None: - cur_tile.overlap.left = max(0, left_neighbor_tile.coords.right - cur_tile.coords.left) - left_neighbor_tile.overlap.right = cur_tile.overlap.left + return calc_overlap(tiles, num_tiles_x, num_tiles_y) - return tiles + +def calc_tiles_min_overlap( + image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int, round_to_8: bool +) -> list[Tile]: + """Calculate the tile coordinates for a given image shape under a simple tiling scheme with overlaps. + + Args: + image_height (int): The image height in px. + image_width (int): The image width in px. + tile_height (int): The tile height in px. All tiles will have this height. + tile_width (int): The tile width in px. All tiles will have this width. + min_overlap (int): The target minimum overlap between adjacent tiles. If the tiles do not evenly cover the image + shape, then the overlap will be spread between the tiles. + + Returns: + list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. + """ + assert image_height >= tile_height + assert image_width >= tile_width + assert min_overlap < tile_height + assert min_overlap < tile_width + + num_tiles_x = math.ceil((image_width - min_overlap) / (tile_width - min_overlap)) if tile_width < image_width else 1 + num_tiles_y = ( + math.ceil((image_height - min_overlap) / (tile_height - min_overlap)) if tile_height < image_height else 1 + ) + + # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. + tiles: list[Tile] = [] + + # Calculate tile coordinates. (Ignore overlap values for now.) + for tile_idx_y in range(num_tiles_y): + top = (tile_idx_y * (image_height - tile_height)) // (num_tiles_y - 1) if num_tiles_y > 1 else 0 + if round_to_8: + top = 8 * (top // 8) + bottom = top + tile_height + + for tile_idx_x in range(num_tiles_x): + left = (tile_idx_x * (image_width - tile_width)) // (num_tiles_x - 1) if num_tiles_x > 1 else 0 + if round_to_8: + left = 8 * (left // 8) + right = left + tile_width + + tile = Tile( + coords=TBLR(top=top, bottom=bottom, left=left, right=right), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) + + tiles.append(tile) + + return calc_overlap(tiles, num_tiles_x, num_tiles_y) def merge_tiles_with_linear_blending( @@ -199,3 +318,91 @@ def merge_tiles_with_linear_blending( ), mask=mask, ) + + +def merge_tiles_with_seam_blending( + dst_image: np.ndarray, tiles: list[Tile], tile_images: list[np.ndarray], blend_amount: int +): + """Merge a set of image tiles into `dst_image` with seam blending between the tiles. + + We expect every tile edge to either: + 1) have an overlap of 0, because it is aligned with the image edge, or + 2) have an overlap >= blend_amount. + If neither of these conditions are satisfied, we raise an exception. + + The seam blending is centered on a seam of least energy of the overlap between adjacent tiles. + + Args: + dst_image (np.ndarray): The destination image. Shape: (H, W, C). + tiles (list[Tile]): The list of tiles describing the locations of the respective `tile_images`. + tile_images (list[np.ndarray]): The tile images to merge into `dst_image`. + blend_amount (int): The amount of blending (in px) between adjacent overlapping tiles. + """ + # Sort tiles and images first by left x coordinate, then by top y coordinate. During tile processing, we want to + # iterate over tiles left-to-right, top-to-bottom. + tiles_and_images = list(zip(tiles, tile_images, strict=True)) + tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.left) + tiles_and_images = sorted(tiles_and_images, key=lambda x: x[0].coords.top) + + # Organize tiles into rows. + tile_and_image_rows: list[list[tuple[Tile, np.ndarray]]] = [] + cur_tile_and_image_row: list[tuple[Tile, np.ndarray]] = [] + first_tile_in_cur_row, _ = tiles_and_images[0] + for tile_and_image in tiles_and_images: + tile, _ = tile_and_image + if not ( + tile.coords.top == first_tile_in_cur_row.coords.top + and tile.coords.bottom == first_tile_in_cur_row.coords.bottom + ): + # Store the previous row, and start a new one. + tile_and_image_rows.append(cur_tile_and_image_row) + cur_tile_and_image_row = [] + first_tile_in_cur_row, _ = tile_and_image + + cur_tile_and_image_row.append(tile_and_image) + tile_and_image_rows.append(cur_tile_and_image_row) + + for tile_and_image_row in tile_and_image_rows: + first_tile_in_row, _ = tile_and_image_row[0] + row_height = first_tile_in_row.coords.bottom - first_tile_in_row.coords.top + row_image = np.zeros((row_height, dst_image.shape[1], dst_image.shape[2]), dtype=dst_image.dtype) + + # Blend the tiles in the row horizontally. + for tile, tile_image in tile_and_image_row: + # We expect the tiles to be ordered left-to-right. + # For each tile: + # - extract the overlap regions and pass to seam_blend() + # - apply blended region to the row_image + # - apply the un-blended region to the row_image + tile_height, tile_width, _ = tile_image.shape + overlap_size = tile.overlap.left + # Left blending: + if overlap_size > 0: + assert overlap_size >= blend_amount + + overlap_coord_right = tile.coords.left + overlap_size + src_overlap = row_image[:, tile.coords.left : overlap_coord_right] + dst_overlap = tile_image[:, :overlap_size] + blended_overlap = seam_blend(src_overlap, dst_overlap, blend_amount, x_seam=False) + row_image[:, tile.coords.left : overlap_coord_right] = blended_overlap + row_image[:, overlap_coord_right : tile.coords.right] = tile_image[:, overlap_size:] + else: + # no overlap just paste the tile + row_image[:, tile.coords.left : tile.coords.right] = tile_image + + # Blend the row into the dst_image + # We assume that the entire row has the same vertical overlaps as the first_tile_in_row. + # Rows are processed in the same way as tiles (extract overlap, blend, apply) + row_overlap_size = first_tile_in_row.overlap.top + if row_overlap_size > 0: + assert row_overlap_size >= blend_amount + + overlap_coords_bottom = first_tile_in_row.coords.top + row_overlap_size + src_overlap = dst_image[first_tile_in_row.coords.top : overlap_coords_bottom, :] + dst_overlap = row_image[:row_overlap_size, :] + blended_overlap = seam_blend(src_overlap, dst_overlap, blend_amount, x_seam=True) + dst_image[first_tile_in_row.coords.top : overlap_coords_bottom, :] = blended_overlap + dst_image[overlap_coords_bottom : first_tile_in_row.coords.bottom, :] = row_image[row_overlap_size:, :] + else: + # no overlap just paste the row + dst_image[first_tile_in_row.coords.top:first_tile_in_row.coords.bottom, :] = row_image diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py index 4ad40ffa35..9df63a601e 100644 --- a/invokeai/backend/tiles/utils.py +++ b/invokeai/backend/tiles/utils.py @@ -1,5 +1,7 @@ +import math from typing import Optional +import cv2 import numpy as np from pydantic import BaseModel, Field @@ -45,3 +47,101 @@ def paste(dst_image: np.ndarray, src_image: np.ndarray, box: TBLR, mask: Optiona mask = np.expand_dims(mask, -1) dst_image_box = dst_image[box.top : box.bottom, box.left : box.right] dst_image[box.top : box.bottom, box.left : box.right] = src_image * mask + dst_image_box * (1.0 - mask) + + +def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool,) -> np.ndarray: + """Blend two overlapping tile sections using a seams to find a path. + + It is assumed that input images will be RGB np arrays and are the same size. + + Args: + ia1 (torch.Tensor): Image array 1 Shape: (H, W, C). + ia2 (torch.Tensor): Image array 2 Shape: (H, W, C). + x_seam (bool): If the images should be blended on the x axis or not. + blend_amount (int): The size of the blur to use on the seam. Half of this value will be used to avoid the edges of the image. + """ + assert ia1.shape == ia2.shape + assert ia2.size == ia2.size + + def shift(arr, num, fill_value=255.0): + result = np.full_like(arr, fill_value) + if num > 0: + result[num:] = arr[:-num] + elif num < 0: + result[:num] = arr[-num:] + else: + result[:] = arr + return result + + # Assume RGB and convert to grey + iag1 = np.dot(ia1, [0.2989, 0.5870, 0.1140]) + iag2 = np.dot(ia2, [0.2989, 0.5870, 0.1140]) + + # Calc Difference between the images + ia = iag2 - iag1 + + # If the seam is on the X-axis rotate the array so we can treat it like a vertical seam + if x_seam: + ia = np.rot90(ia, 1) + + # Calc max and min X & Y limits + # gutter is used to avoid the blur hitting the edge of the image + gutter = math.ceil(blend_amount / 2) if blend_amount > 0 else 0 + max_y, max_x = ia.shape + max_x -= gutter + min_x = gutter + + # Calc the energy in the difference + energy = np.abs(np.gradient(ia, axis=0)) + np.abs(np.gradient(ia, axis=1)) + + #Find the starting position of the seam + res = np.copy(energy) + for y in range(1, max_y): + row = res[y, :] + rowl = shift(row, -1) + rowr = shift(row, 1) + res[y, :] = res[y - 1, :] + np.min([row, rowl, rowr], axis=0) + + # create an array max_y long + lowest_energy_line = np.empty([max_y], dtype="uint16") + lowest_energy_line[max_y - 1] = np.argmin(res[max_y - 1, min_x : max_x - 1]) + + #Calc the path of the seam + for ypos in range(max_y - 2, -1, -1): + lowest_pos = lowest_energy_line[ypos + 1] + lpos = lowest_pos - 1 + rpos = lowest_pos + 1 + lpos = np.clip(lpos, min_x, max_x - 1) + rpos = np.clip(rpos, min_x, max_x - 1) + lowest_energy_line[ypos] = np.argmin(energy[ypos, lpos : rpos + 1]) + lpos + + # Draw the mask + mask = np.zeros_like(ia) + for ypos in range(0, max_y): + to_fill = lowest_energy_line[ypos] + mask[ypos, :to_fill] = 1 + + # If the seam is on the X-axis rotate the array back + if x_seam: + mask = np.rot90(mask, 3) + + # blur the seam mask if required + if blend_amount > 0: + mask = cv2.blur(mask, (blend_amount, blend_amount)) + + # for visual debugging + #from PIL import Image + #m_image = Image.fromarray((mask * 255.0).astype("uint8")) + + # copy ia2 over ia1 while applying the seam mask + mask = np.expand_dims(mask, -1) + blended_image = ia1 * mask + ia2 * (1.0 - mask) + + # for visual debugging + #i1 = Image.fromarray(ia1.astype("uint8")) + #i2 = Image.fromarray(ia2.astype("uint8")) + #b_image = Image.fromarray(blended_image.astype("uint8")) + #print(f"{ia1.shape}, {ia2.shape}, {mask.shape}, {blended_image.shape}") + #print(f"{i1.size}, {i2.size}, {m_image.size}, {b_image.size}") + + return blended_image From cd15d8b7a9ada59cf46abf011081393be7e0b69b Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Wed, 6 Dec 2023 08:10:22 +0000 Subject: [PATCH 18/33] ruff formatting reformatted due to ruff errors --- invokeai/backend/tiles/tiles.py | 3 ++- invokeai/backend/tiles/utils.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 33c5f5c445..2ae62e241d 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -14,6 +14,7 @@ def calc_overlap(tiles: list[Tile], num_tiles_x, num_tiles_y) -> list[Tile]: num_tiles_x: the number of tiles on the x axis. num_tiles_y: the number of tiles on the y axis. """ + def get_tile_or_none(idx_y: int, idx_x: int) -> Union[Tile, None]: if idx_y < 0 or idx_y > num_tiles_y or idx_x < 0 or idx_x > num_tiles_x: return None @@ -405,4 +406,4 @@ def merge_tiles_with_seam_blending( dst_image[overlap_coords_bottom : first_tile_in_row.coords.bottom, :] = row_image[row_overlap_size:, :] else: # no overlap just paste the row - dst_image[first_tile_in_row.coords.top:first_tile_in_row.coords.bottom, :] = row_image + dst_image[first_tile_in_row.coords.top : first_tile_in_row.coords.bottom, :] = row_image diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py index 9df63a601e..c906983587 100644 --- a/invokeai/backend/tiles/utils.py +++ b/invokeai/backend/tiles/utils.py @@ -49,7 +49,7 @@ def paste(dst_image: np.ndarray, src_image: np.ndarray, box: TBLR, mask: Optiona dst_image[box.top : box.bottom, box.left : box.right] = src_image * mask + dst_image_box * (1.0 - mask) -def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool,) -> np.ndarray: +def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool) -> np.ndarray: """Blend two overlapping tile sections using a seams to find a path. It is assumed that input images will be RGB np arrays and are the same size. @@ -94,7 +94,7 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool # Calc the energy in the difference energy = np.abs(np.gradient(ia, axis=0)) + np.abs(np.gradient(ia, axis=1)) - #Find the starting position of the seam + # Find the starting position of the seam res = np.copy(energy) for y in range(1, max_y): row = res[y, :] @@ -106,7 +106,7 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool lowest_energy_line = np.empty([max_y], dtype="uint16") lowest_energy_line[max_y - 1] = np.argmin(res[max_y - 1, min_x : max_x - 1]) - #Calc the path of the seam + # Calc the path of the seam for ypos in range(max_y - 2, -1, -1): lowest_pos = lowest_energy_line[ypos + 1] lpos = lowest_pos - 1 @@ -130,18 +130,18 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool mask = cv2.blur(mask, (blend_amount, blend_amount)) # for visual debugging - #from PIL import Image - #m_image = Image.fromarray((mask * 255.0).astype("uint8")) + # from PIL import Image + # m_image = Image.fromarray((mask * 255.0).astype("uint8")) # copy ia2 over ia1 while applying the seam mask mask = np.expand_dims(mask, -1) blended_image = ia1 * mask + ia2 * (1.0 - mask) # for visual debugging - #i1 = Image.fromarray(ia1.astype("uint8")) - #i2 = Image.fromarray(ia2.astype("uint8")) - #b_image = Image.fromarray(blended_image.astype("uint8")) - #print(f"{ia1.shape}, {ia2.shape}, {mask.shape}, {blended_image.shape}") - #print(f"{i1.size}, {i2.size}, {m_image.size}, {b_image.size}") + # i1 = Image.fromarray(ia1.astype("uint8")) + # i2 = Image.fromarray(ia2.astype("uint8")) + # b_image = Image.fromarray(blended_image.astype("uint8")) + # print(f"{ia1.shape}, {ia2.shape}, {mask.shape}, {blended_image.shape}") + # print(f"{i1.size}, {i2.size}, {m_image.size}, {b_image.size}") return blended_image From fed2bdafeb6599078cecf200390e2c7cc45654ef Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Fri, 8 Dec 2023 18:16:13 +0000 Subject: [PATCH 19/33] Added Defaults to calc_tiles_min_overlap for overlap and round Added tests for min_overlap and even_split tile gen --- invokeai/backend/tiles/tiles.py | 2 +- tests/backend/tiles/test_tiles.py | 238 +++++++++++++++++++++++++++++- 2 files changed, 238 insertions(+), 2 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 2ae62e241d..9715fc4950 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -159,7 +159,7 @@ def calc_tiles_even_split( def calc_tiles_min_overlap( - image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int, round_to_8: bool + image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int = 0, round_to_8: bool = False ) -> list[Tile]: """Calculate the tile coordinates for a given image shape under a simple tiling scheme with overlaps. diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 353e65d336..a930f2f829 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -1,7 +1,12 @@ import numpy as np import pytest -from invokeai.backend.tiles.tiles import calc_tiles_with_overlap, merge_tiles_with_linear_blending +from invokeai.backend.tiles.tiles import ( + calc_tiles_even_split, + calc_tiles_min_overlap, + calc_tiles_with_overlap, + merge_tiles_with_linear_blending, +) from invokeai.backend.tiles.utils import TBLR, Tile #################################### @@ -85,6 +90,237 @@ def test_calc_tiles_with_overlap_input_validation( calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) +#################################### +# Test calc_tiles_min_overlap(...) +#################################### + + +def test_calc_tiles_min_overlap_single_tile(): + """Test calc_tiles_min_overlap() behavior when a single tile covers the image.""" + tiles = calc_tiles_min_overlap( + image_height=512, image_width=1024, tile_height=512, tile_width=1024, min_overlap=64, round_to_8=False + ) + + expected_tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=1024), overlap=TBLR(top=0, bottom=0, left=0, right=0)) + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_min_overlap_evenly_divisible(): + """Test calc_tiles_min_overlap() behavior when the image is evenly covered by multiple tiles.""" + # Parameters mimic roughly the same output as the original tile generations of the same test name + tiles = calc_tiles_min_overlap( + image_height=576, image_width=1600, tile_height=320, tile_width=576, min_overlap=64, round_to_8=False + ) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=320, left=0, right=576), overlap=TBLR(top=0, bottom=64, left=0, right=64)), + Tile(coords=TBLR(top=0, bottom=320, left=512, right=1088), overlap=TBLR(top=0, bottom=64, left=64, right=64)), + Tile(coords=TBLR(top=0, bottom=320, left=1024, right=1600), overlap=TBLR(top=0, bottom=64, left=64, right=0)), + # Row 1 + Tile(coords=TBLR(top=256, bottom=576, left=0, right=576), overlap=TBLR(top=64, bottom=0, left=0, right=64)), + Tile(coords=TBLR(top=256, bottom=576, left=512, right=1088), overlap=TBLR(top=64, bottom=0, left=64, right=64)), + Tile(coords=TBLR(top=256, bottom=576, left=1024, right=1600), overlap=TBLR(top=64, bottom=0, left=64, right=0)), + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_min_overlap_not_evenly_divisible(): + """Test calc_tiles_min_overlap() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" + # Parameters mimic roughly the same output as the original tile generations of the same test name + tiles = calc_tiles_min_overlap( + image_height=400, image_width=1200, tile_height=256, tile_width=512, min_overlap=64, round_to_8=False + ) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=256, left=0, right=512), overlap=TBLR(top=0, bottom=112, left=0, right=168)), + Tile(coords=TBLR(top=0, bottom=256, left=344, right=856), overlap=TBLR(top=0, bottom=112, left=168, right=168)), + Tile(coords=TBLR(top=0, bottom=256, left=688, right=1200), overlap=TBLR(top=0, bottom=112, left=168, right=0)), + # Row 1 + Tile(coords=TBLR(top=144, bottom=400, left=0, right=512), overlap=TBLR(top=112, bottom=0, left=0, right=168)), + Tile( + coords=TBLR(top=144, bottom=400, left=448, right=960), overlap=TBLR(top=112, bottom=0, left=168, right=168) + ), + Tile( + coords=TBLR(top=144, bottom=400, left=688, right=1200), overlap=TBLR(top=112, bottom=0, left=168, right=0) + ), + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_min_overlap_difficult_size(): + """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" + # Parameters are a difficult size for other tile gen routines to calculate + tiles = calc_tiles_min_overlap( + image_height=1000, image_width=1000, tile_height=256, tile_width=512, min_overlap=64, round_to_8=False + ) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=268, left=0, right=268)), + Tile(coords=TBLR(top=0, bottom=512, left=244, right=756), overlap=TBLR(top=0, bottom=268, left=268, right=268)), + Tile(coords=TBLR(top=0, bottom=512, left=488, right=1000), overlap=TBLR(top=0, bottom=268, left=268, right=0)), + # Row 1 + Tile(coords=TBLR(top=244, bottom=756, left=0, right=512), overlap=TBLR(top=268, bottom=268, left=0, right=0)), + Tile(coords=TBLR(top=244, bottom=756, left=244, right=756),overlap=TBLR(top=268, bottom=268, left=268, right=268)), + Tile(coords=TBLR(top=244, bottom=756, left=488, right=1000), overlap=TBLR(top=268, bottom=268, left=268, right=0)), + # Row 2 + Tile(coords=TBLR(top=488, bottom=1000, left=0, right=512), overlap=TBLR(top=268, bottom=0, left=0, right=268)), + Tile(coords=TBLR(top=488, bottom=1000, left=244, right=756),overlap=TBLR(top=268, bottom=0, left=268, right=268)), + Tile(coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=268, bottom=0, left=268, right=0)), + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_min_overlap_difficult_size_div8(): + """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" + # Parameters are a difficult size for other tile gen routines to calculate + tiles = calc_tiles_min_overlap( + image_height=1000, image_width=1000, tile_height=256, tile_width=512, min_overlap=64, round_to_8=True + ) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=512, left=0, right=560), overlap=TBLR(top=0, bottom=272, left=0, right=272)), + Tile(coords=TBLR(top=0, bottom=512, left=240, right=752), overlap=TBLR(top=0, bottom=272, left=272, right=264)), + Tile(coords=TBLR(top=0, bottom=512, left=488, right=1000), overlap=TBLR(top=0, bottom=272, left=264, right=0)), + # Row 1 + Tile(coords=TBLR(top=240, bottom=752, left=0, right=512), overlap=TBLR(top=272, bottom=264, left=0, right=272)), + Tile(coords=TBLR(top=240, bottom=752, left=240, right=752),overlap=TBLR(top=272, bottom=264, left=272, right=264)), + Tile(coords=TBLR(top=240, bottom=752, left=488, right=1000), overlap=TBLR(top=272, bottom=264, left=264, right=0)), + # Row 2 + Tile(coords=TBLR(top=488, bottom=1000, left=0, right=512), overlap=TBLR(top=264, bottom=0, left=0, right=272)), + Tile(coords=TBLR(top=488, bottom=1000, left=240, right=752),overlap=TBLR(top=264, bottom=0, left=272, right=264)), + Tile(coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=264, bottom=0, left=264, right=0)), + ] + + assert tiles == expected_tiles + + +@pytest.mark.parametrize( + ["image_height", "image_width", "tile_height", "tile_width", "overlap", "raises"], + [ + (128, 128, 128, 128, 127, False), # OK + (128, 128, 128, 128, 0, False), # OK + (128, 128, 64, 64, 0, False), # OK + (128, 128, 129, 128, 0, True), # tile_height exceeds image_height. + (128, 128, 128, 129, 0, True), # tile_width exceeds image_width. + (128, 128, 64, 128, 64, True), # overlap equals tile_height. + (128, 128, 128, 64, 64, True), # overlap equals tile_width. + ], +) +def test_calc_tiles_min_overlap_input_validation( + image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int, round_to_8: bool , raises: bool +): + """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" + if raises: + with pytest.raises(AssertionError): + calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap, round_to_8) + else: + calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap, round_to_8) + +#################################### +# Test calc_tiles_even_split(...) +#################################### + + +def test_calc_tiles_even_split_single_tile(): + """Test calc_tiles_even_split() behavior when a single tile covers the image.""" + tiles = calc_tiles_even_split(image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=0.25) + + expected_tiles = [ + Tile(coords=TBLR(top=0, bottom=512, left=0, right=1024), overlap=TBLR(top=0, bottom=0, left=0, right=0)) + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_even_split_evenly_divisible(): + """Test calc_tiles_even_split() behavior when the image is evenly covered by multiple tiles.""" + # Parameters mimic roughly the same output as the original tile generations of the same test name + tiles = calc_tiles_even_split(image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=0.25) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=320, left=0, right=624), overlap=TBLR(top=0, bottom=72, left=0, right=136)), + Tile(coords=TBLR(top=0, bottom=320, left=488, right=1112), overlap=TBLR(top=0, bottom=72, left=136, right=136)), + Tile(coords=TBLR(top=0, bottom=320, left=976, right=1600), overlap=TBLR(top=0, bottom=72, left=136, right=0)), + # Row 1 + Tile(coords=TBLR(top=248, bottom=576, left=0, right=624), overlap=TBLR(top=72, bottom=0, left=0, right=136)), + Tile( + coords=TBLR(top=248, bottom=576, left=488, right=1112), overlap=TBLR(top=72, bottom=0, left=136, right=136) + ), + Tile(coords=TBLR(top=248, bottom=576, left=976, right=1600), overlap=TBLR(top=72, bottom=0, left=136, right=0)), + ] + assert tiles == expected_tiles + + +def test_calc_tiles_even_split_not_evenly_divisible(): + """Test calc_tiles_even_split() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" + # Parameters mimic roughly the same output as the original tile generations of the same test name + tiles = calc_tiles_even_split(image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=0.25) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=224, left=0, right=464), overlap=TBLR(top=0, bottom=56, left=0, right=104)), + Tile(coords=TBLR(top=0, bottom=224, left=360, right=824), overlap=TBLR(top=0, bottom=56, left=104, right=104)), + Tile(coords=TBLR(top=0, bottom=224, left=720, right=1200), overlap=TBLR(top=0, bottom=56, left=104, right=0)), + # Row 1 + Tile(coords=TBLR(top=168, bottom=400, left=0, right=464), overlap=TBLR(top=56, bottom=0, left=0, right=104)), + Tile( + coords=TBLR(top=168, bottom=400, left=360, right=824), overlap=TBLR(top=56, bottom=0, left=104, right=104) + ), + Tile(coords=TBLR(top=168, bottom=400, left=720, right=1200), overlap=TBLR(top=56, bottom=0, left=104, right=0)), + ] + + assert tiles == expected_tiles + + +def test_calc_tiles_even_split_difficult_size(): + """Test calc_tiles_even_split() behavior when the image is a difficult size to spilt evenly and keep div8.""" + # Parameters are a difficult size for other tile gen routines to calculate + tiles = calc_tiles_even_split(image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=0.25) + + expected_tiles = [ + # Row 0 + Tile(coords=TBLR(top=0, bottom=560, left=0, right=560), overlap=TBLR(top=0, bottom=128, left=0, right=128)), + Tile(coords=TBLR(top=0, bottom=560, left=432, right=1000), overlap=TBLR(top=0, bottom=128, left=128, right=0)), + # Row 1 + Tile(coords=TBLR(top=432, bottom=1000, left=0, right=560), overlap=TBLR(top=128, bottom=0, left=0, right=128)), + Tile( + coords=TBLR(top=432, bottom=1000, left=432, right=1000), overlap=TBLR(top=128, bottom=0, left=128, right=0) + ), + ] + + assert tiles == expected_tiles + +@pytest.mark.parametrize( + ["image_height", "image_width", "num_tiles_x", "num_tiles_y", "overlap", "raises"], + [ + (128, 128, 1, 1, 0.25, False), # OK + (128, 128, 1, 1, 0, False), # OK + (128, 128, 2, 1, 0, False), # OK + (127, 127, 1, 1, 0, True), # image size must be drivable by 8 + ], +) +def test_calc_tiles_even_split_input_validation( + image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: float, raises: bool +): + """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" + if raises: + with pytest.raises(AssertionError): + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) + else: + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) + + ############################################# # Test merge_tiles_with_linear_blending(...) ############################################# From 8cda42ab0a8fa0cf20c5e2cdd5fbb919726e0d91 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Fri, 8 Dec 2023 18:17:40 +0000 Subject: [PATCH 20/33] ruff formatting --- tests/backend/tiles/test_tiles.py | 44 ++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index a930f2f829..0f8998f7ed 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -168,12 +168,21 @@ def test_calc_tiles_min_overlap_difficult_size(): Tile(coords=TBLR(top=0, bottom=512, left=488, right=1000), overlap=TBLR(top=0, bottom=268, left=268, right=0)), # Row 1 Tile(coords=TBLR(top=244, bottom=756, left=0, right=512), overlap=TBLR(top=268, bottom=268, left=0, right=0)), - Tile(coords=TBLR(top=244, bottom=756, left=244, right=756),overlap=TBLR(top=268, bottom=268, left=268, right=268)), - Tile(coords=TBLR(top=244, bottom=756, left=488, right=1000), overlap=TBLR(top=268, bottom=268, left=268, right=0)), + Tile( + coords=TBLR(top=244, bottom=756, left=244, right=756), + overlap=TBLR(top=268, bottom=268, left=268, right=268), + ), + Tile( + coords=TBLR(top=244, bottom=756, left=488, right=1000), overlap=TBLR(top=268, bottom=268, left=268, right=0) + ), # Row 2 Tile(coords=TBLR(top=488, bottom=1000, left=0, right=512), overlap=TBLR(top=268, bottom=0, left=0, right=268)), - Tile(coords=TBLR(top=488, bottom=1000, left=244, right=756),overlap=TBLR(top=268, bottom=0, left=268, right=268)), - Tile(coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=268, bottom=0, left=268, right=0)), + Tile( + coords=TBLR(top=488, bottom=1000, left=244, right=756), overlap=TBLR(top=268, bottom=0, left=268, right=268) + ), + Tile( + coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=268, bottom=0, left=268, right=0) + ), ] assert tiles == expected_tiles @@ -193,12 +202,21 @@ def test_calc_tiles_min_overlap_difficult_size_div8(): Tile(coords=TBLR(top=0, bottom=512, left=488, right=1000), overlap=TBLR(top=0, bottom=272, left=264, right=0)), # Row 1 Tile(coords=TBLR(top=240, bottom=752, left=0, right=512), overlap=TBLR(top=272, bottom=264, left=0, right=272)), - Tile(coords=TBLR(top=240, bottom=752, left=240, right=752),overlap=TBLR(top=272, bottom=264, left=272, right=264)), - Tile(coords=TBLR(top=240, bottom=752, left=488, right=1000), overlap=TBLR(top=272, bottom=264, left=264, right=0)), + Tile( + coords=TBLR(top=240, bottom=752, left=240, right=752), + overlap=TBLR(top=272, bottom=264, left=272, right=264), + ), + Tile( + coords=TBLR(top=240, bottom=752, left=488, right=1000), overlap=TBLR(top=272, bottom=264, left=264, right=0) + ), # Row 2 Tile(coords=TBLR(top=488, bottom=1000, left=0, right=512), overlap=TBLR(top=264, bottom=0, left=0, right=272)), - Tile(coords=TBLR(top=488, bottom=1000, left=240, right=752),overlap=TBLR(top=264, bottom=0, left=272, right=264)), - Tile(coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=264, bottom=0, left=264, right=0)), + Tile( + coords=TBLR(top=488, bottom=1000, left=240, right=752), overlap=TBLR(top=264, bottom=0, left=272, right=264) + ), + Tile( + coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=264, bottom=0, left=264, right=0) + ), ] assert tiles == expected_tiles @@ -217,7 +235,13 @@ def test_calc_tiles_min_overlap_difficult_size_div8(): ], ) def test_calc_tiles_min_overlap_input_validation( - image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int, round_to_8: bool , raises: bool + image_height: int, + image_width: int, + tile_height: int, + tile_width: int, + min_overlap: int, + round_to_8: bool, + raises: bool, ): """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" if raises: @@ -226,6 +250,7 @@ def test_calc_tiles_min_overlap_input_validation( else: calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap, round_to_8) + #################################### # Test calc_tiles_even_split(...) #################################### @@ -301,6 +326,7 @@ def test_calc_tiles_even_split_difficult_size(): assert tiles == expected_tiles + @pytest.mark.parametrize( ["image_height", "image_width", "num_tiles_x", "num_tiles_y", "overlap", "raises"], [ From d3ad356c6ac3a15b4314f8f8a684f304f26b8dc0 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Fri, 8 Dec 2023 18:31:33 +0000 Subject: [PATCH 21/33] Ruff Formatting Fix pyTest issues --- invokeai/backend/tiles/tiles.py | 7 ++++++- tests/backend/tiles/test_tiles.py | 13 ++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 9715fc4950..166e45d57c 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -159,7 +159,12 @@ def calc_tiles_even_split( def calc_tiles_min_overlap( - image_height: int, image_width: int, tile_height: int, tile_width: int, min_overlap: int = 0, round_to_8: bool = False + image_height: int, + image_width: int, + tile_height: int, + tile_width: int, + min_overlap: int = 0, + round_to_8: bool = False, ) -> list[Tile]: """Calculate the tile coordinates for a given image shape under a simple tiling scheme with overlaps. diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 0f8998f7ed..5757105982 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -223,7 +223,7 @@ def test_calc_tiles_min_overlap_difficult_size_div8(): @pytest.mark.parametrize( - ["image_height", "image_width", "tile_height", "tile_width", "overlap", "raises"], + ["image_height", "image_width", "tile_height", "tile_width", "min_overlap", "raises"], [ (128, 128, 128, 128, 127, False), # OK (128, 128, 128, 128, 0, False), # OK @@ -240,15 +240,14 @@ def test_calc_tiles_min_overlap_input_validation( tile_height: int, tile_width: int, min_overlap: int, - round_to_8: bool, raises: bool, ): - """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" + """Test that calc_tiles_min_overlap() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(AssertionError): - calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap, round_to_8) + calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap) else: - calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap, round_to_8) + calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap) #################################### @@ -333,13 +332,13 @@ def test_calc_tiles_even_split_difficult_size(): (128, 128, 1, 1, 0.25, False), # OK (128, 128, 1, 1, 0, False), # OK (128, 128, 2, 1, 0, False), # OK - (127, 127, 1, 1, 0, True), # image size must be drivable by 8 + (127, 127, 1, 1, 0, True), # image size must be dividable by 8 ], ) def test_calc_tiles_even_split_input_validation( image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: float, raises: bool ): - """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" + """Test that calc_tiles_even_split() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(AssertionError): calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) From b7ba426249252e0e1197387459b63b47e6a85540 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Fri, 8 Dec 2023 18:53:28 +0000 Subject: [PATCH 22/33] Fixed some params on tile gen tests on tests --- tests/backend/tiles/test_tiles.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 5757105982..519ca8586e 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -133,7 +133,7 @@ def test_calc_tiles_min_overlap_not_evenly_divisible(): """Test calc_tiles_min_overlap() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters mimic roughly the same output as the original tile generations of the same test name tiles = calc_tiles_min_overlap( - image_height=400, image_width=1200, tile_height=256, tile_width=512, min_overlap=64, round_to_8=False + image_height=400, image_width=1200, tile_height=512, tile_width=512, min_overlap=64, round_to_8=False ) expected_tiles = [ @@ -144,7 +144,7 @@ def test_calc_tiles_min_overlap_not_evenly_divisible(): # Row 1 Tile(coords=TBLR(top=144, bottom=400, left=0, right=512), overlap=TBLR(top=112, bottom=0, left=0, right=168)), Tile( - coords=TBLR(top=144, bottom=400, left=448, right=960), overlap=TBLR(top=112, bottom=0, left=168, right=168) + coords=TBLR(top=144, bottom=400, left=344, right=856), overlap=TBLR(top=112, bottom=0, left=168, right=168) ), Tile( coords=TBLR(top=144, bottom=400, left=688, right=1200), overlap=TBLR(top=112, bottom=0, left=168, right=0) @@ -158,7 +158,7 @@ def test_calc_tiles_min_overlap_difficult_size(): """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate tiles = calc_tiles_min_overlap( - image_height=1000, image_width=1000, tile_height=256, tile_width=512, min_overlap=64, round_to_8=False + image_height=1000, image_width=1000, tile_height=512, tile_width=512, min_overlap=64, round_to_8=False ) expected_tiles = [ @@ -340,7 +340,7 @@ def test_calc_tiles_even_split_input_validation( ): """Test that calc_tiles_even_split() raises an exception if the inputs are invalid.""" if raises: - with pytest.raises(AssertionError): + with pytest.raises(ValueError): calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) else: calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) From 375a91db3257474499f915c1a6bbea8a52c13899 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Fri, 8 Dec 2023 19:38:16 +0000 Subject: [PATCH 23/33] further updated tests --- tests/backend/tiles/test_tiles.py | 485 +++++++++++++++++++++++------- 1 file changed, 380 insertions(+), 105 deletions(-) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 519ca8586e..850d6ca6ca 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -16,10 +16,15 @@ from invokeai.backend.tiles.utils import TBLR, Tile def test_calc_tiles_with_overlap_single_tile(): """Test calc_tiles_with_overlap() behavior when a single tile covers the image.""" - tiles = calc_tiles_with_overlap(image_height=512, image_width=1024, tile_height=512, tile_width=1024, overlap=64) + tiles = calc_tiles_with_overlap( + image_height=512, image_width=1024, tile_height=512, tile_width=1024, overlap=64 + ) expected_tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=1024), overlap=TBLR(top=0, bottom=0, left=0, right=0)) + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=1024), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) ] assert tiles == expected_tiles @@ -28,17 +33,37 @@ def test_calc_tiles_with_overlap_single_tile(): def test_calc_tiles_with_overlap_evenly_divisible(): """Test calc_tiles_with_overlap() behavior when the image is evenly covered by multiple tiles.""" # Parameters chosen so that image is evenly covered by 2 rows, 3 columns of tiles. - tiles = calc_tiles_with_overlap(image_height=576, image_width=1600, tile_height=320, tile_width=576, overlap=64) + tiles = calc_tiles_with_overlap( + image_height=576, image_width=1600, tile_height=320, tile_width=576, overlap=64 + ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=320, left=0, right=576), overlap=TBLR(top=0, bottom=64, left=0, right=64)), - Tile(coords=TBLR(top=0, bottom=320, left=512, right=1088), overlap=TBLR(top=0, bottom=64, left=64, right=64)), - Tile(coords=TBLR(top=0, bottom=320, left=1024, right=1600), overlap=TBLR(top=0, bottom=64, left=64, right=0)), + Tile( + coords=TBLR(top=0, bottom=320, left=0, right=576), + overlap=TBLR(top=0, bottom=64, left=0, right=64), + ), + Tile( + coords=TBLR(top=0, bottom=320, left=512, right=1088), + overlap=TBLR(top=0, bottom=64, left=64, right=64), + ), + Tile( + coords=TBLR(top=0, bottom=320, left=1024, right=1600), + overlap=TBLR(top=0, bottom=64, left=64, right=0), + ), # Row 1 - Tile(coords=TBLR(top=256, bottom=576, left=0, right=576), overlap=TBLR(top=64, bottom=0, left=0, right=64)), - Tile(coords=TBLR(top=256, bottom=576, left=512, right=1088), overlap=TBLR(top=64, bottom=0, left=64, right=64)), - Tile(coords=TBLR(top=256, bottom=576, left=1024, right=1600), overlap=TBLR(top=64, bottom=0, left=64, right=0)), + Tile( + coords=TBLR(top=256, bottom=576, left=0, right=576), + overlap=TBLR(top=64, bottom=0, left=0, right=64), + ), + Tile( + coords=TBLR(top=256, bottom=576, left=512, right=1088), + overlap=TBLR(top=64, bottom=0, left=64, right=64), + ), + Tile( + coords=TBLR(top=256, bottom=576, left=1024, right=1600), + overlap=TBLR(top=64, bottom=0, left=64, right=0), + ), ] assert tiles == expected_tiles @@ -47,20 +72,36 @@ def test_calc_tiles_with_overlap_evenly_divisible(): def test_calc_tiles_with_overlap_not_evenly_divisible(): """Test calc_tiles_with_overlap() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters chosen so that image is covered by 2 rows and 3 columns of tiles, with uneven overlaps. - tiles = calc_tiles_with_overlap(image_height=400, image_width=1200, tile_height=256, tile_width=512, overlap=64) + tiles = calc_tiles_with_overlap( + image_height=400, image_width=1200, tile_height=256, tile_width=512, overlap=64 + ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=256, left=0, right=512), overlap=TBLR(top=0, bottom=112, left=0, right=64)), - Tile(coords=TBLR(top=0, bottom=256, left=448, right=960), overlap=TBLR(top=0, bottom=112, left=64, right=272)), - Tile(coords=TBLR(top=0, bottom=256, left=688, right=1200), overlap=TBLR(top=0, bottom=112, left=272, right=0)), - # Row 1 - Tile(coords=TBLR(top=144, bottom=400, left=0, right=512), overlap=TBLR(top=112, bottom=0, left=0, right=64)), Tile( - coords=TBLR(top=144, bottom=400, left=448, right=960), overlap=TBLR(top=112, bottom=0, left=64, right=272) + coords=TBLR(top=0, bottom=256, left=0, right=512), + overlap=TBLR(top=0, bottom=112, left=0, right=64), ), Tile( - coords=TBLR(top=144, bottom=400, left=688, right=1200), overlap=TBLR(top=112, bottom=0, left=272, right=0) + coords=TBLR(top=0, bottom=256, left=448, right=960), + overlap=TBLR(top=0, bottom=112, left=64, right=272), + ), + Tile( + coords=TBLR(top=0, bottom=256, left=688, right=1200), + overlap=TBLR(top=0, bottom=112, left=272, right=0), + ), + # Row 1 + Tile( + coords=TBLR(top=144, bottom=400, left=0, right=512), + overlap=TBLR(top=112, bottom=0, left=0, right=64), + ), + Tile( + coords=TBLR(top=144, bottom=400, left=448, right=960), + overlap=TBLR(top=112, bottom=0, left=64, right=272), + ), + Tile( + coords=TBLR(top=144, bottom=400, left=688, right=1200), + overlap=TBLR(top=112, bottom=0, left=272, right=0), ), ] @@ -80,14 +121,23 @@ def test_calc_tiles_with_overlap_not_evenly_divisible(): ], ) def test_calc_tiles_with_overlap_input_validation( - image_height: int, image_width: int, tile_height: int, tile_width: int, overlap: int, raises: bool + image_height: int, + image_width: int, + tile_height: int, + tile_width: int, + overlap: int, + raises: bool, ): """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(AssertionError): - calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) + calc_tiles_with_overlap( + image_height, image_width, tile_height, tile_width, overlap + ) else: - calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) + calc_tiles_with_overlap( + image_height, image_width, tile_height, tile_width, overlap + ) #################################### @@ -98,11 +148,19 @@ def test_calc_tiles_with_overlap_input_validation( def test_calc_tiles_min_overlap_single_tile(): """Test calc_tiles_min_overlap() behavior when a single tile covers the image.""" tiles = calc_tiles_min_overlap( - image_height=512, image_width=1024, tile_height=512, tile_width=1024, min_overlap=64, round_to_8=False + image_height=512, + image_width=1024, + tile_height=512, + tile_width=1024, + min_overlap=64, + round_to_8=False, ) expected_tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=1024), overlap=TBLR(top=0, bottom=0, left=0, right=0)) + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=1024), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) ] assert tiles == expected_tiles @@ -112,18 +170,41 @@ def test_calc_tiles_min_overlap_evenly_divisible(): """Test calc_tiles_min_overlap() behavior when the image is evenly covered by multiple tiles.""" # Parameters mimic roughly the same output as the original tile generations of the same test name tiles = calc_tiles_min_overlap( - image_height=576, image_width=1600, tile_height=320, tile_width=576, min_overlap=64, round_to_8=False + image_height=576, + image_width=1600, + tile_height=320, + tile_width=576, + min_overlap=64, + round_to_8=False, ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=320, left=0, right=576), overlap=TBLR(top=0, bottom=64, left=0, right=64)), - Tile(coords=TBLR(top=0, bottom=320, left=512, right=1088), overlap=TBLR(top=0, bottom=64, left=64, right=64)), - Tile(coords=TBLR(top=0, bottom=320, left=1024, right=1600), overlap=TBLR(top=0, bottom=64, left=64, right=0)), + Tile( + coords=TBLR(top=0, bottom=320, left=0, right=576), + overlap=TBLR(top=0, bottom=64, left=0, right=64), + ), + Tile( + coords=TBLR(top=0, bottom=320, left=512, right=1088), + overlap=TBLR(top=0, bottom=64, left=64, right=64), + ), + Tile( + coords=TBLR(top=0, bottom=320, left=1024, right=1600), + overlap=TBLR(top=0, bottom=64, left=64, right=0), + ), # Row 1 - Tile(coords=TBLR(top=256, bottom=576, left=0, right=576), overlap=TBLR(top=64, bottom=0, left=0, right=64)), - Tile(coords=TBLR(top=256, bottom=576, left=512, right=1088), overlap=TBLR(top=64, bottom=0, left=64, right=64)), - Tile(coords=TBLR(top=256, bottom=576, left=1024, right=1600), overlap=TBLR(top=64, bottom=0, left=64, right=0)), + Tile( + coords=TBLR(top=256, bottom=576, left=0, right=576), + overlap=TBLR(top=64, bottom=0, left=0, right=64), + ), + Tile( + coords=TBLR(top=256, bottom=576, left=512, right=1088), + overlap=TBLR(top=64, bottom=0, left=64, right=64), + ), + Tile( + coords=TBLR(top=256, bottom=576, left=1024, right=1600), + overlap=TBLR(top=64, bottom=0, left=64, right=0), + ), ] assert tiles == expected_tiles @@ -133,21 +214,40 @@ def test_calc_tiles_min_overlap_not_evenly_divisible(): """Test calc_tiles_min_overlap() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters mimic roughly the same output as the original tile generations of the same test name tiles = calc_tiles_min_overlap( - image_height=400, image_width=1200, tile_height=512, tile_width=512, min_overlap=64, round_to_8=False + image_height=400, + image_width=1200, + tile_height=256, + tile_width=512, + min_overlap=64, + round_to_8=False, ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=256, left=0, right=512), overlap=TBLR(top=0, bottom=112, left=0, right=168)), - Tile(coords=TBLR(top=0, bottom=256, left=344, right=856), overlap=TBLR(top=0, bottom=112, left=168, right=168)), - Tile(coords=TBLR(top=0, bottom=256, left=688, right=1200), overlap=TBLR(top=0, bottom=112, left=168, right=0)), - # Row 1 - Tile(coords=TBLR(top=144, bottom=400, left=0, right=512), overlap=TBLR(top=112, bottom=0, left=0, right=168)), Tile( - coords=TBLR(top=144, bottom=400, left=344, right=856), overlap=TBLR(top=112, bottom=0, left=168, right=168) + coords=TBLR(top=0, bottom=256, left=0, right=512), + overlap=TBLR(top=0, bottom=112, left=0, right=168), ), Tile( - coords=TBLR(top=144, bottom=400, left=688, right=1200), overlap=TBLR(top=112, bottom=0, left=168, right=0) + coords=TBLR(top=0, bottom=256, left=344, right=856), + overlap=TBLR(top=0, bottom=112, left=168, right=168), + ), + Tile( + coords=TBLR(top=0, bottom=256, left=688, right=1200), + overlap=TBLR(top=0, bottom=112, left=168, right=0), + ), + # Row 1 + Tile( + coords=TBLR(top=144, bottom=400, left=0, right=512), + overlap=TBLR(top=112, bottom=0, left=0, right=168), + ), + Tile( + coords=TBLR(top=144, bottom=400, left=344, right=856), + overlap=TBLR(top=112, bottom=0, left=168, right=168), + ), + Tile( + coords=TBLR(top=144, bottom=400, left=688, right=1200), + overlap=TBLR(top=112, bottom=0, left=168, right=0), ), ] @@ -158,30 +258,53 @@ def test_calc_tiles_min_overlap_difficult_size(): """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate tiles = calc_tiles_min_overlap( - image_height=1000, image_width=1000, tile_height=512, tile_width=512, min_overlap=64, round_to_8=False + image_height=1000, + image_width=1000, + tile_height=512, + tile_width=512, + min_overlap=64, + round_to_8=False, ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=268, left=0, right=268)), - Tile(coords=TBLR(top=0, bottom=512, left=244, right=756), overlap=TBLR(top=0, bottom=268, left=268, right=268)), - Tile(coords=TBLR(top=0, bottom=512, left=488, right=1000), overlap=TBLR(top=0, bottom=268, left=268, right=0)), + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=268, left=0, right=268), + ), + Tile( + coords=TBLR(top=0, bottom=512, left=244, right=756), + overlap=TBLR(top=0, bottom=268, left=268, right=268), + ), + Tile( + coords=TBLR(top=0, bottom=512, left=488, right=1000), + overlap=TBLR(top=0, bottom=268, left=268, right=0), + ), # Row 1 - Tile(coords=TBLR(top=244, bottom=756, left=0, right=512), overlap=TBLR(top=268, bottom=268, left=0, right=0)), + Tile( + coords=TBLR(top=244, bottom=756, left=0, right=512), + overlap=TBLR(top=268, bottom=268, left=0, right=268), + ), Tile( coords=TBLR(top=244, bottom=756, left=244, right=756), overlap=TBLR(top=268, bottom=268, left=268, right=268), ), Tile( - coords=TBLR(top=244, bottom=756, left=488, right=1000), overlap=TBLR(top=268, bottom=268, left=268, right=0) + coords=TBLR(top=244, bottom=756, left=488, right=1000), + overlap=TBLR(top=268, bottom=268, left=268, right=0), ), # Row 2 - Tile(coords=TBLR(top=488, bottom=1000, left=0, right=512), overlap=TBLR(top=268, bottom=0, left=0, right=268)), Tile( - coords=TBLR(top=488, bottom=1000, left=244, right=756), overlap=TBLR(top=268, bottom=0, left=268, right=268) + coords=TBLR(top=488, bottom=1000, left=0, right=512), + overlap=TBLR(top=268, bottom=0, left=0, right=268), ), Tile( - coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=268, bottom=0, left=268, right=0) + coords=TBLR(top=488, bottom=1000, left=244, right=756), + overlap=TBLR(top=268, bottom=0, left=268, right=268), + ), + Tile( + coords=TBLR(top=488, bottom=1000, left=488, right=1000), + overlap=TBLR(top=268, bottom=0, left=268, right=0), ), ] @@ -192,30 +315,53 @@ def test_calc_tiles_min_overlap_difficult_size_div8(): """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate tiles = calc_tiles_min_overlap( - image_height=1000, image_width=1000, tile_height=256, tile_width=512, min_overlap=64, round_to_8=True + image_height=1000, + image_width=1000, + tile_height=512, + tile_width=512, + min_overlap=64, + round_to_8=True, ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=512, left=0, right=560), overlap=TBLR(top=0, bottom=272, left=0, right=272)), - Tile(coords=TBLR(top=0, bottom=512, left=240, right=752), overlap=TBLR(top=0, bottom=272, left=272, right=264)), - Tile(coords=TBLR(top=0, bottom=512, left=488, right=1000), overlap=TBLR(top=0, bottom=272, left=264, right=0)), + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=272, left=0, right=272), + ), + Tile( + coords=TBLR(top=0, bottom=512, left=240, right=752), + overlap=TBLR(top=0, bottom=272, left=272, right=264), + ), + Tile( + coords=TBLR(top=0, bottom=512, left=488, right=1000), + overlap=TBLR(top=0, bottom=272, left=264, right=0), + ), # Row 1 - Tile(coords=TBLR(top=240, bottom=752, left=0, right=512), overlap=TBLR(top=272, bottom=264, left=0, right=272)), + Tile( + coords=TBLR(top=240, bottom=752, left=0, right=512), + overlap=TBLR(top=272, bottom=264, left=0, right=272), + ), Tile( coords=TBLR(top=240, bottom=752, left=240, right=752), overlap=TBLR(top=272, bottom=264, left=272, right=264), ), Tile( - coords=TBLR(top=240, bottom=752, left=488, right=1000), overlap=TBLR(top=272, bottom=264, left=264, right=0) + coords=TBLR(top=240, bottom=752, left=488, right=1000), + overlap=TBLR(top=272, bottom=264, left=264, right=0), ), # Row 2 - Tile(coords=TBLR(top=488, bottom=1000, left=0, right=512), overlap=TBLR(top=264, bottom=0, left=0, right=272)), Tile( - coords=TBLR(top=488, bottom=1000, left=240, right=752), overlap=TBLR(top=264, bottom=0, left=272, right=264) + coords=TBLR(top=488, bottom=1000, left=0, right=512), + overlap=TBLR(top=264, bottom=0, left=0, right=272), ), Tile( - coords=TBLR(top=488, bottom=1000, left=488, right=1000), overlap=TBLR(top=264, bottom=0, left=264, right=0) + coords=TBLR(top=488, bottom=1000, left=240, right=752), + overlap=TBLR(top=264, bottom=0, left=272, right=264), + ), + Tile( + coords=TBLR(top=488, bottom=1000, left=488, right=1000), + overlap=TBLR(top=264, bottom=0, left=264, right=0), ), ] @@ -223,7 +369,14 @@ def test_calc_tiles_min_overlap_difficult_size_div8(): @pytest.mark.parametrize( - ["image_height", "image_width", "tile_height", "tile_width", "min_overlap", "raises"], + [ + "image_height", + "image_width", + "tile_height", + "tile_width", + "min_overlap", + "raises", + ], [ (128, 128, 128, 128, 127, False), # OK (128, 128, 128, 128, 0, False), # OK @@ -245,9 +398,13 @@ def test_calc_tiles_min_overlap_input_validation( """Test that calc_tiles_min_overlap() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(AssertionError): - calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap) + calc_tiles_min_overlap( + image_height, image_width, tile_height, tile_width, min_overlap + ) else: - calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap) + calc_tiles_min_overlap( + image_height, image_width, tile_height, tile_width, min_overlap + ) #################################### @@ -257,10 +414,15 @@ def test_calc_tiles_min_overlap_input_validation( def test_calc_tiles_even_split_single_tile(): """Test calc_tiles_even_split() behavior when a single tile covers the image.""" - tiles = calc_tiles_even_split(image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=0.25 + ) expected_tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=1024), overlap=TBLR(top=0, bottom=0, left=0, right=0)) + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=1024), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) ] assert tiles == expected_tiles @@ -269,19 +431,37 @@ def test_calc_tiles_even_split_single_tile(): def test_calc_tiles_even_split_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image is evenly covered by multiple tiles.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split(image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=0.25 + ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=320, left=0, right=624), overlap=TBLR(top=0, bottom=72, left=0, right=136)), - Tile(coords=TBLR(top=0, bottom=320, left=488, right=1112), overlap=TBLR(top=0, bottom=72, left=136, right=136)), - Tile(coords=TBLR(top=0, bottom=320, left=976, right=1600), overlap=TBLR(top=0, bottom=72, left=136, right=0)), - # Row 1 - Tile(coords=TBLR(top=248, bottom=576, left=0, right=624), overlap=TBLR(top=72, bottom=0, left=0, right=136)), Tile( - coords=TBLR(top=248, bottom=576, left=488, right=1112), overlap=TBLR(top=72, bottom=0, left=136, right=136) + coords=TBLR(top=0, bottom=320, left=0, right=624), + overlap=TBLR(top=0, bottom=72, left=0, right=136), + ), + Tile( + coords=TBLR(top=0, bottom=320, left=488, right=1112), + overlap=TBLR(top=0, bottom=72, left=136, right=136), + ), + Tile( + coords=TBLR(top=0, bottom=320, left=976, right=1600), + overlap=TBLR(top=0, bottom=72, left=136, right=0), + ), + # Row 1 + Tile( + coords=TBLR(top=248, bottom=576, left=0, right=624), + overlap=TBLR(top=72, bottom=0, left=0, right=136), + ), + Tile( + coords=TBLR(top=248, bottom=576, left=488, right=1112), + overlap=TBLR(top=72, bottom=0, left=136, right=136), + ), + Tile( + coords=TBLR(top=248, bottom=576, left=976, right=1600), + overlap=TBLR(top=72, bottom=0, left=136, right=0), ), - Tile(coords=TBLR(top=248, bottom=576, left=976, right=1600), overlap=TBLR(top=72, bottom=0, left=136, right=0)), ] assert tiles == expected_tiles @@ -289,19 +469,37 @@ def test_calc_tiles_even_split_evenly_divisible(): def test_calc_tiles_even_split_not_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split(image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=0.25 + ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=224, left=0, right=464), overlap=TBLR(top=0, bottom=56, left=0, right=104)), - Tile(coords=TBLR(top=0, bottom=224, left=360, right=824), overlap=TBLR(top=0, bottom=56, left=104, right=104)), - Tile(coords=TBLR(top=0, bottom=224, left=720, right=1200), overlap=TBLR(top=0, bottom=56, left=104, right=0)), - # Row 1 - Tile(coords=TBLR(top=168, bottom=400, left=0, right=464), overlap=TBLR(top=56, bottom=0, left=0, right=104)), Tile( - coords=TBLR(top=168, bottom=400, left=360, right=824), overlap=TBLR(top=56, bottom=0, left=104, right=104) + coords=TBLR(top=0, bottom=224, left=0, right=464), + overlap=TBLR(top=0, bottom=56, left=0, right=104), + ), + Tile( + coords=TBLR(top=0, bottom=224, left=360, right=824), + overlap=TBLR(top=0, bottom=56, left=104, right=104), + ), + Tile( + coords=TBLR(top=0, bottom=224, left=720, right=1200), + overlap=TBLR(top=0, bottom=56, left=104, right=0), + ), + # Row 1 + Tile( + coords=TBLR(top=168, bottom=400, left=0, right=464), + overlap=TBLR(top=56, bottom=0, left=0, right=104), + ), + Tile( + coords=TBLR(top=168, bottom=400, left=360, right=824), + overlap=TBLR(top=56, bottom=0, left=104, right=104), + ), + Tile( + coords=TBLR(top=168, bottom=400, left=720, right=1200), + overlap=TBLR(top=56, bottom=0, left=104, right=0), ), - Tile(coords=TBLR(top=168, bottom=400, left=720, right=1200), overlap=TBLR(top=56, bottom=0, left=104, right=0)), ] assert tiles == expected_tiles @@ -310,16 +508,28 @@ def test_calc_tiles_even_split_not_evenly_divisible(): def test_calc_tiles_even_split_difficult_size(): """Test calc_tiles_even_split() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate - tiles = calc_tiles_even_split(image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=0.25 + ) expected_tiles = [ # Row 0 - Tile(coords=TBLR(top=0, bottom=560, left=0, right=560), overlap=TBLR(top=0, bottom=128, left=0, right=128)), - Tile(coords=TBLR(top=0, bottom=560, left=432, right=1000), overlap=TBLR(top=0, bottom=128, left=128, right=0)), - # Row 1 - Tile(coords=TBLR(top=432, bottom=1000, left=0, right=560), overlap=TBLR(top=128, bottom=0, left=0, right=128)), Tile( - coords=TBLR(top=432, bottom=1000, left=432, right=1000), overlap=TBLR(top=128, bottom=0, left=128, right=0) + coords=TBLR(top=0, bottom=560, left=0, right=560), + overlap=TBLR(top=0, bottom=128, left=0, right=128), + ), + Tile( + coords=TBLR(top=0, bottom=560, left=432, right=1000), + overlap=TBLR(top=0, bottom=128, left=128, right=0), + ), + # Row 1 + Tile( + coords=TBLR(top=432, bottom=1000, left=0, right=560), + overlap=TBLR(top=128, bottom=0, left=0, right=128), + ), + Tile( + coords=TBLR(top=432, bottom=1000, left=432, right=1000), + overlap=TBLR(top=128, bottom=0, left=128, right=0), ), ] @@ -336,14 +546,23 @@ def test_calc_tiles_even_split_difficult_size(): ], ) def test_calc_tiles_even_split_input_validation( - image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: float, raises: bool + image_height: int, + image_width: int, + num_tiles_x: int, + num_tiles_y: int, + overlap: float, + raises: bool, ): """Test that calc_tiles_even_split() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(ValueError): - calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) + calc_tiles_even_split( + image_height, image_width, num_tiles_x, num_tiles_y, overlap + ) else: - calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) + calc_tiles_even_split( + image_height, image_width, num_tiles_x, num_tiles_y, overlap + ) ############################################# @@ -356,8 +575,14 @@ def test_merge_tiles_with_linear_blending_horizontal(blend_amount: int): """Test merge_tiles_with_linear_blending(...) behavior when merging horizontally.""" # Initialize 2 tiles side-by-side. tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=64)), - Tile(coords=TBLR(top=0, bottom=512, left=448, right=960), overlap=TBLR(top=0, bottom=0, left=64, right=0)), + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=0, left=0, right=64), + ), + Tile( + coords=TBLR(top=0, bottom=512, left=448, right=960), + overlap=TBLR(top=0, bottom=0, left=64, right=0), + ), ] dst_image = np.zeros((512, 960, 3), dtype=np.uint8) @@ -372,12 +597,19 @@ def test_merge_tiles_with_linear_blending_horizontal(blend_amount: int): expected_output = np.zeros((512, 960, 3), dtype=np.uint8) expected_output[:, : 480 - (blend_amount // 2), :] = 64 if blend_amount > 0: - gradient = np.linspace(start=64, stop=128, num=blend_amount, dtype=np.uint8).reshape((1, blend_amount, 1)) - expected_output[:, 480 - (blend_amount // 2) : 480 + (blend_amount // 2), :] = gradient + gradient = np.linspace( + start=64, stop=128, num=blend_amount, dtype=np.uint8 + ).reshape((1, blend_amount, 1)) + expected_output[ + :, 480 - (blend_amount // 2) : 480 + (blend_amount // 2), : + ] = gradient expected_output[:, 480 + (blend_amount // 2) :, :] = 128 merge_tiles_with_linear_blending( - dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=blend_amount + dst_image=dst_image, + tiles=tiles, + tile_images=tile_images, + blend_amount=blend_amount, ) np.testing.assert_array_equal(dst_image, expected_output, strict=True) @@ -388,8 +620,14 @@ def test_merge_tiles_with_linear_blending_vertical(blend_amount: int): """Test merge_tiles_with_linear_blending(...) behavior when merging vertically.""" # Initialize 2 tiles stacked vertically. tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=64, left=0, right=0)), - Tile(coords=TBLR(top=448, bottom=960, left=0, right=512), overlap=TBLR(top=64, bottom=0, left=0, right=0)), + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=64, left=0, right=0), + ), + Tile( + coords=TBLR(top=448, bottom=960, left=0, right=512), + overlap=TBLR(top=64, bottom=0, left=0, right=0), + ), ] dst_image = np.zeros((960, 512, 3), dtype=np.uint8) @@ -404,12 +642,19 @@ def test_merge_tiles_with_linear_blending_vertical(blend_amount: int): expected_output = np.zeros((960, 512, 3), dtype=np.uint8) expected_output[: 480 - (blend_amount // 2), :, :] = 64 if blend_amount > 0: - gradient = np.linspace(start=64, stop=128, num=blend_amount, dtype=np.uint8).reshape((blend_amount, 1, 1)) - expected_output[480 - (blend_amount // 2) : 480 + (blend_amount // 2), :, :] = gradient + gradient = np.linspace( + start=64, stop=128, num=blend_amount, dtype=np.uint8 + ).reshape((blend_amount, 1, 1)) + expected_output[ + 480 - (blend_amount // 2) : 480 + (blend_amount // 2), :, : + ] = gradient expected_output[480 + (blend_amount // 2) :, :, :] = 128 merge_tiles_with_linear_blending( - dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=blend_amount + dst_image=dst_image, + tiles=tiles, + tile_images=tile_images, + blend_amount=blend_amount, ) np.testing.assert_array_equal(dst_image, expected_output, strict=True) @@ -421,8 +666,14 @@ def test_merge_tiles_with_linear_blending_blend_amount_exceeds_vertical_overlap( """ # Initialize 2 tiles stacked vertically. tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=64, left=0, right=0)), - Tile(coords=TBLR(top=448, bottom=960, left=0, right=512), overlap=TBLR(top=64, bottom=0, left=0, right=0)), + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=64, left=0, right=0), + ), + Tile( + coords=TBLR(top=448, bottom=960, left=0, right=512), + overlap=TBLR(top=64, bottom=0, left=0, right=0), + ), ] dst_image = np.zeros((960, 512, 3), dtype=np.uint8) @@ -432,7 +683,9 @@ def test_merge_tiles_with_linear_blending_blend_amount_exceeds_vertical_overlap( # blend_amount=128 exceeds overlap of 64, so should raise exception. with pytest.raises(AssertionError): - merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128) + merge_tiles_with_linear_blending( + dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128 + ) def test_merge_tiles_with_linear_blending_blend_amount_exceeds_horizontal_overlap(): @@ -441,8 +694,14 @@ def test_merge_tiles_with_linear_blending_blend_amount_exceeds_horizontal_overla """ # Initialize 2 tiles side-by-side. tiles = [ - Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=64)), - Tile(coords=TBLR(top=0, bottom=512, left=448, right=960), overlap=TBLR(top=0, bottom=0, left=64, right=0)), + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=0, left=0, right=64), + ), + Tile( + coords=TBLR(top=0, bottom=512, left=448, right=960), + overlap=TBLR(top=0, bottom=0, left=64, right=0), + ), ] dst_image = np.zeros((512, 960, 3), dtype=np.uint8) @@ -452,14 +711,21 @@ def test_merge_tiles_with_linear_blending_blend_amount_exceeds_horizontal_overla # blend_amount=128 exceeds overlap of 64, so should raise exception. with pytest.raises(AssertionError): - merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128) + merge_tiles_with_linear_blending( + dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128 + ) def test_merge_tiles_with_linear_blending_tiles_overflow_dst_image(): """Test that merge_tiles_with_linear_blending(...) raises an exception if any of the tiles overflows the dst_image. """ - tiles = [Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=0))] + tiles = [ + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) + ] dst_image = np.zeros((256, 512, 3), dtype=np.uint8) @@ -467,14 +733,21 @@ def test_merge_tiles_with_linear_blending_tiles_overflow_dst_image(): tile_images = [np.zeros((512, 512, 3))] with pytest.raises(ValueError): - merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0) + merge_tiles_with_linear_blending( + dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0 + ) def test_merge_tiles_with_linear_blending_mismatched_list_lengths(): """Test that merge_tiles_with_linear_blending(...) raises an exception if the lengths of 'tiles' and 'tile_images' do not match. """ - tiles = [Tile(coords=TBLR(top=0, bottom=512, left=0, right=512), overlap=TBLR(top=0, bottom=0, left=0, right=0))] + tiles = [ + Tile( + coords=TBLR(top=0, bottom=512, left=0, right=512), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ) + ] dst_image = np.zeros((256, 512, 3), dtype=np.uint8) @@ -482,4 +755,6 @@ def test_merge_tiles_with_linear_blending_mismatched_list_lengths(): tile_images = [np.zeros((512, 512, 3)), np.zeros((512, 512, 3))] with pytest.raises(ValueError): - merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0) + merge_tiles_with_linear_blending( + dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0 + ) From 5f371769383098196af439c3b5aa59d5a0c94e7b Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Fri, 8 Dec 2023 19:40:10 +0000 Subject: [PATCH 24/33] ruff formatting --- tests/backend/tiles/test_tiles.py | 84 ++++++++----------------------- 1 file changed, 21 insertions(+), 63 deletions(-) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 850d6ca6ca..87700f8d09 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -16,9 +16,7 @@ from invokeai.backend.tiles.utils import TBLR, Tile def test_calc_tiles_with_overlap_single_tile(): """Test calc_tiles_with_overlap() behavior when a single tile covers the image.""" - tiles = calc_tiles_with_overlap( - image_height=512, image_width=1024, tile_height=512, tile_width=1024, overlap=64 - ) + tiles = calc_tiles_with_overlap(image_height=512, image_width=1024, tile_height=512, tile_width=1024, overlap=64) expected_tiles = [ Tile( @@ -33,9 +31,7 @@ def test_calc_tiles_with_overlap_single_tile(): def test_calc_tiles_with_overlap_evenly_divisible(): """Test calc_tiles_with_overlap() behavior when the image is evenly covered by multiple tiles.""" # Parameters chosen so that image is evenly covered by 2 rows, 3 columns of tiles. - tiles = calc_tiles_with_overlap( - image_height=576, image_width=1600, tile_height=320, tile_width=576, overlap=64 - ) + tiles = calc_tiles_with_overlap(image_height=576, image_width=1600, tile_height=320, tile_width=576, overlap=64) expected_tiles = [ # Row 0 @@ -72,9 +68,7 @@ def test_calc_tiles_with_overlap_evenly_divisible(): def test_calc_tiles_with_overlap_not_evenly_divisible(): """Test calc_tiles_with_overlap() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters chosen so that image is covered by 2 rows and 3 columns of tiles, with uneven overlaps. - tiles = calc_tiles_with_overlap( - image_height=400, image_width=1200, tile_height=256, tile_width=512, overlap=64 - ) + tiles = calc_tiles_with_overlap(image_height=400, image_width=1200, tile_height=256, tile_width=512, overlap=64) expected_tiles = [ # Row 0 @@ -131,13 +125,9 @@ def test_calc_tiles_with_overlap_input_validation( """Test that calc_tiles_with_overlap() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(AssertionError): - calc_tiles_with_overlap( - image_height, image_width, tile_height, tile_width, overlap - ) + calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) else: - calc_tiles_with_overlap( - image_height, image_width, tile_height, tile_width, overlap - ) + calc_tiles_with_overlap(image_height, image_width, tile_height, tile_width, overlap) #################################### @@ -398,13 +388,9 @@ def test_calc_tiles_min_overlap_input_validation( """Test that calc_tiles_min_overlap() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(AssertionError): - calc_tiles_min_overlap( - image_height, image_width, tile_height, tile_width, min_overlap - ) + calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap) else: - calc_tiles_min_overlap( - image_height, image_width, tile_height, tile_width, min_overlap - ) + calc_tiles_min_overlap(image_height, image_width, tile_height, tile_width, min_overlap) #################################### @@ -414,9 +400,7 @@ def test_calc_tiles_min_overlap_input_validation( def test_calc_tiles_even_split_single_tile(): """Test calc_tiles_even_split() behavior when a single tile covers the image.""" - tiles = calc_tiles_even_split( - image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=0.25 - ) + tiles = calc_tiles_even_split(image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=0.25) expected_tiles = [ Tile( @@ -431,9 +415,7 @@ def test_calc_tiles_even_split_single_tile(): def test_calc_tiles_even_split_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image is evenly covered by multiple tiles.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split( - image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=0.25 - ) + tiles = calc_tiles_even_split(image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=0.25) expected_tiles = [ # Row 0 @@ -469,9 +451,7 @@ def test_calc_tiles_even_split_evenly_divisible(): def test_calc_tiles_even_split_not_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split( - image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=0.25 - ) + tiles = calc_tiles_even_split(image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=0.25) expected_tiles = [ # Row 0 @@ -508,9 +488,7 @@ def test_calc_tiles_even_split_not_evenly_divisible(): def test_calc_tiles_even_split_difficult_size(): """Test calc_tiles_even_split() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate - tiles = calc_tiles_even_split( - image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=0.25 - ) + tiles = calc_tiles_even_split(image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=0.25) expected_tiles = [ # Row 0 @@ -556,13 +534,9 @@ def test_calc_tiles_even_split_input_validation( """Test that calc_tiles_even_split() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(ValueError): - calc_tiles_even_split( - image_height, image_width, num_tiles_x, num_tiles_y, overlap - ) + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) else: - calc_tiles_even_split( - image_height, image_width, num_tiles_x, num_tiles_y, overlap - ) + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) ############################################# @@ -597,12 +571,8 @@ def test_merge_tiles_with_linear_blending_horizontal(blend_amount: int): expected_output = np.zeros((512, 960, 3), dtype=np.uint8) expected_output[:, : 480 - (blend_amount // 2), :] = 64 if blend_amount > 0: - gradient = np.linspace( - start=64, stop=128, num=blend_amount, dtype=np.uint8 - ).reshape((1, blend_amount, 1)) - expected_output[ - :, 480 - (blend_amount // 2) : 480 + (blend_amount // 2), : - ] = gradient + gradient = np.linspace(start=64, stop=128, num=blend_amount, dtype=np.uint8).reshape((1, blend_amount, 1)) + expected_output[:, 480 - (blend_amount // 2) : 480 + (blend_amount // 2), :] = gradient expected_output[:, 480 + (blend_amount // 2) :, :] = 128 merge_tiles_with_linear_blending( @@ -642,12 +612,8 @@ def test_merge_tiles_with_linear_blending_vertical(blend_amount: int): expected_output = np.zeros((960, 512, 3), dtype=np.uint8) expected_output[: 480 - (blend_amount // 2), :, :] = 64 if blend_amount > 0: - gradient = np.linspace( - start=64, stop=128, num=blend_amount, dtype=np.uint8 - ).reshape((blend_amount, 1, 1)) - expected_output[ - 480 - (blend_amount // 2) : 480 + (blend_amount // 2), :, : - ] = gradient + gradient = np.linspace(start=64, stop=128, num=blend_amount, dtype=np.uint8).reshape((blend_amount, 1, 1)) + expected_output[480 - (blend_amount // 2) : 480 + (blend_amount // 2), :, :] = gradient expected_output[480 + (blend_amount // 2) :, :, :] = 128 merge_tiles_with_linear_blending( @@ -683,9 +649,7 @@ def test_merge_tiles_with_linear_blending_blend_amount_exceeds_vertical_overlap( # blend_amount=128 exceeds overlap of 64, so should raise exception. with pytest.raises(AssertionError): - merge_tiles_with_linear_blending( - dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128 - ) + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128) def test_merge_tiles_with_linear_blending_blend_amount_exceeds_horizontal_overlap(): @@ -711,9 +675,7 @@ def test_merge_tiles_with_linear_blending_blend_amount_exceeds_horizontal_overla # blend_amount=128 exceeds overlap of 64, so should raise exception. with pytest.raises(AssertionError): - merge_tiles_with_linear_blending( - dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128 - ) + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=128) def test_merge_tiles_with_linear_blending_tiles_overflow_dst_image(): @@ -733,9 +695,7 @@ def test_merge_tiles_with_linear_blending_tiles_overflow_dst_image(): tile_images = [np.zeros((512, 512, 3))] with pytest.raises(ValueError): - merge_tiles_with_linear_blending( - dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0 - ) + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0) def test_merge_tiles_with_linear_blending_mismatched_list_lengths(): @@ -755,6 +715,4 @@ def test_merge_tiles_with_linear_blending_mismatched_list_lengths(): tile_images = [np.zeros((512, 512, 3)), np.zeros((512, 512, 3))] with pytest.raises(ValueError): - merge_tiles_with_linear_blending( - dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0 - ) + merge_tiles_with_linear_blending(dst_image=dst_image, tiles=tiles, tile_images=tile_images, blend_amount=0) From 494c2a9b0508def12d082e650a5d0cf8388daa47 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Sat, 9 Dec 2023 18:38:07 +0000 Subject: [PATCH 25/33] Updates based on code review by @RyanJDick --- invokeai/app/invocations/tiles.py | 11 +++++++---- invokeai/backend/tiles/tiles.py | 23 ++++++++++++++--------- invokeai/backend/tiles/utils.py | 12 ++++++------ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 609b539610..4e34cc4359 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -66,7 +66,7 @@ class CalculateImageTilesInvocation(BaseInvocation): @invocation( - "calculate_image_tiles_Even_Split", + "calculate_image_tiles_even_split", title="Calculate Image Tiles Even Split", tags=["tiles"], category="tiles", @@ -93,7 +93,7 @@ class CalculateImageTilesEvenSplitInvocation(BaseInvocation): default=0.25, ge=0, lt=1, - description="Overlap amount of tile size (0-1)", + description="Overlap between adjacent tiles as a fraction of the tile's dimensions (0-1)", ) def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: @@ -126,7 +126,8 @@ class CalculateImageTilesMinimumOverlapInvocation(BaseInvocation): min_overlap: int = InputField( default=128, ge=0, - description="minimum tile overlap size (must be a multiple of 8)", + multiple_of=8, + description="Minimum overlap between adjacent tiles, in pixels(must be a multiple of 8).", ) round_to_8: bool = InputField( default=False, @@ -260,10 +261,12 @@ class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): merge_tiles_with_linear_blending( dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount ) - else: + elif self.blend_mode == "Seam": merge_tiles_with_seam_blending( dst_image=np_image, tiles=tiles, tile_images=tile_np_images, blend_amount=self.blend_amount ) + else: + raise ValueError(f"Unsupported blend mode: '{self.blend_mode}'.") # Convert into a PIL image and save pil_image = Image.fromarray(np_image) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 166e45d57c..2cd87da303 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -2,11 +2,12 @@ import math from typing import Union import numpy as np +from invokeai.app.invocations.latent import LATENT_SCALE_FACTOR from invokeai.backend.tiles.utils import TBLR, Tile, paste, seam_blend -def calc_overlap(tiles: list[Tile], num_tiles_x, num_tiles_y) -> list[Tile]: +def calc_overlap(tiles: list[Tile], num_tiles_x: int, num_tiles_y: int) -> list[Tile]: """Calculate and update the overlap of a list of tiles. Args: @@ -110,23 +111,27 @@ def calc_tiles_even_split( image_width (int): The image width in px. num_x_tiles (int): The number of tile to split the image into on the X-axis. num_y_tiles (int): The number of tile to split the image into on the Y-axis. - overlap (int, optional): The target overlap amount of the tiles size. Defaults to 0. + overlap (float, optional): The target overlap amount of the tiles size. Defaults to 0. Returns: list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. """ # Ensure tile size is divisible by 8 - if image_width % 8 != 0 or image_height % 8 != 0: + if image_width % LATENT_SCALE_FACTOR != 0 or image_height % LATENT_SCALE_FACTOR != 0: raise ValueError(f"image size (({image_width}, {image_height})) must be divisible by 8") # Calculate the overlap size based on the percentage and adjust it to be divisible by 8 (rounding up) - overlap_x = 8 * math.ceil(int((image_width / num_tiles_x) * overlap) / 8) - overlap_y = 8 * math.ceil(int((image_height / num_tiles_y) * overlap) / 8) + overlap_x = LATENT_SCALE_FACTOR * math.ceil(int((image_width / num_tiles_x) * overlap) / LATENT_SCALE_FACTOR) + overlap_y = LATENT_SCALE_FACTOR * math.ceil(int((image_height / num_tiles_y) * overlap) / LATENT_SCALE_FACTOR) # Calculate the tile size based on the number of tiles and overlap, and ensure it's divisible by 8 (rounding down) - tile_size_x = 8 * math.floor(((image_width + overlap_x * (num_tiles_x - 1)) // num_tiles_x) / 8) - tile_size_y = 8 * math.floor(((image_height + overlap_y * (num_tiles_y - 1)) // num_tiles_y) / 8) + tile_size_x = LATENT_SCALE_FACTOR * math.floor( + ((image_width + overlap_x * (num_tiles_x - 1)) // num_tiles_x) / LATENT_SCALE_FACTOR + ) + tile_size_y = LATENT_SCALE_FACTOR * math.floor( + ((image_height + overlap_y * (num_tiles_y - 1)) // num_tiles_y) / LATENT_SCALE_FACTOR + ) # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. tiles: list[Tile] = [] @@ -196,13 +201,13 @@ def calc_tiles_min_overlap( for tile_idx_y in range(num_tiles_y): top = (tile_idx_y * (image_height - tile_height)) // (num_tiles_y - 1) if num_tiles_y > 1 else 0 if round_to_8: - top = 8 * (top // 8) + top = LATENT_SCALE_FACTOR * (top // LATENT_SCALE_FACTOR) bottom = top + tile_height for tile_idx_x in range(num_tiles_x): left = (tile_idx_x * (image_width - tile_width)) // (num_tiles_x - 1) if num_tiles_x > 1 else 0 if round_to_8: - left = 8 * (left // 8) + left = LATENT_SCALE_FACTOR * (left // LATENT_SCALE_FACTOR) right = left + tile_width tile = Tile( diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py index c906983587..8596d25840 100644 --- a/invokeai/backend/tiles/utils.py +++ b/invokeai/backend/tiles/utils.py @@ -33,10 +33,10 @@ def paste(dst_image: np.ndarray, src_image: np.ndarray, box: TBLR, mask: Optiona """Paste a source image into a destination image. Args: - dst_image (torch.Tensor): The destination image to paste into. Shape: (H, W, C). - src_image (torch.Tensor): The source image to paste. Shape: (H, W, C). H and W must be compatible with 'box'. + dst_image (np.array): The destination image to paste into. Shape: (H, W, C). + src_image (np.array): The source image to paste. Shape: (H, W, C). H and W must be compatible with 'box'. box (TBLR): Box defining the region in the 'dst_image' where 'src_image' will be pasted. - mask (Optional[torch.Tensor]): A mask that defines the blending between 'src_image' and 'dst_image'. + mask (Optional[np.array]): A mask that defines the blending between 'src_image' and 'dst_image'. Range: [0.0, 1.0], Shape: (H, W). The output is calculate per-pixel according to `src * mask + dst * (1 - mask)`. """ @@ -55,8 +55,8 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool It is assumed that input images will be RGB np arrays and are the same size. Args: - ia1 (torch.Tensor): Image array 1 Shape: (H, W, C). - ia2 (torch.Tensor): Image array 2 Shape: (H, W, C). + ia1 (np.array): Image array 1 Shape: (H, W, C). + ia2 (np.array): Image array 2 Shape: (H, W, C). x_seam (bool): If the images should be blended on the x axis or not. blend_amount (int): The size of the blur to use on the seam. Half of this value will be used to avoid the edges of the image. """ @@ -74,7 +74,7 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool return result # Assume RGB and convert to grey - iag1 = np.dot(ia1, [0.2989, 0.5870, 0.1140]) + iag1 = np.dot(ia1, [0.2989, 0.5870, 0.1140]) # BT.601 perceived brightness iag2 = np.dot(ia2, [0.2989, 0.5870, 0.1140]) # Calc Difference between the images From e656768eb2181da42a7a658716623b412beed9ce Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Sat, 9 Dec 2023 21:56:31 +0000 Subject: [PATCH 26/33] more fixes from code review --- invokeai/app/invocations/tiles.py | 4 ++-- invokeai/backend/tiles/tiles.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 4e34cc4359..f9dd7b3e69 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -121,8 +121,8 @@ class CalculateImageTilesMinimumOverlapInvocation(BaseInvocation): image_height: int = InputField( ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." ) - tile_width: int = InputField(ge=1, default=576, description="The tile width, in pixels.") - tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") + tile_width: int = InputField(ge=1, default=576, multiple_of=8, description="The tile width, in pixels.") + tile_height: int = InputField(ge=1, default=576, multiple_of=8, description="The tile height, in pixels.") min_overlap: int = InputField( default=128, ge=0, diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 2cd87da303..2f71db03cb 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -2,8 +2,8 @@ import math from typing import Union import numpy as np -from invokeai.app.invocations.latent import LATENT_SCALE_FACTOR +from invokeai.app.invocations.latent import LATENT_SCALE_FACTOR from invokeai.backend.tiles.utils import TBLR, Tile, paste, seam_blend From 4c97b619fbc0fc3fb76bffa26af44af99db2295c Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Sat, 9 Dec 2023 22:05:23 +0000 Subject: [PATCH 27/33] Update tiles.py merge with main --- invokeai/app/invocations/tiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index 0e280159e9..beec084b7d 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -215,7 +215,7 @@ BLEND_MODES = Literal["Linear", "Seam"] @invocation("merge_tiles_to_image", title="Merge Tiles to Image", tags=["tiles"], category="tiles", version="1.1.0") -class MergeTilesToImageInvocation(BaseInvocation, WithMetadata, WithWorkflow): +class MergeTilesToImageInvocation(BaseInvocation, WithMetadata): """Merge multiple tile images into a single image.""" # Inputs From fefb78795f810f707bc77ded6d5107e5e1428cb6 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Mon, 11 Dec 2023 16:55:27 +0000 Subject: [PATCH 28/33] - Even_spilt overlap renamed to overlap_fraction - min_overlap removed * restrictions and round_to_8 - min_overlap handles tile size > image size by clipping the num tiles to 1. - Updated assert test on min_overlap. --- invokeai/app/invocations/tiles.py | 20 +++++--------------- invokeai/backend/tiles/tiles.py | 22 +++++++++++----------- invokeai/backend/tiles/utils.py | 5 +++++ tests/backend/tiles/test_tiles.py | 4 ++-- 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index beec084b7d..e368976b4b 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -88,7 +88,7 @@ class CalculateImageTilesEvenSplitInvocation(BaseInvocation): ge=1, description="Number of tiles to divide image into on the y axis", ) - overlap: float = InputField( + overlap_fraction: float = InputField( default=0.25, ge=0, lt=1, @@ -101,7 +101,7 @@ class CalculateImageTilesEvenSplitInvocation(BaseInvocation): image_width=self.image_width, num_tiles_x=self.num_tiles_x, num_tiles_y=self.num_tiles_y, - overlap=self.overlap, + overlap_fraction=self.overlap_fraction, ) return CalculateImageTilesOutput(tiles=tiles) @@ -120,18 +120,9 @@ class CalculateImageTilesMinimumOverlapInvocation(BaseInvocation): image_height: int = InputField( ge=1, default=1024, description="The image height, in pixels, to calculate tiles for." ) - tile_width: int = InputField(ge=1, default=576, multiple_of=8, description="The tile width, in pixels.") - tile_height: int = InputField(ge=1, default=576, multiple_of=8, description="The tile height, in pixels.") - min_overlap: int = InputField( - default=128, - ge=0, - multiple_of=8, - description="Minimum overlap between adjacent tiles, in pixels(must be a multiple of 8).", - ) - round_to_8: bool = InputField( - default=False, - description="Round outputs down to the nearest 8 (for pulling from a large noise field)", - ) + tile_width: int = InputField(ge=1, default=576, description="The tile width, in pixels.") + tile_height: int = InputField(ge=1, default=576, description="The tile height, in pixels.") + min_overlap: int = InputField(default=128, ge=0, description="Minimum overlap between adjacent tiles, in pixels.") def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: tiles = calc_tiles_min_overlap( @@ -140,7 +131,6 @@ class CalculateImageTilesMinimumOverlapInvocation(BaseInvocation): tile_height=self.tile_height, tile_width=self.tile_width, min_overlap=self.min_overlap, - round_to_8=self.round_to_8, ) return CalculateImageTilesOutput(tiles=tiles) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 2f71db03cb..499558ce9d 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -102,7 +102,7 @@ def calc_tiles_with_overlap( def calc_tiles_even_split( - image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: float = 0 + image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap_fraction: float = 0 ) -> list[Tile]: """Calculate the tile coordinates for a given image shape with the number of tiles requested. @@ -111,7 +111,7 @@ def calc_tiles_even_split( image_width (int): The image width in px. num_x_tiles (int): The number of tile to split the image into on the X-axis. num_y_tiles (int): The number of tile to split the image into on the Y-axis. - overlap (float, optional): The target overlap amount of the tiles size. Defaults to 0. + overlap_fraction (float, optional): The target overlap as fraction of the tiles size. Defaults to 0. Returns: list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. @@ -119,11 +119,15 @@ def calc_tiles_even_split( # Ensure tile size is divisible by 8 if image_width % LATENT_SCALE_FACTOR != 0 or image_height % LATENT_SCALE_FACTOR != 0: - raise ValueError(f"image size (({image_width}, {image_height})) must be divisible by 8") + raise ValueError(f"image size (({image_width}, {image_height})) must be divisible by {LATENT_SCALE_FACTOR}") # Calculate the overlap size based on the percentage and adjust it to be divisible by 8 (rounding up) - overlap_x = LATENT_SCALE_FACTOR * math.ceil(int((image_width / num_tiles_x) * overlap) / LATENT_SCALE_FACTOR) - overlap_y = LATENT_SCALE_FACTOR * math.ceil(int((image_height / num_tiles_y) * overlap) / LATENT_SCALE_FACTOR) + overlap_x = LATENT_SCALE_FACTOR * math.ceil( + int((image_width / num_tiles_x) * overlap_fraction) / LATENT_SCALE_FACTOR + ) + overlap_y = LATENT_SCALE_FACTOR * math.ceil( + int((image_height / num_tiles_y) * overlap_fraction) / LATENT_SCALE_FACTOR + ) # Calculate the tile size based on the number of tiles and overlap, and ensure it's divisible by 8 (rounding down) tile_size_x = LATENT_SCALE_FACTOR * math.floor( @@ -184,11 +188,11 @@ def calc_tiles_min_overlap( Returns: list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. """ - assert image_height >= tile_height - assert image_width >= tile_width + assert min_overlap < tile_height assert min_overlap < tile_width + # The If Else catches the case when the tile size is larger than the images size and just clips the number of tiles to 1 num_tiles_x = math.ceil((image_width - min_overlap) / (tile_width - min_overlap)) if tile_width < image_width else 1 num_tiles_y = ( math.ceil((image_height - min_overlap) / (tile_height - min_overlap)) if tile_height < image_height else 1 @@ -200,14 +204,10 @@ def calc_tiles_min_overlap( # Calculate tile coordinates. (Ignore overlap values for now.) for tile_idx_y in range(num_tiles_y): top = (tile_idx_y * (image_height - tile_height)) // (num_tiles_y - 1) if num_tiles_y > 1 else 0 - if round_to_8: - top = LATENT_SCALE_FACTOR * (top // LATENT_SCALE_FACTOR) bottom = top + tile_height for tile_idx_x in range(num_tiles_x): left = (tile_idx_x * (image_width - tile_width)) // (num_tiles_x - 1) if num_tiles_x > 1 else 0 - if round_to_8: - left = LATENT_SCALE_FACTOR * (left // LATENT_SCALE_FACTOR) right = left + tile_width tile = Tile( diff --git a/invokeai/backend/tiles/utils.py b/invokeai/backend/tiles/utils.py index 8596d25840..dc6d914170 100644 --- a/invokeai/backend/tiles/utils.py +++ b/invokeai/backend/tiles/utils.py @@ -74,6 +74,9 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool return result # Assume RGB and convert to grey + # Could offer other options for the luminance conversion + # BT.709 [0.2126, 0.7152, 0.0722], BT.2020 [0.2627, 0.6780, 0.0593]) + # it might not have a huge impact due to the blur that is applied over the seam iag1 = np.dot(ia1, [0.2989, 0.5870, 0.1140]) # BT.601 perceived brightness iag2 = np.dot(ia2, [0.2989, 0.5870, 0.1140]) @@ -92,6 +95,7 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool min_x = gutter # Calc the energy in the difference + # Could offer different energy calculations e.g. Sobel or Scharr energy = np.abs(np.gradient(ia, axis=0)) + np.abs(np.gradient(ia, axis=1)) # Find the starting position of the seam @@ -107,6 +111,7 @@ def seam_blend(ia1: np.ndarray, ia2: np.ndarray, blend_amount: int, x_seam: bool lowest_energy_line[max_y - 1] = np.argmin(res[max_y - 1, min_x : max_x - 1]) # Calc the path of the seam + # could offer options for larger search than just 1 pixel by adjusting lpos and rpos for ypos in range(max_y - 2, -1, -1): lowest_pos = lowest_energy_line[ypos + 1] lpos = lowest_pos - 1 diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 87700f8d09..3ae7e2b91d 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -371,8 +371,8 @@ def test_calc_tiles_min_overlap_difficult_size_div8(): (128, 128, 128, 128, 127, False), # OK (128, 128, 128, 128, 0, False), # OK (128, 128, 64, 64, 0, False), # OK - (128, 128, 129, 128, 0, True), # tile_height exceeds image_height. - (128, 128, 128, 129, 0, True), # tile_width exceeds image_width. + (128, 128, 129, 128, 0, False), # tile_height exceeds image_height defaults to 1 tile. + (128, 128, 128, 129, 0, False), # tile_width exceeds image_width defaults to 1 tile. (128, 128, 64, 128, 64, True), # overlap equals tile_height. (128, 128, 128, 64, 64, True), # overlap equals tile_width. ], From c84526fae5f14e1faf26c890c504543b10dcf61f Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Mon, 11 Dec 2023 17:05:45 +0000 Subject: [PATCH 29/33] Fixed Tests that where using round_to_8 and removed redundant tests --- invokeai/backend/tiles/tiles.py | 1 - tests/backend/tiles/test_tiles.py | 117 ------------------------------ 2 files changed, 118 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 499558ce9d..1948f6624e 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -173,7 +173,6 @@ def calc_tiles_min_overlap( tile_height: int, tile_width: int, min_overlap: int = 0, - round_to_8: bool = False, ) -> list[Tile]: """Calculate the tile coordinates for a given image shape under a simple tiling scheme with overlaps. diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 3ae7e2b91d..3cda7f1f38 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -143,7 +143,6 @@ def test_calc_tiles_min_overlap_single_tile(): tile_height=512, tile_width=1024, min_overlap=64, - round_to_8=False, ) expected_tiles = [ @@ -165,7 +164,6 @@ def test_calc_tiles_min_overlap_evenly_divisible(): tile_height=320, tile_width=576, min_overlap=64, - round_to_8=False, ) expected_tiles = [ @@ -209,7 +207,6 @@ def test_calc_tiles_min_overlap_not_evenly_divisible(): tile_height=256, tile_width=512, min_overlap=64, - round_to_8=False, ) expected_tiles = [ @@ -244,120 +241,6 @@ def test_calc_tiles_min_overlap_not_evenly_divisible(): assert tiles == expected_tiles -def test_calc_tiles_min_overlap_difficult_size(): - """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" - # Parameters are a difficult size for other tile gen routines to calculate - tiles = calc_tiles_min_overlap( - image_height=1000, - image_width=1000, - tile_height=512, - tile_width=512, - min_overlap=64, - round_to_8=False, - ) - - expected_tiles = [ - # Row 0 - Tile( - coords=TBLR(top=0, bottom=512, left=0, right=512), - overlap=TBLR(top=0, bottom=268, left=0, right=268), - ), - Tile( - coords=TBLR(top=0, bottom=512, left=244, right=756), - overlap=TBLR(top=0, bottom=268, left=268, right=268), - ), - Tile( - coords=TBLR(top=0, bottom=512, left=488, right=1000), - overlap=TBLR(top=0, bottom=268, left=268, right=0), - ), - # Row 1 - Tile( - coords=TBLR(top=244, bottom=756, left=0, right=512), - overlap=TBLR(top=268, bottom=268, left=0, right=268), - ), - Tile( - coords=TBLR(top=244, bottom=756, left=244, right=756), - overlap=TBLR(top=268, bottom=268, left=268, right=268), - ), - Tile( - coords=TBLR(top=244, bottom=756, left=488, right=1000), - overlap=TBLR(top=268, bottom=268, left=268, right=0), - ), - # Row 2 - Tile( - coords=TBLR(top=488, bottom=1000, left=0, right=512), - overlap=TBLR(top=268, bottom=0, left=0, right=268), - ), - Tile( - coords=TBLR(top=488, bottom=1000, left=244, right=756), - overlap=TBLR(top=268, bottom=0, left=268, right=268), - ), - Tile( - coords=TBLR(top=488, bottom=1000, left=488, right=1000), - overlap=TBLR(top=268, bottom=0, left=268, right=0), - ), - ] - - assert tiles == expected_tiles - - -def test_calc_tiles_min_overlap_difficult_size_div8(): - """Test calc_tiles_min_overlap() behavior when the image is a difficult size to spilt evenly and keep div8.""" - # Parameters are a difficult size for other tile gen routines to calculate - tiles = calc_tiles_min_overlap( - image_height=1000, - image_width=1000, - tile_height=512, - tile_width=512, - min_overlap=64, - round_to_8=True, - ) - - expected_tiles = [ - # Row 0 - Tile( - coords=TBLR(top=0, bottom=512, left=0, right=512), - overlap=TBLR(top=0, bottom=272, left=0, right=272), - ), - Tile( - coords=TBLR(top=0, bottom=512, left=240, right=752), - overlap=TBLR(top=0, bottom=272, left=272, right=264), - ), - Tile( - coords=TBLR(top=0, bottom=512, left=488, right=1000), - overlap=TBLR(top=0, bottom=272, left=264, right=0), - ), - # Row 1 - Tile( - coords=TBLR(top=240, bottom=752, left=0, right=512), - overlap=TBLR(top=272, bottom=264, left=0, right=272), - ), - Tile( - coords=TBLR(top=240, bottom=752, left=240, right=752), - overlap=TBLR(top=272, bottom=264, left=272, right=264), - ), - Tile( - coords=TBLR(top=240, bottom=752, left=488, right=1000), - overlap=TBLR(top=272, bottom=264, left=264, right=0), - ), - # Row 2 - Tile( - coords=TBLR(top=488, bottom=1000, left=0, right=512), - overlap=TBLR(top=264, bottom=0, left=0, right=272), - ), - Tile( - coords=TBLR(top=488, bottom=1000, left=240, right=752), - overlap=TBLR(top=264, bottom=0, left=272, right=264), - ), - Tile( - coords=TBLR(top=488, bottom=1000, left=488, right=1000), - overlap=TBLR(top=264, bottom=0, left=264, right=0), - ), - ] - - assert tiles == expected_tiles - - @pytest.mark.parametrize( [ "image_height", From 0852fd4e88d1ca6094aa84a16cb52d6b00f9c6f3 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Mon, 11 Dec 2023 17:17:29 +0000 Subject: [PATCH 30/33] Updated tests for even_split overlap renamed to overlap_fraction --- tests/backend/tiles/test_tiles.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 3cda7f1f38..8ddd44f70b 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -283,7 +283,9 @@ def test_calc_tiles_min_overlap_input_validation( def test_calc_tiles_even_split_single_tile(): """Test calc_tiles_even_split() behavior when a single tile covers the image.""" - tiles = calc_tiles_even_split(image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap_fraction=0.25 + ) expected_tiles = [ Tile( @@ -298,7 +300,9 @@ def test_calc_tiles_even_split_single_tile(): def test_calc_tiles_even_split_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image is evenly covered by multiple tiles.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split(image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap_fraction=0.25 + ) expected_tiles = [ # Row 0 @@ -334,7 +338,9 @@ def test_calc_tiles_even_split_evenly_divisible(): def test_calc_tiles_even_split_not_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split(image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap_fraction=0.25 + ) expected_tiles = [ # Row 0 @@ -371,7 +377,9 @@ def test_calc_tiles_even_split_not_evenly_divisible(): def test_calc_tiles_even_split_difficult_size(): """Test calc_tiles_even_split() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate - tiles = calc_tiles_even_split(image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=0.25) + tiles = calc_tiles_even_split( + image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap_fraction=0.25 + ) expected_tiles = [ # Row 0 @@ -411,15 +419,15 @@ def test_calc_tiles_even_split_input_validation( image_width: int, num_tiles_x: int, num_tiles_y: int, - overlap: float, + overlap_fraction: float, raises: bool, ): """Test that calc_tiles_even_split() raises an exception if the inputs are invalid.""" if raises: with pytest.raises(ValueError): - calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap_fraction) else: - calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap_fraction) ############################################# From fbbc1037cdc91408a9212e38a54e24b3d766429b Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Mon, 11 Dec 2023 17:23:28 +0000 Subject: [PATCH 31/33] missed a rename of overlap to overlap_fraction in test for even_spilt --- tests/backend/tiles/test_tiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 8ddd44f70b..0b18f9ed54 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -406,7 +406,7 @@ def test_calc_tiles_even_split_difficult_size(): @pytest.mark.parametrize( - ["image_height", "image_width", "num_tiles_x", "num_tiles_y", "overlap", "raises"], + ["image_height", "image_width", "num_tiles_x", "num_tiles_y", "overlap_fraction", "raises"], [ (128, 128, 1, 1, 0.25, False), # OK (128, 128, 1, 1, 0, False), # OK From 18093c4f1d257ce22c509f306cad62f61bdbfe3b Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 11 Dec 2023 21:08:03 -0500 Subject: [PATCH 32/33] split installer zipfile script from tagging script; add make commands --- Makefile | 33 +++++++++++++++++++++++- installer/create_installer.sh | 47 ----------------------------------- 2 files changed, 32 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 24722e2264..309aeee4d5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,20 @@ # simple Makefile with scripts that are otherwise hard to remember # to use, run from the repo root `make ` +default: help + +help: + @echo Developer commands: + @echo + @echo "ruff Run ruff, fixing any safely-fixable errors and formatting" + @echo "ruff-unsafe Run ruff, fixing all fixable errors and formatting" + @echo "mypy Run mypy using the config in pyproject.toml to identify type mismatches and other coding errors" + @echo "mypy-all Run mypy ignoring the config in pyproject.tom but still ignoring missing imports" + @echo "frontend-build Build the frontend in order to run on localhost:9090" + @echo "frontend-dev Run the frontend in developer mode on localhost:5173" + @echo "installer-zip Build the installer .zip file for the current version" + @echo "tag-release Tag the GitHub repository with the current version (use at release time only!)" + # Runs ruff, fixing any safely-fixable errors and formatting ruff: ruff check . --fix @@ -18,4 +32,21 @@ mypy: # Runs mypy, ignoring the config in pyproject.toml but still ignoring missing (untyped) imports # (many files are ignored by the config, so this is useful for checking all files) mypy-all: - mypy scripts/invokeai-web.py --config-file= --ignore-missing-imports \ No newline at end of file + mypy scripts/invokeai-web.py --config-file= --ignore-missing-imports + +# Build the frontend +frontend-build: + cd invokeai/frontend/web && pnpm build + +# Run the frontend in dev mode +frontend-dev: + cd invokeai/frontend/web && pnpm dev + +# Installer zip file +installer-zip: + cd installer && ./create_installer.sh + +# Tag the release +tag-release: + cd installer && ./tag-release.sh + diff --git a/installer/create_installer.sh b/installer/create_installer.sh index ef489af751..ed8cbb0d0a 100755 --- a/installer/create_installer.sh +++ b/installer/create_installer.sh @@ -13,14 +13,6 @@ function is_bin_in_path { builtin type -P "$1" &>/dev/null } -function does_tag_exist { - git rev-parse --quiet --verify "refs/tags/$1" >/dev/null -} - -function git_show_ref { - git show-ref --dereference $1 --abbrev 7 -} - function git_show { git show -s --format='%h %s' $1 } @@ -53,50 +45,11 @@ VERSION=$( ) PATCH="" VERSION="v${VERSION}${PATCH}" -LATEST_TAG="v3-latest" - -echo "Building installer for version $VERSION..." -echo - -if does_tag_exist $VERSION; then - echo -e "${BCYAN}${VERSION}${RESET} already exists:" - git_show_ref tags/$VERSION - echo -fi -if does_tag_exist $LATEST_TAG; then - echo -e "${BCYAN}${LATEST_TAG}${RESET} already exists:" - git_show_ref tags/$LATEST_TAG - echo -fi echo -e "${BGREEN}HEAD${RESET}:" git_show echo -echo -e -n "Create tags ${BCYAN}${VERSION}${RESET} and ${BCYAN}${LATEST_TAG}${RESET} @ ${BGREEN}HEAD${RESET}, ${RED}deleting existing tags on remote${RESET}? " -read -e -p 'y/n [n]: ' input -RESPONSE=${input:='n'} -if [ "$RESPONSE" == 'y' ]; then - echo - echo -e "Deleting ${BCYAN}${VERSION}${RESET} tag on remote..." - git push origin :refs/tags/$VERSION - - echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${VERSION}${RESET} locally..." - if ! git tag -fa $VERSION; then - echo "Existing/invalid tag" - exit -1 - fi - - echo -e "Deleting ${BCYAN}${LATEST_TAG}${RESET} tag on remote..." - git push origin :refs/tags/$LATEST_TAG - - echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${LATEST_TAG}${RESET} locally..." - git tag -fa $LATEST_TAG - - echo - echo -e "${BYELLOW}Remember to 'git push origin --tags'!${RESET}" -fi - # ---------------------- FRONTEND ---------------------- pushd ../invokeai/frontend/web >/dev/null From f3a97e06ecd63c4978343e05dfacb47df549212e Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 11 Dec 2023 21:11:37 -0500 Subject: [PATCH 33/33] add the tag_release.sh script --- Makefile | 2 +- installer/tag_release.sh | 71 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100755 installer/tag_release.sh diff --git a/Makefile b/Makefile index 309aeee4d5..10d7a257c5 100644 --- a/Makefile +++ b/Makefile @@ -48,5 +48,5 @@ installer-zip: # Tag the release tag-release: - cd installer && ./tag-release.sh + cd installer && ./tag_release.sh diff --git a/installer/tag_release.sh b/installer/tag_release.sh new file mode 100755 index 0000000000..a914c1a505 --- /dev/null +++ b/installer/tag_release.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +BCYAN="\e[1;36m" +BYELLOW="\e[1;33m" +BGREEN="\e[1;32m" +BRED="\e[1;31m" +RED="\e[31m" +RESET="\e[0m" + +function does_tag_exist { + git rev-parse --quiet --verify "refs/tags/$1" >/dev/null +} + +function git_show_ref { + git show-ref --dereference $1 --abbrev 7 +} + +function git_show { + git show -s --format='%h %s' $1 +} + +VERSION=$( + cd .. + python -c "from invokeai.version import __version__ as version; print(version)" +) +PATCH="" +MAJOR_VERSION=$(echo $VERSION | sed 's/\..*$//') +VERSION="v${VERSION}${PATCH}" +LATEST_TAG="v${MAJOR_VERSION}-latest" + +if does_tag_exist $VERSION; then + echo -e "${BCYAN}${VERSION}${RESET} already exists:" + git_show_ref tags/$VERSION + echo +fi +if does_tag_exist $LATEST_TAG; then + echo -e "${BCYAN}${LATEST_TAG}${RESET} already exists:" + git_show_ref tags/$LATEST_TAG + echo +fi + +echo -e "${BGREEN}HEAD${RESET}:" +git_show +echo + +echo -e -n "Create tags ${BCYAN}${VERSION}${RESET} and ${BCYAN}${LATEST_TAG}${RESET} @ ${BGREEN}HEAD${RESET}, ${RED}deleting existing tags on remote${RESET}? " +read -e -p 'y/n [n]: ' input +RESPONSE=${input:='n'} +if [ "$RESPONSE" == 'y' ]; then + echo + echo -e "Deleting ${BCYAN}${VERSION}${RESET} tag on remote..." + git push --delete origin $VERSION + + echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${VERSION}${RESET} locally..." + if ! git tag -fa $VERSION; then + echo "Existing/invalid tag" + exit -1 + fi + + echo -e "Deleting ${BCYAN}${LATEST_TAG}${RESET} tag on remote..." + git push --delete origin $LATEST_TAG + + echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${LATEST_TAG}${RESET} locally..." + git tag -fa $LATEST_TAG + + echo -e "Pushing updated tags to remote..." + git push origin --tags +fi +exit 0