From 87ff380fe482cd2426776bb9642f821500f5a05b Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Tue, 12 Dec 2023 13:40:28 +0000 Subject: [PATCH 01/12] fix for calc_tiles_min_overlap when tile size is bigger than image size --- invokeai/backend/tiles/tiles.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 1948f6624e..cdc94bf760 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -191,7 +191,14 @@ def calc_tiles_min_overlap( assert min_overlap < tile_height assert min_overlap < tile_width - # The If Else catches the case when the tile size is larger than the images size and just clips the number of tiles to 1 + # catches the cases when the tile size is larger than the images size and just clips the number of tiles to 1 + + if image_width < tile_width: + tile_width = image_width + + if image_height < tile_height: + tile_height = image_height + num_tiles_x = math.ceil((image_width - min_overlap) / (tile_width - min_overlap)) if tile_width < image_width else 1 num_tiles_y = ( math.ceil((image_height - min_overlap) / (tile_height - min_overlap)) if tile_height < image_height else 1 From 0b860582f0add6fc69374142f23d3744d1a8a8e6 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Tue, 12 Dec 2023 14:00:06 +0000 Subject: [PATCH 02/12] remove unneeded if else --- invokeai/backend/tiles/tiles.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index cdc94bf760..9f5817a4ca 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -199,10 +199,8 @@ def calc_tiles_min_overlap( if image_height < tile_height: tile_height = image_height - num_tiles_x = math.ceil((image_width - min_overlap) / (tile_width - min_overlap)) if tile_width < image_width else 1 - num_tiles_y = ( - math.ceil((image_height - min_overlap) / (tile_height - min_overlap)) if tile_height < image_height else 1 - ) + num_tiles_x = math.ceil((image_width - min_overlap) / (tile_width - min_overlap)) + num_tiles_y = math.ceil((image_height - min_overlap) / (tile_height - min_overlap)) # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. tiles: list[Tile] = [] From bca237228016db3ddb9674da07f63262a1abd218 Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Tue, 12 Dec 2023 14:02:28 +0000 Subject: [PATCH 03/12] updated comment --- invokeai/backend/tiles/tiles.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 9f5817a4ca..11d0c86c5c 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -191,8 +191,7 @@ def calc_tiles_min_overlap( assert min_overlap < tile_height assert min_overlap < tile_width - # catches the cases when the tile size is larger than the images size and just clips the number of tiles to 1 - + # catches the cases when the tile size is larger than the images size and adjusts the tile size if image_width < tile_width: tile_width = image_width From 612912a6c980d13e905a4a6086a722342fba735f Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Tue, 12 Dec 2023 14:12:22 +0000 Subject: [PATCH 04/12] updated tests with a test for tile > image for calc_tiles_min_overlap() --- tests/backend/tiles/test_tiles.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 0b18f9ed54..114ff4a5e0 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -241,6 +241,28 @@ def test_calc_tiles_min_overlap_not_evenly_divisible(): assert tiles == expected_tiles +def test_calc_tiles_min_overlap_tile_bigger_than_image(): + """Test calc_tiles_min_overlap() behavior when the tile is nigger than the image""" + # Parameters mimic roughly the same output as the original tile generations of the same test name + tiles = calc_tiles_min_overlap( + image_height=1024, + image_width=1024, + tile_height=1408, + tile_width=1408, + min_overlap=128, + ) + + expected_tiles = [ + # single tile + Tile( + coords=TBLR(top=0, bottom=1024, left=0, right=1024), + overlap=TBLR(top=0, bottom=0, left=0, right=0), + ), + ] + + assert tiles == expected_tiles + + @pytest.mark.parametrize( [ "image_height", From 96a717c4ba4e88fc9fb25adfb1ae0ccd31dbf5ef Mon Sep 17 00:00:00 2001 From: skunkworxdark Date: Sun, 17 Dec 2023 15:10:50 +0000 Subject: [PATCH 05/12] In CalculateImageTilesEvenSplitInvocation to have overlap_fraction becomes just overlap. This is now in pixels rather than as a fraction of the tile size. Update calc_tiles_even_split() with the same change. Ensuring Overlap is within allowed size Update even_split tests --- invokeai/app/invocations/tiles.py | 12 ++-- invokeai/backend/tiles/tiles.py | 44 +++++++------- tests/backend/tiles/test_tiles.py | 96 +++++++++++++++---------------- 3 files changed, 75 insertions(+), 77 deletions(-) diff --git a/invokeai/app/invocations/tiles.py b/invokeai/app/invocations/tiles.py index cbf63ab169..e51f891a8d 100644 --- a/invokeai/app/invocations/tiles.py +++ b/invokeai/app/invocations/tiles.py @@ -77,7 +77,7 @@ class CalculateImageTilesInvocation(BaseInvocation): title="Calculate Image Tiles Even Split", tags=["tiles"], category="tiles", - version="1.0.0", + version="1.1.0", classification=Classification.Beta, ) class CalculateImageTilesEvenSplitInvocation(BaseInvocation): @@ -97,11 +97,11 @@ class CalculateImageTilesEvenSplitInvocation(BaseInvocation): ge=1, description="Number of tiles to divide image into on the y axis", ) - overlap_fraction: float = InputField( - default=0.25, + overlap: int = InputField( + default=128, ge=0, - lt=1, - description="Overlap between adjacent tiles as a fraction of the tile's dimensions (0-1)", + multiple_of=8, + description="The overlap, in pixels, between adjacent tiles.", ) def invoke(self, context: InvocationContext) -> CalculateImageTilesOutput: @@ -110,7 +110,7 @@ class CalculateImageTilesEvenSplitInvocation(BaseInvocation): image_width=self.image_width, num_tiles_x=self.num_tiles_x, num_tiles_y=self.num_tiles_y, - overlap_fraction=self.overlap_fraction, + overlap=self.overlap, ) return CalculateImageTilesOutput(tiles=tiles) diff --git a/invokeai/backend/tiles/tiles.py b/invokeai/backend/tiles/tiles.py index 11d0c86c5c..3c400fc87c 100644 --- a/invokeai/backend/tiles/tiles.py +++ b/invokeai/backend/tiles/tiles.py @@ -102,7 +102,7 @@ def calc_tiles_with_overlap( def calc_tiles_even_split( - image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap_fraction: float = 0 + image_height: int, image_width: int, num_tiles_x: int, num_tiles_y: int, overlap: int = 0 ) -> list[Tile]: """Calculate the tile coordinates for a given image shape with the number of tiles requested. @@ -111,31 +111,35 @@ def calc_tiles_even_split( image_width (int): The image width in px. num_x_tiles (int): The number of tile to split the image into on the X-axis. num_y_tiles (int): The number of tile to split the image into on the Y-axis. - overlap_fraction (float, optional): The target overlap as fraction of the tiles size. Defaults to 0. + overlap (int, optional): The overlap between adjacent tiles in pixels. Defaults to 0. Returns: list[Tile]: A list of tiles that cover the image shape. Ordered from left-to-right, top-to-bottom. """ - - # Ensure tile size is divisible by 8 + # Ensure the image is divisible by LATENT_SCALE_FACTOR if image_width % LATENT_SCALE_FACTOR != 0 or image_height % LATENT_SCALE_FACTOR != 0: raise ValueError(f"image size (({image_width}, {image_height})) must be divisible by {LATENT_SCALE_FACTOR}") - # Calculate the overlap size based on the percentage and adjust it to be divisible by 8 (rounding up) - overlap_x = LATENT_SCALE_FACTOR * math.ceil( - int((image_width / num_tiles_x) * overlap_fraction) / LATENT_SCALE_FACTOR - ) - overlap_y = LATENT_SCALE_FACTOR * math.ceil( - int((image_height / num_tiles_y) * overlap_fraction) / LATENT_SCALE_FACTOR - ) - # Calculate the tile size based on the number of tiles and overlap, and ensure it's divisible by 8 (rounding down) - tile_size_x = LATENT_SCALE_FACTOR * math.floor( - ((image_width + overlap_x * (num_tiles_x - 1)) // num_tiles_x) / LATENT_SCALE_FACTOR - ) - tile_size_y = LATENT_SCALE_FACTOR * math.floor( - ((image_height + overlap_y * (num_tiles_y - 1)) // num_tiles_y) / LATENT_SCALE_FACTOR - ) + if num_tiles_x > 1: + # ensure the overlap is not more than the maximum overlap if we only have 1 tile then we dont care about overlap + assert overlap <= image_width - (LATENT_SCALE_FACTOR * (num_tiles_x - 1)) + tile_size_x = LATENT_SCALE_FACTOR * math.floor( + ((image_width + overlap * (num_tiles_x - 1)) // num_tiles_x) / LATENT_SCALE_FACTOR + ) + assert overlap < tile_size_x + else: + tile_size_x = image_width + + if num_tiles_y > 1: + # ensure the overlap is not more than the maximum overlap if we only have 1 tile then we dont care about overlap + assert overlap <= image_height - (LATENT_SCALE_FACTOR * (num_tiles_y - 1)) + tile_size_y = LATENT_SCALE_FACTOR * math.floor( + ((image_height + overlap * (num_tiles_y - 1)) // num_tiles_y) / LATENT_SCALE_FACTOR + ) + assert overlap < tile_size_y + else: + tile_size_y = image_height # tiles[y * num_tiles_x + x] is the tile for the y'th row, x'th column. tiles: list[Tile] = [] @@ -143,7 +147,7 @@ def calc_tiles_even_split( # Calculate tile coordinates. (Ignore overlap values for now.) for tile_idx_y in range(num_tiles_y): # Calculate the top and bottom of the row - top = tile_idx_y * (tile_size_y - overlap_y) + top = tile_idx_y * (tile_size_y - overlap) bottom = min(top + tile_size_y, image_height) # For the last row adjust bottom to be the height of the image if tile_idx_y == num_tiles_y - 1: @@ -151,7 +155,7 @@ def calc_tiles_even_split( for tile_idx_x in range(num_tiles_x): # Calculate the left & right coordinate of each tile - left = tile_idx_x * (tile_size_x - overlap_x) + left = tile_idx_x * (tile_size_x - overlap) right = min(left + tile_size_x, image_width) # For the last tile in the row adjust right to be the width of the image if tile_idx_x == num_tiles_x - 1: diff --git a/tests/backend/tiles/test_tiles.py b/tests/backend/tiles/test_tiles.py index 114ff4a5e0..32c4bd34c8 100644 --- a/tests/backend/tiles/test_tiles.py +++ b/tests/backend/tiles/test_tiles.py @@ -305,9 +305,7 @@ def test_calc_tiles_min_overlap_input_validation( def test_calc_tiles_even_split_single_tile(): """Test calc_tiles_even_split() behavior when a single tile covers the image.""" - tiles = calc_tiles_even_split( - image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap_fraction=0.25 - ) + tiles = calc_tiles_even_split(image_height=512, image_width=1024, num_tiles_x=1, num_tiles_y=1, overlap=64) expected_tiles = [ Tile( @@ -322,36 +320,34 @@ def test_calc_tiles_even_split_single_tile(): def test_calc_tiles_even_split_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image is evenly covered by multiple tiles.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split( - image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap_fraction=0.25 - ) + tiles = calc_tiles_even_split(image_height=576, image_width=1600, num_tiles_x=3, num_tiles_y=2, overlap=64) expected_tiles = [ # Row 0 Tile( - coords=TBLR(top=0, bottom=320, left=0, right=624), - overlap=TBLR(top=0, bottom=72, left=0, right=136), + coords=TBLR(top=0, bottom=320, left=0, right=576), + overlap=TBLR(top=0, bottom=64, left=0, right=64), ), Tile( - coords=TBLR(top=0, bottom=320, left=488, right=1112), - overlap=TBLR(top=0, bottom=72, left=136, right=136), + coords=TBLR(top=0, bottom=320, left=512, right=1088), + overlap=TBLR(top=0, bottom=64, left=64, right=64), ), Tile( - coords=TBLR(top=0, bottom=320, left=976, right=1600), - overlap=TBLR(top=0, bottom=72, left=136, right=0), + coords=TBLR(top=0, bottom=320, left=1024, right=1600), + overlap=TBLR(top=0, bottom=64, left=64, right=0), ), # Row 1 Tile( - coords=TBLR(top=248, bottom=576, left=0, right=624), - overlap=TBLR(top=72, bottom=0, left=0, right=136), + coords=TBLR(top=256, bottom=576, left=0, right=576), + overlap=TBLR(top=64, bottom=0, left=0, right=64), ), Tile( - coords=TBLR(top=248, bottom=576, left=488, right=1112), - overlap=TBLR(top=72, bottom=0, left=136, right=136), + coords=TBLR(top=256, bottom=576, left=512, right=1088), + overlap=TBLR(top=64, bottom=0, left=64, right=64), ), Tile( - coords=TBLR(top=248, bottom=576, left=976, right=1600), - overlap=TBLR(top=72, bottom=0, left=136, right=0), + coords=TBLR(top=256, bottom=576, left=1024, right=1600), + overlap=TBLR(top=64, bottom=0, left=64, right=0), ), ] assert tiles == expected_tiles @@ -360,36 +356,34 @@ def test_calc_tiles_even_split_evenly_divisible(): def test_calc_tiles_even_split_not_evenly_divisible(): """Test calc_tiles_even_split() behavior when the image requires 'uneven' overlaps to achieve proper coverage.""" # Parameters mimic roughly the same output as the original tile generations of the same test name - tiles = calc_tiles_even_split( - image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap_fraction=0.25 - ) + tiles = calc_tiles_even_split(image_height=400, image_width=1200, num_tiles_x=3, num_tiles_y=2, overlap=64) expected_tiles = [ # Row 0 Tile( - coords=TBLR(top=0, bottom=224, left=0, right=464), - overlap=TBLR(top=0, bottom=56, left=0, right=104), + coords=TBLR(top=0, bottom=232, left=0, right=440), + overlap=TBLR(top=0, bottom=64, left=0, right=64), ), Tile( - coords=TBLR(top=0, bottom=224, left=360, right=824), - overlap=TBLR(top=0, bottom=56, left=104, right=104), + coords=TBLR(top=0, bottom=232, left=376, right=816), + overlap=TBLR(top=0, bottom=64, left=64, right=64), ), Tile( - coords=TBLR(top=0, bottom=224, left=720, right=1200), - overlap=TBLR(top=0, bottom=56, left=104, right=0), + coords=TBLR(top=0, bottom=232, left=752, right=1200), + overlap=TBLR(top=0, bottom=64, left=64, right=0), ), # Row 1 Tile( - coords=TBLR(top=168, bottom=400, left=0, right=464), - overlap=TBLR(top=56, bottom=0, left=0, right=104), + coords=TBLR(top=168, bottom=400, left=0, right=440), + overlap=TBLR(top=64, bottom=0, left=0, right=64), ), Tile( - coords=TBLR(top=168, bottom=400, left=360, right=824), - overlap=TBLR(top=56, bottom=0, left=104, right=104), + coords=TBLR(top=168, bottom=400, left=376, right=816), + overlap=TBLR(top=64, bottom=0, left=64, right=64), ), Tile( - coords=TBLR(top=168, bottom=400, left=720, right=1200), - overlap=TBLR(top=56, bottom=0, left=104, right=0), + coords=TBLR(top=168, bottom=400, left=752, right=1200), + overlap=TBLR(top=64, bottom=0, left=64, right=0), ), ] @@ -399,28 +393,26 @@ def test_calc_tiles_even_split_not_evenly_divisible(): def test_calc_tiles_even_split_difficult_size(): """Test calc_tiles_even_split() behavior when the image is a difficult size to spilt evenly and keep div8.""" # Parameters are a difficult size for other tile gen routines to calculate - tiles = calc_tiles_even_split( - image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap_fraction=0.25 - ) + tiles = calc_tiles_even_split(image_height=1000, image_width=1000, num_tiles_x=2, num_tiles_y=2, overlap=64) expected_tiles = [ # Row 0 Tile( - coords=TBLR(top=0, bottom=560, left=0, right=560), - overlap=TBLR(top=0, bottom=128, left=0, right=128), + coords=TBLR(top=0, bottom=528, left=0, right=528), + overlap=TBLR(top=0, bottom=64, left=0, right=64), ), Tile( - coords=TBLR(top=0, bottom=560, left=432, right=1000), - overlap=TBLR(top=0, bottom=128, left=128, right=0), + coords=TBLR(top=0, bottom=528, left=464, right=1000), + overlap=TBLR(top=0, bottom=64, left=64, right=0), ), # Row 1 Tile( - coords=TBLR(top=432, bottom=1000, left=0, right=560), - overlap=TBLR(top=128, bottom=0, left=0, right=128), + coords=TBLR(top=464, bottom=1000, left=0, right=528), + overlap=TBLR(top=64, bottom=0, left=0, right=64), ), Tile( - coords=TBLR(top=432, bottom=1000, left=432, right=1000), - overlap=TBLR(top=128, bottom=0, left=128, right=0), + coords=TBLR(top=464, bottom=1000, left=464, right=1000), + overlap=TBLR(top=64, bottom=0, left=64, right=0), ), ] @@ -428,11 +420,13 @@ def test_calc_tiles_even_split_difficult_size(): @pytest.mark.parametrize( - ["image_height", "image_width", "num_tiles_x", "num_tiles_y", "overlap_fraction", "raises"], + ["image_height", "image_width", "num_tiles_x", "num_tiles_y", "overlap", "raises"], [ - (128, 128, 1, 1, 0.25, False), # OK + (128, 128, 1, 1, 127, False), # OK (128, 128, 1, 1, 0, False), # OK - (128, 128, 2, 1, 0, False), # OK + (128, 128, 2, 2, 0, False), # OK + (128, 128, 2, 1, 120, True), # overlap equals tile_height. + (128, 128, 1, 2, 120, True), # overlap equals tile_width. (127, 127, 1, 1, 0, True), # image size must be dividable by 8 ], ) @@ -441,15 +435,15 @@ def test_calc_tiles_even_split_input_validation( image_width: int, num_tiles_x: int, num_tiles_y: int, - overlap_fraction: float, + overlap: int, raises: bool, ): """Test that calc_tiles_even_split() raises an exception if the inputs are invalid.""" if raises: - with pytest.raises(ValueError): - calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap_fraction) + with pytest.raises((AssertionError, ValueError)): + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) else: - calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap_fraction) + calc_tiles_even_split(image_height, image_width, num_tiles_x, num_tiles_y, overlap) ############################################# From 74ea592d022ce0cd2c0f2e3fb0e0e475959cedbb Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 17 Dec 2023 14:16:45 -0500 Subject: [PATCH 06/12] tag model manager v2 api as unstable --- invokeai/app/api/routers/model_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/app/api/routers/model_records.py b/invokeai/app/api/routers/model_records.py index 934a7d15b3..f71f4433f4 100644 --- a/invokeai/app/api/routers/model_records.py +++ b/invokeai/app/api/routers/model_records.py @@ -26,7 +26,7 @@ from invokeai.backend.model_manager.config import ( from ..dependencies import ApiDependencies -model_records_router = APIRouter(prefix="/v1/model/record", tags=["model_manager_v2"]) +model_records_router = APIRouter(prefix="/v1/model/record", tags=["model_manager_v2_unstable"]) class ModelsList(BaseModel): From cb698ff1fb2d6d55470fece8abf886a927afe72c Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Fri, 15 Dec 2023 17:56:49 -0500 Subject: [PATCH 07/12] Update model_probe to work with diffuser-format SD TI embeddings. --- invokeai/backend/model_management/model_probe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/backend/model_management/model_probe.py b/invokeai/backend/model_management/model_probe.py index f0af93294f..d312c6a0b4 100644 --- a/invokeai/backend/model_management/model_probe.py +++ b/invokeai/backend/model_management/model_probe.py @@ -389,7 +389,7 @@ class TextualInversionCheckpointProbe(CheckpointProbeBase): elif "clip_g" in checkpoint: token_dim = checkpoint["clip_g"].shape[-1] else: - token_dim = list(checkpoint.values())[0].shape[0] + token_dim = list(checkpoint.values())[0].shape[-1] if token_dim == 768: return BaseModelType.StableDiffusion1 elif token_dim == 1024: From 42be78d32839ac9c420bdb35d5b3b7375fcde552 Mon Sep 17 00:00:00 2001 From: Riccardo Giovanetti Date: Mon, 18 Dec 2023 11:09:23 +0000 Subject: [PATCH 08/12] translationBot(ui): update translation (Italian) Currently translated at 97.2% (1327 of 1365 strings) Co-authored-by: Riccardo Giovanetti Translate-URL: https://hosted.weblate.org/projects/invokeai/web-ui/it/ Translation: InvokeAI/Web UI --- invokeai/frontend/web/public/locales/it.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/invokeai/frontend/web/public/locales/it.json b/invokeai/frontend/web/public/locales/it.json index b816858bd8..6f912d27c0 100644 --- a/invokeai/frontend/web/public/locales/it.json +++ b/invokeai/frontend/web/public/locales/it.json @@ -1109,7 +1109,10 @@ "deletedInvalidEdge": "Eliminata connessione non valida {{source}} -> {{target}}", "unknownInput": "Input sconosciuto: {{name}}", "prototypeDesc": "Questa invocazione è un prototipo. Potrebbe subire modifiche sostanziali durante gli aggiornamenti dell'app e potrebbe essere rimossa in qualsiasi momento.", - "betaDesc": "Questa invocazione è in versione beta. Fino a quando non sarà stabile, potrebbe subire modifiche importanti durante gli aggiornamenti dell'app. Abbiamo intenzione di supportare questa invocazione a lungo termine." + "betaDesc": "Questa invocazione è in versione beta. Fino a quando non sarà stabile, potrebbe subire modifiche importanti durante gli aggiornamenti dell'app. Abbiamo intenzione di supportare questa invocazione a lungo termine.", + "newWorkflow": "Nuovo flusso di lavoro", + "newWorkflowDesc": "Creare un nuovo flusso di lavoro?", + "newWorkflowDesc2": "Il flusso di lavoro attuale presenta modifiche non salvate." }, "boards": { "autoAddBoard": "Aggiungi automaticamente bacheca", @@ -1629,7 +1632,10 @@ "deleteWorkflow": "Elimina flusso di lavoro", "workflows": "Flussi di lavoro", "noDescription": "Nessuna descrizione", - "userWorkflows": "I miei flussi di lavoro" + "userWorkflows": "I miei flussi di lavoro", + "newWorkflowCreated": "Nuovo flusso di lavoro creato", + "downloadWorkflow": "Salva su file", + "uploadWorkflow": "Carica da file" }, "app": { "storeNotInitialized": "Il negozio non è inizializzato" From 16b7246412a42a043ced1cfd3d2323bc6a297124 Mon Sep 17 00:00:00 2001 From: Millun Atluri Date: Tue, 19 Dec 2023 09:30:40 +1100 Subject: [PATCH 09/12] (feat) updater installs from PyPi instead of GitHub releases --- invokeai/frontend/install/invokeai_update.py | 57 ++++++++++---------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/invokeai/frontend/install/invokeai_update.py b/invokeai/frontend/install/invokeai_update.py index 551f2acdf2..4ec4421ec8 100644 --- a/invokeai/frontend/install/invokeai_update.py +++ b/invokeai/frontend/install/invokeai_update.py @@ -8,6 +8,7 @@ import platform import pkg_resources import psutil import requests +from distutils.version import LooseVersion from rich import box, print from rich.console import Console, group from rich.panel import Panel @@ -31,10 +32,6 @@ else: console = Console(style=Style(color="grey74", bgcolor="grey19")) -def get_versions() -> dict: - return requests.get(url=INVOKE_AI_REL).json() - - def invokeai_is_running() -> bool: for p in psutil.process_iter(): try: @@ -50,6 +47,21 @@ def invokeai_is_running() -> bool: return False +def get_pypi_versions(): + url = f"https://pypi.org/pypi/invokeai/json" + try: + data = requests.get(url).json() + except: + raise Exception("Unable to fetch version information from PyPi") + + versions = list(data["releases"].keys()) + versions.sort(key=LooseVersion, reverse=True) + latest_version = [v for v in versions if 'rc' not in v][0] + latest_release_candidate = [v for v in versions if 'rc' in v][0] + return latest_version, latest_release_candidate + + + def welcome(latest_release: str, latest_prerelease: str): @group() def text(): @@ -63,8 +75,7 @@ def welcome(latest_release: str, latest_prerelease: str): yield "[bold yellow]Options:" yield f"""[1] Update to the latest [bold]official release[/bold] ([italic]{latest_release}[/italic]) [2] Update to the latest [bold]pre-release[/bold] (may be buggy; caveat emptor!) ([italic]{latest_prerelease}[/italic]) -[3] Manually enter the [bold]tag name[/bold] for the version you wish to update to -[4] Manually enter the [bold]branch name[/bold] for the version you wish to update to""" +[3] Manually enter the [bold]version[/bold] you wish to update to""" console.rule() print( @@ -91,45 +102,37 @@ def get_extras(): return extras + + + def main(): - versions = get_versions() - released_versions = [x for x in versions if not (x["draft"] or x["prerelease"])] - prerelease_versions = [x for x in versions if not x["draft"] and x["prerelease"]] - latest_release = released_versions[0]["tag_name"] if len(released_versions) else None - latest_prerelease = prerelease_versions[0]["tag_name"] if len(prerelease_versions) else None if invokeai_is_running(): print(":exclamation: [bold red]Please terminate all running instances of InvokeAI before updating.[/red bold]") input("Press any key to continue...") return + latest_release, latest_prerelease = get_pypi_versions() + welcome(latest_release, latest_prerelease) - tag = None - branch = None - release = None - choice = Prompt.ask("Choice:", choices=["1", "2", "3", "4"], default="1") + + release = latest_release + choice = Prompt.ask("Choice:", choices=["1", "2", "3"], default="1") if choice == "1": release = latest_release elif choice == "2": release = latest_prerelease elif choice == "3": - while not tag: - tag = Prompt.ask("Enter an InvokeAI tag name") - elif choice == "4": - while not branch: - branch = Prompt.ask("Enter an InvokeAI branch name") + release = Prompt.ask("Enter an InvokeAI version name") extras = get_extras() - print(f":crossed_fingers: Upgrading to [yellow]{tag or release or branch}[/yellow]") - if release: - cmd = f'pip install "invokeai{extras} @ {INVOKE_AI_SRC}/{release}.zip" --use-pep517 --upgrade' - elif tag: - cmd = f'pip install "invokeai{extras} @ {INVOKE_AI_TAG}/{tag}.zip" --use-pep517 --upgrade' - else: - cmd = f'pip install "invokeai{extras} @ {INVOKE_AI_BRANCH}/{branch}.zip" --use-pep517 --upgrade' + print(f":crossed_fingers: Upgrading to [yellow]{release}[/yellow]") + cmd = f'pip install "invokeai{extras}=={release}" --use-pep517 --upgrade' + + print("") print("") if os.system(cmd) == 0: From cd3111c32460c170b4e46282cada1ab9846463bf Mon Sep 17 00:00:00 2001 From: Millun Atluri Date: Tue, 19 Dec 2023 09:58:10 +1100 Subject: [PATCH 10/12] fix ruff errors --- invokeai/frontend/install/invokeai_update.py | 21 +++++++------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/invokeai/frontend/install/invokeai_update.py b/invokeai/frontend/install/invokeai_update.py index 4ec4421ec8..cde9d8fd1e 100644 --- a/invokeai/frontend/install/invokeai_update.py +++ b/invokeai/frontend/install/invokeai_update.py @@ -4,11 +4,11 @@ pip install . """ import os import platform +from distutils.version import LooseVersion import pkg_resources import psutil import requests -from distutils.version import LooseVersion from rich import box, print from rich.console import Console, group from rich.panel import Panel @@ -48,20 +48,19 @@ def invokeai_is_running() -> bool: def get_pypi_versions(): - url = f"https://pypi.org/pypi/invokeai/json" - try: + url = "https://pypi.org/pypi/invokeai/json" + try: data = requests.get(url).json() - except: + except Exception: raise Exception("Unable to fetch version information from PyPi") versions = list(data["releases"].keys()) versions.sort(key=LooseVersion, reverse=True) - latest_version = [v for v in versions if 'rc' not in v][0] - latest_release_candidate = [v for v in versions if 'rc' in v][0] + latest_version = [v for v in versions if "rc" not in v][0] + latest_release_candidate = [v for v in versions if "rc" in v][0] return latest_version, latest_release_candidate - def welcome(latest_release: str, latest_prerelease: str): @group() def text(): @@ -102,11 +101,7 @@ def get_extras(): return extras - - - def main(): - if invokeai_is_running(): print(":exclamation: [bold red]Please terminate all running instances of InvokeAI before updating.[/red bold]") input("Press any key to continue...") @@ -116,7 +111,6 @@ def main(): welcome(latest_release, latest_prerelease) - release = latest_release choice = Prompt.ask("Choice:", choices=["1", "2", "3"], default="1") @@ -125,14 +119,13 @@ def main(): elif choice == "2": release = latest_prerelease elif choice == "3": - release = Prompt.ask("Enter an InvokeAI version name") + release = Prompt.ask("Enter an InvokeAI version name") extras = get_extras() print(f":crossed_fingers: Upgrading to [yellow]{release}[/yellow]") cmd = f'pip install "invokeai{extras}=={release}" --use-pep517 --upgrade' - print("") print("") if os.system(cmd) == 0: From 2f438431bd00dcae9336df8769554a79d0c3ab7a Mon Sep 17 00:00:00 2001 From: Millun Atluri Date: Tue, 19 Dec 2023 11:04:21 +1100 Subject: [PATCH 11/12] (fix) update logic for installing specific version --- invokeai/frontend/install/invokeai_update.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/invokeai/frontend/install/invokeai_update.py b/invokeai/frontend/install/invokeai_update.py index cde9d8fd1e..3e453a90fd 100644 --- a/invokeai/frontend/install/invokeai_update.py +++ b/invokeai/frontend/install/invokeai_update.py @@ -58,7 +58,7 @@ def get_pypi_versions(): versions.sort(key=LooseVersion, reverse=True) latest_version = [v for v in versions if "rc" not in v][0] latest_release_candidate = [v for v in versions if "rc" in v][0] - return latest_version, latest_release_candidate + return latest_version, latest_release_candidate, versions def welcome(latest_release: str, latest_prerelease: str): @@ -107,7 +107,7 @@ def main(): input("Press any key to continue...") return - latest_release, latest_prerelease = get_pypi_versions() + latest_release, latest_prerelease, versions = get_pypi_versions() welcome(latest_release, latest_prerelease) @@ -119,7 +119,12 @@ def main(): elif choice == "2": release = latest_prerelease elif choice == "3": - release = Prompt.ask("Enter an InvokeAI version name") + while True: + release = Prompt.ask("Enter an InvokeAI version") + release.strip() + if release in versions: + break + print(f":exclamation: [bold red]'{release}' is not a recognized InvokeAI release.[/red bold]") extras = get_extras() From fa3f1b6e415bc7bccd28f2ff7cf27e933ab7c3c5 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 19 Dec 2023 17:01:47 -0500 Subject: [PATCH 12/12] [Feat] reimport model config records after schema migration (#5281) * add code to repopulate model config records after schema update * reformat for ruff * migrate model records using db cursor rather than the ModelRecordConfigService * ruff fixes * tweak exception reporting * fix: build frontend in pypi-release workflow This was missing, resulting in the 3.5.0rc1 having no frontend. * fix: use node 18, set working directory - Node 20 has a problem with `pnpm`; set it to Node 18 - Set the working directory for the frontend commands * Don't copy extraneous paths into installer .zip * feat(installer): delete frontend build after creating installer This prevents an empty `dist/` from breaking the app on startup. * feat: add python dist as release artifact, as input to enable publish to pypi - The release workflow never runs automatically. It must be manually kicked off. - The release workflow has an input. When running it from the GH actions UI, you will see a "Publish build on PyPi" prompt. If this value is "true", the workflow will upload the build to PyPi, releasing it. If this is anything else (e.g. "false", the default), the workflow will build but not upload to PyPi. - The `dist/` folder (where the python package is built) is uploaded as a workflow artifact as a zip file. This can be downloaded and inspected. This allows "dry" runs of the workflow. - The workflow job and some steps have been renamed to clarify what they do * translationBot(ui): update translation files Updated by "Cleanup translation files" hook in Weblate. Co-authored-by: Hosted Weblate Translate-URL: https://hosted.weblate.org/projects/invokeai/web-ui/ Translation: InvokeAI/Web UI * freeze yaml migration logic at upgrade to 3.5 * moved migration code to migration_3 --------- Co-authored-by: Lincoln Stein Co-authored-by: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Co-authored-by: Hosted Weblate --- .../app/services/shared/sqlite/sqlite_util.py | 2 + .../sqlite_migrator/migrations/migration_2.py | 8 ++ .../sqlite_migrator/migrations/migration_3.py | 75 ++++++++++++++++ .../migrations/util/migrate_yaml_config_1.py} | 85 ++++++++++++++----- pyproject.toml | 1 - 5 files changed, 148 insertions(+), 23 deletions(-) create mode 100644 invokeai/app/services/shared/sqlite_migrator/migrations/migration_3.py rename invokeai/{backend/model_manager/migrate_to_db.py => app/services/shared/sqlite_migrator/migrations/util/migrate_yaml_config_1.py} (56%) diff --git a/invokeai/app/services/shared/sqlite/sqlite_util.py b/invokeai/app/services/shared/sqlite/sqlite_util.py index 83a42917a7..43c3edd5f5 100644 --- a/invokeai/app/services/shared/sqlite/sqlite_util.py +++ b/invokeai/app/services/shared/sqlite/sqlite_util.py @@ -5,6 +5,7 @@ from invokeai.app.services.image_files.image_files_base import ImageFileStorageB from invokeai.app.services.shared.sqlite.sqlite_database import SqliteDatabase from invokeai.app.services.shared.sqlite_migrator.migrations.migration_1 import build_migration_1 from invokeai.app.services.shared.sqlite_migrator.migrations.migration_2 import build_migration_2 +from invokeai.app.services.shared.sqlite_migrator.migrations.migration_3 import build_migration_3 from invokeai.app.services.shared.sqlite_migrator.sqlite_migrator_impl import SqliteMigrator @@ -27,6 +28,7 @@ def init_db(config: InvokeAIAppConfig, logger: Logger, image_files: ImageFileSto migrator = SqliteMigrator(db=db) migrator.register_migration(build_migration_1()) migrator.register_migration(build_migration_2(image_files=image_files, logger=logger)) + migrator.register_migration(build_migration_3()) migrator.run_migrations() return db diff --git a/invokeai/app/services/shared/sqlite_migrator/migrations/migration_2.py b/invokeai/app/services/shared/sqlite_migrator/migrations/migration_2.py index 9b9dedcc58..99922e2fc1 100644 --- a/invokeai/app/services/shared/sqlite_migrator/migrations/migration_2.py +++ b/invokeai/app/services/shared/sqlite_migrator/migrations/migration_2.py @@ -11,6 +11,8 @@ from invokeai.app.services.workflow_records.workflow_records_common import ( UnsafeWorkflowWithVersionValidator, ) +from .util.migrate_yaml_config_1 import MigrateModelYamlToDb1 + class Migration2Callback: def __init__(self, image_files: ImageFileStorageBase, logger: Logger): @@ -24,6 +26,7 @@ class Migration2Callback: self._add_workflow_library(cursor) self._drop_model_manager_metadata(cursor) self._recreate_model_config(cursor) + self._migrate_model_config_records(cursor) self._migrate_embedded_workflows(cursor) def _add_images_has_workflow(self, cursor: sqlite3.Cursor) -> None: @@ -131,6 +134,11 @@ class Migration2Callback: """ ) + def _migrate_model_config_records(self, cursor: sqlite3.Cursor) -> None: + """After updating the model config table, we repopulate it.""" + model_record_migrator = MigrateModelYamlToDb1(cursor) + model_record_migrator.migrate() + def _migrate_embedded_workflows(self, cursor: sqlite3.Cursor) -> None: """ In the v3.5.0 release, InvokeAI changed how it handles embedded workflows. The `images` table in diff --git a/invokeai/app/services/shared/sqlite_migrator/migrations/migration_3.py b/invokeai/app/services/shared/sqlite_migrator/migrations/migration_3.py new file mode 100644 index 0000000000..2ffef13dd4 --- /dev/null +++ b/invokeai/app/services/shared/sqlite_migrator/migrations/migration_3.py @@ -0,0 +1,75 @@ +import sqlite3 + +from invokeai.app.services.shared.sqlite_migrator.sqlite_migrator_common import Migration + +from .util.migrate_yaml_config_1 import MigrateModelYamlToDb1 + + +class Migration3Callback: + def __init__(self) -> None: + pass + + def __call__(self, cursor: sqlite3.Cursor) -> None: + self._drop_model_manager_metadata(cursor) + self._recreate_model_config(cursor) + self._migrate_model_config_records(cursor) + + def _drop_model_manager_metadata(self, cursor: sqlite3.Cursor) -> None: + """Drops the `model_manager_metadata` table.""" + cursor.execute("DROP TABLE IF EXISTS model_manager_metadata;") + + def _recreate_model_config(self, cursor: sqlite3.Cursor) -> None: + """ + Drops the `model_config` table, recreating it. + + In 3.4.0, this table used explicit columns but was changed to use json_extract 3.5.0. + + Because this table is not used in production, we are able to simply drop it and recreate it. + """ + + cursor.execute("DROP TABLE IF EXISTS model_config;") + + cursor.execute( + """--sql + CREATE TABLE IF NOT EXISTS model_config ( + id TEXT NOT NULL PRIMARY KEY, + -- The next 3 fields are enums in python, unrestricted string here + base TEXT GENERATED ALWAYS as (json_extract(config, '$.base')) VIRTUAL NOT NULL, + type TEXT GENERATED ALWAYS as (json_extract(config, '$.type')) VIRTUAL NOT NULL, + name TEXT GENERATED ALWAYS as (json_extract(config, '$.name')) VIRTUAL NOT NULL, + path TEXT GENERATED ALWAYS as (json_extract(config, '$.path')) VIRTUAL NOT NULL, + format TEXT GENERATED ALWAYS as (json_extract(config, '$.format')) VIRTUAL NOT NULL, + original_hash TEXT, -- could be null + -- Serialized JSON representation of the whole config object, + -- which will contain additional fields from subclasses + config TEXT NOT NULL, + created_at DATETIME NOT NULL DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), + -- Updated via trigger + updated_at DATETIME NOT NULL DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), + -- unique constraint on combo of name, base and type + UNIQUE(name, base, type) + ); + """ + ) + + def _migrate_model_config_records(self, cursor: sqlite3.Cursor) -> None: + """After updating the model config table, we repopulate it.""" + model_record_migrator = MigrateModelYamlToDb1(cursor) + model_record_migrator.migrate() + + +def build_migration_3() -> Migration: + """ + Build the migration from database version 2 to 3. + + This migration does the following: + - Drops the `model_config` table, recreating it + - Migrates data from `models.yaml` into the `model_config` table + """ + migration_3 = Migration( + from_version=2, + to_version=3, + callback=Migration3Callback(), + ) + + return migration_3 diff --git a/invokeai/backend/model_manager/migrate_to_db.py b/invokeai/app/services/shared/sqlite_migrator/migrations/util/migrate_yaml_config_1.py similarity index 56% rename from invokeai/backend/model_manager/migrate_to_db.py rename to invokeai/app/services/shared/sqlite_migrator/migrations/util/migrate_yaml_config_1.py index e68a2eab36..f2476ed0f6 100644 --- a/invokeai/backend/model_manager/migrate_to_db.py +++ b/invokeai/app/services/shared/sqlite_migrator/migrations/util/migrate_yaml_config_1.py @@ -1,8 +1,12 @@ # Copyright (c) 2023 Lincoln D. Stein """Migrate from the InvokeAI v2 models.yaml format to the v3 sqlite format.""" +import json +import sqlite3 from hashlib import sha1 from logging import Logger +from pathlib import Path +from typing import Optional from omegaconf import DictConfig, OmegaConf from pydantic import TypeAdapter @@ -10,13 +14,12 @@ from pydantic import TypeAdapter from invokeai.app.services.config import InvokeAIAppConfig from invokeai.app.services.model_records import ( DuplicateModelException, - ModelRecordServiceSQL, UnknownModelException, ) -from invokeai.app.services.shared.sqlite.sqlite_database import SqliteDatabase from invokeai.backend.model_manager.config import ( AnyModelConfig, BaseModelType, + ModelConfigFactory, ModelType, ) from invokeai.backend.model_manager.hash import FastModelHash @@ -25,9 +28,9 @@ from invokeai.backend.util.logging import InvokeAILogger ModelsValidator = TypeAdapter(AnyModelConfig) -class MigrateModelYamlToDb: +class MigrateModelYamlToDb1: """ - Migrate the InvokeAI models.yaml format (VERSION 3.0.0) to SQL3 database format (VERSION 3.2.0) + Migrate the InvokeAI models.yaml format (VERSION 3.0.0) to SQL3 database format (VERSION 3.5.0). The class has one externally useful method, migrate(), which scans the currently models.yaml file and imports all its entries into invokeai.db. @@ -41,17 +44,13 @@ class MigrateModelYamlToDb: config: InvokeAIAppConfig logger: Logger + cursor: sqlite3.Cursor - def __init__(self) -> None: + def __init__(self, cursor: sqlite3.Cursor = None) -> None: self.config = InvokeAIAppConfig.get_config() self.config.parse_args() self.logger = InvokeAILogger.get_logger() - - def get_db(self) -> ModelRecordServiceSQL: - """Fetch the sqlite3 database for this installation.""" - db_path = None if self.config.use_memory_db else self.config.db_path - db = SqliteDatabase(db_path=db_path, logger=self.logger, verbose=self.config.log_sql) - return ModelRecordServiceSQL(db) + self.cursor = cursor def get_yaml(self) -> DictConfig: """Fetch the models.yaml DictConfig for this installation.""" @@ -62,8 +61,10 @@ class MigrateModelYamlToDb: def migrate(self) -> None: """Do the migration from models.yaml to invokeai.db.""" - db = self.get_db() - yaml = self.get_yaml() + try: + yaml = self.get_yaml() + except OSError: + return for model_key, stanza in yaml.items(): if model_key == "__metadata__": @@ -86,22 +87,62 @@ class MigrateModelYamlToDb: new_config: AnyModelConfig = ModelsValidator.validate_python(stanza) # type: ignore # see https://github.com/pydantic/pydantic/discussions/7094 try: - if original_record := db.search_by_path(stanza.path): - key = original_record[0].key + if original_record := self._search_by_path(stanza.path): + key = original_record.key self.logger.info(f"Updating model {model_name} with information from models.yaml using key {key}") - db.update_model(key, new_config) + self._update_model(key, new_config) else: self.logger.info(f"Adding model {model_name} with key {model_key}") - db.add_model(new_key, new_config) + self._add_model(new_key, new_config) except DuplicateModelException: self.logger.warning(f"Model {model_name} is already in the database") except UnknownModelException: self.logger.warning(f"Model at {stanza.path} could not be found in database") + def _search_by_path(self, path: Path) -> Optional[AnyModelConfig]: + self.cursor.execute( + """--sql + SELECT config FROM model_config + WHERE path=?; + """, + (str(path),), + ) + results = [ModelConfigFactory.make_config(json.loads(x[0])) for x in self.cursor.fetchall()] + return results[0] if results else None -def main(): - MigrateModelYamlToDb().migrate() + def _update_model(self, key: str, config: AnyModelConfig) -> None: + record = ModelConfigFactory.make_config(config, key=key) # ensure it is a valid config obect + json_serialized = record.model_dump_json() # and turn it into a json string. + self.cursor.execute( + """--sql + UPDATE model_config + SET + config=? + WHERE id=?; + """, + (json_serialized, key), + ) + if self.cursor.rowcount == 0: + raise UnknownModelException("model not found") - -if __name__ == "__main__": - main() + def _add_model(self, key: str, config: AnyModelConfig) -> None: + record = ModelConfigFactory.make_config(config, key=key) # ensure it is a valid config obect. + json_serialized = record.model_dump_json() # and turn it into a json string. + try: + self.cursor.execute( + """--sql + INSERT INTO model_config ( + id, + original_hash, + config + ) + VALUES (?,?,?); + """, + ( + key, + record.original_hash, + json_serialized, + ), + ) + except sqlite3.IntegrityError as exc: + raise DuplicateModelException(f"{record.name}: model is already in database") from exc diff --git a/pyproject.toml b/pyproject.toml index 0b8d258e7d..98018dc7cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,7 +138,6 @@ dependencies = [ "invokeai-node-web" = "invokeai.app.api_app:invoke_api" "invokeai-import-images" = "invokeai.frontend.install.import_images:main" "invokeai-db-maintenance" = "invokeai.backend.util.db_maintenance:main" -"invokeai-migrate-models-to-db" = "invokeai.backend.model_manager.migrate_to_db:main" [project.urls] "Homepage" = "https://invoke-ai.github.io/InvokeAI/"