From 83f75750a9979f2a25bf55a749b0a9ff49304ebf Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:19:24 -0400 Subject: [PATCH 01/10] add techjedi's import script, with some filecompletion tweaks --- invokeai/frontend/install/import_images.py | 739 +++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 740 insertions(+) create mode 100644 invokeai/frontend/install/import_images.py diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py new file mode 100644 index 0000000000..1870fafbf8 --- /dev/null +++ b/invokeai/frontend/install/import_images.py @@ -0,0 +1,739 @@ +# Copyright (c) 2023 - The InvokeAI Team +# Author: techjedi + +# pylint: disable=line-too-long +# pylint: disable=broad-exception-caught +"""Script to import images into the new database system for 3.0.0""" + +import os +import datetime +import shutil +import locale +import sqlite3 +import json +import glob +import re +import uuid +import yaml +import PIL +import PIL.ImageOps +import PIL.PngImagePlugin + +from pathlib import Path +from prompt_toolkit import prompt +from prompt_toolkit.shortcuts import yes_no_dialog, message_dialog, input_dialog, button_dialog +from prompt_toolkit.completion import PathCompleter +from prompt_toolkit.key_binding import KeyBindings + +from invokeai.app.services.config import InvokeAIAppConfig +app_config = InvokeAIAppConfig.get_config() + +bindings = KeyBindings() +@bindings.add('c-c') +def _(event): + raise KeyboardInterrupt + +# release notes +# "Use All" with size dimensions not selectable in the UI will not load dimensions + +class Config: + """Configuration loader.""" + def __init__(self): + pass + + TIMESTAMP_STRING= datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + + INVOKE_DIRNAME = "invokeai" + YAML_FILENAME = "invokeai.yaml" + DATABASE_FILENAME = "invokeai.db" + + database_path = None + database_backup_dir = None + outputs_path = None + thumbnail_path = None + + def find_and_load(self): + """find the yaml config file and load""" + root = app_config.root_path + if not self.confirm_and_load(os.path.abspath(root)): + print ("\r\nSpecify custom database and outputs paths:") + self.confirm_and_load_from_user() + + self.database_backup_dir = os.path.join(os.path.dirname(self.database_path),"backup") + self.thumbnail_path = os.path.join(self.outputs_path,"thumbnails") + + def confirm_and_load(self, invoke_root): + """Validates a yaml path exists, confirms the user wants to use it and loads config.""" + yaml_path = os.path.join(invoke_root,self.YAML_FILENAME) + if os.path.exists(yaml_path): + db_dir, outdir = self.load_paths_from_yaml(yaml_path) + if os.path.isabs(db_dir): + database_path = os.path.join(db_dir,self.DATABASE_FILENAME) + else: + database_path = os.path.join(invoke_root,db_dir,self.DATABASE_FILENAME) + + if os.path.isabs(outdir): + outputs_path = os.path.join(outdir, "images") + else: + outputs_path = os.path.join(invoke_root, outdir, "images") + + db_exists = os.path.exists(database_path) + outdir_exists = os.path.exists(outputs_path) + + text = f"Found {self.YAML_FILENAME} file at {yaml_path}:" + text += f"\n Database : {database_path}" + text += f"\n Outputs : {outputs_path}" + text += "\n\nUse these paths for import (yes) or choose different ones (no) [Yn]: " + + if db_exists and outdir_exists: + if (prompt(text).strip() or 'Y').upper().startswith('Y'): + self.database_path = database_path + self.outputs_path = outputs_path + return True + else: + return False + else: + print(" Invalid: One or more paths in this config did not exist and cannot be used.") + + else: + message_dialog( + title="Path not found", + text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified." + ).run() + return False + + def confirm_and_load_from_user(self): + default = '' + while True: + database_path = os.path.expanduser( + prompt( + "Database: Specify absolute path to the database to import into: ", + completer=PathCompleter(expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith(('.db'))), + default = default, + )) + if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): + break + default = database_path + '/' if Path(database_path).is_dir() else database_path + + default = '' + while True: + outputs_path = os.path.expanduser( + prompt( + "Outputs: Specify absolute path to outputs/images directory to import into: ", + completer = PathCompleter(expanduser=True, only_directories=True), + default = default, + )) + + if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): + break + default = outputs_path + '/' if Path(outputs_path).is_dir() else outputs_path + + self.database_path = database_path + self.outputs_path = outputs_path + + return + + def load_paths_from_yaml(self, yaml_path): + """Load an Invoke AI yaml file and get the database and outputs paths.""" + try: + with open(yaml_path, 'rt', encoding=locale.getpreferredencoding()) as file: + yamlinfo = yaml.safe_load(file) + db_dir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("db_dir", None) + outdir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("outdir", None) + return db_dir, outdir + except Exception: + print(f"Failed to load paths from yaml file! {yaml_path}!") + return None, None + +class ImportStats: + """DTO for tracking work progress.""" + def __init__(self): + pass + + time_start = datetime.datetime.utcnow() + count_source_files = 0 + count_skipped_file_exists = 0 + count_skipped_db_exists = 0 + count_imported = 0 + count_imported_by_version = {} + count_file_errors = 0 + + @staticmethod + def get_elapsed_time_string(): + """Get a friendly time string for the time elapsed since processing start.""" + time_now = datetime.datetime.utcnow() + total_seconds = (time_now - ImportStats.time_start).total_seconds() + hours = int((total_seconds) / 3600) + minutes = int(((total_seconds) % 3600) / 60) + seconds = total_seconds % 60 + out_str = f"{hours} hour(s) -" if hours > 0 else "" + out_str += f"{minutes} minute(s) -" if minutes > 0 else "" + out_str += f"{seconds:.2f} second(s)" + return out_str + +class InvokeAIMetadata: + """DTO for core Invoke AI generation properties parsed from metadata.""" + def __init__(self): + pass + + def __str__(self): + formatted_str = f"{self.generation_mode}~{self.steps}~{self.cfg_scale}~{self.model_name}~{self.scheduler}~{self.seed}~{self.width}~{self.height}~{self.rand_device}~{self.strength}~{self.init_image}" + formatted_str += f"\r\npositive_prompt: {self.positive_prompt}" + formatted_str += f"\r\nnegative_prompt: {self.negative_prompt}" + return formatted_str + + generation_mode = None + steps = None + cfg_scale = None + model_name = None + scheduler = None + seed = None + width = None + height = None + rand_device = None + strength = None + init_image = None + positive_prompt = None + negative_prompt = None + imported_app_version = None + + def to_json(self): + """Convert the active instance to json format.""" + prop_dict = {} + prop_dict["generation_mode"] = self.generation_mode + prop_dict["positive_prompt"] = self.positive_prompt + prop_dict["negative_prompt"] = self.negative_prompt + prop_dict["width"] = self.width + prop_dict["height"] = self.height + prop_dict["seed"] = self.seed + prop_dict["rand_device"] = self.rand_device + prop_dict["cfg_scale"] = self.cfg_scale + prop_dict["steps"] = self.steps + prop_dict["scheduler"] = self.scheduler + prop_dict["clip_skip"] = 0 + prop_dict["model"] = {} + prop_dict["model"]["model_name"] = self.model_name + prop_dict["model"]["base_model"] = None + prop_dict["controlnets"] = [] + prop_dict["loras"] = [] + prop_dict["vae"] = None + prop_dict["strength"] = self.strength + prop_dict["init_image"] = self.init_image + prop_dict["positive_style_prompt"] = None + prop_dict["negative_style_prompt"] = None + prop_dict["refiner_model"] = None + prop_dict["refiner_cfg_scale"] = None + prop_dict["refiner_steps"] = None + prop_dict["refiner_scheduler"] = None + prop_dict["refiner_aesthetic_store"] = None + prop_dict["refiner_start"] = None + prop_dict["imported_app_version"] = self.imported_app_version + + return json.dumps(prop_dict) + + +class InvokeAIMetadataParser: + """Parses strings with json data to find Invoke AI core metadata properties.""" + def __init__(self): + pass + + def parse_meta_tag_dream(self, dream_string): + """Take as input an png metadata json node for the 'dream' field variant from prior to 1.15""" + props = InvokeAIMetadata() + + props.imported_app_version = "pre1.15" + seed_match = re.search("-S\\s*(\\d+)", dream_string) + if seed_match is not None: + try: + props.seed = int(seed_match[1]) + except ValueError: + props.seed = None + raw_prompt = re.sub("(-S\\s*\\d+)", "", dream_string) + else: + raw_prompt = dream_string + + pos_prompt, neg_prompt = self.split_prompt(raw_prompt) + + props.positive_prompt = pos_prompt + props.negative_prompt = neg_prompt + + return props + + def parse_meta_tag_sd_metadata(self, tag_value): + """Take as input an png metadata json node for the 'sd-metadata' field variant from 1.15 through 2.3.5 post 2""" + props = InvokeAIMetadata() + + props.imported_app_version = tag_value.get("app_version") + props.model_name = tag_value.get("model_weights") + img_node = tag_value.get("image") + if img_node is not None: + props.generation_mode = img_node.get("type") + props.width = img_node.get("width") + props.height = img_node.get("height") + props.seed = img_node.get("seed") + props.rand_device = "cuda" #hardcoded since all generations pre 3.0 used cuda random noise instead of cpu + props.cfg_scale = img_node.get("cfg_scale") + props.steps = img_node.get("steps") + props.scheduler = self.map_scheduler(img_node.get("sampler")) + props.strength = img_node.get("strength") + if props.strength is None: + props.strength = img_node.get("strength_steps") # try second name for this property + props.init_image = img_node.get("init_image_path") + if props.init_image is None: # try second name for this property + props.init_image = img_node.get("init_img") + # remove the path info from init_image so if we move the init image, it will be correctly relative in the new location + if props.init_image is not None: + props.init_image = os.path.basename(props.init_image) + raw_prompt = img_node.get("prompt") + if isinstance(raw_prompt, list): + raw_prompt = raw_prompt[0].get("prompt") + + props.positive_prompt, props.negative_prompt = self.split_prompt(raw_prompt) + + return props + + def parse_meta_tag_invokeai(self, tag_value): + """Take as input an png metadata json node for the 'invokeai' field variant from 3.0.0 beta 1 through 5""" + props = InvokeAIMetadata() + + props.imported_app_version = "3.0.0 or later" + props.generation_mode = tag_value.get("type") + if props.generation_mode is not None: + props.generation_mode = props.generation_mode.replace("t2l","txt2img").replace("l2l","img2img") + + props.width = tag_value.get("width") + props.height = tag_value.get("height") + props.seed = tag_value.get("seed") + props.cfg_scale = tag_value.get("cfg_scale") + props.steps = tag_value.get("steps") + props.scheduler = tag_value.get("scheduler") + props.strength = tag_value.get("strength") + props.positive_prompt = tag_value.get("positive_conditioning") + props.negative_prompt = tag_value.get("negative_conditioning") + + return props + + def map_scheduler(self, old_scheduler): + """Convert the legacy sampler names to matching 3.0 schedulers""" + if old_scheduler is None: + return None + + match(old_scheduler): + case "ddim" : return "ddim" + case "plms" : return "pnmd" + case "k_lms" : return "lms" + case "k_dpm_2" : return "kdpm_2" + case "k_dpm_2_a" : return "kdpm_2_a" + case "dpmpp_2" : return "dpmpp_2s" + case "k_dpmpp_2" : return "dpmpp_2m" + case "k_dpmpp_2_a" : return None #invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session + case "k_euler" : return "euler" + case "k_euler_a" : return "euler_a" + case "k_heun" : return "heun" + return None + + def split_prompt(self, raw_prompt: str): + """Split the unified prompt strings by extracting all negative prompt blocks out into the negative prompt.""" + if raw_prompt is None: + return "", "" + raw_prompt_search = raw_prompt.replace("\r","").replace("\n","") + matches = re.findall(r"\[(.+?)\]", raw_prompt_search) + if len(matches) > 0: + negative_prompt = "" + if len(matches) == 1: + negative_prompt = matches[0].strip().strip(',') + else: + for match in matches: + negative_prompt += f"({match.strip().strip(',')})" + positive_prompt = re.sub(r"(\[.+?\])", "", raw_prompt_search).strip() + else: + positive_prompt = raw_prompt_search.strip() + negative_prompt = "" + + return positive_prompt,negative_prompt + +class DatabaseMapper: + """Class to abstract database functionality.""" + def __init__(self, database_path, database_backup_dir): + self.database_path = database_path + self.database_backup_dir = database_backup_dir + self.connection = None + self.cursor = None + + def connect(self): + """Open connection to the database.""" + self.connection = sqlite3.connect(self.database_path) + self.cursor = self.connection.cursor() + + def get_board_names(self): + """Get a list of the current board names from the database.""" + sql_get_board_name = "SELECT board_name FROM boards" + self.cursor.execute(sql_get_board_name) + rows = self.cursor.fetchall() + return [row[0] for row in rows] + + def does_image_exist(self, image_name): + """Check database if a image name already exists and return a boolean.""" + sql_get_image_by_name = f"SELECT image_name FROM images WHERE image_name='{image_name}'" + self.cursor.execute(sql_get_image_by_name) + rows = self.cursor.fetchall() + return True if len(rows) > 0 else False + + def add_new_image_to_database(self, filename, width, height, metadata, modified_date_string): + """Add an image to the database.""" + sql_add_image = f"""INSERT INTO images (image_name, image_origin, image_category, width, height, session_id, node_id, metadata, is_intermediate, created_at, updated_at) +VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{metadata}', 0, '{modified_date_string}', '{modified_date_string}')""" + self.cursor.execute(sql_add_image) + self.connection.commit() + + def get_board_id_with_create(self, board_name): + """Get the board id for supplied name, and create the board if one does not exist.""" + sql_find_board = f"SELECT board_id FROM boards WHERE board_name='{board_name}' COLLATE NOCASE" + self.cursor.execute(sql_find_board) + rows = self.cursor.fetchall() + if len(rows)>0: + return rows[0][0] + else: + board_date_string = datetime.datetime.utcnow().date().isoformat() + new_board_id = str(uuid.uuid4()) + sql_insert_board = f"INSERT INTO boards (board_id, board_name, created_at, updated_at) VALUES ('{new_board_id}', '{board_name}', '{board_date_string}', '{board_date_string}')" + self.cursor.execute(sql_insert_board) + self.connection.commit() + return new_board_id + + def add_image_to_board(self, filename, board_id): + """Add an image mapping to a board.""" + add_datetime_str = datetime.datetime.utcnow().isoformat() + sql_add_image_to_board = f"""INSERT INTO board_images (board_id, image_name, created_at, updated_at) + VALUES ('{board_id}', '{filename}', '{add_datetime_str}', '{add_datetime_str}')""" + self.cursor.execute(sql_add_image_to_board) + self.connection.commit() + + def disconnect(self): + """Disconnect from the db, cleaning up connections and cursors.""" + if self.cursor is not None: + self.cursor.close() + if self.connection is not None: + self.connection.close() + + def backup(self, timestamp_string): + """Take a backup of the database.""" + if not os.path.exists(self.database_backup_dir): + print (f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") + os.makedirs(self.database_backup_dir) + print ("Done!") + database_backup_path = os.path.join(self.database_backup_dir, f"backup-{timestamp_string}-invokeai.db") + print (f"Making DB Backup at {database_backup_path}...", end="") + shutil.copy2(self.database_path, database_backup_path) + print ("Done!") + +class MediaImportProcessor: + """Containing class for script functionality.""" + def __init__(self): + pass + + board_name_id_map = {} + + def get_import_file_list(self): + """Ask the user for the import folder and scan for the list of files to return.""" + while True: + default = '' + while True: + import_dir = os.path.expanduser( + prompt( + "Inputs: Specify absolute path containing InvokeAI .png images to import: ", + completer = PathCompleter(expanduser=True, only_directories=True), + default = default, + )) + if len(import_dir)>0 and Path(import_dir).is_dir(): + break + default = import_dir + + recurse_directories = (prompt("Include files from subfolders recursively [yN]? ").strip() or 'N').upper().startswith('N') + if recurse_directories: + is_recurse = False + matching_file_list = glob.glob(import_dir + '/*.png', recursive=False) + else: + is_recurse = True + matching_file_list = glob.glob(import_dir + '/**/*.png', recursive=True) + + if len(matching_file_list) > 0: + return import_dir, is_recurse, matching_file_list + else: + print(f"The specific path {import_dir} exists, but does not contain .png files!") + + def get_file_details(self, filepath): + """Retrieve the embedded metedata fields and dimensions from an image file.""" + with PIL.Image.open(filepath) as img: + img.load() + png_width, png_height = img.size + img_info = img.info + return img_info, png_width, png_height + + def select_board_option(self, board_names, timestamp_string): + """Allow the user to choose how a board is selected for imported files.""" + while True: + print("\r\nOptions for board selection for imported images:") + print(f"1) Select an existing board name. (found {len(board_names)})") + print( "2) Specify a board name to create/add to.") + print( "3) Create/add to board named 'IMPORT'.") + print(f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string}).") + print( "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5).") + input_option = input("Specify desired board option: ") + match(input_option): + case "1" : + if len(board_names) < 1: + print("\r\nThere are no existing board names to choose from. Select another option!") + continue + board_name = self.select_item_from_list(board_names, "board name", True, "Cancel, go back and choose a different board option.") + if board_name is not None: + return board_name + case "2": + while True: + board_name = input("Specify new/existing board name: ") + if board_name: + return board_name + case "3" : + return "IMPORT" + case "4": + return f"IMPORT_{timestamp_string}" + case "5": + return "IMPORT_APPVERSION" + + def select_item_from_list(self, items, entity_name, allow_cancel, cancel_string): + """A general function to render a list of items to select in the console, prompt the user for a selection and ensure a valid entry is selected.""" + print (f"Select a {entity_name.lower()} from the following list:") + index = 1 + for item in items: + print(f"{index}) {item}") + index += 1 + if allow_cancel: + print(f"{index}) {cancel_string}") + while True: + try: + option_number = int(input("Specify number of selection: ")) + except ValueError: + continue + if allow_cancel and option_number == index: + return None + if option_number >=1 and option_number <= len(items): + return items[option_number-1] + + def import_image(self, filepath: str, board_name_option: str, db_mapper: DatabaseMapper, config: Config): + """Import a single file by its path""" + parser = InvokeAIMetadataParser() + file_name = os.path.basename(filepath) + file_destination_path = os.path.join(config.outputs_path, file_name) + + print("===============================================================================") + print(f"Importing {filepath}") + + # check destination to see if the file was previously imported + if os.path.exists(file_destination_path): + print("File already exists in the destination, skipping!") + ImportStats.count_skipped_file_exists += 1 + return + + # check if file name is already referenced in the database + if db_mapper.does_image_exist(file_name): + print("A reference to a file with this name already exists in the database, skipping!") + ImportStats.count_skipped_db_exists += 1 + return + + # load image info and dimensions + img_info, png_width, png_height = self.get_file_details(filepath) + + # parse metadata + destination_needs_meta_update = True + log_version_note = "(Unknown)" + if "invokeai_metadata" in img_info: + # for the latest, we will just re-emit the same json, no need to parse/modify + converted_field = None + latest_json_string = img_info.get("invokeai_metadata") + log_version_note = "3.0.0+" + destination_needs_meta_update = False + else: + if "sd-metadata" in img_info: + converted_field = parser.parse_meta_tag_sd_metadata(json.loads(img_info.get("sd-metadata"))) + elif "invokeai" in img_info: + converted_field = parser.parse_meta_tag_invokeai(json.loads(img_info.get("invokeai"))) + elif "dream" in img_info: + converted_field = parser.parse_meta_tag_dream(img_info.get("dream")) + elif "Dream" in img_info: + converted_field = parser.parse_meta_tag_dream(img_info.get("Dream")) + else: + converted_field = InvokeAIMetadata() + destination_needs_meta_update = False + print("File does not have metadata from known Invoke AI versions, add only, no update!") + + # use the loaded img dimensions if the metadata didnt have them + if converted_field.width is None: + converted_field.width = png_width + if converted_field.height is None: + converted_field.height = png_height + + log_version_note = converted_field.imported_app_version if converted_field else "NoVersion" + log_version_note = log_version_note or "NoVersion" + + latest_json_string = converted_field.to_json() + + print (f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") + + # if metadata needs update, then update metdata and copy in one shot + if destination_needs_meta_update: + print("Updating metadata while copying...", end="") + self.update_file_metadata_while_copying(filepath, file_destination_path, "invokeai_metadata", latest_json_string) + print("Done!") + else: + print("No metadata update necessary, copying only...", end="") + shutil.copy2(filepath, file_destination_path) + print("Done!") + + # create thumbnail + print("Creating thumbnail...", end="") + thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" + thumbnail_size = 256, 256 + with PIL.Image.open(filepath) as source_image: + source_image.thumbnail(thumbnail_size) + source_image.save(thumbnail_path, "webp") + print("Done!") + + # finalize the dynamic board name if there is an APPVERSION token in it. + if converted_field is not None: + board_name = board_name_option.replace("APPVERSION", converted_field.imported_app_version or "NoVersion") + else: + board_name = board_name_option.replace("APPVERSION", "Latest") + + # maintain a map of alrady created/looked up ids to avoid DB queries + print("Finding/Creating board...", end="") + if board_name in self.board_name_id_map: + board_id = self.board_name_id_map[board_name] + else: + board_id = db_mapper.get_board_id_with_create(board_name) + self.board_name_id_map[board_name] = board_id + print("Done!") + + # add image to db + print("Adding image to database......", end="") + modified_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(filepath)) + db_mapper.add_new_image_to_database(file_name, png_width, png_height, latest_json_string, modified_time) + print("Done!") + + #add image to board + print("Adding image to board......", end="") + db_mapper.add_image_to_board(file_name, board_id) + print("Done!") + + ImportStats.count_imported += 1 + if log_version_note in ImportStats.count_imported_by_version: + ImportStats.count_imported_by_version[log_version_note] += 1 + else: + ImportStats.count_imported_by_version[log_version_note] = 1 + + def update_file_metadata_while_copying(self, filepath, file_destination_path, tag_name, tag_value): + """Perform a metadata update with save to a new destination which accomplishes a copy while updating metadata.""" + with PIL.Image.open(filepath) as target_image: + existing_img_info = target_image.info + metadata = PIL.PngImagePlugin.PngInfo() + # re-add any existing invoke ai tags unless they are the one we are trying to add + for key in existing_img_info: + if key != tag_name and key in ("dream", "Dream", "sd-metadata", "invokeai", "invokeai_metadata"): + metadata.add_text(key,existing_img_info[key]) + metadata.add_text(tag_name, tag_value) + target_image.save(file_destination_path, pnginfo=metadata) + + def process(self): + """Begin main processing.""" + + print("===============================================================================") + print("This script will import images generated by earlier versions of") + print("InvokeAI into the currently installed root directory:") + print(f' {app_config.root_path}') + print("If this is not what you want to do, type ctrl-C now to cancel.") + + # load config + print("===============================================================================") + print("= Configuration & Settings") + + config = Config() + config.find_and_load() + db_mapper = DatabaseMapper(config.database_path, config.database_backup_dir) + db_mapper.connect() + + import_dir, is_recurse, import_file_list = self.get_import_file_list() + ImportStats.count_source_files = len(import_file_list) + + board_names = db_mapper.get_board_names() + board_name_option = self.select_board_option(board_names, config.TIMESTAMP_STRING) + + print("\r\n===============================================================================") + print("= Import Settings Confirmation") + + print() + print(f"Database File Path : {config.database_path}") + print(f"Outputs/Images Directory : {config.outputs_path}") + print(f"Import Image Source Directory : {import_dir}") + print(f" Recurse Source SubDirectories : {'Yes' if is_recurse else 'No'}") + print(f"Count of .png file(s) found : {len(import_file_list)}") + print(f"Board name option specified : {board_name_option}") + print(f"Database backup will be taken at : {config.database_backup_dir}") + + print("\r\nNotes about the import process:") + print("- Source image files will not be modified, only copied to the outputs directory.") + print("- If the same file name already exists in the destination, the file will be skipped.") + print("- If the same file name already has a record in the database, the file will be skipped.") + print("- Invoke AI metadata tags will be updated/written into the imported copy only.") + print("- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)") + print("- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer.") + print("- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder.") + + while True: + should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or 'y' + if should_continue=='n': + print("\r\nCancelling Import") + return + elif should_continue=='y': + print() + break + + db_mapper.backup(config.TIMESTAMP_STRING) + + print() + ImportStats.time_start = datetime.datetime.utcnow() + + for filepath in import_file_list: + try: + self.import_image(filepath, board_name_option, db_mapper, config) + except sqlite3.Error as sql_ex: + print(f"A database related exception was found processing {filepath}, will continue to next file. ") + print("Exception detail:") + print(sql_ex) + ImportStats.count_file_errors += 1 + except Exception as ex: + print(f"Exception processing {filepath}, will continue to next file. ") + print("Exception detail:") + print(ex) + ImportStats.count_file_errors += 1 + + print("\r\n===============================================================================") + print(f"= Import Complete - Elpased Time: {ImportStats.get_elapsed_time_string()}") + print() + print(f"Source File(s) : {ImportStats.count_source_files}") + print(f"Total Imported : {ImportStats.count_imported}") + print(f"Skipped b/c file already exists on disk : {ImportStats.count_skipped_file_exists}") + print(f"Skipped b/c file already exists in db : {ImportStats.count_skipped_db_exists}") + print(f"Errors during import : {ImportStats.count_file_errors}") + if ImportStats.count_imported > 0: + print("\r\nBreakdown of imported files by version:") + for key,version in ImportStats.count_imported_by_version.items(): + print(f" {key:20} : {version}") + +def main(): + try: + processor = MediaImportProcessor() + processor.process() + except KeyboardInterrupt: + print("\r\n\r\nUser cancelled execution.") + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index b3f12481a8..d5aa3f37aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,7 @@ dependencies = [ "invokeai-metadata" = "invokeai.frontend.CLI.sd_metadata:print_metadata" "invokeai-node-cli" = "invokeai.app.cli_app:invoke_cli" "invokeai-node-web" = "invokeai.app.api_app:invoke_api" +"invokeai-import-images" = "invokeai.frontend.install.import_images:main" [project.urls] "Homepage" = "https://invoke-ai.github.io/InvokeAI/" From c8ceb9609147390e5e5407ad7ebf0d5148a639ce Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:26:52 -0400 Subject: [PATCH 02/10] add docs --- README.md | 23 +++++++++++++++++++--- docs/installation/010_INSTALL_AUTOMATED.md | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2efaacd031..7b3f95ab10 100644 --- a/README.md +++ b/README.md @@ -306,13 +306,30 @@ InvokeAI. The second will prepare the 2.3 directory for use with 3.0. You may now launch the WebUI in the usual way, by selecting option [1] from the launcher script -#### Migration Caveats +#### Migrating Images The migration script will migrate your invokeai settings and models, including textual inversion models, LoRAs and merges that you may have installed previously. However it does **not** migrate the generated -images stored in your 2.3-format outputs directory. You will need to -manually import selected images into the 3.0 gallery via drag-and-drop. +images stored in your 2.3-format outputs directory. To do this, you +need to run an additional step: + +1. From a working InvokeAI 3.0 root directory, start the launcher and +enter menu option [8] to open the "developer's console". + +2. At the developer's console command line, type the command: + +```bash +invokeai-import-images +``` + +3. This will lead you through the process of confirming the desired + source and destination for the imported images. The images will + appear in the gallery board of your choice, and contain the + original prompt, model name, and other parameters used to generate + the image. + +(Many kudos to **techjedi** for contributing this script.) ## Hardware Requirements diff --git a/docs/installation/010_INSTALL_AUTOMATED.md b/docs/installation/010_INSTALL_AUTOMATED.md index 9b66fcbd83..07a88d5d1b 100644 --- a/docs/installation/010_INSTALL_AUTOMATED.md +++ b/docs/installation/010_INSTALL_AUTOMATED.md @@ -264,7 +264,7 @@ experimental versions later. you can create several levels of subfolders and drop your models into whichever ones you want. - - ***Autoimport FolderLICENSE*** + - ***LICENSE*** At the bottom of the screen you will see a checkbox for accepting the CreativeML Responsible AI Licenses. You need to accept the license From 4043a4c21cde6156b9c6710e81da79b741bd8b19 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:44:58 -0400 Subject: [PATCH 03/10] blackified --- invokeai/frontend/install/import_images.py | 213 +++++++++++++-------- 1 file changed, 131 insertions(+), 82 deletions(-) diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py index 1870fafbf8..fe0f20c830 100644 --- a/invokeai/frontend/install/import_images.py +++ b/invokeai/frontend/install/import_images.py @@ -26,22 +26,28 @@ from prompt_toolkit.completion import PathCompleter from prompt_toolkit.key_binding import KeyBindings from invokeai.app.services.config import InvokeAIAppConfig + app_config = InvokeAIAppConfig.get_config() bindings = KeyBindings() -@bindings.add('c-c') + + +@bindings.add("c-c") def _(event): raise KeyboardInterrupt + # release notes # "Use All" with size dimensions not selectable in the UI will not load dimensions + class Config: """Configuration loader.""" + def __init__(self): pass - TIMESTAMP_STRING= datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + TIMESTAMP_STRING = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") INVOKE_DIRNAME = "invokeai" YAML_FILENAME = "invokeai.yaml" @@ -56,21 +62,21 @@ class Config: """find the yaml config file and load""" root = app_config.root_path if not self.confirm_and_load(os.path.abspath(root)): - print ("\r\nSpecify custom database and outputs paths:") + print("\r\nSpecify custom database and outputs paths:") self.confirm_and_load_from_user() - self.database_backup_dir = os.path.join(os.path.dirname(self.database_path),"backup") - self.thumbnail_path = os.path.join(self.outputs_path,"thumbnails") + self.database_backup_dir = os.path.join(os.path.dirname(self.database_path), "backup") + self.thumbnail_path = os.path.join(self.outputs_path, "thumbnails") def confirm_and_load(self, invoke_root): """Validates a yaml path exists, confirms the user wants to use it and loads config.""" - yaml_path = os.path.join(invoke_root,self.YAML_FILENAME) + yaml_path = os.path.join(invoke_root, self.YAML_FILENAME) if os.path.exists(yaml_path): db_dir, outdir = self.load_paths_from_yaml(yaml_path) if os.path.isabs(db_dir): - database_path = os.path.join(db_dir,self.DATABASE_FILENAME) + database_path = os.path.join(db_dir, self.DATABASE_FILENAME) else: - database_path = os.path.join(invoke_root,db_dir,self.DATABASE_FILENAME) + database_path = os.path.join(invoke_root, db_dir, self.DATABASE_FILENAME) if os.path.isabs(outdir): outputs_path = os.path.join(outdir, "images") @@ -86,7 +92,7 @@ class Config: text += "\n\nUse these paths for import (yes) or choose different ones (no) [Yn]: " if db_exists and outdir_exists: - if (prompt(text).strip() or 'Y').upper().startswith('Y'): + if (prompt(text).strip() or "Y").upper().startswith("Y"): self.database_path = database_path self.outputs_path = outputs_path return True @@ -98,36 +104,40 @@ class Config: else: message_dialog( title="Path not found", - text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified." + text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified.", ).run() return False def confirm_and_load_from_user(self): - default = '' + default = "" while True: database_path = os.path.expanduser( prompt( "Database: Specify absolute path to the database to import into: ", - completer=PathCompleter(expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith(('.db'))), - default = default, - )) + completer=PathCompleter( + expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith((".db")) + ), + default=default, + ) + ) if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): break - default = database_path + '/' if Path(database_path).is_dir() else database_path + default = database_path + "/" if Path(database_path).is_dir() else database_path - default = '' + default = "" while True: outputs_path = os.path.expanduser( prompt( "Outputs: Specify absolute path to outputs/images directory to import into: ", - completer = PathCompleter(expanduser=True, only_directories=True), - default = default, - )) + completer=PathCompleter(expanduser=True, only_directories=True), + default=default, + ) + ) if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): break - default = outputs_path + '/' if Path(outputs_path).is_dir() else outputs_path - + default = outputs_path + "/" if Path(outputs_path).is_dir() else outputs_path + self.database_path = database_path self.outputs_path = outputs_path @@ -136,17 +146,19 @@ class Config: def load_paths_from_yaml(self, yaml_path): """Load an Invoke AI yaml file and get the database and outputs paths.""" try: - with open(yaml_path, 'rt', encoding=locale.getpreferredencoding()) as file: + with open(yaml_path, "rt", encoding=locale.getpreferredencoding()) as file: yamlinfo = yaml.safe_load(file) - db_dir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("db_dir", None) - outdir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("outdir", None) + db_dir = yamlinfo.get("InvokeAI", {}).get("Paths", {}).get("db_dir", None) + outdir = yamlinfo.get("InvokeAI", {}).get("Paths", {}).get("outdir", None) return db_dir, outdir except Exception: print(f"Failed to load paths from yaml file! {yaml_path}!") return None, None + class ImportStats: """DTO for tracking work progress.""" + def __init__(self): pass @@ -171,8 +183,10 @@ class ImportStats: out_str += f"{seconds:.2f} second(s)" return out_str + class InvokeAIMetadata: """DTO for core Invoke AI generation properties parsed from metadata.""" + def __init__(self): pass @@ -234,6 +248,7 @@ class InvokeAIMetadata: class InvokeAIMetadataParser: """Parses strings with json data to find Invoke AI core metadata properties.""" + def __init__(self): pass @@ -271,15 +286,15 @@ class InvokeAIMetadataParser: props.width = img_node.get("width") props.height = img_node.get("height") props.seed = img_node.get("seed") - props.rand_device = "cuda" #hardcoded since all generations pre 3.0 used cuda random noise instead of cpu + props.rand_device = "cuda" # hardcoded since all generations pre 3.0 used cuda random noise instead of cpu props.cfg_scale = img_node.get("cfg_scale") props.steps = img_node.get("steps") props.scheduler = self.map_scheduler(img_node.get("sampler")) props.strength = img_node.get("strength") if props.strength is None: - props.strength = img_node.get("strength_steps") # try second name for this property + props.strength = img_node.get("strength_steps") # try second name for this property props.init_image = img_node.get("init_image_path") - if props.init_image is None: # try second name for this property + if props.init_image is None: # try second name for this property props.init_image = img_node.get("init_img") # remove the path info from init_image so if we move the init image, it will be correctly relative in the new location if props.init_image is not None: @@ -299,7 +314,7 @@ class InvokeAIMetadataParser: props.imported_app_version = "3.0.0 or later" props.generation_mode = tag_value.get("type") if props.generation_mode is not None: - props.generation_mode = props.generation_mode.replace("t2l","txt2img").replace("l2l","img2img") + props.generation_mode = props.generation_mode.replace("t2l", "txt2img").replace("l2l", "img2img") props.width = tag_value.get("width") props.height = tag_value.get("height") @@ -318,30 +333,41 @@ class InvokeAIMetadataParser: if old_scheduler is None: return None - match(old_scheduler): - case "ddim" : return "ddim" - case "plms" : return "pnmd" - case "k_lms" : return "lms" - case "k_dpm_2" : return "kdpm_2" - case "k_dpm_2_a" : return "kdpm_2_a" - case "dpmpp_2" : return "dpmpp_2s" - case "k_dpmpp_2" : return "dpmpp_2m" - case "k_dpmpp_2_a" : return None #invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session - case "k_euler" : return "euler" - case "k_euler_a" : return "euler_a" - case "k_heun" : return "heun" + match (old_scheduler): + case "ddim": + return "ddim" + case "plms": + return "pnmd" + case "k_lms": + return "lms" + case "k_dpm_2": + return "kdpm_2" + case "k_dpm_2_a": + return "kdpm_2_a" + case "dpmpp_2": + return "dpmpp_2s" + case "k_dpmpp_2": + return "dpmpp_2m" + case "k_dpmpp_2_a": + return None # invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session + case "k_euler": + return "euler" + case "k_euler_a": + return "euler_a" + case "k_heun": + return "heun" return None def split_prompt(self, raw_prompt: str): """Split the unified prompt strings by extracting all negative prompt blocks out into the negative prompt.""" if raw_prompt is None: return "", "" - raw_prompt_search = raw_prompt.replace("\r","").replace("\n","") + raw_prompt_search = raw_prompt.replace("\r", "").replace("\n", "") matches = re.findall(r"\[(.+?)\]", raw_prompt_search) if len(matches) > 0: negative_prompt = "" if len(matches) == 1: - negative_prompt = matches[0].strip().strip(',') + negative_prompt = matches[0].strip().strip(",") else: for match in matches: negative_prompt += f"({match.strip().strip(',')})" @@ -350,10 +376,12 @@ class InvokeAIMetadataParser: positive_prompt = raw_prompt_search.strip() negative_prompt = "" - return positive_prompt,negative_prompt + return positive_prompt, negative_prompt + class DatabaseMapper: """Class to abstract database functionality.""" + def __init__(self, database_path, database_backup_dir): self.database_path = database_path self.database_backup_dir = database_backup_dir @@ -380,7 +408,7 @@ class DatabaseMapper: return True if len(rows) > 0 else False def add_new_image_to_database(self, filename, width, height, metadata, modified_date_string): - """Add an image to the database.""" + """Add an image to the database.""" sql_add_image = f"""INSERT INTO images (image_name, image_origin, image_category, width, height, session_id, node_id, metadata, is_intermediate, created_at, updated_at) VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{metadata}', 0, '{modified_date_string}', '{modified_date_string}')""" self.cursor.execute(sql_add_image) @@ -391,7 +419,7 @@ VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{me sql_find_board = f"SELECT board_id FROM boards WHERE board_name='{board_name}' COLLATE NOCASE" self.cursor.execute(sql_find_board) rows = self.cursor.fetchall() - if len(rows)>0: + if len(rows) > 0: return rows[0][0] else: board_date_string = datetime.datetime.utcnow().date().isoformat() @@ -417,18 +445,20 @@ VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{me self.connection.close() def backup(self, timestamp_string): - """Take a backup of the database.""" + """Take a backup of the database.""" if not os.path.exists(self.database_backup_dir): - print (f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") + print(f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") os.makedirs(self.database_backup_dir) - print ("Done!") + print("Done!") database_backup_path = os.path.join(self.database_backup_dir, f"backup-{timestamp_string}-invokeai.db") - print (f"Making DB Backup at {database_backup_path}...", end="") + print(f"Making DB Backup at {database_backup_path}...", end="") shutil.copy2(self.database_path, database_backup_path) - print ("Done!") + print("Done!") + class MediaImportProcessor: """Containing class for script functionality.""" + def __init__(self): pass @@ -437,25 +467,28 @@ class MediaImportProcessor: def get_import_file_list(self): """Ask the user for the import folder and scan for the list of files to return.""" while True: - default = '' + default = "" while True: import_dir = os.path.expanduser( prompt( "Inputs: Specify absolute path containing InvokeAI .png images to import: ", - completer = PathCompleter(expanduser=True, only_directories=True), - default = default, - )) - if len(import_dir)>0 and Path(import_dir).is_dir(): + completer=PathCompleter(expanduser=True, only_directories=True), + default=default, + ) + ) + if len(import_dir) > 0 and Path(import_dir).is_dir(): break default = import_dir - recurse_directories = (prompt("Include files from subfolders recursively [yN]? ").strip() or 'N').upper().startswith('N') + recurse_directories = ( + (prompt("Include files from subfolders recursively [yN]? ").strip() or "N").upper().startswith("N") + ) if recurse_directories: is_recurse = False - matching_file_list = glob.glob(import_dir + '/*.png', recursive=False) + matching_file_list = glob.glob(import_dir + "/*.png", recursive=False) else: is_recurse = True - matching_file_list = glob.glob(import_dir + '/**/*.png', recursive=True) + matching_file_list = glob.glob(import_dir + "/**/*.png", recursive=True) if len(matching_file_list) > 0: return import_dir, is_recurse, matching_file_list @@ -475,17 +508,23 @@ class MediaImportProcessor: while True: print("\r\nOptions for board selection for imported images:") print(f"1) Select an existing board name. (found {len(board_names)})") - print( "2) Specify a board name to create/add to.") - print( "3) Create/add to board named 'IMPORT'.") - print(f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string}).") - print( "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5).") + print("2) Specify a board name to create/add to.") + print("3) Create/add to board named 'IMPORT'.") + print( + f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string})." + ) + print( + "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5)." + ) input_option = input("Specify desired board option: ") - match(input_option): - case "1" : + match (input_option): + case "1": if len(board_names) < 1: print("\r\nThere are no existing board names to choose from. Select another option!") continue - board_name = self.select_item_from_list(board_names, "board name", True, "Cancel, go back and choose a different board option.") + board_name = self.select_item_from_list( + board_names, "board name", True, "Cancel, go back and choose a different board option." + ) if board_name is not None: return board_name case "2": @@ -493,7 +532,7 @@ class MediaImportProcessor: board_name = input("Specify new/existing board name: ") if board_name: return board_name - case "3" : + case "3": return "IMPORT" case "4": return f"IMPORT_{timestamp_string}" @@ -502,7 +541,7 @@ class MediaImportProcessor: def select_item_from_list(self, items, entity_name, allow_cancel, cancel_string): """A general function to render a list of items to select in the console, prompt the user for a selection and ensure a valid entry is selected.""" - print (f"Select a {entity_name.lower()} from the following list:") + print(f"Select a {entity_name.lower()} from the following list:") index = 1 for item in items: print(f"{index}) {item}") @@ -516,8 +555,8 @@ class MediaImportProcessor: continue if allow_cancel and option_number == index: return None - if option_number >=1 and option_number <= len(items): - return items[option_number-1] + if option_number >= 1 and option_number <= len(items): + return items[option_number - 1] def import_image(self, filepath: str, board_name_option: str, db_mapper: DatabaseMapper, config: Config): """Import a single file by its path""" @@ -577,12 +616,14 @@ class MediaImportProcessor: latest_json_string = converted_field.to_json() - print (f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") + print(f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") # if metadata needs update, then update metdata and copy in one shot if destination_needs_meta_update: print("Updating metadata while copying...", end="") - self.update_file_metadata_while_copying(filepath, file_destination_path, "invokeai_metadata", latest_json_string) + self.update_file_metadata_while_copying( + filepath, file_destination_path, "invokeai_metadata", latest_json_string + ) print("Done!") else: print("No metadata update necessary, copying only...", end="") @@ -591,7 +632,7 @@ class MediaImportProcessor: # create thumbnail print("Creating thumbnail...", end="") - thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" + thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" thumbnail_size = 256, 256 with PIL.Image.open(filepath) as source_image: source_image.thumbnail(thumbnail_size) @@ -619,7 +660,7 @@ class MediaImportProcessor: db_mapper.add_new_image_to_database(file_name, png_width, png_height, latest_json_string, modified_time) print("Done!") - #add image to board + # add image to board print("Adding image to board......", end="") db_mapper.add_image_to_board(file_name, board_id) print("Done!") @@ -638,7 +679,7 @@ class MediaImportProcessor: # re-add any existing invoke ai tags unless they are the one we are trying to add for key in existing_img_info: if key != tag_name and key in ("dream", "Dream", "sd-metadata", "invokeai", "invokeai_metadata"): - metadata.add_text(key,existing_img_info[key]) + metadata.add_text(key, existing_img_info[key]) metadata.add_text(tag_name, tag_value) target_image.save(file_destination_path, pnginfo=metadata) @@ -648,7 +689,7 @@ class MediaImportProcessor: print("===============================================================================") print("This script will import images generated by earlier versions of") print("InvokeAI into the currently installed root directory:") - print(f' {app_config.root_path}') + print(f" {app_config.root_path}") print("If this is not what you want to do, type ctrl-C now to cancel.") # load config @@ -683,16 +724,22 @@ class MediaImportProcessor: print("- If the same file name already exists in the destination, the file will be skipped.") print("- If the same file name already has a record in the database, the file will be skipped.") print("- Invoke AI metadata tags will be updated/written into the imported copy only.") - print("- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)") - print("- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer.") - print("- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder.") + print( + "- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)" + ) + print( + "- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer." + ) + print( + "- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder." + ) while True: - should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or 'y' - if should_continue=='n': + should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or "y" + if should_continue == "n": print("\r\nCancelling Import") return - elif should_continue=='y': + elif should_continue == "y": print() break @@ -725,9 +772,10 @@ class MediaImportProcessor: print(f"Errors during import : {ImportStats.count_file_errors}") if ImportStats.count_imported > 0: print("\r\nBreakdown of imported files by version:") - for key,version in ImportStats.count_imported_by_version.items(): + for key, version in ImportStats.count_imported_by_version.items(): print(f" {key:20} : {version}") + def main(): try: processor = MediaImportProcessor() @@ -735,5 +783,6 @@ def main(): except KeyboardInterrupt: print("\r\n\r\nUser cancelled execution.") + if __name__ == "__main__": main() From 0ccc3b509e252b15f628bb0494ac01727a8d3372 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:19:24 -0400 Subject: [PATCH 04/10] add techjedi's import script, with some filecompletion tweaks --- invokeai/frontend/install/import_images.py | 739 +++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 740 insertions(+) create mode 100644 invokeai/frontend/install/import_images.py diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py new file mode 100644 index 0000000000..1870fafbf8 --- /dev/null +++ b/invokeai/frontend/install/import_images.py @@ -0,0 +1,739 @@ +# Copyright (c) 2023 - The InvokeAI Team +# Author: techjedi + +# pylint: disable=line-too-long +# pylint: disable=broad-exception-caught +"""Script to import images into the new database system for 3.0.0""" + +import os +import datetime +import shutil +import locale +import sqlite3 +import json +import glob +import re +import uuid +import yaml +import PIL +import PIL.ImageOps +import PIL.PngImagePlugin + +from pathlib import Path +from prompt_toolkit import prompt +from prompt_toolkit.shortcuts import yes_no_dialog, message_dialog, input_dialog, button_dialog +from prompt_toolkit.completion import PathCompleter +from prompt_toolkit.key_binding import KeyBindings + +from invokeai.app.services.config import InvokeAIAppConfig +app_config = InvokeAIAppConfig.get_config() + +bindings = KeyBindings() +@bindings.add('c-c') +def _(event): + raise KeyboardInterrupt + +# release notes +# "Use All" with size dimensions not selectable in the UI will not load dimensions + +class Config: + """Configuration loader.""" + def __init__(self): + pass + + TIMESTAMP_STRING= datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + + INVOKE_DIRNAME = "invokeai" + YAML_FILENAME = "invokeai.yaml" + DATABASE_FILENAME = "invokeai.db" + + database_path = None + database_backup_dir = None + outputs_path = None + thumbnail_path = None + + def find_and_load(self): + """find the yaml config file and load""" + root = app_config.root_path + if not self.confirm_and_load(os.path.abspath(root)): + print ("\r\nSpecify custom database and outputs paths:") + self.confirm_and_load_from_user() + + self.database_backup_dir = os.path.join(os.path.dirname(self.database_path),"backup") + self.thumbnail_path = os.path.join(self.outputs_path,"thumbnails") + + def confirm_and_load(self, invoke_root): + """Validates a yaml path exists, confirms the user wants to use it and loads config.""" + yaml_path = os.path.join(invoke_root,self.YAML_FILENAME) + if os.path.exists(yaml_path): + db_dir, outdir = self.load_paths_from_yaml(yaml_path) + if os.path.isabs(db_dir): + database_path = os.path.join(db_dir,self.DATABASE_FILENAME) + else: + database_path = os.path.join(invoke_root,db_dir,self.DATABASE_FILENAME) + + if os.path.isabs(outdir): + outputs_path = os.path.join(outdir, "images") + else: + outputs_path = os.path.join(invoke_root, outdir, "images") + + db_exists = os.path.exists(database_path) + outdir_exists = os.path.exists(outputs_path) + + text = f"Found {self.YAML_FILENAME} file at {yaml_path}:" + text += f"\n Database : {database_path}" + text += f"\n Outputs : {outputs_path}" + text += "\n\nUse these paths for import (yes) or choose different ones (no) [Yn]: " + + if db_exists and outdir_exists: + if (prompt(text).strip() or 'Y').upper().startswith('Y'): + self.database_path = database_path + self.outputs_path = outputs_path + return True + else: + return False + else: + print(" Invalid: One or more paths in this config did not exist and cannot be used.") + + else: + message_dialog( + title="Path not found", + text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified." + ).run() + return False + + def confirm_and_load_from_user(self): + default = '' + while True: + database_path = os.path.expanduser( + prompt( + "Database: Specify absolute path to the database to import into: ", + completer=PathCompleter(expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith(('.db'))), + default = default, + )) + if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): + break + default = database_path + '/' if Path(database_path).is_dir() else database_path + + default = '' + while True: + outputs_path = os.path.expanduser( + prompt( + "Outputs: Specify absolute path to outputs/images directory to import into: ", + completer = PathCompleter(expanduser=True, only_directories=True), + default = default, + )) + + if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): + break + default = outputs_path + '/' if Path(outputs_path).is_dir() else outputs_path + + self.database_path = database_path + self.outputs_path = outputs_path + + return + + def load_paths_from_yaml(self, yaml_path): + """Load an Invoke AI yaml file and get the database and outputs paths.""" + try: + with open(yaml_path, 'rt', encoding=locale.getpreferredencoding()) as file: + yamlinfo = yaml.safe_load(file) + db_dir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("db_dir", None) + outdir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("outdir", None) + return db_dir, outdir + except Exception: + print(f"Failed to load paths from yaml file! {yaml_path}!") + return None, None + +class ImportStats: + """DTO for tracking work progress.""" + def __init__(self): + pass + + time_start = datetime.datetime.utcnow() + count_source_files = 0 + count_skipped_file_exists = 0 + count_skipped_db_exists = 0 + count_imported = 0 + count_imported_by_version = {} + count_file_errors = 0 + + @staticmethod + def get_elapsed_time_string(): + """Get a friendly time string for the time elapsed since processing start.""" + time_now = datetime.datetime.utcnow() + total_seconds = (time_now - ImportStats.time_start).total_seconds() + hours = int((total_seconds) / 3600) + minutes = int(((total_seconds) % 3600) / 60) + seconds = total_seconds % 60 + out_str = f"{hours} hour(s) -" if hours > 0 else "" + out_str += f"{minutes} minute(s) -" if minutes > 0 else "" + out_str += f"{seconds:.2f} second(s)" + return out_str + +class InvokeAIMetadata: + """DTO for core Invoke AI generation properties parsed from metadata.""" + def __init__(self): + pass + + def __str__(self): + formatted_str = f"{self.generation_mode}~{self.steps}~{self.cfg_scale}~{self.model_name}~{self.scheduler}~{self.seed}~{self.width}~{self.height}~{self.rand_device}~{self.strength}~{self.init_image}" + formatted_str += f"\r\npositive_prompt: {self.positive_prompt}" + formatted_str += f"\r\nnegative_prompt: {self.negative_prompt}" + return formatted_str + + generation_mode = None + steps = None + cfg_scale = None + model_name = None + scheduler = None + seed = None + width = None + height = None + rand_device = None + strength = None + init_image = None + positive_prompt = None + negative_prompt = None + imported_app_version = None + + def to_json(self): + """Convert the active instance to json format.""" + prop_dict = {} + prop_dict["generation_mode"] = self.generation_mode + prop_dict["positive_prompt"] = self.positive_prompt + prop_dict["negative_prompt"] = self.negative_prompt + prop_dict["width"] = self.width + prop_dict["height"] = self.height + prop_dict["seed"] = self.seed + prop_dict["rand_device"] = self.rand_device + prop_dict["cfg_scale"] = self.cfg_scale + prop_dict["steps"] = self.steps + prop_dict["scheduler"] = self.scheduler + prop_dict["clip_skip"] = 0 + prop_dict["model"] = {} + prop_dict["model"]["model_name"] = self.model_name + prop_dict["model"]["base_model"] = None + prop_dict["controlnets"] = [] + prop_dict["loras"] = [] + prop_dict["vae"] = None + prop_dict["strength"] = self.strength + prop_dict["init_image"] = self.init_image + prop_dict["positive_style_prompt"] = None + prop_dict["negative_style_prompt"] = None + prop_dict["refiner_model"] = None + prop_dict["refiner_cfg_scale"] = None + prop_dict["refiner_steps"] = None + prop_dict["refiner_scheduler"] = None + prop_dict["refiner_aesthetic_store"] = None + prop_dict["refiner_start"] = None + prop_dict["imported_app_version"] = self.imported_app_version + + return json.dumps(prop_dict) + + +class InvokeAIMetadataParser: + """Parses strings with json data to find Invoke AI core metadata properties.""" + def __init__(self): + pass + + def parse_meta_tag_dream(self, dream_string): + """Take as input an png metadata json node for the 'dream' field variant from prior to 1.15""" + props = InvokeAIMetadata() + + props.imported_app_version = "pre1.15" + seed_match = re.search("-S\\s*(\\d+)", dream_string) + if seed_match is not None: + try: + props.seed = int(seed_match[1]) + except ValueError: + props.seed = None + raw_prompt = re.sub("(-S\\s*\\d+)", "", dream_string) + else: + raw_prompt = dream_string + + pos_prompt, neg_prompt = self.split_prompt(raw_prompt) + + props.positive_prompt = pos_prompt + props.negative_prompt = neg_prompt + + return props + + def parse_meta_tag_sd_metadata(self, tag_value): + """Take as input an png metadata json node for the 'sd-metadata' field variant from 1.15 through 2.3.5 post 2""" + props = InvokeAIMetadata() + + props.imported_app_version = tag_value.get("app_version") + props.model_name = tag_value.get("model_weights") + img_node = tag_value.get("image") + if img_node is not None: + props.generation_mode = img_node.get("type") + props.width = img_node.get("width") + props.height = img_node.get("height") + props.seed = img_node.get("seed") + props.rand_device = "cuda" #hardcoded since all generations pre 3.0 used cuda random noise instead of cpu + props.cfg_scale = img_node.get("cfg_scale") + props.steps = img_node.get("steps") + props.scheduler = self.map_scheduler(img_node.get("sampler")) + props.strength = img_node.get("strength") + if props.strength is None: + props.strength = img_node.get("strength_steps") # try second name for this property + props.init_image = img_node.get("init_image_path") + if props.init_image is None: # try second name for this property + props.init_image = img_node.get("init_img") + # remove the path info from init_image so if we move the init image, it will be correctly relative in the new location + if props.init_image is not None: + props.init_image = os.path.basename(props.init_image) + raw_prompt = img_node.get("prompt") + if isinstance(raw_prompt, list): + raw_prompt = raw_prompt[0].get("prompt") + + props.positive_prompt, props.negative_prompt = self.split_prompt(raw_prompt) + + return props + + def parse_meta_tag_invokeai(self, tag_value): + """Take as input an png metadata json node for the 'invokeai' field variant from 3.0.0 beta 1 through 5""" + props = InvokeAIMetadata() + + props.imported_app_version = "3.0.0 or later" + props.generation_mode = tag_value.get("type") + if props.generation_mode is not None: + props.generation_mode = props.generation_mode.replace("t2l","txt2img").replace("l2l","img2img") + + props.width = tag_value.get("width") + props.height = tag_value.get("height") + props.seed = tag_value.get("seed") + props.cfg_scale = tag_value.get("cfg_scale") + props.steps = tag_value.get("steps") + props.scheduler = tag_value.get("scheduler") + props.strength = tag_value.get("strength") + props.positive_prompt = tag_value.get("positive_conditioning") + props.negative_prompt = tag_value.get("negative_conditioning") + + return props + + def map_scheduler(self, old_scheduler): + """Convert the legacy sampler names to matching 3.0 schedulers""" + if old_scheduler is None: + return None + + match(old_scheduler): + case "ddim" : return "ddim" + case "plms" : return "pnmd" + case "k_lms" : return "lms" + case "k_dpm_2" : return "kdpm_2" + case "k_dpm_2_a" : return "kdpm_2_a" + case "dpmpp_2" : return "dpmpp_2s" + case "k_dpmpp_2" : return "dpmpp_2m" + case "k_dpmpp_2_a" : return None #invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session + case "k_euler" : return "euler" + case "k_euler_a" : return "euler_a" + case "k_heun" : return "heun" + return None + + def split_prompt(self, raw_prompt: str): + """Split the unified prompt strings by extracting all negative prompt blocks out into the negative prompt.""" + if raw_prompt is None: + return "", "" + raw_prompt_search = raw_prompt.replace("\r","").replace("\n","") + matches = re.findall(r"\[(.+?)\]", raw_prompt_search) + if len(matches) > 0: + negative_prompt = "" + if len(matches) == 1: + negative_prompt = matches[0].strip().strip(',') + else: + for match in matches: + negative_prompt += f"({match.strip().strip(',')})" + positive_prompt = re.sub(r"(\[.+?\])", "", raw_prompt_search).strip() + else: + positive_prompt = raw_prompt_search.strip() + negative_prompt = "" + + return positive_prompt,negative_prompt + +class DatabaseMapper: + """Class to abstract database functionality.""" + def __init__(self, database_path, database_backup_dir): + self.database_path = database_path + self.database_backup_dir = database_backup_dir + self.connection = None + self.cursor = None + + def connect(self): + """Open connection to the database.""" + self.connection = sqlite3.connect(self.database_path) + self.cursor = self.connection.cursor() + + def get_board_names(self): + """Get a list of the current board names from the database.""" + sql_get_board_name = "SELECT board_name FROM boards" + self.cursor.execute(sql_get_board_name) + rows = self.cursor.fetchall() + return [row[0] for row in rows] + + def does_image_exist(self, image_name): + """Check database if a image name already exists and return a boolean.""" + sql_get_image_by_name = f"SELECT image_name FROM images WHERE image_name='{image_name}'" + self.cursor.execute(sql_get_image_by_name) + rows = self.cursor.fetchall() + return True if len(rows) > 0 else False + + def add_new_image_to_database(self, filename, width, height, metadata, modified_date_string): + """Add an image to the database.""" + sql_add_image = f"""INSERT INTO images (image_name, image_origin, image_category, width, height, session_id, node_id, metadata, is_intermediate, created_at, updated_at) +VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{metadata}', 0, '{modified_date_string}', '{modified_date_string}')""" + self.cursor.execute(sql_add_image) + self.connection.commit() + + def get_board_id_with_create(self, board_name): + """Get the board id for supplied name, and create the board if one does not exist.""" + sql_find_board = f"SELECT board_id FROM boards WHERE board_name='{board_name}' COLLATE NOCASE" + self.cursor.execute(sql_find_board) + rows = self.cursor.fetchall() + if len(rows)>0: + return rows[0][0] + else: + board_date_string = datetime.datetime.utcnow().date().isoformat() + new_board_id = str(uuid.uuid4()) + sql_insert_board = f"INSERT INTO boards (board_id, board_name, created_at, updated_at) VALUES ('{new_board_id}', '{board_name}', '{board_date_string}', '{board_date_string}')" + self.cursor.execute(sql_insert_board) + self.connection.commit() + return new_board_id + + def add_image_to_board(self, filename, board_id): + """Add an image mapping to a board.""" + add_datetime_str = datetime.datetime.utcnow().isoformat() + sql_add_image_to_board = f"""INSERT INTO board_images (board_id, image_name, created_at, updated_at) + VALUES ('{board_id}', '{filename}', '{add_datetime_str}', '{add_datetime_str}')""" + self.cursor.execute(sql_add_image_to_board) + self.connection.commit() + + def disconnect(self): + """Disconnect from the db, cleaning up connections and cursors.""" + if self.cursor is not None: + self.cursor.close() + if self.connection is not None: + self.connection.close() + + def backup(self, timestamp_string): + """Take a backup of the database.""" + if not os.path.exists(self.database_backup_dir): + print (f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") + os.makedirs(self.database_backup_dir) + print ("Done!") + database_backup_path = os.path.join(self.database_backup_dir, f"backup-{timestamp_string}-invokeai.db") + print (f"Making DB Backup at {database_backup_path}...", end="") + shutil.copy2(self.database_path, database_backup_path) + print ("Done!") + +class MediaImportProcessor: + """Containing class for script functionality.""" + def __init__(self): + pass + + board_name_id_map = {} + + def get_import_file_list(self): + """Ask the user for the import folder and scan for the list of files to return.""" + while True: + default = '' + while True: + import_dir = os.path.expanduser( + prompt( + "Inputs: Specify absolute path containing InvokeAI .png images to import: ", + completer = PathCompleter(expanduser=True, only_directories=True), + default = default, + )) + if len(import_dir)>0 and Path(import_dir).is_dir(): + break + default = import_dir + + recurse_directories = (prompt("Include files from subfolders recursively [yN]? ").strip() or 'N').upper().startswith('N') + if recurse_directories: + is_recurse = False + matching_file_list = glob.glob(import_dir + '/*.png', recursive=False) + else: + is_recurse = True + matching_file_list = glob.glob(import_dir + '/**/*.png', recursive=True) + + if len(matching_file_list) > 0: + return import_dir, is_recurse, matching_file_list + else: + print(f"The specific path {import_dir} exists, but does not contain .png files!") + + def get_file_details(self, filepath): + """Retrieve the embedded metedata fields and dimensions from an image file.""" + with PIL.Image.open(filepath) as img: + img.load() + png_width, png_height = img.size + img_info = img.info + return img_info, png_width, png_height + + def select_board_option(self, board_names, timestamp_string): + """Allow the user to choose how a board is selected for imported files.""" + while True: + print("\r\nOptions for board selection for imported images:") + print(f"1) Select an existing board name. (found {len(board_names)})") + print( "2) Specify a board name to create/add to.") + print( "3) Create/add to board named 'IMPORT'.") + print(f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string}).") + print( "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5).") + input_option = input("Specify desired board option: ") + match(input_option): + case "1" : + if len(board_names) < 1: + print("\r\nThere are no existing board names to choose from. Select another option!") + continue + board_name = self.select_item_from_list(board_names, "board name", True, "Cancel, go back and choose a different board option.") + if board_name is not None: + return board_name + case "2": + while True: + board_name = input("Specify new/existing board name: ") + if board_name: + return board_name + case "3" : + return "IMPORT" + case "4": + return f"IMPORT_{timestamp_string}" + case "5": + return "IMPORT_APPVERSION" + + def select_item_from_list(self, items, entity_name, allow_cancel, cancel_string): + """A general function to render a list of items to select in the console, prompt the user for a selection and ensure a valid entry is selected.""" + print (f"Select a {entity_name.lower()} from the following list:") + index = 1 + for item in items: + print(f"{index}) {item}") + index += 1 + if allow_cancel: + print(f"{index}) {cancel_string}") + while True: + try: + option_number = int(input("Specify number of selection: ")) + except ValueError: + continue + if allow_cancel and option_number == index: + return None + if option_number >=1 and option_number <= len(items): + return items[option_number-1] + + def import_image(self, filepath: str, board_name_option: str, db_mapper: DatabaseMapper, config: Config): + """Import a single file by its path""" + parser = InvokeAIMetadataParser() + file_name = os.path.basename(filepath) + file_destination_path = os.path.join(config.outputs_path, file_name) + + print("===============================================================================") + print(f"Importing {filepath}") + + # check destination to see if the file was previously imported + if os.path.exists(file_destination_path): + print("File already exists in the destination, skipping!") + ImportStats.count_skipped_file_exists += 1 + return + + # check if file name is already referenced in the database + if db_mapper.does_image_exist(file_name): + print("A reference to a file with this name already exists in the database, skipping!") + ImportStats.count_skipped_db_exists += 1 + return + + # load image info and dimensions + img_info, png_width, png_height = self.get_file_details(filepath) + + # parse metadata + destination_needs_meta_update = True + log_version_note = "(Unknown)" + if "invokeai_metadata" in img_info: + # for the latest, we will just re-emit the same json, no need to parse/modify + converted_field = None + latest_json_string = img_info.get("invokeai_metadata") + log_version_note = "3.0.0+" + destination_needs_meta_update = False + else: + if "sd-metadata" in img_info: + converted_field = parser.parse_meta_tag_sd_metadata(json.loads(img_info.get("sd-metadata"))) + elif "invokeai" in img_info: + converted_field = parser.parse_meta_tag_invokeai(json.loads(img_info.get("invokeai"))) + elif "dream" in img_info: + converted_field = parser.parse_meta_tag_dream(img_info.get("dream")) + elif "Dream" in img_info: + converted_field = parser.parse_meta_tag_dream(img_info.get("Dream")) + else: + converted_field = InvokeAIMetadata() + destination_needs_meta_update = False + print("File does not have metadata from known Invoke AI versions, add only, no update!") + + # use the loaded img dimensions if the metadata didnt have them + if converted_field.width is None: + converted_field.width = png_width + if converted_field.height is None: + converted_field.height = png_height + + log_version_note = converted_field.imported_app_version if converted_field else "NoVersion" + log_version_note = log_version_note or "NoVersion" + + latest_json_string = converted_field.to_json() + + print (f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") + + # if metadata needs update, then update metdata and copy in one shot + if destination_needs_meta_update: + print("Updating metadata while copying...", end="") + self.update_file_metadata_while_copying(filepath, file_destination_path, "invokeai_metadata", latest_json_string) + print("Done!") + else: + print("No metadata update necessary, copying only...", end="") + shutil.copy2(filepath, file_destination_path) + print("Done!") + + # create thumbnail + print("Creating thumbnail...", end="") + thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" + thumbnail_size = 256, 256 + with PIL.Image.open(filepath) as source_image: + source_image.thumbnail(thumbnail_size) + source_image.save(thumbnail_path, "webp") + print("Done!") + + # finalize the dynamic board name if there is an APPVERSION token in it. + if converted_field is not None: + board_name = board_name_option.replace("APPVERSION", converted_field.imported_app_version or "NoVersion") + else: + board_name = board_name_option.replace("APPVERSION", "Latest") + + # maintain a map of alrady created/looked up ids to avoid DB queries + print("Finding/Creating board...", end="") + if board_name in self.board_name_id_map: + board_id = self.board_name_id_map[board_name] + else: + board_id = db_mapper.get_board_id_with_create(board_name) + self.board_name_id_map[board_name] = board_id + print("Done!") + + # add image to db + print("Adding image to database......", end="") + modified_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(filepath)) + db_mapper.add_new_image_to_database(file_name, png_width, png_height, latest_json_string, modified_time) + print("Done!") + + #add image to board + print("Adding image to board......", end="") + db_mapper.add_image_to_board(file_name, board_id) + print("Done!") + + ImportStats.count_imported += 1 + if log_version_note in ImportStats.count_imported_by_version: + ImportStats.count_imported_by_version[log_version_note] += 1 + else: + ImportStats.count_imported_by_version[log_version_note] = 1 + + def update_file_metadata_while_copying(self, filepath, file_destination_path, tag_name, tag_value): + """Perform a metadata update with save to a new destination which accomplishes a copy while updating metadata.""" + with PIL.Image.open(filepath) as target_image: + existing_img_info = target_image.info + metadata = PIL.PngImagePlugin.PngInfo() + # re-add any existing invoke ai tags unless they are the one we are trying to add + for key in existing_img_info: + if key != tag_name and key in ("dream", "Dream", "sd-metadata", "invokeai", "invokeai_metadata"): + metadata.add_text(key,existing_img_info[key]) + metadata.add_text(tag_name, tag_value) + target_image.save(file_destination_path, pnginfo=metadata) + + def process(self): + """Begin main processing.""" + + print("===============================================================================") + print("This script will import images generated by earlier versions of") + print("InvokeAI into the currently installed root directory:") + print(f' {app_config.root_path}') + print("If this is not what you want to do, type ctrl-C now to cancel.") + + # load config + print("===============================================================================") + print("= Configuration & Settings") + + config = Config() + config.find_and_load() + db_mapper = DatabaseMapper(config.database_path, config.database_backup_dir) + db_mapper.connect() + + import_dir, is_recurse, import_file_list = self.get_import_file_list() + ImportStats.count_source_files = len(import_file_list) + + board_names = db_mapper.get_board_names() + board_name_option = self.select_board_option(board_names, config.TIMESTAMP_STRING) + + print("\r\n===============================================================================") + print("= Import Settings Confirmation") + + print() + print(f"Database File Path : {config.database_path}") + print(f"Outputs/Images Directory : {config.outputs_path}") + print(f"Import Image Source Directory : {import_dir}") + print(f" Recurse Source SubDirectories : {'Yes' if is_recurse else 'No'}") + print(f"Count of .png file(s) found : {len(import_file_list)}") + print(f"Board name option specified : {board_name_option}") + print(f"Database backup will be taken at : {config.database_backup_dir}") + + print("\r\nNotes about the import process:") + print("- Source image files will not be modified, only copied to the outputs directory.") + print("- If the same file name already exists in the destination, the file will be skipped.") + print("- If the same file name already has a record in the database, the file will be skipped.") + print("- Invoke AI metadata tags will be updated/written into the imported copy only.") + print("- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)") + print("- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer.") + print("- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder.") + + while True: + should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or 'y' + if should_continue=='n': + print("\r\nCancelling Import") + return + elif should_continue=='y': + print() + break + + db_mapper.backup(config.TIMESTAMP_STRING) + + print() + ImportStats.time_start = datetime.datetime.utcnow() + + for filepath in import_file_list: + try: + self.import_image(filepath, board_name_option, db_mapper, config) + except sqlite3.Error as sql_ex: + print(f"A database related exception was found processing {filepath}, will continue to next file. ") + print("Exception detail:") + print(sql_ex) + ImportStats.count_file_errors += 1 + except Exception as ex: + print(f"Exception processing {filepath}, will continue to next file. ") + print("Exception detail:") + print(ex) + ImportStats.count_file_errors += 1 + + print("\r\n===============================================================================") + print(f"= Import Complete - Elpased Time: {ImportStats.get_elapsed_time_string()}") + print() + print(f"Source File(s) : {ImportStats.count_source_files}") + print(f"Total Imported : {ImportStats.count_imported}") + print(f"Skipped b/c file already exists on disk : {ImportStats.count_skipped_file_exists}") + print(f"Skipped b/c file already exists in db : {ImportStats.count_skipped_db_exists}") + print(f"Errors during import : {ImportStats.count_file_errors}") + if ImportStats.count_imported > 0: + print("\r\nBreakdown of imported files by version:") + for key,version in ImportStats.count_imported_by_version.items(): + print(f" {key:20} : {version}") + +def main(): + try: + processor = MediaImportProcessor() + processor.process() + except KeyboardInterrupt: + print("\r\n\r\nUser cancelled execution.") + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index b3f12481a8..d5aa3f37aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,7 @@ dependencies = [ "invokeai-metadata" = "invokeai.frontend.CLI.sd_metadata:print_metadata" "invokeai-node-cli" = "invokeai.app.cli_app:invoke_cli" "invokeai-node-web" = "invokeai.app.api_app:invoke_api" +"invokeai-import-images" = "invokeai.frontend.install.import_images:main" [project.urls] "Homepage" = "https://invoke-ai.github.io/InvokeAI/" From b2eb83deff8d525ca9799b0c0251aa0f26b9b5d2 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:26:52 -0400 Subject: [PATCH 05/10] add docs --- README.md | 23 +++++++++++++++++++--- docs/installation/010_INSTALL_AUTOMATED.md | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2efaacd031..7b3f95ab10 100644 --- a/README.md +++ b/README.md @@ -306,13 +306,30 @@ InvokeAI. The second will prepare the 2.3 directory for use with 3.0. You may now launch the WebUI in the usual way, by selecting option [1] from the launcher script -#### Migration Caveats +#### Migrating Images The migration script will migrate your invokeai settings and models, including textual inversion models, LoRAs and merges that you may have installed previously. However it does **not** migrate the generated -images stored in your 2.3-format outputs directory. You will need to -manually import selected images into the 3.0 gallery via drag-and-drop. +images stored in your 2.3-format outputs directory. To do this, you +need to run an additional step: + +1. From a working InvokeAI 3.0 root directory, start the launcher and +enter menu option [8] to open the "developer's console". + +2. At the developer's console command line, type the command: + +```bash +invokeai-import-images +``` + +3. This will lead you through the process of confirming the desired + source and destination for the imported images. The images will + appear in the gallery board of your choice, and contain the + original prompt, model name, and other parameters used to generate + the image. + +(Many kudos to **techjedi** for contributing this script.) ## Hardware Requirements diff --git a/docs/installation/010_INSTALL_AUTOMATED.md b/docs/installation/010_INSTALL_AUTOMATED.md index 9b66fcbd83..07a88d5d1b 100644 --- a/docs/installation/010_INSTALL_AUTOMATED.md +++ b/docs/installation/010_INSTALL_AUTOMATED.md @@ -264,7 +264,7 @@ experimental versions later. you can create several levels of subfolders and drop your models into whichever ones you want. - - ***Autoimport FolderLICENSE*** + - ***LICENSE*** At the bottom of the screen you will see a checkbox for accepting the CreativeML Responsible AI Licenses. You need to accept the license From 12e51c84ae290fe4fec2a3343e448c06841a8faf Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:44:58 -0400 Subject: [PATCH 06/10] blackified --- invokeai/frontend/install/import_images.py | 213 +++++++++++++-------- 1 file changed, 131 insertions(+), 82 deletions(-) diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py index 1870fafbf8..fe0f20c830 100644 --- a/invokeai/frontend/install/import_images.py +++ b/invokeai/frontend/install/import_images.py @@ -26,22 +26,28 @@ from prompt_toolkit.completion import PathCompleter from prompt_toolkit.key_binding import KeyBindings from invokeai.app.services.config import InvokeAIAppConfig + app_config = InvokeAIAppConfig.get_config() bindings = KeyBindings() -@bindings.add('c-c') + + +@bindings.add("c-c") def _(event): raise KeyboardInterrupt + # release notes # "Use All" with size dimensions not selectable in the UI will not load dimensions + class Config: """Configuration loader.""" + def __init__(self): pass - TIMESTAMP_STRING= datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + TIMESTAMP_STRING = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") INVOKE_DIRNAME = "invokeai" YAML_FILENAME = "invokeai.yaml" @@ -56,21 +62,21 @@ class Config: """find the yaml config file and load""" root = app_config.root_path if not self.confirm_and_load(os.path.abspath(root)): - print ("\r\nSpecify custom database and outputs paths:") + print("\r\nSpecify custom database and outputs paths:") self.confirm_and_load_from_user() - self.database_backup_dir = os.path.join(os.path.dirname(self.database_path),"backup") - self.thumbnail_path = os.path.join(self.outputs_path,"thumbnails") + self.database_backup_dir = os.path.join(os.path.dirname(self.database_path), "backup") + self.thumbnail_path = os.path.join(self.outputs_path, "thumbnails") def confirm_and_load(self, invoke_root): """Validates a yaml path exists, confirms the user wants to use it and loads config.""" - yaml_path = os.path.join(invoke_root,self.YAML_FILENAME) + yaml_path = os.path.join(invoke_root, self.YAML_FILENAME) if os.path.exists(yaml_path): db_dir, outdir = self.load_paths_from_yaml(yaml_path) if os.path.isabs(db_dir): - database_path = os.path.join(db_dir,self.DATABASE_FILENAME) + database_path = os.path.join(db_dir, self.DATABASE_FILENAME) else: - database_path = os.path.join(invoke_root,db_dir,self.DATABASE_FILENAME) + database_path = os.path.join(invoke_root, db_dir, self.DATABASE_FILENAME) if os.path.isabs(outdir): outputs_path = os.path.join(outdir, "images") @@ -86,7 +92,7 @@ class Config: text += "\n\nUse these paths for import (yes) or choose different ones (no) [Yn]: " if db_exists and outdir_exists: - if (prompt(text).strip() or 'Y').upper().startswith('Y'): + if (prompt(text).strip() or "Y").upper().startswith("Y"): self.database_path = database_path self.outputs_path = outputs_path return True @@ -98,36 +104,40 @@ class Config: else: message_dialog( title="Path not found", - text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified." + text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified.", ).run() return False def confirm_and_load_from_user(self): - default = '' + default = "" while True: database_path = os.path.expanduser( prompt( "Database: Specify absolute path to the database to import into: ", - completer=PathCompleter(expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith(('.db'))), - default = default, - )) + completer=PathCompleter( + expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith((".db")) + ), + default=default, + ) + ) if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): break - default = database_path + '/' if Path(database_path).is_dir() else database_path + default = database_path + "/" if Path(database_path).is_dir() else database_path - default = '' + default = "" while True: outputs_path = os.path.expanduser( prompt( "Outputs: Specify absolute path to outputs/images directory to import into: ", - completer = PathCompleter(expanduser=True, only_directories=True), - default = default, - )) + completer=PathCompleter(expanduser=True, only_directories=True), + default=default, + ) + ) if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): break - default = outputs_path + '/' if Path(outputs_path).is_dir() else outputs_path - + default = outputs_path + "/" if Path(outputs_path).is_dir() else outputs_path + self.database_path = database_path self.outputs_path = outputs_path @@ -136,17 +146,19 @@ class Config: def load_paths_from_yaml(self, yaml_path): """Load an Invoke AI yaml file and get the database and outputs paths.""" try: - with open(yaml_path, 'rt', encoding=locale.getpreferredencoding()) as file: + with open(yaml_path, "rt", encoding=locale.getpreferredencoding()) as file: yamlinfo = yaml.safe_load(file) - db_dir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("db_dir", None) - outdir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("outdir", None) + db_dir = yamlinfo.get("InvokeAI", {}).get("Paths", {}).get("db_dir", None) + outdir = yamlinfo.get("InvokeAI", {}).get("Paths", {}).get("outdir", None) return db_dir, outdir except Exception: print(f"Failed to load paths from yaml file! {yaml_path}!") return None, None + class ImportStats: """DTO for tracking work progress.""" + def __init__(self): pass @@ -171,8 +183,10 @@ class ImportStats: out_str += f"{seconds:.2f} second(s)" return out_str + class InvokeAIMetadata: """DTO for core Invoke AI generation properties parsed from metadata.""" + def __init__(self): pass @@ -234,6 +248,7 @@ class InvokeAIMetadata: class InvokeAIMetadataParser: """Parses strings with json data to find Invoke AI core metadata properties.""" + def __init__(self): pass @@ -271,15 +286,15 @@ class InvokeAIMetadataParser: props.width = img_node.get("width") props.height = img_node.get("height") props.seed = img_node.get("seed") - props.rand_device = "cuda" #hardcoded since all generations pre 3.0 used cuda random noise instead of cpu + props.rand_device = "cuda" # hardcoded since all generations pre 3.0 used cuda random noise instead of cpu props.cfg_scale = img_node.get("cfg_scale") props.steps = img_node.get("steps") props.scheduler = self.map_scheduler(img_node.get("sampler")) props.strength = img_node.get("strength") if props.strength is None: - props.strength = img_node.get("strength_steps") # try second name for this property + props.strength = img_node.get("strength_steps") # try second name for this property props.init_image = img_node.get("init_image_path") - if props.init_image is None: # try second name for this property + if props.init_image is None: # try second name for this property props.init_image = img_node.get("init_img") # remove the path info from init_image so if we move the init image, it will be correctly relative in the new location if props.init_image is not None: @@ -299,7 +314,7 @@ class InvokeAIMetadataParser: props.imported_app_version = "3.0.0 or later" props.generation_mode = tag_value.get("type") if props.generation_mode is not None: - props.generation_mode = props.generation_mode.replace("t2l","txt2img").replace("l2l","img2img") + props.generation_mode = props.generation_mode.replace("t2l", "txt2img").replace("l2l", "img2img") props.width = tag_value.get("width") props.height = tag_value.get("height") @@ -318,30 +333,41 @@ class InvokeAIMetadataParser: if old_scheduler is None: return None - match(old_scheduler): - case "ddim" : return "ddim" - case "plms" : return "pnmd" - case "k_lms" : return "lms" - case "k_dpm_2" : return "kdpm_2" - case "k_dpm_2_a" : return "kdpm_2_a" - case "dpmpp_2" : return "dpmpp_2s" - case "k_dpmpp_2" : return "dpmpp_2m" - case "k_dpmpp_2_a" : return None #invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session - case "k_euler" : return "euler" - case "k_euler_a" : return "euler_a" - case "k_heun" : return "heun" + match (old_scheduler): + case "ddim": + return "ddim" + case "plms": + return "pnmd" + case "k_lms": + return "lms" + case "k_dpm_2": + return "kdpm_2" + case "k_dpm_2_a": + return "kdpm_2_a" + case "dpmpp_2": + return "dpmpp_2s" + case "k_dpmpp_2": + return "dpmpp_2m" + case "k_dpmpp_2_a": + return None # invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session + case "k_euler": + return "euler" + case "k_euler_a": + return "euler_a" + case "k_heun": + return "heun" return None def split_prompt(self, raw_prompt: str): """Split the unified prompt strings by extracting all negative prompt blocks out into the negative prompt.""" if raw_prompt is None: return "", "" - raw_prompt_search = raw_prompt.replace("\r","").replace("\n","") + raw_prompt_search = raw_prompt.replace("\r", "").replace("\n", "") matches = re.findall(r"\[(.+?)\]", raw_prompt_search) if len(matches) > 0: negative_prompt = "" if len(matches) == 1: - negative_prompt = matches[0].strip().strip(',') + negative_prompt = matches[0].strip().strip(",") else: for match in matches: negative_prompt += f"({match.strip().strip(',')})" @@ -350,10 +376,12 @@ class InvokeAIMetadataParser: positive_prompt = raw_prompt_search.strip() negative_prompt = "" - return positive_prompt,negative_prompt + return positive_prompt, negative_prompt + class DatabaseMapper: """Class to abstract database functionality.""" + def __init__(self, database_path, database_backup_dir): self.database_path = database_path self.database_backup_dir = database_backup_dir @@ -380,7 +408,7 @@ class DatabaseMapper: return True if len(rows) > 0 else False def add_new_image_to_database(self, filename, width, height, metadata, modified_date_string): - """Add an image to the database.""" + """Add an image to the database.""" sql_add_image = f"""INSERT INTO images (image_name, image_origin, image_category, width, height, session_id, node_id, metadata, is_intermediate, created_at, updated_at) VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{metadata}', 0, '{modified_date_string}', '{modified_date_string}')""" self.cursor.execute(sql_add_image) @@ -391,7 +419,7 @@ VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{me sql_find_board = f"SELECT board_id FROM boards WHERE board_name='{board_name}' COLLATE NOCASE" self.cursor.execute(sql_find_board) rows = self.cursor.fetchall() - if len(rows)>0: + if len(rows) > 0: return rows[0][0] else: board_date_string = datetime.datetime.utcnow().date().isoformat() @@ -417,18 +445,20 @@ VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{me self.connection.close() def backup(self, timestamp_string): - """Take a backup of the database.""" + """Take a backup of the database.""" if not os.path.exists(self.database_backup_dir): - print (f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") + print(f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") os.makedirs(self.database_backup_dir) - print ("Done!") + print("Done!") database_backup_path = os.path.join(self.database_backup_dir, f"backup-{timestamp_string}-invokeai.db") - print (f"Making DB Backup at {database_backup_path}...", end="") + print(f"Making DB Backup at {database_backup_path}...", end="") shutil.copy2(self.database_path, database_backup_path) - print ("Done!") + print("Done!") + class MediaImportProcessor: """Containing class for script functionality.""" + def __init__(self): pass @@ -437,25 +467,28 @@ class MediaImportProcessor: def get_import_file_list(self): """Ask the user for the import folder and scan for the list of files to return.""" while True: - default = '' + default = "" while True: import_dir = os.path.expanduser( prompt( "Inputs: Specify absolute path containing InvokeAI .png images to import: ", - completer = PathCompleter(expanduser=True, only_directories=True), - default = default, - )) - if len(import_dir)>0 and Path(import_dir).is_dir(): + completer=PathCompleter(expanduser=True, only_directories=True), + default=default, + ) + ) + if len(import_dir) > 0 and Path(import_dir).is_dir(): break default = import_dir - recurse_directories = (prompt("Include files from subfolders recursively [yN]? ").strip() or 'N').upper().startswith('N') + recurse_directories = ( + (prompt("Include files from subfolders recursively [yN]? ").strip() or "N").upper().startswith("N") + ) if recurse_directories: is_recurse = False - matching_file_list = glob.glob(import_dir + '/*.png', recursive=False) + matching_file_list = glob.glob(import_dir + "/*.png", recursive=False) else: is_recurse = True - matching_file_list = glob.glob(import_dir + '/**/*.png', recursive=True) + matching_file_list = glob.glob(import_dir + "/**/*.png", recursive=True) if len(matching_file_list) > 0: return import_dir, is_recurse, matching_file_list @@ -475,17 +508,23 @@ class MediaImportProcessor: while True: print("\r\nOptions for board selection for imported images:") print(f"1) Select an existing board name. (found {len(board_names)})") - print( "2) Specify a board name to create/add to.") - print( "3) Create/add to board named 'IMPORT'.") - print(f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string}).") - print( "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5).") + print("2) Specify a board name to create/add to.") + print("3) Create/add to board named 'IMPORT'.") + print( + f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string})." + ) + print( + "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5)." + ) input_option = input("Specify desired board option: ") - match(input_option): - case "1" : + match (input_option): + case "1": if len(board_names) < 1: print("\r\nThere are no existing board names to choose from. Select another option!") continue - board_name = self.select_item_from_list(board_names, "board name", True, "Cancel, go back and choose a different board option.") + board_name = self.select_item_from_list( + board_names, "board name", True, "Cancel, go back and choose a different board option." + ) if board_name is not None: return board_name case "2": @@ -493,7 +532,7 @@ class MediaImportProcessor: board_name = input("Specify new/existing board name: ") if board_name: return board_name - case "3" : + case "3": return "IMPORT" case "4": return f"IMPORT_{timestamp_string}" @@ -502,7 +541,7 @@ class MediaImportProcessor: def select_item_from_list(self, items, entity_name, allow_cancel, cancel_string): """A general function to render a list of items to select in the console, prompt the user for a selection and ensure a valid entry is selected.""" - print (f"Select a {entity_name.lower()} from the following list:") + print(f"Select a {entity_name.lower()} from the following list:") index = 1 for item in items: print(f"{index}) {item}") @@ -516,8 +555,8 @@ class MediaImportProcessor: continue if allow_cancel and option_number == index: return None - if option_number >=1 and option_number <= len(items): - return items[option_number-1] + if option_number >= 1 and option_number <= len(items): + return items[option_number - 1] def import_image(self, filepath: str, board_name_option: str, db_mapper: DatabaseMapper, config: Config): """Import a single file by its path""" @@ -577,12 +616,14 @@ class MediaImportProcessor: latest_json_string = converted_field.to_json() - print (f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") + print(f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") # if metadata needs update, then update metdata and copy in one shot if destination_needs_meta_update: print("Updating metadata while copying...", end="") - self.update_file_metadata_while_copying(filepath, file_destination_path, "invokeai_metadata", latest_json_string) + self.update_file_metadata_while_copying( + filepath, file_destination_path, "invokeai_metadata", latest_json_string + ) print("Done!") else: print("No metadata update necessary, copying only...", end="") @@ -591,7 +632,7 @@ class MediaImportProcessor: # create thumbnail print("Creating thumbnail...", end="") - thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" + thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" thumbnail_size = 256, 256 with PIL.Image.open(filepath) as source_image: source_image.thumbnail(thumbnail_size) @@ -619,7 +660,7 @@ class MediaImportProcessor: db_mapper.add_new_image_to_database(file_name, png_width, png_height, latest_json_string, modified_time) print("Done!") - #add image to board + # add image to board print("Adding image to board......", end="") db_mapper.add_image_to_board(file_name, board_id) print("Done!") @@ -638,7 +679,7 @@ class MediaImportProcessor: # re-add any existing invoke ai tags unless they are the one we are trying to add for key in existing_img_info: if key != tag_name and key in ("dream", "Dream", "sd-metadata", "invokeai", "invokeai_metadata"): - metadata.add_text(key,existing_img_info[key]) + metadata.add_text(key, existing_img_info[key]) metadata.add_text(tag_name, tag_value) target_image.save(file_destination_path, pnginfo=metadata) @@ -648,7 +689,7 @@ class MediaImportProcessor: print("===============================================================================") print("This script will import images generated by earlier versions of") print("InvokeAI into the currently installed root directory:") - print(f' {app_config.root_path}') + print(f" {app_config.root_path}") print("If this is not what you want to do, type ctrl-C now to cancel.") # load config @@ -683,16 +724,22 @@ class MediaImportProcessor: print("- If the same file name already exists in the destination, the file will be skipped.") print("- If the same file name already has a record in the database, the file will be skipped.") print("- Invoke AI metadata tags will be updated/written into the imported copy only.") - print("- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)") - print("- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer.") - print("- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder.") + print( + "- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)" + ) + print( + "- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer." + ) + print( + "- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder." + ) while True: - should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or 'y' - if should_continue=='n': + should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or "y" + if should_continue == "n": print("\r\nCancelling Import") return - elif should_continue=='y': + elif should_continue == "y": print() break @@ -725,9 +772,10 @@ class MediaImportProcessor: print(f"Errors during import : {ImportStats.count_file_errors}") if ImportStats.count_imported > 0: print("\r\nBreakdown of imported files by version:") - for key,version in ImportStats.count_imported_by_version.items(): + for key, version in ImportStats.count_imported_by_version.items(): print(f" {key:20} : {version}") + def main(): try: processor = MediaImportProcessor() @@ -735,5 +783,6 @@ def main(): except KeyboardInterrupt: print("\r\n\r\nUser cancelled execution.") + if __name__ == "__main__": main() From 1a6f5f0860169569115f689350f5247f1787877c Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 6 Aug 2023 12:29:31 -0400 Subject: [PATCH 07/10] use backslash on Windows systems for autoadded delimiter --- invokeai/frontend/install/import_images.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py index fe0f20c830..c56dd2e105 100644 --- a/invokeai/frontend/install/import_images.py +++ b/invokeai/frontend/install/import_images.py @@ -6,6 +6,7 @@ """Script to import images into the new database system for 3.0.0""" import os +import platform import datetime import shutil import locale @@ -21,21 +22,14 @@ import PIL.PngImagePlugin from pathlib import Path from prompt_toolkit import prompt -from prompt_toolkit.shortcuts import yes_no_dialog, message_dialog, input_dialog, button_dialog +from prompt_toolkit.shortcuts import message_dialog from prompt_toolkit.completion import PathCompleter -from prompt_toolkit.key_binding import KeyBindings from invokeai.app.services.config import InvokeAIAppConfig app_config = InvokeAIAppConfig.get_config() - -bindings = KeyBindings() - - -@bindings.add("c-c") -def _(event): - raise KeyboardInterrupt - +path_delimiter = '\\' if platform.uname().system=='Windows' else '/' +print(path_delimiter) # release notes # "Use All" with size dimensions not selectable in the UI will not load dimensions @@ -122,7 +116,7 @@ class Config: ) if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): break - default = database_path + "/" if Path(database_path).is_dir() else database_path + default = database_path + path_delimiter if Path(database_path).is_dir() else database_path default = "" while True: @@ -136,7 +130,7 @@ class Config: if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): break - default = outputs_path + "/" if Path(outputs_path).is_dir() else outputs_path + default = outputs_path + path_delimiter if Path(outputs_path).is_dir() else outputs_path self.database_path = database_path self.outputs_path = outputs_path From 8469d3e95a7c5533508094ff1fcfa20ab1bcd22d Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Mon, 7 Aug 2023 10:05:52 +1000 Subject: [PATCH 08/10] chore: black --- invokeai/frontend/install/import_images.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py index 71bef34f26..befef58744 100644 --- a/invokeai/frontend/install/import_images.py +++ b/invokeai/frontend/install/import_images.py @@ -116,7 +116,11 @@ class Config: ) if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): break - default = database_path + path_delimiter if Path(database_path).is_dir() and not database_path.endswith(('\\','/')) else database_path + default = ( + database_path + path_delimiter + if Path(database_path).is_dir() and not database_path.endswith(("\\", "/")) + else database_path + ) default = "" while True: @@ -130,7 +134,11 @@ class Config: if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): break - default = outputs_path + path_delimiter if Path(outputs_path).is_dir() and not database_path.endswith(('\\','/')) else outputs_path + default = ( + outputs_path + path_delimiter + if Path(outputs_path).is_dir() and not database_path.endswith(("\\", "/")) + else outputs_path + ) self.database_path = database_path self.outputs_path = outputs_path From a846d82fa1e0de8525088db40f847379d07777f6 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 7 Aug 2023 16:29:46 -0400 Subject: [PATCH 09/10] Add techedi code to avoid rendering prompt/seed with null - Added techjedi github and real names --- invokeai/frontend/install/import_images.py | 39 ++++++++++++---------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py index befef58744..ea5a9d2d97 100644 --- a/invokeai/frontend/install/import_images.py +++ b/invokeai/frontend/install/import_images.py @@ -1,12 +1,12 @@ # Copyright (c) 2023 - The InvokeAI Team -# Author: techjedi +# Primary Author: David Lovell (github @f412design, discord @techedi) +# co-author, minor tweaks - Lincoln Stein # pylint: disable=line-too-long # pylint: disable=broad-exception-caught """Script to import images into the new database system for 3.0.0""" import os -import platform import datetime import shutil import locale @@ -24,12 +24,19 @@ from pathlib import Path from prompt_toolkit import prompt from prompt_toolkit.shortcuts import message_dialog from prompt_toolkit.completion import PathCompleter +from prompt_toolkit.key_binding import KeyBindings from invokeai.app.services.config import InvokeAIAppConfig app_config = InvokeAIAppConfig.get_config() -path_delimiter = "\\" if platform.uname().system == "Windows" else "/" -print(path_delimiter) + +bindings = KeyBindings() + + +@bindings.add("c-c") +def _(event): + raise KeyboardInterrupt + # release notes # "Use All" with size dimensions not selectable in the UI will not load dimensions @@ -116,11 +123,7 @@ class Config: ) if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): break - default = ( - database_path + path_delimiter - if Path(database_path).is_dir() and not database_path.endswith(("\\", "/")) - else database_path - ) + default = database_path + "/" if Path(database_path).is_dir() else database_path default = "" while True: @@ -134,11 +137,7 @@ class Config: if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): break - default = ( - outputs_path + path_delimiter - if Path(outputs_path).is_dir() and not database_path.endswith(("\\", "/")) - else outputs_path - ) + default = outputs_path + "/" if Path(outputs_path).is_dir() else outputs_path self.database_path = database_path self.outputs_path = outputs_path @@ -217,11 +216,17 @@ class InvokeAIMetadata: """Convert the active instance to json format.""" prop_dict = {} prop_dict["generation_mode"] = self.generation_mode - prop_dict["positive_prompt"] = self.positive_prompt - prop_dict["negative_prompt"] = self.negative_prompt + # dont render prompt nodes if neither are set to avoid the ui thinking it can set them + # if at least one exists, render them both, but use empty string instead of None if one of them is empty + # this allows the field that is empty to actually be cleared byt he UI instead of leaving the previous value + if self.positive_prompt or self.negative_prompt: + prop_dict["positive_prompt"] = "" if self.positive_prompt is None else self.positive_prompt + prop_dict["negative_prompt"] = "" if self.negative_prompt is None else self.negative_prompt prop_dict["width"] = self.width prop_dict["height"] = self.height - prop_dict["seed"] = self.seed + # only render seed if it has a value to avoid ui thinking it can set this and then error + if self.seed: + prop_dict["seed"] = self.seed prop_dict["rand_device"] = self.rand_device prop_dict["cfg_scale"] = self.cfg_scale prop_dict["steps"] = self.steps From 2c2b731386979e467f50089939bc1dea583322a4 Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Wed, 9 Aug 2023 13:08:59 -0400 Subject: [PATCH 10/10] fix typo --- invokeai/frontend/install/import_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py index ea5a9d2d97..1f68f23607 100644 --- a/invokeai/frontend/install/import_images.py +++ b/invokeai/frontend/install/import_images.py @@ -1,5 +1,5 @@ # Copyright (c) 2023 - The InvokeAI Team -# Primary Author: David Lovell (github @f412design, discord @techedi) +# Primary Author: David Lovell (github @f412design, discord @techjedi) # co-author, minor tweaks - Lincoln Stein # pylint: disable=line-too-long