From 0ccc3b509e252b15f628bb0494ac01727a8d3372 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 5 Aug 2023 12:19:24 -0400 Subject: [PATCH] add techjedi's import script, with some filecompletion tweaks --- invokeai/frontend/install/import_images.py | 739 +++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 740 insertions(+) create mode 100644 invokeai/frontend/install/import_images.py diff --git a/invokeai/frontend/install/import_images.py b/invokeai/frontend/install/import_images.py new file mode 100644 index 0000000000..1870fafbf8 --- /dev/null +++ b/invokeai/frontend/install/import_images.py @@ -0,0 +1,739 @@ +# Copyright (c) 2023 - The InvokeAI Team +# Author: techjedi + +# pylint: disable=line-too-long +# pylint: disable=broad-exception-caught +"""Script to import images into the new database system for 3.0.0""" + +import os +import datetime +import shutil +import locale +import sqlite3 +import json +import glob +import re +import uuid +import yaml +import PIL +import PIL.ImageOps +import PIL.PngImagePlugin + +from pathlib import Path +from prompt_toolkit import prompt +from prompt_toolkit.shortcuts import yes_no_dialog, message_dialog, input_dialog, button_dialog +from prompt_toolkit.completion import PathCompleter +from prompt_toolkit.key_binding import KeyBindings + +from invokeai.app.services.config import InvokeAIAppConfig +app_config = InvokeAIAppConfig.get_config() + +bindings = KeyBindings() +@bindings.add('c-c') +def _(event): + raise KeyboardInterrupt + +# release notes +# "Use All" with size dimensions not selectable in the UI will not load dimensions + +class Config: + """Configuration loader.""" + def __init__(self): + pass + + TIMESTAMP_STRING= datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + + INVOKE_DIRNAME = "invokeai" + YAML_FILENAME = "invokeai.yaml" + DATABASE_FILENAME = "invokeai.db" + + database_path = None + database_backup_dir = None + outputs_path = None + thumbnail_path = None + + def find_and_load(self): + """find the yaml config file and load""" + root = app_config.root_path + if not self.confirm_and_load(os.path.abspath(root)): + print ("\r\nSpecify custom database and outputs paths:") + self.confirm_and_load_from_user() + + self.database_backup_dir = os.path.join(os.path.dirname(self.database_path),"backup") + self.thumbnail_path = os.path.join(self.outputs_path,"thumbnails") + + def confirm_and_load(self, invoke_root): + """Validates a yaml path exists, confirms the user wants to use it and loads config.""" + yaml_path = os.path.join(invoke_root,self.YAML_FILENAME) + if os.path.exists(yaml_path): + db_dir, outdir = self.load_paths_from_yaml(yaml_path) + if os.path.isabs(db_dir): + database_path = os.path.join(db_dir,self.DATABASE_FILENAME) + else: + database_path = os.path.join(invoke_root,db_dir,self.DATABASE_FILENAME) + + if os.path.isabs(outdir): + outputs_path = os.path.join(outdir, "images") + else: + outputs_path = os.path.join(invoke_root, outdir, "images") + + db_exists = os.path.exists(database_path) + outdir_exists = os.path.exists(outputs_path) + + text = f"Found {self.YAML_FILENAME} file at {yaml_path}:" + text += f"\n Database : {database_path}" + text += f"\n Outputs : {outputs_path}" + text += "\n\nUse these paths for import (yes) or choose different ones (no) [Yn]: " + + if db_exists and outdir_exists: + if (prompt(text).strip() or 'Y').upper().startswith('Y'): + self.database_path = database_path + self.outputs_path = outputs_path + return True + else: + return False + else: + print(" Invalid: One or more paths in this config did not exist and cannot be used.") + + else: + message_dialog( + title="Path not found", + text=f"Auto-discovery of configuration failed! Could not find ({yaml_path}), Custom paths can be specified." + ).run() + return False + + def confirm_and_load_from_user(self): + default = '' + while True: + database_path = os.path.expanduser( + prompt( + "Database: Specify absolute path to the database to import into: ", + completer=PathCompleter(expanduser=True, file_filter=lambda x: Path(x).is_dir() or x.endswith(('.db'))), + default = default, + )) + if database_path.endswith(".db") and os.path.isabs(database_path) and os.path.exists(database_path): + break + default = database_path + '/' if Path(database_path).is_dir() else database_path + + default = '' + while True: + outputs_path = os.path.expanduser( + prompt( + "Outputs: Specify absolute path to outputs/images directory to import into: ", + completer = PathCompleter(expanduser=True, only_directories=True), + default = default, + )) + + if outputs_path.endswith("images") and os.path.isabs(outputs_path) and os.path.exists(outputs_path): + break + default = outputs_path + '/' if Path(outputs_path).is_dir() else outputs_path + + self.database_path = database_path + self.outputs_path = outputs_path + + return + + def load_paths_from_yaml(self, yaml_path): + """Load an Invoke AI yaml file and get the database and outputs paths.""" + try: + with open(yaml_path, 'rt', encoding=locale.getpreferredencoding()) as file: + yamlinfo = yaml.safe_load(file) + db_dir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("db_dir", None) + outdir = yamlinfo.get("InvokeAI",{}).get("Paths",{}).get("outdir", None) + return db_dir, outdir + except Exception: + print(f"Failed to load paths from yaml file! {yaml_path}!") + return None, None + +class ImportStats: + """DTO for tracking work progress.""" + def __init__(self): + pass + + time_start = datetime.datetime.utcnow() + count_source_files = 0 + count_skipped_file_exists = 0 + count_skipped_db_exists = 0 + count_imported = 0 + count_imported_by_version = {} + count_file_errors = 0 + + @staticmethod + def get_elapsed_time_string(): + """Get a friendly time string for the time elapsed since processing start.""" + time_now = datetime.datetime.utcnow() + total_seconds = (time_now - ImportStats.time_start).total_seconds() + hours = int((total_seconds) / 3600) + minutes = int(((total_seconds) % 3600) / 60) + seconds = total_seconds % 60 + out_str = f"{hours} hour(s) -" if hours > 0 else "" + out_str += f"{minutes} minute(s) -" if minutes > 0 else "" + out_str += f"{seconds:.2f} second(s)" + return out_str + +class InvokeAIMetadata: + """DTO for core Invoke AI generation properties parsed from metadata.""" + def __init__(self): + pass + + def __str__(self): + formatted_str = f"{self.generation_mode}~{self.steps}~{self.cfg_scale}~{self.model_name}~{self.scheduler}~{self.seed}~{self.width}~{self.height}~{self.rand_device}~{self.strength}~{self.init_image}" + formatted_str += f"\r\npositive_prompt: {self.positive_prompt}" + formatted_str += f"\r\nnegative_prompt: {self.negative_prompt}" + return formatted_str + + generation_mode = None + steps = None + cfg_scale = None + model_name = None + scheduler = None + seed = None + width = None + height = None + rand_device = None + strength = None + init_image = None + positive_prompt = None + negative_prompt = None + imported_app_version = None + + def to_json(self): + """Convert the active instance to json format.""" + prop_dict = {} + prop_dict["generation_mode"] = self.generation_mode + prop_dict["positive_prompt"] = self.positive_prompt + prop_dict["negative_prompt"] = self.negative_prompt + prop_dict["width"] = self.width + prop_dict["height"] = self.height + prop_dict["seed"] = self.seed + prop_dict["rand_device"] = self.rand_device + prop_dict["cfg_scale"] = self.cfg_scale + prop_dict["steps"] = self.steps + prop_dict["scheduler"] = self.scheduler + prop_dict["clip_skip"] = 0 + prop_dict["model"] = {} + prop_dict["model"]["model_name"] = self.model_name + prop_dict["model"]["base_model"] = None + prop_dict["controlnets"] = [] + prop_dict["loras"] = [] + prop_dict["vae"] = None + prop_dict["strength"] = self.strength + prop_dict["init_image"] = self.init_image + prop_dict["positive_style_prompt"] = None + prop_dict["negative_style_prompt"] = None + prop_dict["refiner_model"] = None + prop_dict["refiner_cfg_scale"] = None + prop_dict["refiner_steps"] = None + prop_dict["refiner_scheduler"] = None + prop_dict["refiner_aesthetic_store"] = None + prop_dict["refiner_start"] = None + prop_dict["imported_app_version"] = self.imported_app_version + + return json.dumps(prop_dict) + + +class InvokeAIMetadataParser: + """Parses strings with json data to find Invoke AI core metadata properties.""" + def __init__(self): + pass + + def parse_meta_tag_dream(self, dream_string): + """Take as input an png metadata json node for the 'dream' field variant from prior to 1.15""" + props = InvokeAIMetadata() + + props.imported_app_version = "pre1.15" + seed_match = re.search("-S\\s*(\\d+)", dream_string) + if seed_match is not None: + try: + props.seed = int(seed_match[1]) + except ValueError: + props.seed = None + raw_prompt = re.sub("(-S\\s*\\d+)", "", dream_string) + else: + raw_prompt = dream_string + + pos_prompt, neg_prompt = self.split_prompt(raw_prompt) + + props.positive_prompt = pos_prompt + props.negative_prompt = neg_prompt + + return props + + def parse_meta_tag_sd_metadata(self, tag_value): + """Take as input an png metadata json node for the 'sd-metadata' field variant from 1.15 through 2.3.5 post 2""" + props = InvokeAIMetadata() + + props.imported_app_version = tag_value.get("app_version") + props.model_name = tag_value.get("model_weights") + img_node = tag_value.get("image") + if img_node is not None: + props.generation_mode = img_node.get("type") + props.width = img_node.get("width") + props.height = img_node.get("height") + props.seed = img_node.get("seed") + props.rand_device = "cuda" #hardcoded since all generations pre 3.0 used cuda random noise instead of cpu + props.cfg_scale = img_node.get("cfg_scale") + props.steps = img_node.get("steps") + props.scheduler = self.map_scheduler(img_node.get("sampler")) + props.strength = img_node.get("strength") + if props.strength is None: + props.strength = img_node.get("strength_steps") # try second name for this property + props.init_image = img_node.get("init_image_path") + if props.init_image is None: # try second name for this property + props.init_image = img_node.get("init_img") + # remove the path info from init_image so if we move the init image, it will be correctly relative in the new location + if props.init_image is not None: + props.init_image = os.path.basename(props.init_image) + raw_prompt = img_node.get("prompt") + if isinstance(raw_prompt, list): + raw_prompt = raw_prompt[0].get("prompt") + + props.positive_prompt, props.negative_prompt = self.split_prompt(raw_prompt) + + return props + + def parse_meta_tag_invokeai(self, tag_value): + """Take as input an png metadata json node for the 'invokeai' field variant from 3.0.0 beta 1 through 5""" + props = InvokeAIMetadata() + + props.imported_app_version = "3.0.0 or later" + props.generation_mode = tag_value.get("type") + if props.generation_mode is not None: + props.generation_mode = props.generation_mode.replace("t2l","txt2img").replace("l2l","img2img") + + props.width = tag_value.get("width") + props.height = tag_value.get("height") + props.seed = tag_value.get("seed") + props.cfg_scale = tag_value.get("cfg_scale") + props.steps = tag_value.get("steps") + props.scheduler = tag_value.get("scheduler") + props.strength = tag_value.get("strength") + props.positive_prompt = tag_value.get("positive_conditioning") + props.negative_prompt = tag_value.get("negative_conditioning") + + return props + + def map_scheduler(self, old_scheduler): + """Convert the legacy sampler names to matching 3.0 schedulers""" + if old_scheduler is None: + return None + + match(old_scheduler): + case "ddim" : return "ddim" + case "plms" : return "pnmd" + case "k_lms" : return "lms" + case "k_dpm_2" : return "kdpm_2" + case "k_dpm_2_a" : return "kdpm_2_a" + case "dpmpp_2" : return "dpmpp_2s" + case "k_dpmpp_2" : return "dpmpp_2m" + case "k_dpmpp_2_a" : return None #invalid, in 2.3.x, selecting this sample would just fallback to last run or plms if new session + case "k_euler" : return "euler" + case "k_euler_a" : return "euler_a" + case "k_heun" : return "heun" + return None + + def split_prompt(self, raw_prompt: str): + """Split the unified prompt strings by extracting all negative prompt blocks out into the negative prompt.""" + if raw_prompt is None: + return "", "" + raw_prompt_search = raw_prompt.replace("\r","").replace("\n","") + matches = re.findall(r"\[(.+?)\]", raw_prompt_search) + if len(matches) > 0: + negative_prompt = "" + if len(matches) == 1: + negative_prompt = matches[0].strip().strip(',') + else: + for match in matches: + negative_prompt += f"({match.strip().strip(',')})" + positive_prompt = re.sub(r"(\[.+?\])", "", raw_prompt_search).strip() + else: + positive_prompt = raw_prompt_search.strip() + negative_prompt = "" + + return positive_prompt,negative_prompt + +class DatabaseMapper: + """Class to abstract database functionality.""" + def __init__(self, database_path, database_backup_dir): + self.database_path = database_path + self.database_backup_dir = database_backup_dir + self.connection = None + self.cursor = None + + def connect(self): + """Open connection to the database.""" + self.connection = sqlite3.connect(self.database_path) + self.cursor = self.connection.cursor() + + def get_board_names(self): + """Get a list of the current board names from the database.""" + sql_get_board_name = "SELECT board_name FROM boards" + self.cursor.execute(sql_get_board_name) + rows = self.cursor.fetchall() + return [row[0] for row in rows] + + def does_image_exist(self, image_name): + """Check database if a image name already exists and return a boolean.""" + sql_get_image_by_name = f"SELECT image_name FROM images WHERE image_name='{image_name}'" + self.cursor.execute(sql_get_image_by_name) + rows = self.cursor.fetchall() + return True if len(rows) > 0 else False + + def add_new_image_to_database(self, filename, width, height, metadata, modified_date_string): + """Add an image to the database.""" + sql_add_image = f"""INSERT INTO images (image_name, image_origin, image_category, width, height, session_id, node_id, metadata, is_intermediate, created_at, updated_at) +VALUES ('{filename}', 'internal', 'general', {width}, {height}, null, null, '{metadata}', 0, '{modified_date_string}', '{modified_date_string}')""" + self.cursor.execute(sql_add_image) + self.connection.commit() + + def get_board_id_with_create(self, board_name): + """Get the board id for supplied name, and create the board if one does not exist.""" + sql_find_board = f"SELECT board_id FROM boards WHERE board_name='{board_name}' COLLATE NOCASE" + self.cursor.execute(sql_find_board) + rows = self.cursor.fetchall() + if len(rows)>0: + return rows[0][0] + else: + board_date_string = datetime.datetime.utcnow().date().isoformat() + new_board_id = str(uuid.uuid4()) + sql_insert_board = f"INSERT INTO boards (board_id, board_name, created_at, updated_at) VALUES ('{new_board_id}', '{board_name}', '{board_date_string}', '{board_date_string}')" + self.cursor.execute(sql_insert_board) + self.connection.commit() + return new_board_id + + def add_image_to_board(self, filename, board_id): + """Add an image mapping to a board.""" + add_datetime_str = datetime.datetime.utcnow().isoformat() + sql_add_image_to_board = f"""INSERT INTO board_images (board_id, image_name, created_at, updated_at) + VALUES ('{board_id}', '{filename}', '{add_datetime_str}', '{add_datetime_str}')""" + self.cursor.execute(sql_add_image_to_board) + self.connection.commit() + + def disconnect(self): + """Disconnect from the db, cleaning up connections and cursors.""" + if self.cursor is not None: + self.cursor.close() + if self.connection is not None: + self.connection.close() + + def backup(self, timestamp_string): + """Take a backup of the database.""" + if not os.path.exists(self.database_backup_dir): + print (f"Database backup directory {self.database_backup_dir} does not exist -> creating...", end="") + os.makedirs(self.database_backup_dir) + print ("Done!") + database_backup_path = os.path.join(self.database_backup_dir, f"backup-{timestamp_string}-invokeai.db") + print (f"Making DB Backup at {database_backup_path}...", end="") + shutil.copy2(self.database_path, database_backup_path) + print ("Done!") + +class MediaImportProcessor: + """Containing class for script functionality.""" + def __init__(self): + pass + + board_name_id_map = {} + + def get_import_file_list(self): + """Ask the user for the import folder and scan for the list of files to return.""" + while True: + default = '' + while True: + import_dir = os.path.expanduser( + prompt( + "Inputs: Specify absolute path containing InvokeAI .png images to import: ", + completer = PathCompleter(expanduser=True, only_directories=True), + default = default, + )) + if len(import_dir)>0 and Path(import_dir).is_dir(): + break + default = import_dir + + recurse_directories = (prompt("Include files from subfolders recursively [yN]? ").strip() or 'N').upper().startswith('N') + if recurse_directories: + is_recurse = False + matching_file_list = glob.glob(import_dir + '/*.png', recursive=False) + else: + is_recurse = True + matching_file_list = glob.glob(import_dir + '/**/*.png', recursive=True) + + if len(matching_file_list) > 0: + return import_dir, is_recurse, matching_file_list + else: + print(f"The specific path {import_dir} exists, but does not contain .png files!") + + def get_file_details(self, filepath): + """Retrieve the embedded metedata fields and dimensions from an image file.""" + with PIL.Image.open(filepath) as img: + img.load() + png_width, png_height = img.size + img_info = img.info + return img_info, png_width, png_height + + def select_board_option(self, board_names, timestamp_string): + """Allow the user to choose how a board is selected for imported files.""" + while True: + print("\r\nOptions for board selection for imported images:") + print(f"1) Select an existing board name. (found {len(board_names)})") + print( "2) Specify a board name to create/add to.") + print( "3) Create/add to board named 'IMPORT'.") + print(f"4) Create/add to board named 'IMPORT' with the current datetime string appended (.e.g IMPORT_{timestamp_string}).") + print( "5) Create/add to board named 'IMPORT' with a the original file app_version appended (.e.g IMPORT_2.2.5).") + input_option = input("Specify desired board option: ") + match(input_option): + case "1" : + if len(board_names) < 1: + print("\r\nThere are no existing board names to choose from. Select another option!") + continue + board_name = self.select_item_from_list(board_names, "board name", True, "Cancel, go back and choose a different board option.") + if board_name is not None: + return board_name + case "2": + while True: + board_name = input("Specify new/existing board name: ") + if board_name: + return board_name + case "3" : + return "IMPORT" + case "4": + return f"IMPORT_{timestamp_string}" + case "5": + return "IMPORT_APPVERSION" + + def select_item_from_list(self, items, entity_name, allow_cancel, cancel_string): + """A general function to render a list of items to select in the console, prompt the user for a selection and ensure a valid entry is selected.""" + print (f"Select a {entity_name.lower()} from the following list:") + index = 1 + for item in items: + print(f"{index}) {item}") + index += 1 + if allow_cancel: + print(f"{index}) {cancel_string}") + while True: + try: + option_number = int(input("Specify number of selection: ")) + except ValueError: + continue + if allow_cancel and option_number == index: + return None + if option_number >=1 and option_number <= len(items): + return items[option_number-1] + + def import_image(self, filepath: str, board_name_option: str, db_mapper: DatabaseMapper, config: Config): + """Import a single file by its path""" + parser = InvokeAIMetadataParser() + file_name = os.path.basename(filepath) + file_destination_path = os.path.join(config.outputs_path, file_name) + + print("===============================================================================") + print(f"Importing {filepath}") + + # check destination to see if the file was previously imported + if os.path.exists(file_destination_path): + print("File already exists in the destination, skipping!") + ImportStats.count_skipped_file_exists += 1 + return + + # check if file name is already referenced in the database + if db_mapper.does_image_exist(file_name): + print("A reference to a file with this name already exists in the database, skipping!") + ImportStats.count_skipped_db_exists += 1 + return + + # load image info and dimensions + img_info, png_width, png_height = self.get_file_details(filepath) + + # parse metadata + destination_needs_meta_update = True + log_version_note = "(Unknown)" + if "invokeai_metadata" in img_info: + # for the latest, we will just re-emit the same json, no need to parse/modify + converted_field = None + latest_json_string = img_info.get("invokeai_metadata") + log_version_note = "3.0.0+" + destination_needs_meta_update = False + else: + if "sd-metadata" in img_info: + converted_field = parser.parse_meta_tag_sd_metadata(json.loads(img_info.get("sd-metadata"))) + elif "invokeai" in img_info: + converted_field = parser.parse_meta_tag_invokeai(json.loads(img_info.get("invokeai"))) + elif "dream" in img_info: + converted_field = parser.parse_meta_tag_dream(img_info.get("dream")) + elif "Dream" in img_info: + converted_field = parser.parse_meta_tag_dream(img_info.get("Dream")) + else: + converted_field = InvokeAIMetadata() + destination_needs_meta_update = False + print("File does not have metadata from known Invoke AI versions, add only, no update!") + + # use the loaded img dimensions if the metadata didnt have them + if converted_field.width is None: + converted_field.width = png_width + if converted_field.height is None: + converted_field.height = png_height + + log_version_note = converted_field.imported_app_version if converted_field else "NoVersion" + log_version_note = log_version_note or "NoVersion" + + latest_json_string = converted_field.to_json() + + print (f"From Invoke AI Version {log_version_note} with dimensions {png_width} x {png_height}.") + + # if metadata needs update, then update metdata and copy in one shot + if destination_needs_meta_update: + print("Updating metadata while copying...", end="") + self.update_file_metadata_while_copying(filepath, file_destination_path, "invokeai_metadata", latest_json_string) + print("Done!") + else: + print("No metadata update necessary, copying only...", end="") + shutil.copy2(filepath, file_destination_path) + print("Done!") + + # create thumbnail + print("Creating thumbnail...", end="") + thumbnail_path = os.path.join(config.thumbnail_path, os.path.splitext(file_name)[0]) + ".webp" + thumbnail_size = 256, 256 + with PIL.Image.open(filepath) as source_image: + source_image.thumbnail(thumbnail_size) + source_image.save(thumbnail_path, "webp") + print("Done!") + + # finalize the dynamic board name if there is an APPVERSION token in it. + if converted_field is not None: + board_name = board_name_option.replace("APPVERSION", converted_field.imported_app_version or "NoVersion") + else: + board_name = board_name_option.replace("APPVERSION", "Latest") + + # maintain a map of alrady created/looked up ids to avoid DB queries + print("Finding/Creating board...", end="") + if board_name in self.board_name_id_map: + board_id = self.board_name_id_map[board_name] + else: + board_id = db_mapper.get_board_id_with_create(board_name) + self.board_name_id_map[board_name] = board_id + print("Done!") + + # add image to db + print("Adding image to database......", end="") + modified_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(filepath)) + db_mapper.add_new_image_to_database(file_name, png_width, png_height, latest_json_string, modified_time) + print("Done!") + + #add image to board + print("Adding image to board......", end="") + db_mapper.add_image_to_board(file_name, board_id) + print("Done!") + + ImportStats.count_imported += 1 + if log_version_note in ImportStats.count_imported_by_version: + ImportStats.count_imported_by_version[log_version_note] += 1 + else: + ImportStats.count_imported_by_version[log_version_note] = 1 + + def update_file_metadata_while_copying(self, filepath, file_destination_path, tag_name, tag_value): + """Perform a metadata update with save to a new destination which accomplishes a copy while updating metadata.""" + with PIL.Image.open(filepath) as target_image: + existing_img_info = target_image.info + metadata = PIL.PngImagePlugin.PngInfo() + # re-add any existing invoke ai tags unless they are the one we are trying to add + for key in existing_img_info: + if key != tag_name and key in ("dream", "Dream", "sd-metadata", "invokeai", "invokeai_metadata"): + metadata.add_text(key,existing_img_info[key]) + metadata.add_text(tag_name, tag_value) + target_image.save(file_destination_path, pnginfo=metadata) + + def process(self): + """Begin main processing.""" + + print("===============================================================================") + print("This script will import images generated by earlier versions of") + print("InvokeAI into the currently installed root directory:") + print(f' {app_config.root_path}') + print("If this is not what you want to do, type ctrl-C now to cancel.") + + # load config + print("===============================================================================") + print("= Configuration & Settings") + + config = Config() + config.find_and_load() + db_mapper = DatabaseMapper(config.database_path, config.database_backup_dir) + db_mapper.connect() + + import_dir, is_recurse, import_file_list = self.get_import_file_list() + ImportStats.count_source_files = len(import_file_list) + + board_names = db_mapper.get_board_names() + board_name_option = self.select_board_option(board_names, config.TIMESTAMP_STRING) + + print("\r\n===============================================================================") + print("= Import Settings Confirmation") + + print() + print(f"Database File Path : {config.database_path}") + print(f"Outputs/Images Directory : {config.outputs_path}") + print(f"Import Image Source Directory : {import_dir}") + print(f" Recurse Source SubDirectories : {'Yes' if is_recurse else 'No'}") + print(f"Count of .png file(s) found : {len(import_file_list)}") + print(f"Board name option specified : {board_name_option}") + print(f"Database backup will be taken at : {config.database_backup_dir}") + + print("\r\nNotes about the import process:") + print("- Source image files will not be modified, only copied to the outputs directory.") + print("- If the same file name already exists in the destination, the file will be skipped.") + print("- If the same file name already has a record in the database, the file will be skipped.") + print("- Invoke AI metadata tags will be updated/written into the imported copy only.") + print("- On the imported copy, only Invoke AI known tags (latest and legacy) will be retained (dream, sd-metadata, invokeai, invokeai_metadata)") + print("- A property 'imported_app_version' will be added to metadata that can be viewed in the UI's metadata viewer.") + print("- The new 3.x InvokeAI outputs folder structure is flat so recursively found source imges will all be placed into the single outputs/images folder.") + + while True: + should_continue = prompt("\nDo you wish to continue with the import [Yn] ? ").lower() or 'y' + if should_continue=='n': + print("\r\nCancelling Import") + return + elif should_continue=='y': + print() + break + + db_mapper.backup(config.TIMESTAMP_STRING) + + print() + ImportStats.time_start = datetime.datetime.utcnow() + + for filepath in import_file_list: + try: + self.import_image(filepath, board_name_option, db_mapper, config) + except sqlite3.Error as sql_ex: + print(f"A database related exception was found processing {filepath}, will continue to next file. ") + print("Exception detail:") + print(sql_ex) + ImportStats.count_file_errors += 1 + except Exception as ex: + print(f"Exception processing {filepath}, will continue to next file. ") + print("Exception detail:") + print(ex) + ImportStats.count_file_errors += 1 + + print("\r\n===============================================================================") + print(f"= Import Complete - Elpased Time: {ImportStats.get_elapsed_time_string()}") + print() + print(f"Source File(s) : {ImportStats.count_source_files}") + print(f"Total Imported : {ImportStats.count_imported}") + print(f"Skipped b/c file already exists on disk : {ImportStats.count_skipped_file_exists}") + print(f"Skipped b/c file already exists in db : {ImportStats.count_skipped_db_exists}") + print(f"Errors during import : {ImportStats.count_file_errors}") + if ImportStats.count_imported > 0: + print("\r\nBreakdown of imported files by version:") + for key,version in ImportStats.count_imported_by_version.items(): + print(f" {key:20} : {version}") + +def main(): + try: + processor = MediaImportProcessor() + processor.process() + except KeyboardInterrupt: + print("\r\n\r\nUser cancelled execution.") + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index b3f12481a8..d5aa3f37aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,7 @@ dependencies = [ "invokeai-metadata" = "invokeai.frontend.CLI.sd_metadata:print_metadata" "invokeai-node-cli" = "invokeai.app.cli_app:invoke_cli" "invokeai-node-web" = "invokeai.app.api_app:invoke_api" +"invokeai-import-images" = "invokeai.frontend.install.import_images:main" [project.urls] "Homepage" = "https://invoke-ai.github.io/InvokeAI/"