From b856fac7130bf324b24242e63dc1e16737886d1c Mon Sep 17 00:00:00 2001 From: Jonathan <34005131+JPPhoto@users.noreply.github.com> Date: Thu, 23 Mar 2023 14:27:12 -0500 Subject: [PATCH 1/6] Keep torch version at 1.13.1 (#2985) Now that torch 2.0 is out, Invoke 2.3 should lock down its version to 1.13.1 for new installs and upgrades. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d5c8c03163..5394d31354 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ dependencies = [ "taming-transformers-rom1504", "test-tube>=0.7.5", "torch-fidelity", - "torch>=1.13.1", + "torch~=1.13.1", "torchmetrics", "torchvision>=0.14.1", "transformers~=4.26", From 2a8513eee0653493dff211681d45b78c45d72d61 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 23 Mar 2023 19:49:13 -0400 Subject: [PATCH 2/6] adjust textual inversion training parameters according to xformers availability - If xformers is available, then default "use xformers" checkbox to on. - Increase batch size to 8 (from 3). --- ldm/invoke/training/textual_inversion.py | 14 ++++++++------ ldm/invoke/training/textual_inversion_training.py | 7 ++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ldm/invoke/training/textual_inversion.py b/ldm/invoke/training/textual_inversion.py index 2961e4d99c..f620bbf71e 100755 --- a/ldm/invoke/training/textual_inversion.py +++ b/ldm/invoke/training/textual_inversion.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import List, Tuple import npyscreen +from diffusers.utils.import_utils import is_xformers_available from npyscreen import widget from omegaconf import OmegaConf @@ -29,7 +30,7 @@ from ldm.invoke.training.textual_inversion_training import ( TRAINING_DATA = "text-inversion-training-data" TRAINING_DIR = "text-inversion-output" CONF_FILE = "preferences.conf" - +XFORMERS_AVAILABLE = is_xformers_available() class textualInversionForm(npyscreen.FormMultiPageAction): resolutions = [512, 768, 1024] @@ -178,7 +179,7 @@ class textualInversionForm(npyscreen.FormMultiPageAction): out_of=10000, step=500, lowest=1, - value=saved_args.get("max_train_steps", 3000), + value=saved_args.get("max_train_steps", 2500), scroll_exit=True, ) self.train_batch_size = self.add_widget_intelligent( @@ -187,7 +188,7 @@ class textualInversionForm(npyscreen.FormMultiPageAction): out_of=50, step=1, lowest=1, - value=saved_args.get("train_batch_size", 8), + value=saved_args.get("train_batch_size", 8 if XFORMERS_AVAILABLE else 3), scroll_exit=True, ) self.gradient_accumulation_steps = self.add_widget_intelligent( @@ -225,7 +226,7 @@ class textualInversionForm(npyscreen.FormMultiPageAction): self.enable_xformers_memory_efficient_attention = self.add_widget_intelligent( npyscreen.Checkbox, name="Use xformers acceleration", - value=saved_args.get("enable_xformers_memory_efficient_attention", False), + value=saved_args.get("enable_xformers_memory_efficient_attention", XFORMERS_AVAILABLE), scroll_exit=True, ) self.lr_scheduler = self.add_widget_intelligent( @@ -428,11 +429,12 @@ def do_front_end(args: Namespace): print(str(e)) print("** DETAILS:") print(traceback.format_exc()) - - + def main(): args = parse_args() global_set_root(args.root_dir or Globals.root) + print(XFORMERS_AVAILABLE,file=sys.stderr) + sys.exit(0) try: if args.front_end: do_front_end(args) diff --git a/ldm/invoke/training/textual_inversion_training.py b/ldm/invoke/training/textual_inversion_training.py index 7794712bc1..efc0986d6c 100644 --- a/ldm/invoke/training/textual_inversion_training.py +++ b/ldm/invoke/training/textual_inversion_training.py @@ -67,7 +67,7 @@ else: "nearest": PIL.Image.NEAREST, } # ------------------------------------------------------------------------------ - +XFORMERS_AVAILABLE = is_xformers_available # Will error if the minimal version of diffusers is not installed. Remove at your own risks. check_min_version("0.10.0.dev0") @@ -227,7 +227,7 @@ def parse_args(): training_group.add_argument( "--train_batch_size", type=int, - default=16, + default=8 if XFORMERS_AVAILABLE else 3, help="Batch size (per device) for the training dataloader.", ) training_group.add_argument("--num_train_epochs", type=int, default=100) @@ -324,6 +324,7 @@ def parse_args(): parser.add_argument( "--enable_xformers_memory_efficient_attention", action="store_true", + default=XFORMERS_AVAILABLE, help="Whether or not to use xformers.", ) @@ -536,7 +537,7 @@ def do_textual_inversion_training( seed: int = None, resolution: int = 512, center_crop: bool = False, - train_batch_size: int = 16, + train_batch_size: int = 4, num_train_epochs: int = 100, max_train_steps: int = 5000, gradient_accumulation_steps: int = 1, From 4515d52a426ed179949452f8f35e1d4f2f75d000 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 23 Mar 2023 21:00:54 -0400 Subject: [PATCH 3/6] fix textual inversion documentation and code This PR addresses issues raised by #3008. 1. Update documentation to indicate the correct maximum batch size for TI training when xformers is and isn't used. 2. Update textual inversion code so that the default for batch size is aware of xformer availability. 3. Add documentation for how to launch TI with distributed learning. --- docs/features/TEXTUAL_INVERSION.md | 56 ++++++++++++++++++++++-- ldm/invoke/training/textual_inversion.py | 2 - 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/docs/features/TEXTUAL_INVERSION.md b/docs/features/TEXTUAL_INVERSION.md index c0b34e3f7c..ebb09d6ee2 100644 --- a/docs/features/TEXTUAL_INVERSION.md +++ b/docs/features/TEXTUAL_INVERSION.md @@ -154,8 +154,11 @@ training sets will converge with 2000-3000 steps. This adjusts how many training images are processed simultaneously in each step. Higher values will cause the training process to run more -quickly, but use more memory. The default size will run with GPUs with -as little as 12 GB. +quickly, but use more memory. The default size is selected based on +whether you have the `xformers` memory-efficient attention library +installed. If `xformers` is available, the batch size will be 8, +otherwise 3. These values were chosen to allow training to run with +GPUs with as little as 12 GB VRAM. ### Learning rate @@ -172,8 +175,10 @@ learning rate to improve performance. ### Use xformers acceleration -This will activate XFormers memory-efficient attention. You need to -have XFormers installed for this to have an effect. +This will activate XFormers memory-efficient attention, which will +reduce memory requirements by half or more and allow you to select a +higher batch size. You need to have XFormers installed for this to +have an effect. ### Learning rate scheduler @@ -250,6 +255,49 @@ invokeai-ti \ --only_save_embeds ``` +## Using Distributed Training + +If you have multiple GPUs on one machine, or a cluster of GPU-enabled +machines, you can activate distributed training. See the [HuggingFace +Accelerate pages](https://huggingface.co/docs/accelerate/index) for +full information, but the basic recipe is: + +1. Enter the InvokeAI developer's console command line by selecting +option [8] from the `invoke.sh`/`invoke.bat` script. + +2. Configurate Accelerate using `accelerate config`: +```sh +accelerate config +``` +This will guide you through the configuration process, including +specifying how many machines you will run training on and the number +of GPUs pe rmachine. + +You only need to do this once. + +3. Launch training from the command line using `accelerate launch`. Be sure +that your current working directory is the InvokeAI root directory (usually +named `invokeai` in your home directory): + +```sh +accelerate launch .venv/bin/invokeai-ti \ + --model=stable-diffusion-1.5 \ + --resolution=512 \ + --learnable_property=object \ + --initializer_token='*' \ + --placeholder_token='' \ + --train_data_dir=/home/lstein/invokeai/text-inversion-training-data/shraddha \ + --output_dir=/home/lstein/invokeai/text-inversion-training/shraddha \ + --scale_lr \ + --train_batch_size=10 \ + --gradient_accumulation_steps=4 \ + --max_train_steps=2000 \ + --learning_rate=0.0005 \ + --lr_scheduler=constant \ + --mixed_precision=fp16 \ + --only_save_embeds +``` + ## Using Embeddings After training completes, the resultant embeddings will be saved into your `$INVOKEAI_ROOT/embeddings//learned_embeds.bin`. diff --git a/ldm/invoke/training/textual_inversion.py b/ldm/invoke/training/textual_inversion.py index f620bbf71e..f1e8e2d679 100755 --- a/ldm/invoke/training/textual_inversion.py +++ b/ldm/invoke/training/textual_inversion.py @@ -433,8 +433,6 @@ def do_front_end(args: Namespace): def main(): args = parse_args() global_set_root(args.root_dir or Globals.root) - print(XFORMERS_AVAILABLE,file=sys.stderr) - sys.exit(0) try: if args.front_end: do_front_end(args) From a79d40519ca3f07d09607c5a77db64dd56c774be Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 23 Mar 2023 21:43:21 -0400 Subject: [PATCH 4/6] fix batch generation logfile name to be compatible with Windows OS - `invokeai-batch --invoke` was created a time-stamped logfile with colons in its name, which is a Windows no-no. This corrects the problem by writing the timestamp out as "13-06-2023_8-35-10" - Closes #3005 --- ldm/invoke/dynamic_prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/invoke/dynamic_prompts.py b/ldm/invoke/dynamic_prompts.py index c196ce7c33..de02c55b56 100755 --- a/ldm/invoke/dynamic_prompts.py +++ b/ldm/invoke/dynamic_prompts.py @@ -157,7 +157,7 @@ def _run_invoke( ): pid = os.getpid() logdir.mkdir(parents=True, exist_ok=True) - logfile = Path(logdir, f'{time.strftime("%Y-%m-%d-%H:%M:%S")}-pid={pid}.txt') + logfile = Path(logdir, f'{time.strftime("%Y-%m-%d_%H-%M-%S")}-pid={pid}.txt') print( f">> Process {pid} running on GPU {gpu}; logging to {logfile}", file=sys.stderr ) From 9f7c86c33e12d478ee55cc23cc38c86d7356bafe Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 23 Mar 2023 21:47:56 -0400 Subject: [PATCH 5/6] bump version to 2.3.3-rc1 Lots of little bugs have been squashed since 2.3.2 and a new minor point release is imminent. This PR updates the version number in preparation for a RC. --- ldm/invoke/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/invoke/_version.py b/ldm/invoke/_version.py index de365d650b..c2881e4f9e 100644 --- a/ldm/invoke/_version.py +++ b/ldm/invoke/_version.py @@ -1,2 +1,2 @@ -__version__='2.3.2.post1' +__version__='2.3.3-rc1' From 29b348ece1ccc767e08d393e84125c54d2500cb1 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 23 Mar 2023 22:03:02 -0400 Subject: [PATCH 6/6] fix corrupted outputs/.next_prefix file - Since 2.3.2 invokeai stores the next PNG file's numeric prefix in a file named `.next_prefix` in the outputs directory. This avoids the overhead of doing a directory listing to find out what file number comes next. - The code uses advisory locking to prevent corruption of this file in the event that multiple invokeai's try to access it simultaneously, but some users have experienced corruption of the file nevertheless. - This PR addresses the problem by detecting a potentially corrupted `.next_prefix` file and falling back to the directory listing method. A fixed version of the file is then written out. - Closes #3001 --- ldm/invoke/pngwriter.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ldm/invoke/pngwriter.py b/ldm/invoke/pngwriter.py index da5af82aa8..cdd9107f20 100644 --- a/ldm/invoke/pngwriter.py +++ b/ldm/invoke/pngwriter.py @@ -30,14 +30,17 @@ class PngWriter: prefix = self._unused_prefix() else: with open(next_prefix_file,'r') as file: - prefix=int(file.readline() or int(self._unused_prefix())-1) - prefix+=1 + prefix = 0 + try: + prefix=int(file.readline()) + except (TypeError, ValueError): + prefix=self._unused_prefix() with open(next_prefix_file,'w') as file: - file.write(str(prefix)) + file.write(str(prefix+1)) return f'{prefix:06}' # gives the next unique prefix in outdir - def _unused_prefix(self): + def _unused_prefix(self)->int: # sort reverse alphabetically until we find max+1 dirlist = sorted(os.listdir(self.outdir), reverse=True) # find the first filename that matches our pattern or return 000000.0.png @@ -45,8 +48,7 @@ class PngWriter: (f for f in dirlist if re.match('^(\d+)\..*\.png', f)), '0000000.0.png', ) - basecount = int(existing_name.split('.', 1)[0]) + 1 - return f'{basecount:06}' + return int(existing_name.split('.', 1)[0]) + 1 # saves image named _image_ to outdir/name, writing metadata from prompt # returns full path of output