Merge remote-tracking branch 'upstream/development' into development

docker-build/Dockerfile

@@ -47,7 +47,7 @@ RUN         git clone https://github.com/TencentARC/GFPGAN.git
 WORKDIR     /GFPGAN 
 RUN         pip3 install -r requirements.txt \
             && python3 setup.py develop \
-            && ln -s "/data/GFPGANv1.3.pth" experiments/pretrained_models/GFPGANv1.3.pth
+            && ln -s "/data/GFPGANv1.4.pth" experiments/pretrained_models/GFPGANv1.4.pth
 
 WORKDIR     /stable-diffusion 
 RUN         python3 scripts/preload_models.py         

docs/features/CLI.md

@@ -8,20 +8,23 @@ hide:
 
 ## **Interactive Command Line Interface**
 
-The `dream.py` script, located in `scripts/dream.py`, provides an interactive interface to image
+The `dream.py` script, located in `scripts/dream.py`, provides an interactive
-generation similar to the "dream mothership" bot that Stable AI provided on its Discord server.
+interface to image generation similar to the "dream mothership" bot that Stable
+AI provided on its Discord server.
 
 Unlike the `txt2img.py` and `img2img.py` scripts provided in the original
-[CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion) source code repository, the
+[CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion) source
-time-consuming initialization of the AI model initialization only happens once. After that image
+code repository, the time-consuming initialization of the AI model
-generation from the command-line interface is very fast.
+initialization only happens once. After that image generation from the
+command-line interface is very fast.
 
-The script uses the readline library to allow for in-line editing, command history (++up++ and
+The script uses the readline library to allow for in-line editing, command
-++down++), autocompletion, and more. To help keep track of which prompts generated which images, the
+history (++up++ and ++down++), autocompletion, and more. To help keep track of
-script writes a log file of image names and prompts to the selected output directory.
+which prompts generated which images, the script writes a log file of image
+names and prompts to the selected output directory.
 
-In addition, as of version 1.02, it also writes the prompt into the PNG file's metadata where it can
+In addition, as of version 1.02, it also writes the prompt into the PNG file's
-be retrieved using `scripts/images2prompt.py`
+metadata where it can be retrieved using `scripts/images2prompt.py`
 
 The script is confirmed to work on Linux, Windows and Mac systems.
 
@@ -56,21 +59,24 @@ dream> q
 
 ![dream-py-demo](../assets/dream-py-demo.png)
 
-The `dream>` prompt's arguments are pretty much identical to those used in the Discord bot, except
+The `dream>` prompt's arguments are pretty much identical to those used in the
-you don't need to type "!dream" (it doesn't hurt if you do). A significant change is that creation
+Discord bot, except you don't need to type "!dream" (it doesn't hurt if you do).
-of individual images is now the default unless `--grid` (`-g`) is given. A full list is given in
+A significant change is that creation of individual images is now the default
+unless `--grid` (`-g`) is given. A full list is given in
 [List of prompt arguments](#list-of-prompt-arguments).
 
 ## Arguments
 
-The script itself also recognizes a series of command-line switches that will change important
+The script itself also recognizes a series of command-line switches that will
-global defaults, such as the directory for image outputs and the location of the model weight files.
+change important global defaults, such as the directory for image outputs and
+the location of the model weight files.
 
 ### List of arguments recognized at the command line
 
-These command-line arguments can be passed to `dream.py` when you first run it from the Windows, Mac
+These command-line arguments can be passed to `dream.py` when you first run it
-or Linux command line. Some set defaults that can be overridden on a per-prompt basis (see [List of
+from the Windows, Mac or Linux command line. Some set defaults that can be
-prompt arguments] (#list-of-prompt-arguments). Others
+overridden on a per-prompt basis (see [List of prompt arguments]
+(#list-of-prompt-arguments). Others
 
 | Argument <img width="240" align="right"/> | Shortcut <img width="100" align="right"/> | Default <img width="320" align="right"/>       | Description                                                                                          |
 | ----------------------------------------- | ----------------------------------------- | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------- |
@@ -90,7 +96,7 @@ prompt arguments] (#list-of-prompt-arguments). Others
 | `--seamless`                              |                                           | `False`                                        | Create interesting effects by tiling elements of the image.                                          |
 | `--embedding_path <path>`                 |                                           | `None`                                         | Path to pre-trained embedding manager checkpoints, for custom models                                 |
 | `--gfpgan_dir`                            |                                           | `src/gfpgan`                                   | Path to where GFPGAN is installed.                                                                   |
-| `--gfpgan_model_path`                     |                                           | `experiments/pretrained_models/GFPGANv1.3.pth` | Path to GFPGAN model file, relative to `--gfpgan_dir`.                                               |
+| `--gfpgan_model_path`                     |                                           | `experiments/pretrained_models/GFPGANv1.4.pth` | Path to GFPGAN model file, relative to `--gfpgan_dir`.                                               |
 | `--device <device>`                       | `-d<device>`                              | `torch.cuda.current_device()`                  | Device to run SD on, e.g. "cuda:0"                                                                   |
 
 #### deprecated
@@ -115,9 +121,10 @@ These arguments are deprecated but still work:
 
 ### List of prompt arguments
 
-After the `dream.py` script initializes, it will present you with a **`dream>`** prompt. Here you
+After the `dream.py` script initializes, it will present you with a **`dream>`**
-can enter information to generate images from text (txt2img), to embellish an existing image or
+prompt. Here you can enter information to generate images from text (txt2img),
-sketch (img2img), or to selectively alter chosen regions of the image (inpainting).
+to embellish an existing image or sketch (img2img), or to selectively alter
+chosen regions of the image (inpainting).
 
 #### txt2img
 
@@ -171,12 +178,13 @@ Those are the `dream` commands that apply to txt2img:
     than 640x480. Otherwise the image size will be identical to the provided photo and you may run out
     of memory if it is large.
 
-Repeated chaining of img2img on an image can result in significant color shifts in the output,
+Repeated chaining of img2img on an image can result in significant color shifts
-especially if run with lower strength. Color correction can be run against a reference image to fix
+in the output, especially if run with lower strength. Color correction can be
-this issue. Use the original input image to the chain as the the reference image for each step in
+run against a reference image to fix this issue. Use the original input image to
-the chain.
+the chain as the the reference image for each step in the chain.
 
-In addition to the command-line options recognized by txt2img, img2img accepts additional options:
+In addition to the command-line options recognized by txt2img, img2img accepts
+additional options:
 
 | Argument <img width="160" align="right"/> | Shortcut    | Default | Description                                                                                                                                |
 | ----------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
@@ -198,8 +206,8 @@ In addition to the command-line options recognized by txt2img, img2img accepts a
     the areas to overpaint made transparent, but you must be careful not to destroy the pixels
     underneath when you create the transparent areas. See [Inpainting](./INPAINTING.md) for details.
 
-Inpainting accepts all the arguments used for txt2img and img2img, as well as the `--mask` (`-M`)
+Inpainting accepts all the arguments used for txt2img and img2img, as well as
-argument:
+the `--mask` (`-M`) argument:
 
 | Argument <img width="100" align="right"/> | Shortcut   | Default | Description                                                                                      |
 | ----------------------------------------- | ---------- | ------- | ------------------------------------------------------------------------------------------------ |
@@ -207,37 +215,42 @@ argument:
 
 ## Command-line editing and completion
 
-If you are on a Macintosh or Linux machine, the command-line offers convenient history tracking,
+If you are on a Macintosh or Linux machine, the command-line offers convenient
-editing, and command completion.
+history tracking, editing, and command completion.
 
-- To scroll through previous commands and potentially edit/reuse them, use the ++up++ and ++down++
+- To scroll through previous commands and potentially edit/reuse them, use the
-  cursor keys.
+  ++up++ and ++down++ cursor keys.
-- To edit the current command, use the ++left++ and ++right++ cursor keys to position the cursor,
+- To edit the current command, use the ++left++ and ++right++ cursor keys to
-  and then ++backspace++, ++delete++ or ++insert++ characters.
+  position the cursor, and then ++backspace++, ++delete++ or ++insert++
-- To move to the very beginning of the command, type ++ctrl+a++ (or ++command+a++ on the Mac)
+  characters.
+- To move to the very beginning of the command, type ++ctrl+a++ (or
+  ++command+a++ on the Mac)
 - To move to the end of the command, type ++ctrl+e++.
-- To cut a section of the command, position the cursor where you want to start cutting and type
+- To cut a section of the command, position the cursor where you want to start
-  ++ctrl+k++.
+  cutting and type ++ctrl+k++.
-- To paste a cut section back in, position the cursor where you want to paste, and type ++ctrl+y++
+- To paste a cut section back in, position the cursor where you want to paste,
+  and type ++ctrl+y++
 
-Windows users can get similar, but more limited, functionality if they launch `dream.py` with the
+Windows users can get similar, but more limited, functionality if they launch
-"winpty" program:
+`dream.py` with the "winpty" program:
 
 ```batch
 winpty python scripts\dream.py
 ```
 
-On the Mac and Linux platforms, when you exit `dream.py`, the last 1000 lines of your command-line
+On the Mac and Linux platforms, when you exit `dream.py`, the last 1000 lines of
-history will be saved. When you restart `dream.py`, you can access the saved history using the
+your command-line history will be saved. When you restart `dream.py`, you can
-++up++ key.
+access the saved history using the ++up++ key.
 
-In addition, limited command-line completion is installed. In various contexts, you can start typing
+In addition, limited command-line completion is installed. In various contexts,
-your command and press tab. A list of potential completions will be presented to you. You can then
+you can start typing your command and press tab. A list of potential completions
-type a little more, hit tab again, and eventually autocomplete what you want.
+will be presented to you. You can then type a little more, hit tab again, and
+eventually autocomplete what you want.
 
-When specifying file paths using the one-letter shortcuts, the CLI will attempt to complete
+When specifying file paths using the one-letter shortcuts, the CLI will attempt
-pathnames for you. This is most handy for the `-I` (init image) and `-M` (init mask) paths. To
+to complete pathnames for you. This is most handy for the `-I` (init image) and
-initiate completion, start the path with a slash `/` or `./`, for example:
+`-M` (init mask) paths. To initiate completion, start the path with a slash `/`
+or `./`, for example:
 
 ```bash
 dream> "zebra with a mustache" -I./test-pictures<TAB>

docs/features/UPSCALE.md

@@ -4,37 +4,42 @@ title: Upscale
 
 ## Intro
 
-The script provides the ability to restore faces and upscale. You can apply these operations
+The script provides the ability to restore faces and upscale. You can apply
-at the time you generate the images, or at any time to a previously-generated PNG file, using 
+these operations at the time you generate the images, or at any time to a
-the [!fix](#fixing-previously-generated-images) command.
+previously-generated PNG file, using the
+[!fix](#fixing-previously-generated-images) command.
 
 ## Face Fixing
 
-The default face restoration module is GFPGAN. The default upscale is Real-ESRGAN. For an alternative
+The default face restoration module is GFPGAN. The default upscale is
-face restoration module, see [CodeFormer Support] below.
+Real-ESRGAN. For an alternative face restoration module, see [CodeFormer
+Support] below.
 
-As of version 1.14, environment.yaml will install the Real-ESRGAN package into the standard install
+As of version 1.14, environment.yaml will install the Real-ESRGAN package into
-location for python packages, and will put GFPGAN into a subdirectory of "src" in the
+the standard install location for python packages, and will put GFPGAN into a
-stable-diffusion directory. (The reason for this is that the standard GFPGAN distribution has a
+subdirectory of "src" in the stable-diffusion directory. (The reason for this is
-minor bug that adversely affects image color.) Upscaling with Real-ESRGAN should "just work" without
+that the standard GFPGAN distribution has a minor bug that adversely affects
-further intervention. Simply pass the --upscale (-U) option on the dream> command line, or indicate
+image color.) Upscaling with Real-ESRGAN should "just work" without further
-the desired scale on the popup in the Web GUI.
+intervention. Simply pass the --upscale (-U) option on the dream> command line,
+or indicate the desired scale on the popup in the Web GUI.
 
-For **GFPGAN** to work, there is one additional step needed. You will need to download and copy the
+For **GFPGAN** to work, there is one additional step needed. You will need to
-GFPGAN [models file](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth)
+download and copy the GFPGAN
-into **src/gfpgan/experiments/pretrained_models**. On Mac and Linux systems, here's how you'd do it
+[models file](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth)
-using **wget**:
+into **src/gfpgan/experiments/pretrained_models**. On Mac and Linux systems,
+here's how you'd do it using **wget**:
 
 ```bash
-> wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth src/gfpgan/experiments/pretrained_models/
+> wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth src/gfpgan/experiments/pretrained_models/
 ```
 
 Make sure that you're in the stable-diffusion directory when you do this.
 
-Alternatively, if you have GFPGAN installed elsewhere, or if you are using an earlier version of
+Alternatively, if you have GFPGAN installed elsewhere, or if you are using an
-this package which asked you to install GFPGAN in a sibling directory, you may use the
+earlier version of this package which asked you to install GFPGAN in a sibling
-`--gfpgan_dir` argument with `dream.py` to set a custom path to your GFPGAN directory. _There are
+directory, you may use the `--gfpgan_dir` argument with `dream.py` to set a
-other GFPGAN related boot arguments if you wish to customize further._
+custom path to your GFPGAN directory. _There are other GFPGAN related boot
+arguments if you wish to customize further._
 
 !!! warning "Internet connection needed"
 
@@ -52,13 +57,14 @@ You will now have access to two new prompt arguments.
 
 `-U : <upscaling_factor> <upscaling_strength>`
 
-The upscaling prompt argument takes two values. The first value is a scaling factor and should be
+The upscaling prompt argument takes two values. The first value is a scaling
-set to either `2` or `4` only. This will either scale the image 2x or 4x respectively using
+factor and should be set to either `2` or `4` only. This will either scale the
-different models.
+image 2x or 4x respectively using different models.
 
-You can set the scaling stength between `0` and `1.0` to control intensity of the of the scaling.
+You can set the scaling stength between `0` and `1.0` to control intensity of
-This is handy because AI upscalers generally tend to smooth out texture details. If you wish to
+the of the scaling. This is handy because AI upscalers generally tend to smooth
-retain some of those for natural looking results, we recommend using values between `0.5 to 0.8`.
+out texture details. If you wish to retain some of those for natural looking
+results, we recommend using values between `0.5 to 0.8`.
 
 If you do not explicitly specify an upscaling_strength, it will default to 0.75.
 
@@ -66,18 +72,19 @@ If you do not explicitly specify an upscaling_strength, it will default to 0.75.
 
 `-G : <gfpgan_strength>`
 
-This prompt argument controls the strength of the face restoration that is being applied. Similar to
+This prompt argument controls the strength of the face restoration that is being
-upscaling, values between `0.5 to 0.8` are recommended.
+applied. Similar to upscaling, values between `0.5 to 0.8` are recommended.
 
-You can use either one or both without any conflicts. In cases where you use both, the image will be
+You can use either one or both without any conflicts. In cases where you use
-first upscaled and then the face restoration process will be executed to ensure you get the highest
+both, the image will be first upscaled and then the face restoration process
-quality facial features.
+will be executed to ensure you get the highest quality facial features.
 
 `--save_orig`
 
-When you use either `-U` or `-G`, the final result you get is upscaled or face modified. If you want
+When you use either `-U` or `-G`, the final result you get is upscaled or face
-to save the original Stable Diffusion generation, you can use the `-save_orig` prompt argument to
+modified. If you want to save the original Stable Diffusion generation, you can
-save the original unaffected version too.
+use the `-save_orig` prompt argument to save the original unaffected version
+too.
 
 ### Example Usage
 
@@ -102,60 +109,69 @@ dream> a man wearing a pineapple hat -I path/to/your/file.png -U 2 0.5 -G 0.6
     process is complete. While the image generation is taking place, you will still be able to preview
     the base images.
 
-If you wish to stop during the image generation but want to upscale or face restore a particular
+If you wish to stop during the image generation but want to upscale or face
-generated image, pass it again with the same prompt and generated seed along with the `-U` and `-G`
+restore a particular generated image, pass it again with the same prompt and
-prompt arguments to perform those actions.
+generated seed along with the `-U` and `-G` prompt arguments to perform those
+actions.
 
 ## CodeFormer Support
 
 This repo also allows you to perform face restoration using
 [CodeFormer](https://github.com/sczhou/CodeFormer).
 
-In order to setup CodeFormer to work, you need to download the models like with GFPGAN. You can do
+In order to setup CodeFormer to work, you need to download the models like with
-this either by running `preload_models.py` or by manually downloading the
+GFPGAN. You can do this either by running `preload_models.py` or by manually
-[model file](https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth) and
+downloading the
-saving it to `ldm/restoration/codeformer/weights` folder.
+[model file](https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth)
+and saving it to `ldm/restoration/codeformer/weights` folder.
 
-You can use `-ft` prompt argument to swap between CodeFormer and the default GFPGAN. The above
+You can use `-ft` prompt argument to swap between CodeFormer and the default
-mentioned `-G` prompt argument will allow you to control the strength of the restoration effect.
+GFPGAN. The above mentioned `-G` prompt argument will allow you to control the
+strength of the restoration effect.
 
 ### Usage:
 
-The following command will perform face restoration with CodeFormer instead of the default gfpgan.
+The following command will perform face restoration with CodeFormer instead of
+the default gfpgan.
 
 `<prompt> -G 0.8 -ft codeformer`
 
 ### Other Options:
 
-- `-cf` - cf or CodeFormer Fidelity takes values between `0` and `1`. 0 produces high quality
+- `-cf` - cf or CodeFormer Fidelity takes values between `0` and `1`. 0 produces
-  results but low accuracy and 1 produces lower quality results but higher accuacy to your original
+  high quality results but low accuracy and 1 produces lower quality results but
-  face.
+  higher accuacy to your original face.
 
-The following command will perform face restoration with CodeFormer. CodeFormer will output a result
+The following command will perform face restoration with CodeFormer. CodeFormer
-that is closely matching to the input face.
+will output a result that is closely matching to the input face.
 
 `<prompt> -G 1.0 -ft codeformer -cf 0.9`
 
-The following command will perform face restoration with CodeFormer. CodeFormer will output a result
+The following command will perform face restoration with CodeFormer. CodeFormer
-that is the best restoration possible. This may deviate slightly from the original face. This is an
+will output a result that is the best restoration possible. This may deviate
-excellent option to use in situations when there is very little facial data to work with.
+slightly from the original face. This is an excellent option to use in
+situations when there is very little facial data to work with.
 
 `<prompt> -G 1.0 -ft codeformer -cf 0.1`
 
 ## Fixing Previously-Generated Images
 
-It is easy to apply face restoration and/or upscaling to any previously-generated file. Just use the
+It is easy to apply face restoration and/or upscaling to any
-syntax `!fix path/to/file.png <options>`. For example, to apply GFPGAN at strength 0.8 and upscale 2X
+previously-generated file. Just use the syntax
-for a file named `./outputs/img-samples/000044.2945021133.png`, just run:
+`!fix path/to/file.png <options>`. For example, to apply GFPGAN at strength 0.8
+and upscale 2X for a file named `./outputs/img-samples/000044.2945021133.png`,
+just run:
 
-~~~~
+```
 dream> !fix ./outputs/img-samples/000044.2945021133.png -G 0.8 -U 2
-~~~~
+```
 
-A new file named `000044.2945021133.fixed.png` will be created in the output directory. Note that
+A new file named `000044.2945021133.fixed.png` will be created in the output
-the `!fix` command does not replace the original file, unlike the behavior at generate time.
+directory. Note that the `!fix` command does not replace the original file,
+unlike the behavior at generate time.
 
 ### Disabling:
 
-If, for some reason, you do not wish to load the GFPGAN and/or ESRGAN libraries, you can disable them 
+If, for some reason, you do not wish to load the GFPGAN and/or ESRGAN libraries,
-on the dream.py command line with the `--no_restore` and `--no_upscale` options, respectively.
+you can disable them on the dream.py command line with the `--no_restore` and
+`--no_upscale` options, respectively.

docs/help/SAMPLER_CONVERGENCE.md

@@ -0,0 +1,141 @@
+---
+title: SAMPLER CONVERGENCE
+---
+
+## *Sampler Convergence*
+
+As features keep increasing, making the right choices for your needs can become increasingly difficult. What sampler to use? And for how many steps? Do you change the CFG value? Do you use prompt weighting? Do you allow variations?
+
+Even once you have a result, do you blend it with other images? Pass it through `img2img`? With what strength? Do you use inpainting to correct small details? Outpainting to extend cropped sections?
+
+The purpose of this series of documents is to help you better understand these tools, so you can make the best out of them. Feel free to contribute with your own findings!
+
+In this document, we will talk about sampler convergence.
+
+Looking for a short version? Here's a TL;DR in 3 tables.
+
+| Remember  |
+|:---|
+| Results converge as steps (`-s`) are increased (except for `K_DPM_2_A` and `K_EULER_A`). Often at ≥ `-s100`, but may require ≥ `-s700`).  |
+| Producing a batch of candidate images at low (`-s8` to `-s30`) step counts can save you hours of computation.  |
+| `K_HEUN` and `K_DPM_2`  converge in less steps (but are slower).  |
+| `K_DPM_2_A` and `K_EULER_A` incorporate a lot of creativity/variability. |
+
+| Sampler   | (3 sample avg) it/s (M1 Max 64GB, 512x512)  |
+|---|---|
+|  `DDIM` | 1.89  |
+|  `PLMS` | 1.86  |
+|  `K_EULER` | 1.86  |
+|  `K_LMS` | 1.91  |
+|  `K_HEUN` | 0.95 (slower)  |
+|  `K_DPM_2` | 0.95 (slower)  |
+|  `K_DPM_2_A` | 0.95 (slower)  |
+|  `K_EULER_A` | 1.86  |
+
+| Suggestions  |
+|:---|
+| For most use cases, `K_LMS`, `K_HEUN` and `K_DPM_2` are the best choices (the latter 2 run 0.5x as quick, but tend to converge 2x as quick as `K_LMS`). At very low steps (≤ `-s8`), `K_HEUN` and `K_DPM_2` are not recommended. Use `K_LMS` instead.|
+| For variability, use `K_EULER_A` (runs 2x as quick as `K_DPM_2_A`).  |
+
+---
+
+### *Sampler results*
+
+Let's start by choosing a prompt and using it with each of our 8 samplers, running it for 10, 20, 30, 40, 50 and 100 steps.
+
+Anime. `"an anime girl" -W512 -H512 -C7.5 -S3031912972`
+
+![191636411-083c8282-6ed1-4f78-9273-ee87c0a0f1b6-min (1)](https://user-images.githubusercontent.com/50542132/191868725-7f7af991-e254-4c1f-83e7-bed8c9b2d34f.png)
+
+### *Sampler convergence*
+
+Immediately, you can notice results tend to converge -that is, as `-s` (step) values increase, images look more and more similar until there comes a point where the image no longer changes.
+
+You can also notice how `DDIM` and `PLMS` eventually tend to converge to K-sampler results as steps are increased.
+Among K-samplers, `K_HEUN` and `K_DPM_2` seem to require the fewest steps to converge, and even at low step counts they are good indicators of the final result. And finally, `K_DPM_2_A` and `K_EULER_A` seem to do a bit of their own thing and don't keep much similarity with the rest of the samplers.
+
+### *Batch generation speedup*
+
+This realization is very useful because it means you don't need to create a batch of 100 images (`-n100`) at `-s100` to choose your favorite 2 or 3 images.
+You can produce the same 100 images at `-s10` to `-s30` using a K-sampler (since they converge faster), get a rough idea of the final result, choose your 2 or 3 favorite ones, and then run `-s100` on those images to polish some details.
+The latter technique is 3-8x as quick.
+
+Example:
+
+At 60s per 100 steps.
+
+(Option A) 60s * 100 images = 6000s (100 images at `-s100`, manually picking 3 favorites)
+
+(Option B) 6s * 100 images + 60s * 3 images = 780s (100 images at `-s10`, manually picking 3 favorites, and running those 3 at `-s100` to polish details)
+
+The result is 1 hour and 40 minutes (Option A) vs 13 minutes (Option B).
+
+### *Topic convergance*
+
+Now, these results seem interesting, but do they hold for other topics? How about nature? Food? People? Animals? Let's try!
+
+Nature. `"valley landscape wallpaper, d&d art, fantasy, painted, 4k, high detail, sharp focus, washed colors, elaborate excellent painted illustration" -W512 -H512 -C7.5 -S1458228930`
+
+![191736091-dda76929-00d1-4590-bef4-7314ea4ea419-min (1)](https://user-images.githubusercontent.com/50542132/191868763-b151c69e-0a72-4cf1-a151-5a64edd0c93e.png)
+
+With nature, you can see how initial results are even more indicative of final result -more so than with characters/people. `K_HEUN` and `K_DPM_2` are again the quickest indicators, almost right from the start. Results also converge faster (e.g. `K_HEUN` converged at `-s21`).
+
+Food. `"a hamburger with a bowl of french fries" -W512 -H512 -C7.5 -S4053222918`
+
+![191639011-f81d9d38-0a15-45f0-9442-a5e8d5c25f1f-min (1)](https://user-images.githubusercontent.com/50542132/191868898-98801a62-885f-4ea1-aee8-563503522aa9.png)
+
+Again, `K_HEUN` and `K_DPM_2` take the fewest number of steps to be good indicators of the final result. `K_DPM_2_A` and `K_EULER_A` seem to incorporate a lot of creativity/variability, capable of producing rotten hamburgers, but also of adding lettuce to the mix. And they're the only samplers that produced an actual 'bowl of fries'!
+
+Animals. `"grown tiger, full body" -W512 -H512 -C7.5 -S3721629802`
+
+![191771922-6029a4f5-f707-4684-9011-c6f96e25fe56-min (1)](https://user-images.githubusercontent.com/50542132/191868870-9e3b7d82-b909-429f-893a-13f6ec343454.png)
+
+`K_HEUN` and `K_DPM_2` once again require the least number of steps to be indicative of the final result (around `-s30`), while other samplers are still struggling with several tails or malformed back legs.
+
+It also takes longer to converge (for comparison, `K_HEUN` required around 150 steps to converge). This is normal, as producing human/animal faces/bodies is one of the things the model struggles the most with. For these topics, running for more steps will often increase coherence within the composition.
+
+People. `"Ultra realistic photo, (Miranda Bloom-Kerr), young, stunning model, blue eyes, blond hair, beautiful face, intricate, highly detailed, smooth, art by artgerm and greg rutkowski and alphonse mucha, stained glass" -W512 -H512 -C7.5 -S2131956332`. This time, we will go up to 300 steps.
+
+![Screenshot 2022-09-23 at 02 05 48-min (1)](https://user-images.githubusercontent.com/50542132/191871743-6802f199-0ffd-4986-98c5-df2d8db30d18.png)
+
+Observing the results, it again takes longer for all samplers to converge (`K_HEUN` took around 150 steps), but we can observe good indicative results much earlier (see: `K_HEUN`). Conversely, `DDIM` and `PLMS` are still undergoing moderate changes (see: lace around her neck), even at `-s300`.
+
+In fact, as we can see in this other experiment, some samplers can take 700+ steps to converge when generating people.
+
+![191988191-c586b75a-2d7f-4351-b705-83cc1149881a-min (1)](https://user-images.githubusercontent.com/50542132/191992123-7e0759d6-6220-42c4-a961-88c7071c5ee6.png)
+
+Note also the point of convergence may not be the most desirable state (e.g. I prefer an earlier version of the face, more rounded), but it will probably be the most coherent arms/hands/face attributes-wise. You can always merge different images with a photo editing tool and pass it through `img2img` to smoothen the composition.
+
+### *Sampler generation times*
+
+Once we understand the concept of sampler convergence, we must look into the performance of each sampler in terms of steps (iterations) per second, as not all samplers run at the same speed.
+
+On my M1 Max with 64GB of RAM, for a 512x512 image:
+| Sampler   | (3 sample average) it/s |
+|---|---|
+|  `DDIM` | 1.89  |
+|  `PLMS` | 1.86  |
+|  `K_EULER` | 1.86  |
+|  `K_LMS` | 1.91  |
+|  `K_HEUN` | 0.95 (slower)  |
+|  `K_DPM_2` | 0.95 (slower)  |
+|  `K_DPM_2_A` | 0.95 (slower)  |
+|  `K_EULER_A` | 1.86  |
+
+Combining our results with the steps per second of each sampler, three choices come out on top: `K_LMS`, `K_HEUN` and `K_DPM_2` (where the latter two run 0.5x as quick but tend to converge 2x as quick as `K_LMS`). For creativity and a lot of variation between iterations, `K_EULER_A` can be a good choice (which runs 2x as quick as `K_DPM_2_A`).
+
+Additionally, image generation at very low steps (≤ `-s8`) is not recommended for `K_HEUN` and `K_DPM_2`. Use `K_LMS` instead.
+
+<img width="397" alt="192044949-67d5d441-a0d5-4d5a-be30-5dda4fc28a00-min" src="https://user-images.githubusercontent.com/50542132/192046823-2714cb29-bbf3-4eb1-9213-e27a0963905c.png">
+
+### *Three key points*
+
+Finally, it is relevant to mention that, in general, there are 3 important moments in the process of image formation as steps increase:
+
+* The (earliest) point at which an image becomes a good indicator of the final result (useful for batch generation at low step values, to then improve the quality/coherence of the chosen images via running the same prompt and seed for more steps).
+
+* The (earliest) point at which an image becomes coherent, even if different from the result if steps are increased (useful for batch generation at low step values, where quality/coherence is improved via techniques other than increasing the steps -e.g. via inpainting).
+
+* The point at which an image fully converges.
+
+Hence, remember that your workflow/strategy should define your optimal number of steps, even for the same prompt and seed (for example, if you seek full convergence, you may run `K_LMS` for `-s200` in the case of the red-haired girl, but `K_LMS` and `-s20`-taking one tenth the time- may do as well if your workflow includes adding small details, such as the missing shoulder strap, via `img2img`).

docs/installation/INSTALL_DOCKER.md

@@ -1,36 +1,60 @@
 # Before you begin
 
-- For end users: Install Stable Diffusion locally using the instructions for your OS.
+- For end users: Install Stable Diffusion locally using the instructions for
-- For developers: For container-related development tasks or for enabling easy deployment to other environments (on-premises or cloud), follow these instructions. For general use, install locally to leverage your machine's GPU.
+  your OS.
+- For developers: For container-related development tasks or for enabling easy
+  deployment to other environments (on-premises or cloud), follow these
+  instructions. For general use, install locally to leverage your machine's GPU.
 
 # Why containers?
 
-They provide a flexible, reliable way to build and deploy Stable Diffusion. You'll also use a Docker volume to store the largest model files and image outputs as a first step in decoupling storage and compute. Future enhancements can do this for other assets. See [Processes](https://12factor.net/processes) under the Twelve-Factor App methodology for details on why running applications in such a stateless fashion is important.
+They provide a flexible, reliable way to build and deploy Stable Diffusion.
+You'll also use a Docker volume to store the largest model files and image
+outputs as a first step in decoupling storage and compute. Future enhancements
+can do this for other assets. See [Processes](https://12factor.net/processes)
+under the Twelve-Factor App methodology for details on why running applications
+in such a stateless fashion is important.
 
-You can specify the target platform when building the image and running the container. You'll also need to specify the Stable Diffusion requirements file that matches the container's OS and the architecture it will run on.  
+You can specify the target platform when building the image and running the
+container. You'll also need to specify the Stable Diffusion requirements file
+that matches the container's OS and the architecture it will run on.
 
-Developers on Apple silicon (M1/M2): You [can't access your GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to another environment with NVIDIA GPUs on-premises or in the cloud.  
+Developers on Apple silicon (M1/M2): You
+[can't access your GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224)
+and performance is reduced compared with running it directly on macOS but for
+development purposes it's fine. Once you're done with development tasks on your
+laptop you can build for the target platform and architecture and deploy to
+another environment with NVIDIA GPUs on-premises or in the cloud.
 
-# Installation on a Linux container 
+# Installation on a Linux container
 
 ## Prerequisites
 
-### Get the data files  
+### Get the data files
 
-Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download the model file ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free.  
+Go to
+[Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original),
+and click "Access repository" to Download the model file `sd-v1-4.ckpt` (~4 GB)
+to `~/Downloads`. You'll need to create an account but it's quick and free.
 
 Also download the face restoration model.
+
 ```Shell
 cd ~/Downloads
-wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth
+wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth
 ```
 
-### Install [Docker](https://github.com/santisbon/guides#docker)  
+### Install [Docker](https://github.com/santisbon/guides#docker)
-On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/invoke-ai/InvokeAI/issues/342). You may need to increase Swap and Disk image size too.  
+
+On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the
+CPUs and Memory to avoid this
+[Issue](https://github.com/invoke-ai/InvokeAI/issues/342). You may need to
+increase Swap and Disk image size too.
 
 ## Setup
 
-Set the fork you want to use and other variables.  
+Set the fork you want to use and other variables.
+
 ```Shell
 TAG_STABLE_DIFFUSION="santisbon/stable-diffusion"
 PLATFORM="linux/arm64"
@@ -46,21 +70,28 @@ echo $CONDA_SUBDIR
 ```
 
 Create a Docker volume for the downloaded model files.
+
 ```Shell
 docker volume create my-vol
 ```
 
-Copy the data files to the Docker volume using a lightweight Linux container. We'll need the models at run time. You just need to create the container with the mountpoint; no need to run this dummy container.
+Copy the data files to the Docker volume using a lightweight Linux container.
+We'll need the models at run time. You just need to create the container with
+the mountpoint; no need to run this dummy container.
+
 ```Shell
 cd ~/Downloads # or wherever you saved the files
 
-docker create --platform $PLATFORM --name dummy --mount source=my-vol,target=/data alpine 
+docker create --platform $PLATFORM --name dummy --mount source=my-vol,target=/data alpine
 
 docker cp sd-v1-4.ckpt dummy:/data
-docker cp GFPGANv1.3.pth dummy:/data
+docker cp GFPGANv1.4.pth dummy:/data
 ```
 
-Get the repo and download the Miniconda installer (we'll need it at build time). Replace the URL with the version matching your container OS and the architecture it will run on.
+Get the repo and download the Miniconda installer (we'll need it at build time).
+Replace the URL with the version matching your container OS and the architecture
+it will run on.
+
 ```Shell
 cd ~
 git clone $GITHUB_STABLE_DIFFUSION
@@ -70,10 +101,15 @@ chmod +x entrypoint.sh
 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh
 ```
 
-Build the Docker image. Give it any tag ```-t``` that you want.  
+Build the Docker image. Give it any tag `-t` that you want.  
-Choose the Linux container's host platform: x86-64/Intel is ```amd64```. Apple silicon is ```arm64```. If deploying the container to the cloud to leverage powerful GPU instances you'll be on amd64 hardware but if you're just trying this out locally on Apple silicon choose arm64.  
+Choose the Linux container's host platform: x86-64/Intel is `amd64`. Apple
-The application uses libraries that need to match the host environment so use the appropriate requirements file.  
+silicon is `arm64`. If deploying the container to the cloud to leverage powerful
-Tip: Check that your shell session has the env variables set above.  
+GPU instances you'll be on amd64 hardware but if you're just trying this out
+locally on Apple silicon choose arm64.  
+The application uses libraries that need to match the host environment so use
+the appropriate requirements file.  
+Tip: Check that your shell session has the env variables set above.
+
 ```Shell
 docker build -t $TAG_STABLE_DIFFUSION \
 --platform $PLATFORM \
@@ -85,6 +121,7 @@ docker build -t $TAG_STABLE_DIFFUSION \
 
 Run a container using your built image.  
 Tip: Make sure you've created and populated the Docker volume (above).
+
 ```Shell
 docker run -it \
 --rm \
@@ -98,86 +135,121 @@ $TAG_STABLE_DIFFUSION
 # Usage (time to have fun)
 
 ## Startup
-If you're on a **Linux container** the ```dream``` script is **automatically started** and the output dir set to the Docker volume you created earlier. 
+
+If you're on a **Linux container** the `dream` script is **automatically
+started** and the output dir set to the Docker volume you created earlier.
 
 If you're **directly on macOS follow these startup instructions**.  
-With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts ```txt2img``` and ```img2img```:  
+With the Conda environment activated (`conda activate ldm`), run the interactive
-Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work.  
+interface that combines the functionality of the original scripts `txt2img` and
-By default the images are saved in ```outputs/img-samples/```.
+`img2img`:  
+Use the more accurate but VRAM-intensive full precision math because
+half-precision requires autocast and won't work.  
+By default the images are saved in `outputs/img-samples/`.
+
 ```Shell
-python3 scripts/dream.py --full_precision  
+python3 scripts/dream.py --full_precision
 ```
 
 You'll get the script's prompt. You can see available options or quit.
+
 ```Shell
 dream> -h
 dream> q
 ```
 
 ## Text to Image
-For quick (but bad) image results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly.  
+
+For quick (but bad) image results test with 5 steps (default 50) and 1 sample
+image. This will let you know that everything is set up correctly.  
 Then increase steps to 100 or more for good (but slower) results.  
 The prompt can be in quotes or not.
+
 ```Shell
-dream> The hulk fighting with sheldon cooper -s5 -n1 
+dream> The hulk fighting with sheldon cooper -s5 -n1
 dream> "woman closeup highly detailed"  -s 150
 # Reuse previous seed and apply face restoration
-dream> "woman closeup highly detailed"  --steps 150 --seed -1 -G 0.75 
+dream> "woman closeup highly detailed"  --steps 150 --seed -1 -G 0.75
 ```
 
-You'll need to experiment to see if face restoration is making it better or worse for your specific prompt.
+You'll need to experiment to see if face restoration is making it better or
+worse for your specific prompt.
 
-If you're on a container the output is set to the Docker volume. You can copy it wherever you want.  
+If you're on a container the output is set to the Docker volume. You can copy it
+wherever you want.  
 You can download it from the Docker Desktop app, Volumes, my-vol, data.  
-Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name.  
+Or you can copy it from your Mac terminal. Keep in mind `docker cp` can't expand
+`*.png` so you'll need to specify the image file name.
+
+On your host Mac (you can use the name of any container that mounted the
+volume):
 
-On your host Mac (you can use the name of any container that mounted the volume):
 ```Shell
-docker cp dummy:/data/000001.928403745.png /Users/<your-user>/Pictures 
+docker cp dummy:/data/000001.928403745.png /Users/<your-user>/Pictures
 ```
 
 ## Image to Image
-You can also do text-guided image-to-image translation. For example, turning a sketch into a detailed drawing.  
 
-```strength``` is a value between 0.0 and 1.0 that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely.  
+You can also do text-guided image-to-image translation. For example, turning a
+sketch into a detailed drawing.
 
-Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Otherwise you'll get ```Error: product of dimension sizes > 2**31'```. If you still get the error [try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.) like 512x256.  
+`strength` is a value between 0.0 and 1.0 that controls the amount of noise that
+is added to the input image. Values that approach 1.0 allow for lots of
+variations but will also produce images that are not semantically consistent
+with the input. 0.0 preserves image exactly, 1.0 replaces it completely.
+
+Make sure your input image size dimensions are multiples of 64 e.g. 512x512.
+Otherwise you'll get `Error: product of dimension sizes > 2**31'`. If you still
+get the error
+[try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.)
+like 512x256.
 
 If you're on a Docker container, copy your input image into the Docker volume
+
 ```Shell
 docker cp /Users/<your-user>/Pictures/sketch-mountains-input.jpg dummy:/data/
 ```
 
-Try it out generating an image (or more). The ```dream``` script needs absolute paths to find the image so don't use ```~```.  
+Try it out generating an image (or more). The `dream` script needs absolute
+paths to find the image so don't use `~`.
 
 If you're on your Mac
-```Shell 
+
+```Shell
 dream> "A fantasy landscape, trending on artstation" -I /Users/<your-user>/Pictures/sketch-mountains-input.jpg --strength 0.75  --steps 100 -n4
 ```
+
 If you're on a Linux container on your Mac
+
 ```Shell
 dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75  --steps 50 -n1
 ```
 
 ## Web Interface
-You can use the ```dream``` script with a graphical web interface. Start the web server with:
+
+You can use the `dream` script with a graphical web interface. Start the web
+server with:
+
 ```Shell
 python3 scripts/dream.py --full_precision --web
 ```
-If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090  
+
+If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090
 
 Press Control-C at the command line to stop the web server.
 
 ## Notes
 
 Some text you can add at the end of the prompt to make it very pretty:
+
 ```Shell
 cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed, painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida
 ```
 
 The original scripts should work as well.
+
 ```Shell
 python3 scripts/orig_scripts/txt2img.py --help
 python3 scripts/orig_scripts/txt2img.py --ddim_steps 100 --n_iter 1 --n_samples 1  --plms --prompt "new born baby kitten. Hyper Detail, Octane Rendering, Unreal Engine, V-Ray"
 python3 scripts/orig_scripts/txt2img.py --ddim_steps 5   --n_iter 1 --n_samples 1  --plms --prompt "ocean" # or --klms
-```
+```

docs/installation/INSTALL_LINUX.md

@@ -24,40 +24,40 @@ title: Linux
     the installation worked, your command prompt will be prefixed by the name of the
     current anaconda environment - `(base)`.
 
-3. Copy the stable-diffusion source code from GitHub:
+3. Copy the InvokeAI source code from GitHub:
 
-    ```bash
+```
-    (base) ~$ git clone https://github.com/invoke-ai/InvokeAI.git
+(base) ~$ git clone https://github.com/invoke-ai/InvokeAI.git
-    ```
+```
 
-    This will create stable-diffusion folder where you will follow the rest of the
+This will create InvokeAI folder where you will follow the rest of the steps.
-    steps.
 
-4. Enter the newly-created stable-diffusion folder. From this step forward make
+4. Enter the newly-created InvokeAI folder. From this step forward make sure that you are working in the InvokeAI directory!
-   sure that you are working in the stable-diffusion directory!
 
-    ```bash
+```
-    (base) ~$ cd stable-diffusion
+(base) ~$ cd InvokeAI
-    (base) ~/stable-diffusion$
+(base) ~/InvokeAI$
-    ```
+```
 
 5. Use anaconda to copy necessary python packages, create a new python
    environment named `ldm` and activate the environment.
 
-    ```bash
+
-    (base) ~/stable-diffusion$ conda env create -f environment.yaml
+```
-    (base) ~/stable-diffusion$ conda activate ldm
+(base) ~/InvokeAI$ conda env create -f environment.yaml
-    (ldm) ~/stable-diffusion$
+(base) ~/InvokeAI$ conda activate ldm
-    ```
+(ldm) ~/InvokeAI$
+```
 
     After these steps, your command prompt will be prefixed by `(ldm)` as shown
     above.
 
 6. Load a couple of small machine-learning models required by stable diffusion:
 
-    ```bash
+
-    (ldm) ~/stable-diffusion$ python3 scripts/preload_models.py
+```
-    ```
+(ldm) ~/InvokeAI$ python3 scripts/preload_models.py
+```
 
     !!! note
 
@@ -79,38 +79,34 @@ title: Linux
       This will create a symbolic link from the stable-diffusion model.ckpt file, to
       the true location of the `sd-v1-4.ckpt` file.
 
-    ```bash
+
-    (ldm) ~/stable-diffusion$ mkdir -p models/ldm/stable-diffusion-v1
+```
-    (ldm) ~/stable-diffusion$ ln -sf /path/to/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt
+(ldm) ~/InvokeAI$ mkdir -p models/ldm/stable-diffusion-v1
-    ```
+(ldm) ~/InvokeAI$ ln -sf /path/to/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt
+```
 
 8. Start generating images!
 
-    ```bash
+```
-    # for the pre-release weights use the -l or --liaon400m switch
+# for the pre-release weights use the -l or --liaon400m switch
-    (ldm) ~/stable-diffusion$ python3 scripts/dream.py -l
+(ldm) ~/InvokeAI$ python3 scripts/dream.py -l
 
-    # for the post-release weights do not use the switch
+# for the post-release weights do not use the switch
-    (ldm) ~/stable-diffusion$ python3 scripts/dream.py
+(ldm) ~/InvokeAI$ python3 scripts/dream.py
 
-    # for additional configuration switches and arguments, use -h or --help
+# for additional configuration switches and arguments, use -h or --help
-    (ldm) ~/stable-diffusion$ python3 scripts/dream.py -h
+(ldm) ~/InvokeAI$ python3 scripts/dream.py -h
-    ```
+```
 
-9. Subsequently, to relaunch the script, be sure to run "conda activate ldm"
+9. Subsequently, to relaunch the script, be sure to run "conda activate ldm" (step 5, second command), enter the `InvokeAI` directory, and then launch the dream script (step 8). If you forget to activate the ldm environment, the script will fail with multiple `ModuleNotFound` errors.
-   (step 5, second command), enter the `stable-diffusion` directory, and then
-   launch the dream script (step 8). If you forget to activate the ldm
-   environment, the script will fail with multiple `ModuleNotFound` errors.
 
 ## Updating to newer versions of the script
 
-This distribution is changing rapidly. If you used the `git clone` method
-(step 5) to download the stable-diffusion directory, then to update to the
-latest and greatest version, launch the Anaconda window, enter
-`stable-diffusion` and type:
 
-```bash
+This distribution is changing rapidly. If you used the `git clone` method (step 5) to download the InvokeAI directory, then to update to the latest and greatest version, launch the Anaconda window, enter `InvokeAI` and type:
-(ldm) ~/stable-diffusion$ git pull
+
+```
+(ldm) ~/InvokeAI$ git pull
 ```
 
 This will bring your local copy into sync with the remote one.

frontend/src/app/features.ts

@@ -0,0 +1,59 @@
+type FeatureHelpInfo = {
+  text: string;
+  href: string;
+  guideImage: string;
+};
+
+export enum Feature {
+  PROMPT,
+  GALLERY,
+  OUTPUT,
+  SEED_AND_VARIATION,
+  ESRGAN,
+  FACE_CORRECTION,
+  IMAGE_TO_IMAGE,
+  SAMPLER,
+}
+
+export const FEATURES: Record<Feature, FeatureHelpInfo> = {
+  [Feature.PROMPT]: {
+    text: 'This field will take all prompt text, including both content and stylistic terms. CLI Commands will not work in the prompt.',
+    href: 'link/to/docs/feature3.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.GALLERY]: {
+    text: 'As new invocations are generated, files from the output directory will be displayed here. Generations have additional options to configure new generations.',
+    href: 'link/to/docs/feature3.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.OUTPUT]: {
+    text: 'The Height and Width of generations can be controlled here. If you experience errors, you may be generating an image too large for your system. The seamless option will more often result in repeating patterns in outputs.',
+    href: 'link/to/docs/feature3.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.SEED_AND_VARIATION]: {
+    text: 'Seed values provide an initial set of noise which guide the denoising process. Try a variation with an amount of between 0 and 1 to change the output image for that seed.',
+    href: 'link/to/docs/feature3.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.ESRGAN]: {
+    text: 'The ESRGAN setting can be used to increase the output resolution without requiring a higher width/height in the initial generation.',
+    href: 'link/to/docs/feature1.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.FACE_CORRECTION]: {
+    text: 'Using GFPGAN or CodeFormer, Face Correction will attempt to identify faces in outputs, and correct any defects/abnormalities. Higher values will apply a stronger corrective pressure on outputs.',
+    href: 'link/to/docs/feature2.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.IMAGE_TO_IMAGE]: {
+    text: 'ImageToImage allows the upload of an initial image, which InvokeAI will use to guide the generation process, along with a prompt. A lower value for this setting will more closely resemble the original image. Values between 0-1 are accepted, and a range of .25-.75 is recommended ',
+    href: 'link/to/docs/feature3.html',
+    guideImage: 'asset/path.gif',
+  },
+  [Feature.SAMPLER]: {
+    text: 'This setting allows for different denoising samplers to be used, as well as the number of denoising steps used, which will change the resulting output.',
+    href: 'link/to/docs/feature3.html',
+    guideImage: 'asset/path.gif',
+  },
+};

frontend/src/common/components/GuideIcon.tsx

@@ -0,0 +1,22 @@
+import { Box, forwardRef, Icon } from '@chakra-ui/react';
+import { IconType } from 'react-icons';
+import { MdHelp } from 'react-icons/md';
+import { Feature } from '../../app/features';
+import GuidePopover from './GuidePopover';
+
+type GuideIconProps = {
+  feature: Feature;
+  icon?: IconType;
+};
+
+const GuideIcon = forwardRef(
+  ({ feature, icon = MdHelp }: GuideIconProps, ref) => (
+    <GuidePopover feature={feature}>
+      <Box ref={ref}>
+        <Icon as={icon} />
+      </Box>
+    </GuidePopover>
+  )
+);
+
+export default GuideIcon;

frontend/src/common/components/GuidePopover.tsx

@@ -0,0 +1,51 @@
+import {
+  Popover,
+  PopoverArrow,
+  PopoverContent,
+  PopoverTrigger,
+  PopoverHeader,
+  Flex,
+  Box,
+} from '@chakra-ui/react';
+import { SystemState } from '../../features/system/systemSlice';
+import { useAppSelector } from '../../app/store';
+import { RootState } from '../../app/store';
+import { createSelector } from '@reduxjs/toolkit';
+import { ReactElement } from 'react';
+import {  Feature, FEATURES } from '../../app/features';
+
+type GuideProps = {
+  children: ReactElement;
+  feature: Feature;
+};
+
+const systemSelector = createSelector(
+  (state: RootState) => state.system,
+  (system: SystemState) => system.shouldDisplayGuides
+);
+
+const GuidePopover = ({ children, feature }: GuideProps) => {
+  const shouldDisplayGuides = useAppSelector(systemSelector);
+  const { text } = FEATURES[feature];
+  return shouldDisplayGuides ? (
+    <Popover trigger={'hover'}>
+      <PopoverTrigger>
+        <Box>{children}</Box>
+      </PopoverTrigger>
+      <PopoverContent
+        maxWidth="400px"
+        onClick={(e) => e.preventDefault()}
+        cursor={'initial'}
+      >
+        <PopoverArrow />
+        <Flex alignItems={'center'} gap={2} p={4}>
+          {text}
+        </Flex>
+      </PopoverContent>
+    </Popover>
+  ) : (
+    <></>
+  );
+};
+
+export default GuidePopover;

frontend/src/features/options/OptionsAccordion.tsx

@@ -31,6 +31,9 @@ import OutputOptions from './OutputOptions';
 import ImageToImageOptions from './ImageToImageOptions';
 import { ChangeEvent } from 'react';
 
+import GuideIcon from '../../common/components/GuideIcon';
+import { Feature } from '../../app/features';
+
 const optionsSelector = createSelector(
   (state: RootState) => state.options,
   (options: OptionsState) => {
@@ -108,6 +111,7 @@ const OptionsAccordion = () => {
             <Box flex="1" textAlign="left">
               Seed & Variation
             </Box>
+            <GuideIcon feature={Feature.SEED_AND_VARIATION} />
             <AccordionIcon />
           </AccordionButton>
         </h2>
@@ -121,6 +125,7 @@ const OptionsAccordion = () => {
             <Box flex="1" textAlign="left">
               Sampler
             </Box>
+            <GuideIcon feature={Feature.SAMPLER} />
             <AccordionIcon />
           </AccordionButton>
         </h2>
@@ -144,6 +149,7 @@ const OptionsAccordion = () => {
                 onChange={handleChangeShouldRunESRGAN}
               />
             </Flex>
+            <GuideIcon feature={Feature.ESRGAN} />
             <AccordionIcon />
           </AccordionButton>
         </h2>
@@ -160,13 +166,14 @@ const OptionsAccordion = () => {
               width={'100%'}
               mr={2}
             >
-              <Text>Fix Faces (GFPGAN)</Text>
+              <Text>Face Correction</Text>
               <Switch
                 isDisabled={!isGFPGANAvailable}
                 isChecked={shouldRunGFPGAN}
                 onChange={handleChangeShouldRunGFPGAN}
               />
             </Flex>
+            <GuideIcon feature={Feature.FACE_CORRECTION} />
             <AccordionIcon />
           </AccordionButton>
         </h2>
@@ -190,6 +197,7 @@ const OptionsAccordion = () => {
                 onChange={handleChangeShouldUseInitImage}
               />
             </Flex>
+            <GuideIcon feature={Feature.IMAGE_TO_IMAGE} />
             <AccordionIcon />
           </AccordionButton>
         </h2>
@@ -203,6 +211,7 @@ const OptionsAccordion = () => {
             <Box flex="1" textAlign="left">
               Output
             </Box>
+            <GuideIcon feature={Feature.OUTPUT} />
             <AccordionIcon />
           </AccordionButton>
         </h2>

frontend/src/features/system/SettingsModal.tsx

@@ -20,6 +20,7 @@ import { useAppDispatch, useAppSelector } from '../../app/store';
 import {
   setShouldConfirmOnDelete,
   setShouldDisplayInProgress,
+  setShouldDisplayGuides,
   SystemState,
 } from './systemSlice';
 import { RootState } from '../../app/store';
@@ -31,8 +32,8 @@ import { cloneElement, ReactElement } from 'react';
 const systemSelector = createSelector(
   (state: RootState) => state.system,
   (system: SystemState) => {
-    const { shouldDisplayInProgress, shouldConfirmOnDelete } = system;
+    const { shouldDisplayInProgress, shouldConfirmOnDelete, shouldDisplayGuides } = system;
-    return { shouldDisplayInProgress, shouldConfirmOnDelete };
+    return { shouldDisplayInProgress, shouldConfirmOnDelete, shouldDisplayGuides };
   },
   {
     memoizeOptions: { resultEqualityCheck: isEqual },
@@ -63,7 +64,7 @@ const SettingsModal = ({ children }: SettingsModalProps) => {
     onClose: onRefreshModalClose,
   } = useDisclosure();
 
-  const { shouldDisplayInProgress, shouldConfirmOnDelete } =
+  const { shouldDisplayInProgress, shouldConfirmOnDelete, shouldDisplayGuides } =
     useAppSelector(systemSelector);
 
   const dispatch = useAppDispatch();
@@ -116,6 +117,19 @@ const SettingsModal = ({ children }: SettingsModalProps) => {
                   />
                 </HStack>
               </FormControl>
+              <FormControl>
+                <HStack>
+                  <FormLabel marginBottom={1}>
+                    Display help guides in configuration menus
+                  </FormLabel>
+                  <Switch
+                    isChecked={shouldDisplayGuides}
+                    onChange={(e) =>
+                      dispatch(setShouldDisplayGuides(e.target.checked))
+                    }
+                  />
+                </HStack>
+              </FormControl>
 
               <Heading size={'md'}>Reset Web UI</Heading>
               <Text>

frontend/src/features/system/systemSlice.ts

@@ -31,6 +31,8 @@ export interface SystemState extends InvokeAI.SystemStatus, InvokeAI.SystemConfi
   totalIterations: number;
   currentStatus: string;
   currentStatusHasSteps: boolean;
+  
+  shouldDisplayGuides: boolean;
 }
 
 const initialSystemState = {
@@ -39,6 +41,7 @@ const initialSystemState = {
   log: [],
   shouldShowLogViewer: false,
   shouldDisplayInProgress: false,
+  shouldDisplayGuides: true,
   isGFPGANAvailable: true,
   isESRGANAvailable: true,
   socketId: '',
@@ -117,6 +120,9 @@ export const systemSlice = createSlice({
     setSystemConfig: (state, action: PayloadAction<InvokeAI.SystemConfig>) => {
       return { ...state, ...action.payload };
     },
+    setShouldDisplayGuides: (state, action: PayloadAction<boolean>) => {
+      state.shouldDisplayGuides = action.payload;
+    },
   },
 });
 
@@ -132,6 +138,7 @@ export const {
   setSystemStatus,
   setCurrentStatus,
   setSystemConfig,
+  setShouldDisplayGuides,
 } = systemSlice.actions;
 
 export default systemSlice.reducer;

ldm/dream/args.py

@@ -400,7 +400,7 @@ class Args(object):
         postprocessing_group.add_argument(
             '--gfpgan_model_path',
             type=str,
-            default='experiments/pretrained_models/GFPGANv1.3.pth',
+            default='experiments/pretrained_models/GFPGANv1.4.pth',
             help='Indicates the path to the GFPGAN model, relative to --gfpgan_dir.',
         )
         postprocessing_group.add_argument(

ldm/dream/conditioning.py

@@ -79,10 +79,10 @@ def split_weighted_subprompts(text, skip_normalize=False)->list:
     if weight_sum == 0:
         print(
             "Warning: Subprompt weights add up to zero. Discarding and using even weights instead.")
-        equal_weight = 1 / len(parsed_prompts)
+        equal_weight = 1 / max(len(parsed_prompts), 1)
         return [(x[0], equal_weight) for x in parsed_prompts]
     return [(x[0], x[1] / weight_sum) for x in parsed_prompts]
-        
+
 # shows how the prompt is tokenized
 # usually tokens have '</w>' to indicate end-of-word,
 # but for readability it has been replaced with ' '

ldm/dream/generator/inpaint.py

@@ -34,9 +34,9 @@ class Inpaint(Img2Img):
             )
             sampler = DDIMSampler(self.model, device=self.model.device)
 
-            sampler.make_schedule(
+        sampler.make_schedule(
-                ddim_num_steps=steps, ddim_eta=ddim_eta, verbose=False
+            ddim_num_steps=steps, ddim_eta=ddim_eta, verbose=False
-            )
+        )
 
         scope = choose_autocast(self.precision)
         with scope(self.model.device.type):

ldm/dream/restoration/base.py

@@ -1,34 +1,38 @@
 class Restoration():
-    def __init__(self, gfpgan_dir='./src/gfpgan', gfpgan_model_path='experiments/pretrained_models/GFPGANv1.3.pth', esrgan_bg_tile=400) -> None:
+    def __init__(self) -> None:
-        self.gfpgan_dir = gfpgan_dir
+        pass
-        self.gfpgan_model_path = gfpgan_model_path
-        self.esrgan_bg_tile = esrgan_bg_tile
 
-    def load_face_restore_models(self):
+    def load_face_restore_models(self, gfpgan_dir='./src/gfpgan', gfpgan_model_path='experiments/pretrained_models/GFPGANv1.4.pth'):
         # Load GFPGAN
-        gfpgan = self.load_gfpgan()
+        gfpgan = self.load_gfpgan(gfpgan_dir, gfpgan_model_path)
         if gfpgan.gfpgan_model_exists:
             print('>> GFPGAN Initialized')
+        else:
+            print('>> GFPGAN Disabled')
+            gfpgan = None
         
         # Load CodeFormer
         codeformer = self.load_codeformer()
         if codeformer.codeformer_model_exists:
             print('>> CodeFormer Initialized')
+        else:
+            print('>> CodeFormer Disabled')
+            codeformer = None
         
         return gfpgan, codeformer
 
     # Face Restore Models
-    def load_gfpgan(self):
+    def load_gfpgan(self, gfpgan_dir, gfpgan_model_path):
         from ldm.dream.restoration.gfpgan import GFPGAN
-        return GFPGAN(self.gfpgan_dir, self.gfpgan_model_path)
+        return GFPGAN(gfpgan_dir, gfpgan_model_path)
 
     def load_codeformer(self):
         from ldm.dream.restoration.codeformer import CodeFormerRestoration
         return CodeFormerRestoration()
 
     # Upscale Models
-    def load_esrgan(self):
+    def load_esrgan(self, esrgan_bg_tile=400):
         from ldm.dream.restoration.realesrgan import ESRGAN
-        esrgan = ESRGAN(self.esrgan_bg_tile)
+        esrgan = ESRGAN(esrgan_bg_tile)
         print('>> ESRGAN Initialized')
         return esrgan;

ldm/dream/restoration/gfpgan.py

@@ -11,14 +11,14 @@ class GFPGAN():
     def __init__(
             self,
             gfpgan_dir='src/gfpgan',
-            gfpgan_model_path='experiments/pretrained_models/GFPGANv1.3.pth') -> None:
+            gfpgan_model_path='experiments/pretrained_models/GFPGANv1.4.pth') -> None:
 
         self.model_path = os.path.join(gfpgan_dir, gfpgan_model_path)
         self.gfpgan_model_exists = os.path.isfile(self.model_path)
 
         if not self.gfpgan_model_exists:
-            raise Exception(
+            print('## NOT FOUND: GFPGAN model not found at ' + self.model_path)
-                'GFPGAN model not found at path ' + self.model_path)
+            return None
         sys.path.append(os.path.abspath(gfpgan_dir))
 
     def model_exists(self):
@@ -50,7 +50,7 @@ class GFPGAN():
                 f'>> WARNING: GFPGAN not initialized.'
             )
             print(
-                f'>> Download https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth to {self.model_path}, \nor change GFPGAN directory with --gfpgan_dir.'
+                f'>> Download https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth to {self.model_path}, \nor change GFPGAN directory with --gfpgan_dir.'
             )
 
         image = image.convert('RGB')

ldm/dream/server.py

@@ -161,7 +161,7 @@ class DreamServer(BaseHTTPRequestHandler):
         # is complete. The upscaling replaces the original file, so the second
         # entry should not be inserted into the image list.
         # LS: This repeats code in dream.py
-        def image_done(image, seed, upscaled=False):
+        def image_done(image, seed, upscaled=False, first_seed=None):
             name = f'{prefix}.{seed}.png'
             iter_opt  = copy.copy(opt)
             if opt.variation_amount > 0:

ldm/generate.py

@@ -591,8 +591,8 @@ class Generate:
 
     def _make_images(
             self,
-            img_path,
+            img,
-            mask_path,
+            mask,
             width,
             height,
             fit=False,
@@ -600,11 +600,11 @@ class Generate:
     ):
         init_image      = None
         init_mask       = None
-        if not img_path:
+        if not img:
             return None, None
 
         image = self._load_img(
-            img_path,
+            img,
             width,
             height,
             fit=fit
@@ -614,7 +614,7 @@ class Generate:
         init_image   = self._create_init_image(image)                   # this returns a torch tensor
 
         # if image has a transparent area and no mask was provided, then try to generate mask
-        if self._has_transparency(image) and not mask_path:
+        if self._has_transparency(image) and not mask:
             print(
                 '>> Initial image has transparent areas. Will inpaint in these regions.')
             if self._check_for_erasure(image):
@@ -626,13 +626,19 @@ class Generate:
             # this returns a torch tensor
             init_mask = self._create_init_mask(image)
 
-        if mask_path:
+        if mask:
             mask_image = self._load_img(
-                mask_path, width, height, fit=fit)  # this returns an Image
+                mask, width, height, fit=fit)  # this returns an Image
             init_mask = self._create_init_mask(mask_image)
 
         return init_image, init_mask
 
+    def _make_base(self):
+        if not self.generators.get('base'):
+            from ldm.dream.generator import Generator
+            self.generators['base'] = Generator(self.model, self.precision)
+        return self.generators['base']
+
     def _make_img2img(self):
         if not self.generators.get('img2img'):
             from ldm.dream.generator.img2img import Img2Img
@@ -726,11 +732,18 @@ class Generate:
                     else:
                         print(">> ESRGAN is disabled. Image not upscaled.")
                 if strength > 0:
-                    if self.gfpgan is not None and self.codeformer is not None:
+                    if self.gfpgan is not None or self.codeformer is not None:
+                        if facetool == 'gfpgan':
+                            if self.gfpgan is None:
+                                print('>> GFPGAN not found. Face restoration is disabled.')
+                            else:
+                              image = self.gfpgan.process(image, strength, seed)                              
                         if facetool == 'codeformer':
-                            image = self.codeformer.process(image=image, strength=strength, device=self.device, seed=seed, fidelity=codeformer_fidelity)
+                            if self.codeformer is None:
-                        else:
+                                print('>> CodeFormer not found. Face restoration is disabled.')
-                            image = self.gfpgan.process(image, strength, seed)
+                            else:
+                                cf_device = 'cpu' if str(self.device) == 'mps' else self.device
+                                image = self.codeformer.process(image=image, strength=strength, device=cf_device, seed=seed, fidelity=codeformer_fidelity)
                     else:
                         print(">> Face Restoration is disabled.")
             except Exception as e:
@@ -745,13 +758,7 @@ class Generate:
 
     # to help WebGUI - front end to generator util function
     def sample_to_image(self, samples):
-        return self._sample_to_image(samples)
+        return self._make_base().sample_to_image(samples)
-
-    def _sample_to_image(self, samples):
-        if not self.base_generator:
-            from ldm.dream.generator import Generator
-            self.base_generator = Generator(self.model)
-        return self.base_generator.sample_to_image(samples)
 
     def _set_sampler(self):
         msg = f'>> Setting Sampler to {self.sampler_name}'
@@ -828,15 +835,25 @@ class Generate:
 
         return model
 
-    def _load_img(self, path, width, height, fit=False):
+    def _load_img(self, img, width, height, fit=False):
-        assert os.path.exists(path), f'>> {path}: File not found'
+        if isinstance(img, Image.Image):
+            image = img
+            print(
+                f'>> using provided input image of size {image.width}x{image.height}'
+            )
+        elif isinstance(img, str):
+            assert os.path.exists(img), f'>> {img}: File not found'
+
+            image = Image.open(img)
+            print(
+                f'>> loaded input image of size {image.width}x{image.height} from {img}'
+            )
+        else:
+            image = Image.open(img)
+            print(
+                f'>> loaded input image of size {image.width}x{image.height}'
+            )
 
-        #        with Image.open(path) as img:
-        #            image = img.convert('RGBA')
-        image = Image.open(path)
-        print(
-            f'>> loaded input image of size {image.width}x{image.height} from {path}'
-        )
         if fit:
             image = self._fit_image(image, (width, height))
         else:
@@ -922,7 +939,7 @@ class Generate:
         # BUG: We need to use the model's downsample factor rather than hardcoding "8"
         from ldm.dream.generator.base import downsampling
         image = image.resize((image.width//downsampling, image.height //
-                             downsampling), resample=Image.Resampling.LANCZOS)
+                              downsampling), resample=Image.Resampling.NEAREST)
         # print(
         #     f'>> DEBUG: writing the mask to mask.png'
         #     )

scripts/dream.py

@@ -47,16 +47,19 @@ def main():
     # Loading Face Restoration and ESRGAN Modules
     try:
         gfpgan, codeformer, esrgan = None, None, None
-        from ldm.dream.restoration import Restoration
+        if opt.restore or opt.esrgan:
-        restoration = Restoration(opt.gfpgan_dir, opt.gfpgan_model_path, opt.esrgan_bg_tile)
+            from ldm.dream.restoration import Restoration
-        if opt.restore:
+            restoration = Restoration()
-            gfpgan, codeformer = restoration.load_face_restore_models()
+            if opt.restore:
+                gfpgan, codeformer = restoration.load_face_restore_models(opt.gfpgan_dir, opt.gfpgan_model_path)
+            else:
+                print('>> Face restoration disabled')
+            if opt.esrgan:
+                esrgan = restoration.load_esrgan(opt.esrgan_bg_tile)
+            else:
+                print('>> Upscaling disabled')
         else:
-            print('>> Face restoration disabled')
+            print('>> Face restoration and upscaling disabled')
-        if opt.esrgan:
-            esrgan = restoration.load_esrgan()
-        else:
-            print('>> Upscaling disabled')
     except (ModuleNotFoundError, ImportError):
         import traceback
         print(traceback.format_exc(), file=sys.stderr)

scripts/preload_models.py

@@ -87,8 +87,8 @@ if gfpgan:
 
     try:
         import urllib.request
-        model_url  = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth'
+        model_url  = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth'
-        model_dest = 'src/gfpgan/experiments/pretrained_models/GFPGANv1.3.pth'
+        model_dest = 'src/gfpgan/experiments/pretrained_models/GFPGANv1.4.pth'
 
         if not os.path.exists(model_dest):
             print('downloading gfpgan model file...')