Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'development' of github.com:invoke-ai/InvokeAI into development

Browse Source
This commit is contained in:
Lincoln Stein 2022-09-27 18:15:14 -04:00
commit 3491a1688b
49 changed files with 2072 additions and 1323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/create-caches.yml vendored
View File

@ -13,10 +13,10 @@ jobs:
id: vars
run: |
if [ "$RUNNER_OS" = "macOS" ]; then
echo "::set-output name=ENV_FILE::environment-mac.yaml"
echo "::set-output name=ENV_FILE::environment-mac.yml"
echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python"
elif [ "$RUNNER_OS" = "Linux" ]; then
echo "::set-output name=ENV_FILE::environment.yaml"
echo "::set-output name=ENV_FILE::environment.yml"
echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python"
fi
- name: Checkout sources

4
.github/workflows/test-dream-conda.yml vendored
View File

@ -19,10 +19,10 @@ jobs:
run: |
# Note, can't "activate" via github action; specifying the env's python has the same effect
if [ "$RUNNER_OS" = "macOS" ]; then
echo "::set-output name=ENV_FILE::environment-mac.yaml"
echo "::set-output name=ENV_FILE::environment-mac.yml"
echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python"
elif [ "$RUNNER_OS" = "Linux" ]; then
echo "::set-output name=ENV_FILE::environment.yaml"
echo "::set-output name=ENV_FILE::environment.yml"
echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python"
fi
- name: Checkout sources

11
README.md
View File

@ -2,6 +2,15 @@
# InvokeAI: A Stable Diffusion Toolkit
_Note: This fork is rapidly evolving. Please use the
[Issues](https://github.com/invoke-ai/InvokeAI/issues) tab to
report bugs and make feature requests. Be sure to use the provided
templates. They will help aid diagnose issues faster._
_This repository was formally known as lstein/stable-diffusion_
# **Table of Contents**
![project logo](docs/assets/logo.png)
[![discord badge]][discord link]
@ -19,7 +28,7 @@
[discord badge]: https://flat.badgen.net/discord/members/htRgbc7e?icon=discord
[discord link]: https://discord.gg/ZmtBAhwWhy
[github forks badge]: https://flat.badgen.net/github/forks/invoke-ai/InvokeAI?icon=github
[github forks link]: https://useful-forks.github.io/?repo=lstein%2Fstable-diffusion
[github forks link]: https://useful-forks.github.io/?repo=invoke-ai%2FInvokeAI
[github open issues badge]: https://flat.badgen.net/github/open-issues/invoke-ai/InvokeAI?icon=github
[github open issues link]: https://github.com/invoke-ai/InvokeAI/issues?q=is%3Aissue+is%3Aopen
[github open prs badge]: https://flat.badgen.net/github/open-prs/invoke-ai/InvokeAI?icon=github

80
backend/server.py
View File

@ -103,6 +103,8 @@ socketio = SocketIO(
engineio_logger=engineio_logger,
max_http_buffer_size=max_http_buffer_size,
cors_allowed_origins=cors_allowed_origins,
ping_interval=(50, 50),
ping_timeout=60,
)
@ -186,17 +188,50 @@ def handle_request_capabilities():
socketio.emit("systemConfig", config)
@socketio.on("requestAllImages")
def handle_request_all_images():
print(f">> All images requested")
paths = list(filter(os.path.isfile, glob.glob(result_path + "*.png")))
paths.sort(key=lambda x: os.path.getmtime(x))
@socketio.on("requestImages")
def handle_request_images(page=1, offset=0, last_mtime=None):
chunk_size = 50
if last_mtime:
print(f">> Latest images requested")
else:
print(
f">> Page {page} of images requested (page size {chunk_size} offset {offset})"
)
paths = glob.glob(os.path.join(result_path, "*.png"))
sorted_paths = sorted(paths, key=lambda x: os.path.getmtime(x), reverse=True)
if last_mtime:
image_paths = filter(lambda x: os.path.getmtime(x) > last_mtime, sorted_paths)
else:
image_paths = sorted_paths[
slice(chunk_size * (page - 1) + offset, chunk_size * page + offset)
]
page = page + 1
image_array = []
for path in paths:
for path in image_paths:
metadata = retrieve_metadata(path)
image_array.append({"url": path, "metadata": metadata["sd-metadata"]})
socketio.emit("galleryImages", {"images": image_array})
eventlet.sleep(0)
image_array.append(
{
"url": path,
"mtime": os.path.getmtime(path),
"metadata": metadata["sd-metadata"],
}
)
socketio.emit(
"galleryImages",
{
"images": image_array,
"nextPage": page,
"offset": offset,
"onlyNewImages": True if last_mtime else False,
},
)
@socketio.on("generateImage")
@ -275,6 +310,7 @@ def handle_run_esrgan_event(original_image, esrgan_parameters):
"esrganResult",
{
"url": os.path.relpath(path),
"mtime": os.path.getmtime(path),
"metadata": metadata,
},
)
@ -343,6 +379,7 @@ def handle_run_gfpgan_event(original_image, gfpgan_parameters):
"gfpganResult",
{
"url": os.path.relpath(path),
"mtime": os.path.mtime(path),
"metadata": metadata,
},
)
@ -635,14 +672,19 @@ def generate_images(generation_parameters, esrgan_parameters, gfpgan_parameters)
and step < generation_parameters["steps"] - 1
):
image = generate.sample_to_image(sample)
path = save_image(
image, generation_parameters, intermediate_path, step_index
)
metadata = parameters_to_generated_image_metadata(generation_parameters)
command = parameters_to_command(generation_parameters)
path = save_image(image, command, metadata, intermediate_path, step_index=step_index, postprocessing=False)
step_index += 1
socketio.emit(
"intermediateResult",
{"url": os.path.relpath(path), "metadata": generation_parameters},
{
"url": os.path.relpath(path),
"mtime": os.path.getmtime(path),
"metadata": metadata,
},
)
socketio.emit("progressUpdate", progress)
eventlet.sleep(0)
@ -670,6 +712,11 @@ def generate_images(generation_parameters, esrgan_parameters, gfpgan_parameters)
first_seed = first_seed or seed
this_variation = [[seed, all_parameters["variation_amount"]]]
all_parameters["with_variations"] = prior_variations + this_variation
all_parameters["seed"] = first_seed
elif ("with_variations" in all_parameters):
all_parameters["seed"] = first_seed
else:
all_parameters["seed"] = seed
if esrgan_parameters:
progress["currentStatus"] = "Upscaling"
@ -702,7 +749,6 @@ def generate_images(generation_parameters, esrgan_parameters, gfpgan_parameters)
postprocessing = True
all_parameters["gfpgan_strength"] = gfpgan_parameters["strength"]
all_parameters["seed"] = first_seed
progress["currentStatus"] = "Saving image"
socketio.emit("progressUpdate", progress)
eventlet.sleep(0)
@ -735,7 +781,11 @@ def generate_images(generation_parameters, esrgan_parameters, gfpgan_parameters)
socketio.emit(
"generationResult",
{"url": os.path.relpath(path), "metadata": metadata},
{
"url": os.path.relpath(path),
"mtime": os.path.getmtime(path),
"metadata": metadata,
},
)
eventlet.sleep(0)

2
docker-build/Dockerfile
View File

@ -47,7 +47,7 @@ RUN git clone https://github.com/TencentARC/GFPGAN.git
WORKDIR /GFPGAN
RUN pip3 install -r requirements.txt \
&& python3 setup.py develop \
&& ln -s "/data/GFPGANv1.3.pth" experiments/pretrained_models/GFPGANv1.3.pth
&& ln -s "/data/GFPGANv1.4.pth" experiments/pretrained_models/GFPGANv1.4.pth
WORKDIR /stable-diffusion
RUN python3 scripts/preload_models.py

137
docs/CHANGELOG.md Normal file
View File

@ -0,0 +1,137 @@
# **Changelog**
## v1.13 (in process)
- Supports a Google Colab notebook for a standalone server running on Google hardware [Arturo Mendivil](https://github.com/artmen1516)
- WebUI supports GFPGAN/ESRGAN facial reconstruction and upscaling [Kevin Gibbons](https://github.com/bakkot)
- WebUI supports incremental display of in-progress images during generation [Kevin Gibbons](https://github.com/bakkot)
- Output directory can be specified on the dream> command line.
- The grid was displaying duplicated images when not enough images to fill the final row [Muhammad Usama](https://github.com/SMUsamaShah)
- Can specify --grid on dream.py command line as the default.
- Miscellaneous internal bug and stability fixes.
---
## v1.12 (28 August 2022)
- Improved file handling, including ability to read prompts from standard input.
(kudos to [Yunsaki](https://github.com/yunsaki)
- The web server is now integrated with the dream.py script. Invoke by adding --web to
the dream.py command arguments.
- Face restoration and upscaling via GFPGAN and Real-ESGAN are now automatically
enabled if the GFPGAN directory is located as a sibling to Stable Diffusion.
VRAM requirements are modestly reduced. Thanks to both [Blessedcoolant](https://github.com/blessedcoolant) and
[Oceanswave](https://github.com/oceanswave) for their work on this.
- You can now swap samplers on the dream> command line. [Blessedcoolant](https://github.com/blessedcoolant)
---
## v1.11 (26 August 2022)
- NEW FEATURE: Support upscaling and face enhancement using the GFPGAN module. (kudos to [Oceanswave](https://github.com/Oceanswave)
- You now can specify a seed of -1 to use the previous image's seed, -2 to use the seed for the image generated before that, etc.
Seed memory only extends back to the previous command, but will work on all images generated with the -n# switch.
- Variant generation support temporarily disabled pending more general solution.
- Created a feature branch named **yunsaki-morphing-dream** which adds experimental support for
iteratively modifying the prompt and its parameters. Please see[ Pull Request #86](https://github.com/lstein/stable-diffusion/pull/86)
for a synopsis of how this works. Note that when this feature is eventually added to the main branch, it will may be modified
significantly.
---
## v1.10 (25 August 2022)
- A barebones but fully functional interactive web server for online generation of txt2img and img2img.
---
## v1.09 (24 August 2022)
- A new -v option allows you to generate multiple variants of an initial image
in img2img mode. (kudos to [Oceanswave](https://github.com/Oceanswave). [
See this discussion in the PR for examples and details on use](https://github.com/lstein/stable-diffusion/pull/71#issuecomment-1226700810))
- Added ability to personalize text to image generation (kudos to [Oceanswave](https://github.com/Oceanswave) and [nicolai256](https://github.com/nicolai256))
- Enabled all of the samplers from k_diffusion
---
## v1.08 (24 August 2022)
- Escape single quotes on the dream> command before trying to parse. This avoids
parse errors.
- Removed instruction to get Python3.8 as first step in Windows install.
Anaconda3 does it for you.
- Added bounds checks for numeric arguments that could cause crashes.
- Cleaned up the copyright and license agreement files.
---
## v1.07 (23 August 2022)
- Image filenames will now never fill gaps in the sequence, but will be assigned the
next higher name in the chosen directory. This ensures that the alphabetic and chronological
sort orders are the same.
---
## v1.06 (23 August 2022)
- Added weighted prompt support contributed by [xraxra](https://github.com/xraxra)
- Example of using weighted prompts to tweak a demonic figure contributed by [bmaltais](https://github.com/bmaltais)
---
## v1.05 (22 August 2022 - after the drop)
- Filenames now use the following formats:
000010.95183149.png -- Two files produced by the same command (e.g. -n2),
000010.26742632.png -- distinguished by a different seed.
000011.455191342.01.png -- Two files produced by the same command using
000011.455191342.02.png -- a batch size>1 (e.g. -b2). They have the same seed.
000011.4160627868.grid#1-4.png -- a grid of four images (-g); the whole grid can
be regenerated with the indicated key
- It should no longer be possible for one image to overwrite another
- You can use the "cd" and "pwd" commands at the dream> prompt to set and retrieve
the path of the output directory.
---
## v1.04 (22 August 2022 - after the drop)
- Updated README to reflect installation of the released weights.
- Suppressed very noisy and inconsequential warning when loading the frozen CLIP
tokenizer.
---
## v1.03 (22 August 2022)
- The original txt2img and img2img scripts from the CompViz repository have been moved into
a subfolder named "orig_scripts", to reduce confusion.
---
## v1.02 (21 August 2022)
- A copy of the prompt and all of its switches and options is now stored in the corresponding
image in a tEXt metadata field named "Dream". You can read the prompt using scripts/images2prompt.py,
or an image editor that allows you to explore the full metadata.
**Please run "conda env update" to load the k_lms dependencies!!**
---
## v1.01 (21 August 2022)
- added k_lms sampling.
**Please run "conda env update" to load the k_lms dependencies!!**
- use half precision arithmetic by default, resulting in faster execution and lower memory requirements
Pass argument --full_precision to dream.py to get slower but more accurate image generation
---
## Links
- **[Read Me](../readme.md)**

263
docs/features/CLI.md
View File

@ -8,20 +8,23 @@ hide:
## **Interactive Command Line Interface**
The `dream.py` script, located in `scripts/dream.py`, provides an interactive interface to image
generation similar to the "dream mothership" bot that Stable AI provided on its Discord server.
The `dream.py` script, located in `scripts/dream.py`, provides an interactive
interface to image generation similar to the "dream mothership" bot that Stable
AI provided on its Discord server.
Unlike the `txt2img.py` and `img2img.py` scripts provided in the original
[CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion) source code repository, the
time-consuming initialization of the AI model initialization only happens once. After that image
generation from the command-line interface is very fast.
[CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion) source
code repository, the time-consuming initialization of the AI model
initialization only happens once. After that image generation from the
command-line interface is very fast.
The script uses the readline library to allow for in-line editing, command history (++up++ and
++down++), autocompletion, and more. To help keep track of which prompts generated which images, the
script writes a log file of image names and prompts to the selected output directory.
The script uses the readline library to allow for in-line editing, command
history (++up++ and ++down++), autocompletion, and more. To help keep track of
which prompts generated which images, the script writes a log file of image
names and prompts to the selected output directory.
In addition, as of version 1.02, it also writes the prompt into the PNG file's metadata where it can
be retrieved using `scripts/images2prompt.py`
In addition, as of version 1.02, it also writes the prompt into the PNG file's
metadata where it can be retrieved using `scripts/images2prompt.py`
The script is confirmed to work on Linux, Windows and Mac systems.
@ -56,21 +59,24 @@ dream> q
![dream-py-demo](../assets/dream-py-demo.png)
The `dream>` prompt's arguments are pretty much identical to those used in the Discord bot, except
you don't need to type "!dream" (it doesn't hurt if you do). A significant change is that creation
of individual images is now the default unless `--grid` (`-g`) is given. A full list is given in
The `dream>` prompt's arguments are pretty much identical to those used in the
Discord bot, except you don't need to type "!dream" (it doesn't hurt if you do).
A significant change is that creation of individual images is now the default
unless `--grid` (`-g`) is given. A full list is given in
[List of prompt arguments](#list-of-prompt-arguments).
## Arguments
The script itself also recognizes a series of command-line switches that will change important
global defaults, such as the directory for image outputs and the location of the model weight files.
The script itself also recognizes a series of command-line switches that will
change important global defaults, such as the directory for image outputs and
the location of the model weight files.
### List of arguments recognized at the command line
These command-line arguments can be passed to `dream.py` when you first run it from the Windows, Mac
or Linux command line. Some set defaults that can be overridden on a per-prompt basis (see [List of
prompt arguments] (#list-of-prompt-arguments). Others
These command-line arguments can be passed to `dream.py` when you first run it
from the Windows, Mac or Linux command line. Some set defaults that can be
overridden on a per-prompt basis (see [List of prompt arguments]
(#list-of-prompt-arguments). Others
| Argument <img width="240" align="right"/> | Shortcut <img width="100" align="right"/> | Default <img width="320" align="right"/> | Description |
| ----------------------------------------- | ----------------------------------------- | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------- |
@ -90,157 +96,150 @@ prompt arguments] (#list-of-prompt-arguments). Others
| `--seamless` | | `False` | Create interesting effects by tiling elements of the image. |
| `--embedding_path <path>` | | `None` | Path to pre-trained embedding manager checkpoints, for custom models |
| `--gfpgan_dir` | | `src/gfpgan` | Path to where GFPGAN is installed. |
| `--gfpgan_model_path` | | `experiments/pretrained_models/GFPGANv1.3.pth` | Path to GFPGAN model file, relative to `--gfpgan_dir`. |
| `--gfpgan_model_path` | | `experiments/pretrained_models/GFPGANv1.4.pth` | Path to GFPGAN model file, relative to `--gfpgan_dir`. |
| `--device <device>` | `-d<device>` | `torch.cuda.current_device()` | Device to run SD on, e.g. "cuda:0" |
| `--free_gpu_mem` | | `False` | Free GPU memory after sampling, to allow image decoding and saving in low VRAM conditions |
| `--precision` | | `auto` | Set model precision, default is selected by device. Options: auto, float32, float16, autocast |
#### deprecated
These arguments are deprecated but still work:
<figure markdown>
| Argument | Shortcut | Default | Description |
| ------------------ | -------- | ------- | --------------------------------------------------------------- |
| `--weights <path>` | | `None` | Pth to weights file; use `--model stable-diffusion-1.4` instead |
| `--laion400m` | `-l` | `False` | Use older LAION400m weights; use `--model=laion400m` instead |
| Argument | Shortcut | Default | Description |
|--------------------|------------|---------------------|--------------|
| --weights <path> | | None | Pth to weights file; use `--model stable-diffusion-1.4` instead |
| --laion400m | -l | False | Use older LAION400m weights; use `--model=laion400m` instead |
</figure>
**A note on path names:** On Windows systems, you may run into
problems when passing the dream script standard backslashed path
names because the Python interpreter treats "\" as an escape.
You can either double your slashes (ick): C:\\\\path\\\\to\\\\my\\\\file, or
use Linux/Mac style forward slashes (better): C:/path/to/my/file.
!!! note
## List of prompt arguments
On Windows systems, you may run into problems when passing the dream script standard backslashed
path names because the Python interpreter treats `\` as an escape. You can either double your
slashes (ick): `C:\\path\\to\\my\\file`, or use Linux/Mac style forward slashes (better):
`C:/path/to/my/file`.
After the dream.py script initializes, it will present you with a
**dream>** prompt. Here you can enter information to generate images
from text (txt2img), to embellish an existing image or sketch
(img2img), or to selectively alter chosen regions of the image
(inpainting).
### List of prompt arguments
### This is an example of txt2img:
After the `dream.py` script initializes, it will present you with a **`dream>`** prompt. Here you
can enter information to generate images from text (txt2img), to embellish an existing image or
sketch (img2img), or to selectively alter chosen regions of the image (inpainting).
~~~~
dream> waterfall and rainbow -W640 -H480
~~~~
#### txt2img
This will create the requested image with the dimensions 640 (width)
and 480 (height).
!!! example
Here are the dream> command that apply to txt2img:
```bash
dream> "waterfall and rainbow" -W640 -H480
```
| Argument | Shortcut | Default | Description |
|--------------------|------------|---------------------|--------------|
| "my prompt" | | | Text prompt to use. The quotation marks are optional. |
| --width <int> | -W<int> | 512 | Width of generated image |
| --height <int> | -H<int> | 512 | Height of generated image |
| --iterations <int> | -n<int> | 1 | How many images to generate from this prompt |
| --steps <int> | -s<int> | 50 | How many steps of refinement to apply |
| --cfg_scale <float>| -C<float> | 7.5 | How hard to try to match the prompt to the generated image; any number greater than 1.0 works, but the useful range is roughly 5.0 to 20.0 |
| --seed <int> | -S<int> | None | Set the random seed for the next series of images. This can be used to recreate an image generated previously.|
| --sampler <sampler>| -A<sampler>| k_lms | Sampler to use. Use -h to get list of available samplers. |
| --grid | -g | False | Turn on grid mode to return a single image combining all the images generated by this prompt |
| --individual | -i | True | Turn off grid mode (deprecated; leave off --grid instead) |
| --outdir <path> | -o<path> | outputs/img_samples | Temporarily change the location of these images |
| --seamless | | False | Activate seamless tiling for interesting effects |
| --log_tokenization | -t | False | Display a color-coded list of the parsed tokens derived from the prompt |
| --skip_normalization| -x | False | Weighted subprompts will not be normalized. See [Weighted Prompts](./OTHER.md#weighted-prompts) |
| --upscale <int> <float> | -U <int> <float> | -U 1 0.75| Upscale image by magnification factor (2, 4), and set strength of upscaling (0.0-1.0). If strength not set, will default to 0.75. |
| --gfpgan_strength <float> | -G <float> | -G0 | Fix faces using the GFPGAN algorithm; argument indicates how hard the algorithm should try (0.0-1.0) |
| --save_original | -save_orig| False | When upscaling or fixing faces, this will cause the original image to be saved rather than replaced. |
| --variation <float> |-v<float>| 0.0 | Add a bit of noise (0.0=none, 1.0=high) to the image in order to generate a series of variations. Usually used in combination with -S<seed> and -n<int> to generate a series a riffs on a starting image. See [Variations](./VARIATIONS.md). |
| --with_variations <pattern> | -V<pattern>| None | Combine two or more variations. See [Variations](./VARIATIONS.md) for now to use this. |
This will create the requested image with the dimensions 640 (width) and 480 (height).
Note that the width and height of the image must be multiples of
64. You can provide different values, but they will be rounded down to
the nearest multiple of 64.
Those are the `dream` commands that apply to txt2img:
| Argument <img width="680" align="right"/> | Shortcut <img width="420" align="right"/> | Default <img width="480" align="right"/> | Description |
| ----------------------------------------- | ----------------------------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `"my prompt"` | | | Text prompt to use. The quotation marks are optional. |
| `--width <int>` | `-W<int>` | `512` | Width of generated image |
| `--height <int>` | `-H<int>` | `512` | Height of generated image |
| `--iterations <int>` | `-n<int>` | `1` | How many images to generate from this prompt |
| `--steps <int>` | `-s<int>` | `50` | How many steps of refinement to apply |
| `--cfg_scale <float>` | `-C<float>` | `7.5` | How hard to try to match the prompt to the generated image; any number greater than 0.0 works, but the useful range is roughly 5.0 to 20.0 |
| `--seed <int>` | `-S<int>` | `None` | Set the random seed for the next series of images. This can be used to recreate an image generated previously. |
| `--sampler <sampler>` | `-A<sampler>` | `k_lms` | Sampler to use. Use `-h` to get list of available samplers. |
| `--grid` | `-g` | `False` | Turn on grid mode to return a single image combining all the images generated by this prompt |
| `--individual` | `-i` | `True` | Turn off grid mode (deprecated; leave off `--grid` instead) |
| `--outdir <path>` | `-o<path>` | `outputs/img_samples` | Temporarily change the location of these images |
| `--seamless` | | `False` | Activate seamless tiling for interesting effects |
| `--log_tokenization` | `-t` | `False` | Display a color-coded list of the parsed tokens derived from the prompt |
| `--skip_normalization` | `-x` | `False` | Weighted subprompts will not be normalized. See [Weighted Prompts](./OTHER.md#weighted-prompts) |
| `--upscale <int> <float>` | `-U <int> <float>` | `-U 1 0.75` | Upscale image by magnification factor (2, 4), and set strength of upscaling (0.0-1.0). If strength not set, will default to 0.75. |
| `--gfpgan_strength <float>` | `-G <float>` | `-G0` | Fix faces using the GFPGAN algorithm; argument indicates how hard the algorithm should try (0.0-1.0) |
| `--save_original` | `-save_orig` | `False` | When upscaling or fixing faces, this will cause the original image to be saved rather than replaced. |
| `--variation <float>` | `-v<float>` | `0.0` | Add a bit of noise (0.0=none, 1.0=high) to the image in order to generate a series of variations. Usually used in combination with `-S<seed>` and `-n<int>` to generate a series a riffs on a starting image. See [Variations](./VARIATIONS.md). |
| `--with_variations <pattern>` | `-V<pattern>` | `None` | Combine two or more variations. See [Variations](./VARIATIONS.md) for now to use this. |
### This is an example of img2img:
!!! note
~~~~
dream> waterfall and rainbow -I./vacation-photo.png -W640 -H480 --fit
~~~~
The width and height of the image must be multiples of 64. You can provide different
values, but they will be rounded down to the nearest multiple of 64.
This will modify the indicated vacation photograph by making it more
like the prompt. Results will vary greatly depending on what is in the
image. We also ask to --fit the image into a box no bigger than
640x480. Otherwise the image size will be identical to the provided
photo and you may run out of memory if it is large.
#### img2img
In addition to the command-line options recognized by txt2img, img2img
accepts additional options:
!!! example
| Argument | Shortcut | Default | Description |
|--------------------|------------|---------------------|--------------|
| --init_img <path> | -I<path> | None | Path to the initialization image |
| --fit | -F | False | Scale the image to fit into the specified -H and -W dimensions |
| --strength <float> | -s<float> | 0.75 | How hard to try to match the prompt to the initial image. Ranges from 0.0-0.99, with higher values replacing the initial image completely.|
```bash
dream> "waterfall and rainbow" -I./vacation-photo.png -W640 -H480 --fit
```
### This is an example of inpainting:
This will modify the indicated vacation photograph by making it more like the prompt. Results will
vary greatly depending on what is in the image. We also ask to --fit the image into a box no bigger
than 640x480. Otherwise the image size will be identical to the provided photo and you may run out
of memory if it is large.
~~~~
dream> waterfall and rainbow -I./vacation-photo.png -M./vacation-mask.png -W640 -H480 --fit
~~~~
Repeated chaining of img2img on an image can result in significant color shifts in the output,
especially if run with lower strength. Color correction can be run against a reference image to fix
this issue. Use the original input image to the chain as the the reference image for each step in
the chain.
This will do the same thing as img2img, but image alterations will
only occur within transparent areas defined by the mask file specified
by -M. You may also supply just a single initial image with the areas
to overpaint made transparent, but you must be careful not to destroy
the pixels underneath when you create the transparent areas. See
[Inpainting](./INPAINTING.md) for details.
In addition to the command-line options recognized by txt2img, img2img accepts additional options:
inpainting accepts all the arguments used for txt2img and img2img, as
well as the --mask (-M) argument:
| Argument <img width="160" align="right"/> | Shortcut | Default | Description |
| ----------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `--init_img <path>` | `-I<path>` | `None` | Path to the initialization image |
| `--init_color <path>` | | `None` | Path to reference image for color correction |
| `--fit` | `-F` | `False` | Scale the image to fit into the specified -H and -W dimensions |
| `--strength <float>` | `-f<float>` | `0.75` | How hard to try to match the prompt to the initial image. Ranges from 0.0-0.99, with higher values replacing the initial image completely. |
| Argument | Shortcut | Default | Description |
|--------------------|------------|---------------------|--------------|
| --init_mask <path> | -M<path> | None |Path to an image the same size as the initial_image, with areas for inpainting made transparent.|
#### Inpainting
!!! example
# Command-line editing and completion
```bash
dream> "waterfall and rainbow" -I./vacation-photo.png -M./vacation-mask.png -W640 -H480 --fit
```
If you are on a Macintosh or Linux machine, the command-line offers
convenient history tracking, editing, and command completion.
This will do the same thing as img2img, but image alterations will only occur within transparent
areas defined by the mask file specified by `-M`. You may also supply just a single initial image with
the areas to overpaint made transparent, but you must be careful not to destroy the pixels
underneath when you create the transparent areas. See [Inpainting](./INPAINTING.md) for details.
- To scroll through previous commands and potentially edit/reuse them, use the up and down cursor keys.
- To edit the current command, use the left and right cursor keys to position the cursor, and then backspace, delete or insert characters.
- To move to the very beginning of the command, type CTRL-A (or command-A on the Mac)
- To move to the end of the command, type CTRL-E.
- To cut a section of the command, position the cursor where you want to start cutting and type CTRL-K.
- To paste a cut section back in, position the cursor where you want to paste, and type CTRL-Y
Inpainting accepts all the arguments used for txt2img and img2img, as well as the `--mask` (`-M`)
argument:
Windows users can get similar, but more limited, functionality if they
launch dream.py with the "winpty" program:
| Argument <img width="100" align="right"/> | Shortcut | Default | Description |
| ----------------------------------------- | ---------- | ------- | ------------------------------------------------------------------------------------------------ |
| `--init_mask <path>` | `-M<path>` | `None` | Path to an image the same size as the initial_image, with areas for inpainting made transparent. |
~~~
> winpty python scripts\dream.py
~~~
## Command-line editing and completion
On the Mac and Linux platforms, when you exit dream.py, the last 1000
lines of your command-line history will be saved. When you restart
dream.py, you can access the saved history using the up-arrow key.
If you are on a Macintosh or Linux machine, the command-line offers convenient history tracking,
editing, and command completion.
In addition, limited command-line completion is installed. In various
contexts, you can start typing your command and press tab. A list of
potential completions will be presented to you. You can then type a
little more, hit tab again, and eventually autocomplete what you want.
- To scroll through previous commands and potentially edit/reuse them, use the ++up++ and ++down++
cursor keys.
- To edit the current command, use the ++left++ and ++right++ cursor keys to position the cursor,
and then ++backspace++, ++delete++ or ++insert++ characters.
- To move to the very beginning of the command, type ++ctrl+a++ (or ++command+a++ on the Mac)
- To move to the end of the command, type ++ctrl+e++.
- To cut a section of the command, position the cursor where you want to start cutting and type
++ctrl+k++.
- To paste a cut section back in, position the cursor where you want to paste, and type ++ctrl+y++
When specifying file paths using the one-letter shortcuts, the CLI
will attempt to complete pathnames for you. This is most handy for the
-I (init image) and -M (init mask) paths. To initiate completion, start
the path with a slash ("/") or "./". For example:
Windows users can get similar, but more limited, functionality if they launch `dream.py` with the
"winpty" program:
```batch
winpty python scripts\dream.py
```
On the Mac and Linux platforms, when you exit `dream.py`, the last 1000 lines of your command-line
history will be saved. When you restart `dream.py`, you can access the saved history using the
++up++ key.
In addition, limited command-line completion is installed. In various contexts, you can start typing
your command and press tab. A list of potential completions will be presented to you. You can then
type a little more, hit tab again, and eventually autocomplete what you want.
When specifying file paths using the one-letter shortcuts, the CLI will attempt to complete
pathnames for you. This is most handy for the `-I` (init image) and `-M` (init mask) paths. To
initiate completion, start the path with a slash `/` or `./`, for example:
```bash
dream> "zebra with a mustache" -I./test-pictures<TAB>
~~~
dream> zebra with a mustache -I./test-pictures<TAB>
-I./test-pictures/Lincoln-and-Parrot.png -I./test-pictures/zebra.jpg -I./test-pictures/madonna.png
-I./test-pictures/bad-sketch.png -I./test-pictures/man_with_eagle/
```

140
docs/features/UPSCALE.md
View File

@ -4,37 +4,42 @@ title: Upscale
## Intro
The script provides the ability to restore faces and upscale. You can apply these operations
at the time you generate the images, or at any time to a previously-generated PNG file, using
the [!fix](#fixing-previously-generated-images) command.
The script provides the ability to restore faces and upscale. You can apply
these operations at the time you generate the images, or at any time to a
previously-generated PNG file, using the
[!fix](#fixing-previously-generated-images) command.
## Face Fixing
The default face restoration module is GFPGAN. The default upscale is Real-ESRGAN. For an alternative
face restoration module, see [CodeFormer Support] below.
The default face restoration module is GFPGAN. The default upscale is
Real-ESRGAN. For an alternative face restoration module, see [CodeFormer
Support] below.
As of version 1.14, environment.yaml will install the Real-ESRGAN package into the standard install
location for python packages, and will put GFPGAN into a subdirectory of "src" in the
stable-diffusion directory. (The reason for this is that the standard GFPGAN distribution has a
minor bug that adversely affects image color.) Upscaling with Real-ESRGAN should "just work" without
further intervention. Simply pass the --upscale (-U) option on the dream> command line, or indicate
the desired scale on the popup in the Web GUI.
As of version 1.14, environment.yaml will install the Real-ESRGAN package into
the standard install location for python packages, and will put GFPGAN into a
subdirectory of "src" in the InvokeAI directory. (The reason for this is
that the standard GFPGAN distribution has a minor bug that adversely affects
image color.) Upscaling with Real-ESRGAN should "just work" without further
intervention. Simply pass the --upscale (-U) option on the dream> command line,
or indicate the desired scale on the popup in the Web GUI.
For **GFPGAN** to work, there is one additional step needed. You will need to download and copy the
GFPGAN [models file](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth)
into **src/gfpgan/experiments/pretrained_models**. On Mac and Linux systems, here's how you'd do it
using **wget**:
For **GFPGAN** to work, there is one additional step needed. You will need to
download and copy the GFPGAN
[models file](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth)
into **src/gfpgan/experiments/pretrained_models**. On Mac and Linux systems,
here's how you'd do it using **wget**:
```bash
> wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth src/gfpgan/experiments/pretrained_models/
> wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth src/gfpgan/experiments/pretrained_models/
```
Make sure that you're in the stable-diffusion directory when you do this.
Make sure that you're in the InvokeAI directory when you do this.
Alternatively, if you have GFPGAN installed elsewhere, or if you are using an earlier version of
this package which asked you to install GFPGAN in a sibling directory, you may use the
`--gfpgan_dir` argument with `dream.py` to set a custom path to your GFPGAN directory. _There are
other GFPGAN related boot arguments if you wish to customize further._
Alternatively, if you have GFPGAN installed elsewhere, or if you are using an
earlier version of this package which asked you to install GFPGAN in a sibling
directory, you may use the `--gfpgan_dir` argument with `dream.py` to set a
custom path to your GFPGAN directory. _There are other GFPGAN related boot
arguments if you wish to customize further._
!!! warning "Internet connection needed"
@ -52,13 +57,14 @@ You will now have access to two new prompt arguments.
`-U : <upscaling_factor> <upscaling_strength>`
The upscaling prompt argument takes two values. The first value is a scaling factor and should be
set to either `2` or `4` only. This will either scale the image 2x or 4x respectively using
different models.
The upscaling prompt argument takes two values. The first value is a scaling
factor and should be set to either `2` or `4` only. This will either scale the
image 2x or 4x respectively using different models.
You can set the scaling stength between `0` and `1.0` to control intensity of the of the scaling.
This is handy because AI upscalers generally tend to smooth out texture details. If you wish to
retain some of those for natural looking results, we recommend using values between `0.5 to 0.8`.
You can set the scaling stength between `0` and `1.0` to control intensity of
the of the scaling. This is handy because AI upscalers generally tend to smooth
out texture details. If you wish to retain some of those for natural looking
results, we recommend using values between `0.5 to 0.8`.
If you do not explicitly specify an upscaling_strength, it will default to 0.75.
@ -66,18 +72,19 @@ If you do not explicitly specify an upscaling_strength, it will default to 0.75.
`-G : <gfpgan_strength>`
This prompt argument controls the strength of the face restoration that is being applied. Similar to
upscaling, values between `0.5 to 0.8` are recommended.
This prompt argument controls the strength of the face restoration that is being
applied. Similar to upscaling, values between `0.5 to 0.8` are recommended.
You can use either one or both without any conflicts. In cases where you use both, the image will be
first upscaled and then the face restoration process will be executed to ensure you get the highest
quality facial features.
You can use either one or both without any conflicts. In cases where you use
both, the image will be first upscaled and then the face restoration process
will be executed to ensure you get the highest quality facial features.
`--save_orig`
When you use either `-U` or `-G`, the final result you get is upscaled or face modified. If you want
to save the original Stable Diffusion generation, you can use the `-save_orig` prompt argument to
save the original unaffected version too.
When you use either `-U` or `-G`, the final result you get is upscaled or face
modified. If you want to save the original Stable Diffusion generation, you can
use the `-save_orig` prompt argument to save the original unaffected version
too.
### Example Usage
@ -102,60 +109,69 @@ dream> a man wearing a pineapple hat -I path/to/your/file.png -U 2 0.5 -G 0.6
process is complete. While the image generation is taking place, you will still be able to preview
the base images.
If you wish to stop during the image generation but want to upscale or face restore a particular
generated image, pass it again with the same prompt and generated seed along with the `-U` and `-G`
prompt arguments to perform those actions.
If you wish to stop during the image generation but want to upscale or face
restore a particular generated image, pass it again with the same prompt and
generated seed along with the `-U` and `-G` prompt arguments to perform those
actions.
## CodeFormer Support
This repo also allows you to perform face restoration using
[CodeFormer](https://github.com/sczhou/CodeFormer).
In order to setup CodeFormer to work, you need to download the models like with GFPGAN. You can do
this either by running `preload_models.py` or by manually downloading the
[model file](https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth) and
saving it to `ldm/restoration/codeformer/weights` folder.
In order to setup CodeFormer to work, you need to download the models like with
GFPGAN. You can do this either by running `preload_models.py` or by manually
downloading the
[model file](https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth)
and saving it to `ldm/restoration/codeformer/weights` folder.
You can use `-ft` prompt argument to swap between CodeFormer and the default GFPGAN. The above
mentioned `-G` prompt argument will allow you to control the strength of the restoration effect.
You can use `-ft` prompt argument to swap between CodeFormer and the default
GFPGAN. The above mentioned `-G` prompt argument will allow you to control the
strength of the restoration effect.
### Usage:
The following command will perform face restoration with CodeFormer instead of the default gfpgan.
The following command will perform face restoration with CodeFormer instead of
the default gfpgan.
`<prompt> -G 0.8 -ft codeformer`
### Other Options:
- `-cf` - cf or CodeFormer Fidelity takes values between `0` and `1`. 0 produces high quality
results but low accuracy and 1 produces lower quality results but higher accuacy to your original
face.
- `-cf` - cf or CodeFormer Fidelity takes values between `0` and `1`. 0 produces
high quality results but low accuracy and 1 produces lower quality results but
higher accuacy to your original face.
The following command will perform face restoration with CodeFormer. CodeFormer will output a result
that is closely matching to the input face.
The following command will perform face restoration with CodeFormer. CodeFormer
will output a result that is closely matching to the input face.
`<prompt> -G 1.0 -ft codeformer -cf 0.9`
The following command will perform face restoration with CodeFormer. CodeFormer will output a result
that is the best restoration possible. This may deviate slightly from the original face. This is an
excellent option to use in situations when there is very little facial data to work with.
The following command will perform face restoration with CodeFormer. CodeFormer
will output a result that is the best restoration possible. This may deviate
slightly from the original face. This is an excellent option to use in
situations when there is very little facial data to work with.
`<prompt> -G 1.0 -ft codeformer -cf 0.1`
## Fixing Previously-Generated Images
It is easy to apply face restoration and/or upscaling to any previously-generated file. Just use the
syntax `!fix path/to/file.png <options>`. For example, to apply GFPGAN at strength 0.8 and upscale 2X
for a file named `./outputs/img-samples/000044.2945021133.png`, just run:
It is easy to apply face restoration and/or upscaling to any
previously-generated file. Just use the syntax
`!fix path/to/file.png <options>`. For example, to apply GFPGAN at strength 0.8
and upscale 2X for a file named `./outputs/img-samples/000044.2945021133.png`,
just run:
~~~~
```
dream> !fix ./outputs/img-samples/000044.2945021133.png -G 0.8 -U 2
~~~~
```
A new file named `000044.2945021133.fixed.png` will be created in the output directory. Note that
the `!fix` command does not replace the original file, unlike the behavior at generate time.
A new file named `000044.2945021133.fixed.png` will be created in the output
directory. Note that the `!fix` command does not replace the original file,
unlike the behavior at generate time.
### Disabling:
If, for some reason, you do not wish to load the GFPGAN and/or ESRGAN libraries, you can disable them
on the dream.py command line with the `--no_restore` and `--no_upscale` options, respectively.
If, for some reason, you do not wish to load the GFPGAN and/or ESRGAN libraries,
you can disable them on the dream.py command line with the `--no_restore` and
`--no_upscale` options, respectively.

141
docs/help/SAMPLER_CONVERGENCE.md Normal file
View File

@ -0,0 +1,141 @@
---
title: SAMPLER CONVERGENCE
---
## *Sampler Convergence*
As features keep increasing, making the right choices for your needs can become increasingly difficult. What sampler to use? And for how many steps? Do you change the CFG value? Do you use prompt weighting? Do you allow variations?
Even once you have a result, do you blend it with other images? Pass it through `img2img`? With what strength? Do you use inpainting to correct small details? Outpainting to extend cropped sections?
The purpose of this series of documents is to help you better understand these tools, so you can make the best out of them. Feel free to contribute with your own findings!
In this document, we will talk about sampler convergence.
Looking for a short version? Here's a TL;DR in 3 tables.
| Remember |
|:---|
| Results converge as steps (`-s`) are increased (except for `K_DPM_2_A` and `K_EULER_A`). Often at ≥ `-s100`, but may require ≥ `-s700`). |
| Producing a batch of candidate images at low (`-s8` to `-s30`) step counts can save you hours of computation. |
| `K_HEUN` and `K_DPM_2` converge in less steps (but are slower). |
| `K_DPM_2_A` and `K_EULER_A` incorporate a lot of creativity/variability. |
| Sampler | (3 sample avg) it/s (M1 Max 64GB, 512x512) |
|---|---|
| `DDIM` | 1.89 |
| `PLMS` | 1.86 |
| `K_EULER` | 1.86 |
| `K_LMS` | 1.91 |
| `K_HEUN` | 0.95 (slower) |
| `K_DPM_2` | 0.95 (slower) |
| `K_DPM_2_A` | 0.95 (slower) |
| `K_EULER_A` | 1.86 |
| Suggestions |
|:---|
| For most use cases, `K_LMS`, `K_HEUN` and `K_DPM_2` are the best choices (the latter 2 run 0.5x as quick, but tend to converge 2x as quick as `K_LMS`). At very low steps (≤ `-s8`), `K_HEUN` and `K_DPM_2` are not recommended. Use `K_LMS` instead.|
| For variability, use `K_EULER_A` (runs 2x as quick as `K_DPM_2_A`). |
---
### *Sampler results*
Let's start by choosing a prompt and using it with each of our 8 samplers, running it for 10, 20, 30, 40, 50 and 100 steps.
Anime. `"an anime girl" -W512 -H512 -C7.5 -S3031912972`
![191636411-083c8282-6ed1-4f78-9273-ee87c0a0f1b6-min (1)](https://user-images.githubusercontent.com/50542132/191868725-7f7af991-e254-4c1f-83e7-bed8c9b2d34f.png)
### *Sampler convergence*
Immediately, you can notice results tend to converge -that is, as `-s` (step) values increase, images look more and more similar until there comes a point where the image no longer changes.
You can also notice how `DDIM` and `PLMS` eventually tend to converge to K-sampler results as steps are increased.
Among K-samplers, `K_HEUN` and `K_DPM_2` seem to require the fewest steps to converge, and even at low step counts they are good indicators of the final result. And finally, `K_DPM_2_A` and `K_EULER_A` seem to do a bit of their own thing and don't keep much similarity with the rest of the samplers.
### *Batch generation speedup*
This realization is very useful because it means you don't need to create a batch of 100 images (`-n100`) at `-s100` to choose your favorite 2 or 3 images.
You can produce the same 100 images at `-s10` to `-s30` using a K-sampler (since they converge faster), get a rough idea of the final result, choose your 2 or 3 favorite ones, and then run `-s100` on those images to polish some details.
The latter technique is 3-8x as quick.
Example:
At 60s per 100 steps.
(Option A) 60s * 100 images = 6000s (100 images at `-s100`, manually picking 3 favorites)
(Option B) 6s * 100 images + 60s * 3 images = 780s (100 images at `-s10`, manually picking 3 favorites, and running those 3 at `-s100` to polish details)
The result is 1 hour and 40 minutes (Option A) vs 13 minutes (Option B).
### *Topic convergance*
Now, these results seem interesting, but do they hold for other topics? How about nature? Food? People? Animals? Let's try!
Nature. `"valley landscape wallpaper, d&d art, fantasy, painted, 4k, high detail, sharp focus, washed colors, elaborate excellent painted illustration" -W512 -H512 -C7.5 -S1458228930`
![191736091-dda76929-00d1-4590-bef4-7314ea4ea419-min (1)](https://user-images.githubusercontent.com/50542132/191868763-b151c69e-0a72-4cf1-a151-5a64edd0c93e.png)
With nature, you can see how initial results are even more indicative of final result -more so than with characters/people. `K_HEUN` and `K_DPM_2` are again the quickest indicators, almost right from the start. Results also converge faster (e.g. `K_HEUN` converged at `-s21`).
Food. `"a hamburger with a bowl of french fries" -W512 -H512 -C7.5 -S4053222918`
![191639011-f81d9d38-0a15-45f0-9442-a5e8d5c25f1f-min (1)](https://user-images.githubusercontent.com/50542132/191868898-98801a62-885f-4ea1-aee8-563503522aa9.png)
Again, `K_HEUN` and `K_DPM_2` take the fewest number of steps to be good indicators of the final result. `K_DPM_2_A` and `K_EULER_A` seem to incorporate a lot of creativity/variability, capable of producing rotten hamburgers, but also of adding lettuce to the mix. And they're the only samplers that produced an actual 'bowl of fries'!
Animals. `"grown tiger, full body" -W512 -H512 -C7.5 -S3721629802`
![191771922-6029a4f5-f707-4684-9011-c6f96e25fe56-min (1)](https://user-images.githubusercontent.com/50542132/191868870-9e3b7d82-b909-429f-893a-13f6ec343454.png)
`K_HEUN` and `K_DPM_2` once again require the least number of steps to be indicative of the final result (around `-s30`), while other samplers are still struggling with several tails or malformed back legs.
It also takes longer to converge (for comparison, `K_HEUN` required around 150 steps to converge). This is normal, as producing human/animal faces/bodies is one of the things the model struggles the most with. For these topics, running for more steps will often increase coherence within the composition.
People. `"Ultra realistic photo, (Miranda Bloom-Kerr), young, stunning model, blue eyes, blond hair, beautiful face, intricate, highly detailed, smooth, art by artgerm and greg rutkowski and alphonse mucha, stained glass" -W512 -H512 -C7.5 -S2131956332`. This time, we will go up to 300 steps.
![Screenshot 2022-09-23 at 02 05 48-min (1)](https://user-images.githubusercontent.com/50542132/191871743-6802f199-0ffd-4986-98c5-df2d8db30d18.png)
Observing the results, it again takes longer for all samplers to converge (`K_HEUN` took around 150 steps), but we can observe good indicative results much earlier (see: `K_HEUN`). Conversely, `DDIM` and `PLMS` are still undergoing moderate changes (see: lace around her neck), even at `-s300`.
In fact, as we can see in this other experiment, some samplers can take 700+ steps to converge when generating people.
![191988191-c586b75a-2d7f-4351-b705-83cc1149881a-min (1)](https://user-images.githubusercontent.com/50542132/191992123-7e0759d6-6220-42c4-a961-88c7071c5ee6.png)
Note also the point of convergence may not be the most desirable state (e.g. I prefer an earlier version of the face, more rounded), but it will probably be the most coherent arms/hands/face attributes-wise. You can always merge different images with a photo editing tool and pass it through `img2img` to smoothen the composition.
### *Sampler generation times*
Once we understand the concept of sampler convergence, we must look into the performance of each sampler in terms of steps (iterations) per second, as not all samplers run at the same speed.
On my M1 Max with 64GB of RAM, for a 512x512 image:
| Sampler | (3 sample average) it/s |
|---|---|
| `DDIM` | 1.89 |
| `PLMS` | 1.86 |
| `K_EULER` | 1.86 |
| `K_LMS` | 1.91 |
| `K_HEUN` | 0.95 (slower) |
| `K_DPM_2` | 0.95 (slower) |
| `K_DPM_2_A` | 0.95 (slower) |
| `K_EULER_A` | 1.86 |
Combining our results with the steps per second of each sampler, three choices come out on top: `K_LMS`, `K_HEUN` and `K_DPM_2` (where the latter two run 0.5x as quick but tend to converge 2x as quick as `K_LMS`). For creativity and a lot of variation between iterations, `K_EULER_A` can be a good choice (which runs 2x as quick as `K_DPM_2_A`).
Additionally, image generation at very low steps (≤ `-s8`) is not recommended for `K_HEUN` and `K_DPM_2`. Use `K_LMS` instead.
<img width="397" alt="192044949-67d5d441-a0d5-4d5a-be30-5dda4fc28a00-min" src="https://user-images.githubusercontent.com/50542132/192046823-2714cb29-bbf3-4eb1-9213-e27a0963905c.png">
### *Three key points*
Finally, it is relevant to mention that, in general, there are 3 important moments in the process of image formation as steps increase:
* The (earliest) point at which an image becomes a good indicator of the final result (useful for batch generation at low step values, to then improve the quality/coherence of the chosen images via running the same prompt and seed for more steps).
* The (earliest) point at which an image becomes coherent, even if different from the result if steps are increased (useful for batch generation at low step values, where quality/coherence is improved via techniques other than increasing the steps -e.g. via inpainting).
* The point at which an image fully converges.
Hence, remember that your workflow/strategy should define your optimal number of steps, even for the same prompt and seed (for example, if you seek full convergence, you may run `K_LMS` for `-s200` in the case of the red-haired girl, but `K_LMS` and `-s20`-taking one tenth the time- may do as well if your workflow includes adding small details, such as the missing shoulder strap, via `img2img`).

57
docs/help/TROUBLESHOOT.md
View File

@ -13,14 +13,39 @@ incomplete installations or crashes during the install process.
### **QUESTION**
During `conda env create -f environment.yaml`, conda hangs indefinitely.
During `conda env create`, conda hangs indefinitely.
### **SOLUTION**
If it is because of the last PIP step (usually stuck in the Git Clone step, you can check the detailed log by this method):
```bash
export PIP_LOG="/tmp/pip_log.txt"
touch ${PIP_LOG}
tail -f ${PIP_LOG} &
conda env create -f environment-mac.yaml --debug --verbose
killall tail
rm ${PIP_LOG}
```
Enter the stable-diffusion directory and completely remove the `src` directory and all its contents.
The safest way to do this is to enter the stable-diffusion directory and give the command
`git clean -f`. If this still doesn't fix the problem, try "conda clean -all" and then restart at
the `conda env create` step.
**SOLUTION**
Conda sometimes gets stuck at the last PIP step, in which several git repositories are
cloned and built.
Enter the stable-diffusion directory and completely remove the `src`
directory and all its contents. The safest way to do this is to enter
the stable-diffusion directory and give the command `git clean -f`. If
this still doesn't fix the problem, try "conda clean -all" and then
restart at the `conda env create` step.
To further understand the problem to checking the install lot using this method:
```bash
export PIP_LOG="/tmp/pip_log.txt"
touch ${PIP_LOG}
tail -f ${PIP_LOG} &
conda env create -f environment-mac.yaml --debug --verbose
killall tail
rm ${PIP_LOG}
```
---
@ -42,8 +67,8 @@ Reinstall the stable diffusion modules. Enter the `stable-diffusion` directory a
### **SOLUTION**
From within the `stable-diffusion` directory, run `conda env update -f environment.yaml` This is
also frequently the solution to complaints about an unknown function in a module.
From within the `InvokeAI` directory, run `conda env update` This is also frequently the solution to
complaints about an unknown function in a module.
---
@ -58,8 +83,10 @@ There's a feature or bugfix in the Stable Diffusion GitHub that you want to try
If the fix/feature is on the `main` branch, enter the stable-diffusion directory and do a
`git pull`.
Usually this will be sufficient, but if you start to see errors about missing or incorrect modules,
use the command
Usually this will be sufficient, but if you start to see errors about
missing or incorrect modules, use the command `pip install -e .`
and/or `conda env update` (These commands won't break anything.)
`pip install -e .` and/or
@ -89,3 +116,13 @@ branch that contains the pull request:
You will need to go through the install procedure again, but it should be fast because all the
dependencies are already loaded.
---
### **QUESTION**
Image generation crashed with CUDA out of memory error after successful sampling.
### **SOLUTION**
Try to run script with option `--free_gpu_mem` This will free memory before image decoding step.

154
docs/installation/INSTALL_DOCKER.md
View File

@ -1,36 +1,60 @@
# Before you begin
- For end users: Install Stable Diffusion locally using the instructions for your OS.
- For developers: For container-related development tasks or for enabling easy deployment to other environments (on-premises or cloud), follow these instructions. For general use, install locally to leverage your machine's GPU.
- For end users: Install Stable Diffusion locally using the instructions for
your OS.
- For developers: For container-related development tasks or for enabling easy
deployment to other environments (on-premises or cloud), follow these
instructions. For general use, install locally to leverage your machine's GPU.
# Why containers?
They provide a flexible, reliable way to build and deploy Stable Diffusion. You'll also use a Docker volume to store the largest model files and image outputs as a first step in decoupling storage and compute. Future enhancements can do this for other assets. See [Processes](https://12factor.net/processes) under the Twelve-Factor App methodology for details on why running applications in such a stateless fashion is important.
They provide a flexible, reliable way to build and deploy Stable Diffusion.
You'll also use a Docker volume to store the largest model files and image
outputs as a first step in decoupling storage and compute. Future enhancements
can do this for other assets. See [Processes](https://12factor.net/processes)
under the Twelve-Factor App methodology for details on why running applications
in such a stateless fashion is important.
You can specify the target platform when building the image and running the container. You'll also need to specify the Stable Diffusion requirements file that matches the container's OS and the architecture it will run on.
You can specify the target platform when building the image and running the
container. You'll also need to specify the Stable Diffusion requirements file
that matches the container's OS and the architecture it will run on.
Developers on Apple silicon (M1/M2): You [can't access your GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to another environment with NVIDIA GPUs on-premises or in the cloud.
Developers on Apple silicon (M1/M2): You
[can't access your GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224)
and performance is reduced compared with running it directly on macOS but for
development purposes it's fine. Once you're done with development tasks on your
laptop you can build for the target platform and architecture and deploy to
another environment with NVIDIA GPUs on-premises or in the cloud.
# Installation on a Linux container
# Installation on a Linux container
## Prerequisites
### Get the data files
### Get the data files
Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download the model file ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free.
Go to
[Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original),
and click "Access repository" to Download the model file `sd-v1-4.ckpt` (~4 GB)
to `~/Downloads`. You'll need to create an account but it's quick and free.
Also download the face restoration model.
```Shell
cd ~/Downloads
wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth
wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth
```
### Install [Docker](https://github.com/santisbon/guides#docker)
On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/invoke-ai/InvokeAI/issues/342). You may need to increase Swap and Disk image size too.
### Install [Docker](https://github.com/santisbon/guides#docker)
On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the
CPUs and Memory to avoid this
[Issue](https://github.com/invoke-ai/InvokeAI/issues/342). You may need to
increase Swap and Disk image size too.
## Setup
Set the fork you want to use and other variables.
Set the fork you want to use and other variables.
```Shell
TAG_STABLE_DIFFUSION="santisbon/stable-diffusion"
PLATFORM="linux/arm64"
@ -46,21 +70,28 @@ echo $CONDA_SUBDIR
```
Create a Docker volume for the downloaded model files.
```Shell
docker volume create my-vol
```
Copy the data files to the Docker volume using a lightweight Linux container. We'll need the models at run time. You just need to create the container with the mountpoint; no need to run this dummy container.
Copy the data files to the Docker volume using a lightweight Linux container.
We'll need the models at run time. You just need to create the container with
the mountpoint; no need to run this dummy container.
```Shell
cd ~/Downloads # or wherever you saved the files
docker create --platform $PLATFORM --name dummy --mount source=my-vol,target=/data alpine
docker create --platform $PLATFORM --name dummy --mount source=my-vol,target=/data alpine
docker cp sd-v1-4.ckpt dummy:/data
docker cp GFPGANv1.3.pth dummy:/data
docker cp GFPGANv1.4.pth dummy:/data
```
Get the repo and download the Miniconda installer (we'll need it at build time). Replace the URL with the version matching your container OS and the architecture it will run on.
Get the repo and download the Miniconda installer (we'll need it at build time).
Replace the URL with the version matching your container OS and the architecture
it will run on.
```Shell
cd ~
git clone $GITHUB_STABLE_DIFFUSION
@ -70,10 +101,15 @@ chmod +x entrypoint.sh
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh
```
Build the Docker image. Give it any tag ```-t``` that you want.
Choose the Linux container's host platform: x86-64/Intel is ```amd64```. Apple silicon is ```arm64```. If deploying the container to the cloud to leverage powerful GPU instances you'll be on amd64 hardware but if you're just trying this out locally on Apple silicon choose arm64.
The application uses libraries that need to match the host environment so use the appropriate requirements file.
Tip: Check that your shell session has the env variables set above.
Build the Docker image. Give it any tag `-t` that you want.
Choose the Linux container's host platform: x86-64/Intel is `amd64`. Apple
silicon is `arm64`. If deploying the container to the cloud to leverage powerful
GPU instances you'll be on amd64 hardware but if you're just trying this out
locally on Apple silicon choose arm64.
The application uses libraries that need to match the host environment so use
the appropriate requirements file.
Tip: Check that your shell session has the env variables set above.
```Shell
docker build -t $TAG_STABLE_DIFFUSION \
--platform $PLATFORM \
@ -85,6 +121,7 @@ docker build -t $TAG_STABLE_DIFFUSION \
Run a container using your built image.
Tip: Make sure you've created and populated the Docker volume (above).
```Shell
docker run -it \
--rm \
@ -98,86 +135,121 @@ $TAG_STABLE_DIFFUSION
# Usage (time to have fun)
## Startup
If you're on a **Linux container** the ```dream``` script is **automatically started** and the output dir set to the Docker volume you created earlier.
If you're on a **Linux container** the `dream` script is **automatically
started** and the output dir set to the Docker volume you created earlier.
If you're **directly on macOS follow these startup instructions**.
With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts ```txt2img``` and ```img2img```:
Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work.
By default the images are saved in ```outputs/img-samples/```.
With the Conda environment activated (`conda activate ldm`), run the interactive
interface that combines the functionality of the original scripts `txt2img` and
`img2img`:
Use the more accurate but VRAM-intensive full precision math because
half-precision requires autocast and won't work.
By default the images are saved in `outputs/img-samples/`.
```Shell
python3 scripts/dream.py --full_precision
python3 scripts/dream.py --full_precision
```
You'll get the script's prompt. You can see available options or quit.
```Shell
dream> -h
dream> q
```
## Text to Image
For quick (but bad) image results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly.
For quick (but bad) image results test with 5 steps (default 50) and 1 sample
image. This will let you know that everything is set up correctly.
Then increase steps to 100 or more for good (but slower) results.
The prompt can be in quotes or not.
```Shell
dream> The hulk fighting with sheldon cooper -s5 -n1
dream> The hulk fighting with sheldon cooper -s5 -n1
dream> "woman closeup highly detailed" -s 150
# Reuse previous seed and apply face restoration
dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75
dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75
```
You'll need to experiment to see if face restoration is making it better or worse for your specific prompt.
You'll need to experiment to see if face restoration is making it better or
worse for your specific prompt.
If you're on a container the output is set to the Docker volume. You can copy it wherever you want.
If you're on a container the output is set to the Docker volume. You can copy it
wherever you want.
You can download it from the Docker Desktop app, Volumes, my-vol, data.
Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name.
Or you can copy it from your Mac terminal. Keep in mind `docker cp` can't expand
`*.png` so you'll need to specify the image file name.
On your host Mac (you can use the name of any container that mounted the
volume):
On your host Mac (you can use the name of any container that mounted the volume):
```Shell
docker cp dummy:/data/000001.928403745.png /Users/<your-user>/Pictures
docker cp dummy:/data/000001.928403745.png /Users/<your-user>/Pictures
```
## Image to Image
You can also do text-guided image-to-image translation. For example, turning a sketch into a detailed drawing.
```strength``` is a value between 0.0 and 1.0 that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely.
You can also do text-guided image-to-image translation. For example, turning a
sketch into a detailed drawing.
Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Otherwise you'll get ```Error: product of dimension sizes > 2**31'```. If you still get the error [try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.) like 512x256.
`strength` is a value between 0.0 and 1.0 that controls the amount of noise that
is added to the input image. Values that approach 1.0 allow for lots of
variations but will also produce images that are not semantically consistent
with the input. 0.0 preserves image exactly, 1.0 replaces it completely.
Make sure your input image size dimensions are multiples of 64 e.g. 512x512.
Otherwise you'll get `Error: product of dimension sizes > 2**31'`. If you still
get the error
[try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.)
like 512x256.
If you're on a Docker container, copy your input image into the Docker volume
```Shell
docker cp /Users/<your-user>/Pictures/sketch-mountains-input.jpg dummy:/data/
```
Try it out generating an image (or more). The ```dream``` script needs absolute paths to find the image so don't use ```~```.
Try it out generating an image (or more). The `dream` script needs absolute
paths to find the image so don't use `~`.
If you're on your Mac
```Shell
```Shell
dream> "A fantasy landscape, trending on artstation" -I /Users/<your-user>/Pictures/sketch-mountains-input.jpg --strength 0.75 --steps 100 -n4
```
If you're on a Linux container on your Mac
```Shell
dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75 --steps 50 -n1
```
## Web Interface
You can use the ```dream``` script with a graphical web interface. Start the web server with:
You can use the `dream` script with a graphical web interface. Start the web
server with:
```Shell
python3 scripts/dream.py --full_precision --web
```
If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090
If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090
Press Control-C at the command line to stop the web server.
## Notes
Some text you can add at the end of the prompt to make it very pretty:
```Shell
cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed, painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida
```
The original scripts should work as well.
```Shell
python3 scripts/orig_scripts/txt2img.py --help
python3 scripts/orig_scripts/txt2img.py --ddim_steps 100 --n_iter 1 --n_samples 1 --plms --prompt "new born baby kitten. Hyper Detail, Octane Rendering, Unreal Engine, V-Ray"
python3 scripts/orig_scripts/txt2img.py --ddim_steps 5 --n_iter 1 --n_samples 1 --plms --prompt "ocean" # or --klms
```
```

80
docs/installation/INSTALL_LINUX.md
View File

@ -24,40 +24,40 @@ title: Linux
the installation worked, your command prompt will be prefixed by the name of the
current anaconda environment - `(base)`.
3. Copy the stable-diffusion source code from GitHub:
3. Copy the InvokeAI source code from GitHub:
```bash
(base) ~$ git clone https://github.com/invoke-ai/InvokeAI.git
```
```
(base) ~$ git clone https://github.com/invoke-ai/InvokeAI.git
```
This will create stable-diffusion folder where you will follow the rest of the
steps.
This will create InvokeAI folder where you will follow the rest of the steps.
4. Enter the newly-created stable-diffusion folder. From this step forward make
sure that you are working in the stable-diffusion directory!
4. Enter the newly-created InvokeAI folder. From this step forward make sure that you are working in the InvokeAI directory!
```bash
(base) ~$ cd stable-diffusion
(base) ~/stable-diffusion$
```
```
(base) ~$ cd InvokeAI
(base) ~/InvokeAI$
```
5. Use anaconda to copy necessary python packages, create a new python
environment named `ldm` and activate the environment.
```bash
(base) ~/stable-diffusion$ conda env create -f environment.yaml
(base) ~/stable-diffusion$ conda activate ldm
(ldm) ~/stable-diffusion$
```
```
(base) ~/InvokeAI$ conda env create
(base) ~/InvokeAI$ conda activate ldm
(ldm) ~/InvokeAI$
```
After these steps, your command prompt will be prefixed by `(ldm)` as shown
above.
6. Load a couple of small machine-learning models required by stable diffusion:
```bash
(ldm) ~/stable-diffusion$ python3 scripts/preload_models.py
```
```
(ldm) ~/InvokeAI$ python3 scripts/preload_models.py
```
!!! note
@ -79,38 +79,34 @@ title: Linux
This will create a symbolic link from the stable-diffusion model.ckpt file, to
the true location of the `sd-v1-4.ckpt` file.
```bash
(ldm) ~/stable-diffusion$ mkdir -p models/ldm/stable-diffusion-v1
(ldm) ~/stable-diffusion$ ln -sf /path/to/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt
```
```
(ldm) ~/InvokeAI$ mkdir -p models/ldm/stable-diffusion-v1
(ldm) ~/InvokeAI$ ln -sf /path/to/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt
```
8. Start generating images!
```bash
# for the pre-release weights use the -l or --liaon400m switch
(ldm) ~/stable-diffusion$ python3 scripts/dream.py -l
```
# for the pre-release weights use the -l or --liaon400m switch
(ldm) ~/InvokeAI$ python3 scripts/dream.py -l
# for the post-release weights do not use the switch
(ldm) ~/stable-diffusion$ python3 scripts/dream.py
# for the post-release weights do not use the switch
(ldm) ~/InvokeAI$ python3 scripts/dream.py
# for additional configuration switches and arguments, use -h or --help
(ldm) ~/stable-diffusion$ python3 scripts/dream.py -h
```
# for additional configuration switches and arguments, use -h or --help
(ldm) ~/InvokeAI$ python3 scripts/dream.py -h
```
9. Subsequently, to relaunch the script, be sure to run "conda activate ldm"
(step 5, second command), enter the `stable-diffusion` directory, and then
launch the dream script (step 8). If you forget to activate the ldm
environment, the script will fail with multiple `ModuleNotFound` errors.
9. Subsequently, to relaunch the script, be sure to run "conda activate ldm" (step 5, second command), enter the `InvokeAI` directory, and then launch the dream script (step 8). If you forget to activate the ldm environment, the script will fail with multiple `ModuleNotFound` errors.
## Updating to newer versions of the script
This distribution is changing rapidly. If you used the `git clone` method
(step 5) to download the stable-diffusion directory, then to update to the
latest and greatest version, launch the Anaconda window, enter
`stable-diffusion` and type:
```bash
(ldm) ~/stable-diffusion$ git pull
This distribution is changing rapidly. If you used the `git clone` method (step 5) to download the InvokeAI directory, then to update to the latest and greatest version, launch the Anaconda window, enter `InvokeAI` and type:
```
(ldm) ~/InvokeAI$ git pull
```
This will bring your local copy into sync with the remote one.

65
docs/installation/INSTALL_MAC.md
View File

@ -95,7 +95,8 @@ While that is downloading, open a Terminal and run the following commands:
```{.bash .annotate title="local repo setup"}
# clone the repo
git clone https://github.com/invoke-ai/InvokeAI.git
cd stable-diffusion
cd InvokeAI
# wait until the checkpoint file has downloaded, then proceed
@ -124,7 +125,7 @@ ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" \
=== "Intel x86_64"
```bash
PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 \
PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-64 \
conda env create \
-f environment-mac.yaml \
&& conda activate ldm
@ -146,18 +147,15 @@ python scripts/orig_scripts/txt2img.py \
--plms
```
1. half-precision requires autocast which is unfortunatelly incompatible
## Notes
!!! note
1. half-precision requires autocast which is unfortunately incompatible with the
implementation of pytorch on the M1 architecture. On Macs, --full-precision will
default to True.
`#!bash export PIP_EXISTS_ACTION=w` is a precaution to fix a problem where
```bash
conda env create \
-f environment-mac.yaml
```
did never finish in some situations. So it isn't required but wont hurt.
2. `export PIP_EXISTS_ACTION=w` in the commands above, is a precaution to fix `conda env
create -f environment-mac.yml` never finishing in some situations. So
it isn't required but wont hurt.
---
@ -198,30 +196,23 @@ conda install \
-n ldm
```
If it takes forever to run
If it takes forever to run `conda env create -f environment-mac.yml` you could try to run:
```bash
conda env create \
-f environment-mac.yaml
```
you could try to run:
```bash
git clean -f
conda clean \
--yes \
--all
```
```bash
git clean -f
conda clean \
--yes \
--all
```
Or you could try to completley reset Anaconda:
```bash
conda update \
--force-reinstall \
-y \
-n base \
-c defaults conda
```bash
conda update \
--force-reinstall \
-y \
-n base \
-c defaults conda
```
---
@ -246,11 +237,9 @@ There are several causes of these errors:
```bash
conda deactivate
conda env remove -n ldm
PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 \
conda env create \
-f environment-mac.yaml
conda env create -f environment-mac.yml
```
4. If you have activated the ldm virtual environment and tried rebuilding it,
maybe the problem could be that I have something installed that you don't and
you'll just need to manually install it. Make sure you activate the virtual
@ -395,9 +384,7 @@ python scripts/preload_models.py
```
This fork already includes a fix for this in
[environment-mac.yaml](https://github.com/invoke-ai/InvokeAI/blob/main/environment-mac.yaml).
---
[environment-mac.yaml](https://github.com/invoke-ai/InvokeAI/blob/main/environment-mac.yml).
### "Could not build wheels for tokenizers"

60
docs/installation/INSTALL_WINDOWS.md
View File

@ -46,23 +46,26 @@ in the wiki
This will create stable-diffusion folder where you will follow the rest of
the steps.
5. Enter the newly-created stable-diffusion folder. From this step forward make
sure that you are working in the stable-diffusion directory!
5. Enter the newly-created InvokeAI folder. From this step forward make sure that you are working in the InvokeAI directory!
```batch
cd stable-diffusion
cd InvokeAI
```
6. Run the following two commands:
```batch
conda env create -f environment.yaml
conda activate ldm
conda env create (step 6a)
conda activate ldm (step 6b)
```
This will install all python requirements and activate the "ldm" environment
which sets PATH and other environment variables properly.
Note that the long form of the first command is `conda env create -f environment.yml`. If the
environment file isn't specified, conda will default to `environment.yml`. You will need
to provide the `-f` option if you wish to load a different environment file at any point.
7. Run the command:
```batch
@ -77,29 +80,23 @@ in the wiki
8. Now you need to install the weights for the big stable diffusion model.
- For running with the released weights, you will first need to set up an
acount with [Hugging Face](https://huggingface.co).
- Use your credentials to log in, and then point your browser at
[https://huggingface.co/CompVis/stable-diffusion-v-1-4-original](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original).
- You may be asked to sign a license agreement at this point.
- Click on "Files and versions" near the top of the page, and then click on
the file named `sd-v1-4.ckpt`. You'll be taken to a page that prompts you
to click the "download" link. Now save the file somewhere safe on your
local machine.
- The weight file is >4 GB in size, so downloading may take a while.
- For running with the released weights, you will first need to set up an acount with Hugging Face (https://huggingface.co).
- Use your credentials to log in, and then point your browser at https://huggingface.co/CompVis/stable-diffusion-v-1-4-original.
- You may be asked to sign a license agreement at this point.
- Click on "Files and versions" near the top of the page, and then click on the file named `sd-v1-4.ckpt`. You'll be taken to a page that
prompts you to click the "download" link. Now save the file somewhere safe on your local machine.
- The weight file is >4 GB in size, so
downloading may take a while.
Now run the following commands from **within the stable-diffusion directory**
to copy the weights file to the right place:
Now run the following commands from **within the InvokeAI directory** to copy the weights file to the right place:
```batch
mkdir -p models\ldm\stable-diffusion-v1
copy C:\path\to\sd-v1-4.ckpt models\ldm\stable-diffusion-v1\model.ckpt
```
```batch
mkdir -p models\ldm\stable-diffusion-v1
copy C:\path\to\sd-v1-4.ckpt models\ldm\stable-diffusion-v1\model.ckpt
```
Please replace `C:\path\to\sd-v1.4.ckpt` with the correct path to wherever
you stashed this file. If you prefer not to copy or move the .ckpt file, you
may instead create a shortcut to it from within
`models\ldm\stable-diffusion-v1\`.
Please replace `C:\path\to\sd-v1.4.ckpt` with the correct path to wherever you stashed this file. If you prefer not to copy or move the .ckpt file,
you may instead create a shortcut to it from within `models\ldm\stable-diffusion-v1\`.
9. Start generating images!
@ -111,10 +108,7 @@ in the wiki
python scripts\dream.py
```
10. Subsequently, to relaunch the script, first activate the Anaconda command
window (step 3),enter the stable-diffusion directory (step 5,
`cd \path\to\stable-diffusion`), run `conda activate ldm` (step 6b), and
then launch the dream script (step 9).
10. Subsequently, to relaunch the script, first activate the Anaconda command window (step 3),enter the InvokeAI directory (step 5, `cd \path\to\InvokeAI`), run `conda activate ldm` (step 6b), and then launch the dream script (step 9).
**Note:** Tildebyte has written an alternative
["Easy peasy Windows install"](https://github.com/invoke-ai/InvokeAI/wiki/Easy-peasy-Windows-install)
@ -130,9 +124,9 @@ This distribution is changing rapidly. If you used the `git clone` method
latest and greatest version, launch the Anaconda window, enter
`stable-diffusion`, and type:
```batch
git pull
conda env update -f environment.yaml
```
```batch
git pull
conda env update
```
This will bring your local copy into sync with the remote one.

1
docs/other/CONTRIBUTORS.md
View File

@ -57,6 +57,7 @@ We thank them for all of their time and hard work.
- [Kyle Schouviller](https://github.com/kyle0654)
- [rabidcopy](https://github.com/rabidcopy)
- [Dominic Letz](https://github.com/dominicletz)
- [Dmitry T.](https://github.com/ArDiouscuros)
## **Original CompVis Authors:**

7
docs/other/README-CompViz.md
View File

@ -39,11 +39,14 @@ lightweight and runs on a GPU with at least 10GB VRAM. See
A suitable [conda](https://conda.io/) environment named `ldm` can be created and
activated with:
```bash
conda env create -f environment.yaml
```
conda env create -f environment.yml
conda activate ldm
```
Note that the first line may be abbreviated `conda env create`, since conda will
look for `environment.yml` by default.
You can also update an existing
[latent diffusion](https://github.com/CompVis/latent-diffusion) environment by
running

3
environment-mac.yaml → environment-mac.yml
View File

@ -14,7 +14,7 @@ dependencies:
# To determine what the latest versions should be, run:
#
# ```shell
# sed -E 's/ldm/ldm-updated/;20,99s/- ([^=]+)==.+/- \1/' environment-mac.yaml > environment-mac-updated.yml
# sed -E 's/ldm/ldm-updated/;20,99s/- ([^=]+)==.+/- \1/' environment-mac.yml > environment-mac-updated.yml
# CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac-updated.yml && conda list -n ldm-updated | awk ' {print " - " $1 "==" $2;} '
# ```
- albumentations==1.2.1
@ -30,6 +30,7 @@ dependencies:
- nomkl
- numpy==1.23.2
- omegaconf==2.1.1
- openh264==2.3.0
- onnx==1.12.0
- onnxruntime==1.12.1
- protobuf==3.20.1

0
environment.yaml → environment.yml
View File

694
frontend/dist/assets/index.632c341a.js vendored
View File

File diff suppressed because one or more lines are too long

694
frontend/dist/assets/index.66192cce.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

2
frontend/dist/index.html vendored
View File

@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>InvokeAI Stable Diffusion Dream Server</title>
<script type="module" crossorigin src="/assets/index.632c341a.js"></script>
<script type="module" crossorigin src="/assets/index.66192cce.js"></script>
<link rel="stylesheet" href="/assets/index.447eb2a9.css">
</head>
<body>

4
frontend/src/app/App.tsx
View File

@ -10,15 +10,13 @@ import PromptInput from '../features/options/PromptInput';
import LogViewer from '../features/system/LogViewer';
import Loading from '../Loading';
import { useAppDispatch } from './store';
import { requestAllImages, requestSystemConfig } from './socketio/actions';
import { requestSystemConfig } from './socketio/actions';
const App = () => {
const dispatch = useAppDispatch();
const [isReady, setIsReady] = useState<boolean>(false);
// Load images from the gallery once
useEffect(() => {
dispatch(requestAllImages());
dispatch(requestSystemConfig());
setIsReady(true);
}, [dispatch]);

59
frontend/src/app/features.ts Normal file
View File

@ -0,0 +1,59 @@
type FeatureHelpInfo = {
text: string;
href: string;
guideImage: string;
};
export enum Feature {
PROMPT,
GALLERY,
OUTPUT,
SEED_AND_VARIATION,
ESRGAN,
FACE_CORRECTION,
IMAGE_TO_IMAGE,
SAMPLER,
}
export const FEATURES: Record<Feature, FeatureHelpInfo> = {
[Feature.PROMPT]: {
text: 'This field will take all prompt text, including both content and stylistic terms. CLI Commands will not work in the prompt.',
href: 'link/to/docs/feature3.html',
guideImage: 'asset/path.gif',
},
[Feature.GALLERY]: {
text: 'As new invocations are generated, files from the output directory will be displayed here. Generations have additional options to configure new generations.',
href: 'link/to/docs/feature3.html',
guideImage: 'asset/path.gif',
},
[Feature.OUTPUT]: {
text: 'The Height and Width of generations can be controlled here. If you experience errors, you may be generating an image too large for your system. The seamless option will more often result in repeating patterns in outputs.',
href: 'link/to/docs/feature3.html',
guideImage: 'asset/path.gif',
},
[Feature.SEED_AND_VARIATION]: {
text: 'Seed values provide an initial set of noise which guide the denoising process. Try a variation with an amount of between 0 and 1 to change the output image for that seed.',
href: 'link/to/docs/feature3.html',
guideImage: 'asset/path.gif',
},
[Feature.ESRGAN]: {
text: 'The ESRGAN setting can be used to increase the output resolution without requiring a higher width/height in the initial generation.',
href: 'link/to/docs/feature1.html',
guideImage: 'asset/path.gif',
},
[Feature.FACE_CORRECTION]: {
text: 'Using GFPGAN or CodeFormer, Face Correction will attempt to identify faces in outputs, and correct any defects/abnormalities. Higher values will apply a stronger corrective pressure on outputs.',
href: 'link/to/docs/feature2.html',
guideImage: 'asset/path.gif',
},
[Feature.IMAGE_TO_IMAGE]: {
text: 'ImageToImage allows the upload of an initial image, which InvokeAI will use to guide the generation process, along with a prompt. A lower value for this setting will more closely resemble the original image. Values between 0-1 are accepted, and a range of .25-.75 is recommended ',
href: 'link/to/docs/feature3.html',
guideImage: 'asset/path.gif',
},
[Feature.SAMPLER]: {
text: 'This setting allows for different denoising samplers to be used, as well as the number of denoising steps used, which will change the resulting output.',
href: 'link/to/docs/feature3.html',
guideImage: 'asset/path.gif',
},
};

7
frontend/src/app/invokeai.d.ts vendored
View File

@ -107,6 +107,7 @@ export declare type Metadata = SystemConfig & {
export declare type Image = {
uuid: string;
url: string;
mtime: number;
metadata: Metadata;
};
@ -148,6 +149,7 @@ export declare type SystemConfigResponse = SystemConfig;
export declare type ImageResultResponse = {
url: string;
mtime: number;
metadata: Metadata;
};
@ -157,7 +159,10 @@ export declare type ErrorResponse = {
};
export declare type GalleryImagesResponse = {
images: Array<{ url: string; metadata: Metadata }>;
images: Array<Omit<Image, 'uuid'>>;
nextPage: number;
offset: number;
onlyNewImages: boolean;
};
export declare type ImageUrlAndUuidResponse = {

11
frontend/src/app/socketio/actions.ts
View File

@ -12,8 +12,11 @@ export const generateImage = createAction<undefined>('socketio/generateImage');
export const runESRGAN = createAction<InvokeAI.Image>('socketio/runESRGAN');
export const runGFPGAN = createAction<InvokeAI.Image>('socketio/runGFPGAN');
export const deleteImage = createAction<InvokeAI.Image>('socketio/deleteImage');
export const requestAllImages = createAction<undefined>(
'socketio/requestAllImages'
export const requestImages = createAction<undefined>(
'socketio/requestImages'
);
export const requestNewImages = createAction<undefined>(
'socketio/requestNewImages'
);
export const cancelProcessing = createAction<undefined>(
'socketio/cancelProcessing'
@ -23,4 +26,6 @@ export const uploadInitialImage = createAction<File>(
);
export const uploadMaskImage = createAction<File>('socketio/uploadMaskImage');
export const requestSystemConfig = createAction<undefined>('socketio/requestSystemConfig');
export const requestSystemConfig = createAction<undefined>(
'socketio/requestSystemConfig'
);

17
frontend/src/app/socketio/emitters.ts
View File

@ -83,8 +83,17 @@ const makeSocketIOEmitters = (
const { url, uuid } = imageToDelete;
socketio.emit('deleteImage', url, uuid);
},
emitRequestAllImages: () => {
socketio.emit('requestAllImages');
emitRequestImages: () => {
const { nextPage, offset } = getState().gallery;
socketio.emit('requestImages', nextPage, offset);
},
emitRequestNewImages: () => {
const { nextPage, offset, images } = getState().gallery;
if (images.length > 0) {
socketio.emit('requestImages', nextPage, offset, images[0].mtime);
} else {
socketio.emit('requestImages', nextPage, offset);
}
},
emitCancelProcessing: () => {
socketio.emit('cancel');
@ -96,8 +105,8 @@ const makeSocketIOEmitters = (
socketio.emit('uploadMaskImage', file, file.name);
},
emitRequestSystemConfig: () => {
socketio.emit('requestSystemConfig')
}
socketio.emit('requestSystemConfig');
},
};
};

33
frontend/src/app/socketio/listeners.ts
View File

@ -14,10 +14,10 @@ import {
} from '../../features/system/systemSlice';
import {
addGalleryImages,
addImage,
clearIntermediateImage,
removeImage,
setGalleryImages,
setIntermediateImage,
} from '../../features/gallery/gallerySlice';
@ -25,6 +25,7 @@ import {
setInitialImagePath,
setMaskPath,
} from '../../features/options/optionsSlice';
import { requestNewImages } from './actions';
/**
* Returns an object containing listener callbacks for socketio events.
@ -43,6 +44,7 @@ const makeSocketIOListeners = (
try {
dispatch(setIsConnected(true));
dispatch(setCurrentStatus('Connected'));
dispatch(requestNewImages());
} catch (e) {
console.error(e);
}
@ -53,7 +55,6 @@ const makeSocketIOListeners = (
onDisconnect: () => {
try {
dispatch(setIsConnected(false));
dispatch(setIsProcessing(false));
dispatch(setCurrentStatus('Disconnected'));
dispatch(
@ -72,13 +73,14 @@ const makeSocketIOListeners = (
*/
onGenerationResult: (data: InvokeAI.ImageResultResponse) => {
try {
const { url, metadata } = data;
const { url, mtime, metadata } = data;
const newUuid = uuidv4();
dispatch(
addImage({
uuid: newUuid,
url,
mtime,
metadata: metadata,
})
);
@ -99,11 +101,12 @@ const makeSocketIOListeners = (
onIntermediateResult: (data: InvokeAI.ImageResultResponse) => {
try {
const uuid = uuidv4();
const { url, metadata } = data;
const { url, metadata, mtime } = data;
dispatch(
setIntermediateImage({
uuid,
url,
mtime,
metadata,
})
);
@ -123,12 +126,13 @@ const makeSocketIOListeners = (
*/
onESRGANResult: (data: InvokeAI.ImageResultResponse) => {
try {
const { url, metadata } = data;
const { url, metadata, mtime } = data;
dispatch(
addImage({
uuid: uuidv4(),
url,
mtime,
metadata,
})
);
@ -149,12 +153,13 @@ const makeSocketIOListeners = (
*/
onGFPGANResult: (data: InvokeAI.ImageResultResponse) => {
try {
const { url, metadata } = data;
const { url, metadata, mtime } = data;
dispatch(
addImage({
uuid: uuidv4(),
url,
mtime,
metadata,
})
);
@ -209,16 +214,26 @@ const makeSocketIOListeners = (
* Callback to run when we receive a 'galleryImages' event.
*/
onGalleryImages: (data: InvokeAI.GalleryImagesResponse) => {
const { images } = data;
const { images, nextPage, offset } = data;
/**
* the logic here ideally would be in the reducer but we have a side effect:
* generating a uuid. so the logic needs to be here, outside redux.
*/
// Generate a UUID for each image
const preparedImages = images.map((image): InvokeAI.Image => {
const { url, metadata } = image;
const { url, metadata, mtime } = image;
return {
uuid: uuidv4(),
url,
mtime,
metadata,
};
});
dispatch(setGalleryImages(preparedImages));
dispatch(addGalleryImages({ images: preparedImages, nextPage, offset }));
dispatch(
addLogEntry({
timestamp: dateFormat(new Date(), 'isoDateTime'),

17
frontend/src/app/socketio/middleware.ts
View File

@ -24,7 +24,9 @@ import * as InvokeAI from '../invokeai';
export const socketioMiddleware = () => {
const { hostname, port } = new URL(window.location.href);
const socketio = io(`http://${hostname}:9090`);
const socketio = io(`http://${hostname}:9090`, {
timeout: 60000,
});
let areListenersSet = false;
@ -51,7 +53,8 @@ export const socketioMiddleware = () => {
emitRunESRGAN,
emitRunGFPGAN,
emitDeleteImage,
emitRequestAllImages,
emitRequestImages,
emitRequestNewImages,
emitCancelProcessing,
emitUploadInitialImage,
emitUploadMaskImage,
@ -140,11 +143,17 @@ export const socketioMiddleware = () => {
break;
}
case 'socketio/requestAllImages': {
emitRequestAllImages();
case 'socketio/requestImages': {
emitRequestImages();
break;
}
case 'socketio/requestNewImages': {
emitRequestNewImages();
break;
}
case 'socketio/cancelProcessing': {
emitCancelProcessing();
break;

22
frontend/src/common/components/GuideIcon.tsx Normal file
View File

@ -0,0 +1,22 @@
import { Box, forwardRef, Icon } from '@chakra-ui/react';
import { IconType } from 'react-icons';
import { MdHelp } from 'react-icons/md';
import { Feature } from '../../app/features';
import GuidePopover from './GuidePopover';
type GuideIconProps = {
feature: Feature;
icon?: IconType;
};
const GuideIcon = forwardRef(
({ feature, icon = MdHelp }: GuideIconProps, ref) => (
<GuidePopover feature={feature}>
<Box ref={ref}>
<Icon as={icon} />
</Box>
</GuidePopover>
)
);
export default GuideIcon;

51
frontend/src/common/components/GuidePopover.tsx Normal file
View File

@ -0,0 +1,51 @@
import {
Popover,
PopoverArrow,
PopoverContent,
PopoverTrigger,
PopoverHeader,
Flex,
Box,
} from '@chakra-ui/react';
import { SystemState } from '../../features/system/systemSlice';
import { useAppSelector } from '../../app/store';
import { RootState } from '../../app/store';
import { createSelector } from '@reduxjs/toolkit';
import { ReactElement } from 'react';
import { Feature, FEATURES } from '../../app/features';
type GuideProps = {
children: ReactElement;
feature: Feature;
};
const systemSelector = createSelector(
(state: RootState) => state.system,
(system: SystemState) => system.shouldDisplayGuides
);
const GuidePopover = ({ children, feature }: GuideProps) => {
const shouldDisplayGuides = useAppSelector(systemSelector);
const { text } = FEATURES[feature];
return shouldDisplayGuides ? (
<Popover trigger={'hover'}>
<PopoverTrigger>
<Box>{children}</Box>
</PopoverTrigger>
<PopoverContent
maxWidth="400px"
onClick={(e) => e.preventDefault()}
cursor={'initial'}
>
<PopoverArrow />
<Flex alignItems={'center'} gap={2} p={4}>
{text}
</Flex>
</PopoverContent>
</Popover>
) : (
<></>
);
};
export default GuidePopover;

30
frontend/src/features/gallery/ImageGallery.tsx
View File

@ -1,5 +1,6 @@
import { Center, Flex, Text } from '@chakra-ui/react';
import { RootState } from '../../app/store';
import { Button, Center, Flex, Text } from '@chakra-ui/react';
import { requestImages } from '../../app/socketio/actions';
import { RootState, useAppDispatch } from '../../app/store';
import { useAppSelector } from '../../app/store';
import HoverableImage from './HoverableImage';
@ -10,7 +11,7 @@ const ImageGallery = () => {
const { images, currentImageUuid } = useAppSelector(
(state: RootState) => state.gallery
);
const dispatch = useAppDispatch();
/**
* I don't like that this needs to rerender whenever the current image is changed.
* What if we have a large number of images? I suppose pagination (planned) will
@ -19,15 +20,22 @@ const ImageGallery = () => {
* TODO: Refactor if performance complaints, or after migrating to new API which supports pagination.
*/
const handleClickLoadMore = () => {
dispatch(requestImages());
};
return images.length ? (
<Flex gap={2} wrap="wrap" pb={2}>
{[...images].reverse().map((image) => {
const { uuid } = image;
const isSelected = currentImageUuid === uuid;
return (
<HoverableImage key={uuid} image={image} isSelected={isSelected} />
);
})}
<Flex direction={'column'} gap={2} pb={2}>
<Flex gap={2} wrap="wrap">
{images.map((image) => {
const { uuid } = image;
const isSelected = currentImageUuid === uuid;
return (
<HoverableImage key={uuid} image={image} isSelected={isSelected} />
);
})}
</Flex>
<Button onClick={handleClickLoadMore}>Load more...</Button>
</Flex>
) : (
<Center height={'100%'} position={'relative'}>

32
frontend/src/features/gallery/gallerySlice.ts
View File

@ -8,11 +8,15 @@ export interface GalleryState {
currentImageUuid: string;
images: Array<InvokeAI.Image>;
intermediateImage?: InvokeAI.Image;
nextPage: number;
offset: number;
}
const initialState: GalleryState = {
currentImageUuid: '',
images: [],
nextPage: 1,
offset: 0,
};
export const gallerySlice = createSlice({
@ -50,7 +54,7 @@ export const gallerySlice = createSlice({
* Clamp the new index to ensure it is valid..
*/
const newCurrentImageIndex = clamp(
imageToDeleteIndex - 1,
imageToDeleteIndex,
0,
newImages.length - 1
);
@ -67,10 +71,11 @@ export const gallerySlice = createSlice({
state.images = newImages;
},
addImage: (state, action: PayloadAction<InvokeAI.Image>) => {
state.images.push(action.payload);
state.images.unshift(action.payload);
state.currentImageUuid = action.payload.uuid;
state.intermediateImage = undefined;
state.currentImage = action.payload;
state.offset += 1
},
setIntermediateImage: (state, action: PayloadAction<InvokeAI.Image>) => {
state.intermediateImage = action.payload;
@ -78,13 +83,24 @@ export const gallerySlice = createSlice({
clearIntermediateImage: (state) => {
state.intermediateImage = undefined;
},
setGalleryImages: (state, action: PayloadAction<Array<InvokeAI.Image>>) => {
const newImages = action.payload;
if (newImages.length) {
const newCurrentImage = newImages[newImages.length - 1];
state.images = newImages;
addGalleryImages: (
state,
action: PayloadAction<{
images: Array<InvokeAI.Image>;
nextPage: number;
offset: number;
}>
) => {
const { images, nextPage, offset } = action.payload;
if (images.length) {
const newCurrentImage = images[0];
state.images = state.images
.concat(images)
.sort((a, b) => b.mtime - a.mtime);
state.currentImage = newCurrentImage;
state.currentImageUuid = newCurrentImage.uuid;
state.nextPage = nextPage;
state.offset = offset;
}
},
},
@ -95,7 +111,7 @@ export const {
clearIntermediateImage,
removeImage,
setCurrentImage,
setGalleryImages,
addGalleryImages,
setIntermediateImage,
} = gallerySlice.actions;

11
frontend/src/features/options/OptionsAccordion.tsx
View File

@ -31,6 +31,9 @@ import OutputOptions from './OutputOptions';
import ImageToImageOptions from './ImageToImageOptions';
import { ChangeEvent } from 'react';
import GuideIcon from '../../common/components/GuideIcon';
import { Feature } from '../../app/features';
const optionsSelector = createSelector(
(state: RootState) => state.options,
(options: OptionsState) => {
@ -108,6 +111,7 @@ const OptionsAccordion = () => {
<Box flex="1" textAlign="left">
Seed & Variation
</Box>
<GuideIcon feature={Feature.SEED_AND_VARIATION} />
<AccordionIcon />
</AccordionButton>
</h2>
@ -121,6 +125,7 @@ const OptionsAccordion = () => {
<Box flex="1" textAlign="left">
Sampler
</Box>
<GuideIcon feature={Feature.SAMPLER} />
<AccordionIcon />
</AccordionButton>
</h2>
@ -144,6 +149,7 @@ const OptionsAccordion = () => {
onChange={handleChangeShouldRunESRGAN}
/>
</Flex>
<GuideIcon feature={Feature.ESRGAN} />
<AccordionIcon />
</AccordionButton>
</h2>
@ -160,13 +166,14 @@ const OptionsAccordion = () => {
width={'100%'}
mr={2}
>
<Text>Fix Faces (GFPGAN)</Text>
<Text>Face Correction</Text>
<Switch
isDisabled={!isGFPGANAvailable}
isChecked={shouldRunGFPGAN}
onChange={handleChangeShouldRunGFPGAN}
/>
</Flex>
<GuideIcon feature={Feature.FACE_CORRECTION} />
<AccordionIcon />
</AccordionButton>
</h2>
@ -190,6 +197,7 @@ const OptionsAccordion = () => {
onChange={handleChangeShouldUseInitImage}
/>
</Flex>
<GuideIcon feature={Feature.IMAGE_TO_IMAGE} />
<AccordionIcon />
</AccordionButton>
</h2>
@ -203,6 +211,7 @@ const OptionsAccordion = () => {
<Box flex="1" textAlign="left">
Output
</Box>
<GuideIcon feature={Feature.OUTPUT} />
<AccordionIcon />
</AccordionButton>
</h2>

33
frontend/src/features/system/SettingsModal.tsx
View File

@ -20,6 +20,7 @@ import { useAppDispatch, useAppSelector } from '../../app/store';
import {
setShouldConfirmOnDelete,
setShouldDisplayInProgress,
setShouldDisplayGuides,
SystemState,
} from './systemSlice';
import { RootState } from '../../app/store';
@ -31,8 +32,16 @@ import { cloneElement, ReactElement } from 'react';
const systemSelector = createSelector(
(state: RootState) => state.system,
(system: SystemState) => {
const { shouldDisplayInProgress, shouldConfirmOnDelete } = system;
return { shouldDisplayInProgress, shouldConfirmOnDelete };
const {
shouldDisplayInProgress,
shouldConfirmOnDelete,
shouldDisplayGuides,
} = system;
return {
shouldDisplayInProgress,
shouldConfirmOnDelete,
shouldDisplayGuides,
};
},
{
memoizeOptions: { resultEqualityCheck: isEqual },
@ -63,8 +72,11 @@ const SettingsModal = ({ children }: SettingsModalProps) => {
onClose: onRefreshModalClose,
} = useDisclosure();
const { shouldDisplayInProgress, shouldConfirmOnDelete } =
useAppSelector(systemSelector);
const {
shouldDisplayInProgress,
shouldConfirmOnDelete,
shouldDisplayGuides,
} = useAppSelector(systemSelector);
const dispatch = useAppDispatch();
@ -116,6 +128,19 @@ const SettingsModal = ({ children }: SettingsModalProps) => {
/>
</HStack>
</FormControl>
<FormControl>
<HStack>
<FormLabel marginBottom={1}>
Display help guides in configuration menus
</FormLabel>
<Switch
isChecked={shouldDisplayGuides}
onChange={(e) =>
dispatch(setShouldDisplayGuides(e.target.checked))
}
/>
</HStack>
</FormControl>
<Heading size={'md'}>Reset Web UI</Heading>
<Text>

20
frontend/src/features/system/systemSlice.ts
View File

@ -1,7 +1,7 @@
import { createSlice } from '@reduxjs/toolkit';
import type { PayloadAction } from '@reduxjs/toolkit';
import { ExpandedIndex } from '@chakra-ui/react';
import * as InvokeAI from '../../app/invokeai'
import * as InvokeAI from '../../app/invokeai';
export type LogLevel = 'info' | 'warning' | 'error';
@ -15,7 +15,9 @@ export interface Log {
[index: number]: LogEntry;
}
export interface SystemState extends InvokeAI.SystemStatus, InvokeAI.SystemConfig {
export interface SystemState
extends InvokeAI.SystemStatus,
InvokeAI.SystemConfig {
shouldDisplayInProgress: boolean;
log: Array<LogEntry>;
shouldShowLogViewer: boolean;
@ -31,6 +33,7 @@ export interface SystemState extends InvokeAI.SystemStatus, InvokeAI.SystemConfi
totalIterations: number;
currentStatus: string;
currentStatusHasSteps: boolean;
shouldDisplayGuides: boolean;
}
const initialSystemState = {
@ -39,6 +42,7 @@ const initialSystemState = {
log: [],
shouldShowLogViewer: false,
shouldDisplayInProgress: false,
shouldDisplayGuides: true,
isGFPGANAvailable: true,
isESRGANAvailable: true,
socketId: '',
@ -48,7 +52,7 @@ const initialSystemState = {
totalSteps: 0,
currentIteration: 0,
totalIterations: 0,
currentStatus: '',
currentStatus: 'Disconnected',
currentStatusHasSteps: false,
model: '',
model_id: '',
@ -104,6 +108,12 @@ export const systemSlice = createSlice({
},
setIsConnected: (state, action: PayloadAction<boolean>) => {
state.isConnected = action.payload;
state.isProcessing = false;
state.currentStep = 0;
state.totalSteps = 0;
state.currentIteration = 0;
state.totalIterations = 0;
state.currentStatusHasSteps = false;
},
setSocketId: (state, action: PayloadAction<string>) => {
state.socketId = action.payload;
@ -117,6 +127,9 @@ export const systemSlice = createSlice({
setSystemConfig: (state, action: PayloadAction<InvokeAI.SystemConfig>) => {
return { ...state, ...action.payload };
},
setShouldDisplayGuides: (state, action: PayloadAction<boolean>) => {
state.shouldDisplayGuides = action.payload;
},
},
});
@ -132,6 +145,7 @@ export const {
setSystemStatus,
setCurrentStatus,
setSystemConfig,
setShouldDisplayGuides,
} = systemSlice.actions;
export default systemSlice.reducer;

10
ldm/dream/args.py
View File

@ -339,6 +339,12 @@ class Args(object):
action='store_true',
help='Deprecated way to set --precision=float32',
)
model_group.add_argument(
'--free_gpu_mem',
dest='free_gpu_mem',
action='store_true',
help='Force free gpu memory before final decoding',
)
model_group.add_argument(
'--precision',
dest='precision',
@ -400,7 +406,7 @@ class Args(object):
postprocessing_group.add_argument(
'--gfpgan_model_path',
type=str,
default='experiments/pretrained_models/GFPGANv1.3.pth',
default='experiments/pretrained_models/GFPGANv1.4.pth',
help='Indicates the path to the GFPGAN model, relative to --gfpgan_dir.',
)
postprocessing_group.add_argument(
@ -588,7 +594,7 @@ class Args(object):
'--upscale',
nargs='+',
type=float,
help='Scale factor (2, 4) for upscaling final output followed by upscaling strength (0-1.0). If strength not specified, defaults to 0.75',
help='Scale factor (1, 2, 3, 4, etc..) for upscaling final output followed by upscaling strength (0-1.0). If strength not specified, defaults to 0.75',
default=None,
)
postprocessing_group.add_argument(

4
ldm/dream/conditioning.py
View File

@ -79,10 +79,10 @@ def split_weighted_subprompts(text, skip_normalize=False)->list:
if weight_sum == 0:
print(
"Warning: Subprompt weights add up to zero. Discarding and using even weights instead.")
equal_weight = 1 / len(parsed_prompts)
equal_weight = 1 / max(len(parsed_prompts), 1)
return [(x[0], equal_weight) for x in parsed_prompts]
return [(x[0], x[1] / weight_sum) for x in parsed_prompts]
# shows how the prompt is tokenized
# usually tokens have '</w>' to indicate end-of-word,
# but for readability it has been replaced with ' '

134
ldm/dream/generator/embiggen.py
View File

@ -4,18 +4,42 @@ and generates with ldm.dream.generator.img2img
'''
import torch
import numpy as np
from PIL import Image
from ldm.dream.generator.base import Generator
from ldm.models.diffusion.ddim import DDIMSampler
from ldm.dream.generator.img2img import Img2Img
import numpy as np
from tqdm import trange
from PIL import Image
from ldm.dream.generator.base import Generator
from ldm.dream.generator.img2img import Img2Img
from ldm.dream.devices import choose_autocast
class Embiggen(Generator):
def __init__(self, model, precision):
super().__init__(model, precision)
self.init_latent = None
# Replace generate because Embiggen doesn't need/use most of what it does normallly
def generate(self,prompt,iterations=1,seed=None,
image_callback=None, step_callback=None,
**kwargs):
scope = choose_autocast(self.precision)
make_image = self.get_make_image(
prompt,
step_callback = step_callback,
**kwargs
)
results = []
seed = seed if seed else self.new_seed()
# Noise will be generated by the Img2Img generator when called
with scope(self.model.device.type), self.model.ema_scope():
for n in trange(iterations, desc='Generating'):
# make_image will call Img2Img which will do the equivalent of get_noise itself
image = make_image()
results.append([image, seed])
if image_callback is not None:
image_callback(image, seed)
seed = self.new_seed()
return results
@torch.no_grad()
def get_make_image(
self,
@ -151,8 +175,19 @@ class Embiggen(Generator):
# Clamp values to max 255
if distanceToLR > 255:
distanceToLR = 255
# Place the pixel as invert of distance
agradientC.putpixel((x, y), int(255 - distanceToLR))
#Place the pixel as invert of distance
agradientC.putpixel((x, y), round(255 - distanceToLR))
# Create alternative asymmetric diagonal corner to use on "tailing" intersections to prevent hard edges
# Fits for a left-fading gradient on the bottom side and full opacity on the right side.
agradientAsymC = Image.new('L', (256, 256))
for y in range(256):
for x in range(256):
value = round(max(0, x-(255-y)) * (255 / max(1,y)))
#Clamp values
value = max(0, value)
value = min(255, value)
agradientAsymC.putpixel((x, y), value)
# Create alpha layers default fully white
alphaLayerL = Image.new("L", (width, height), 255)
@ -163,8 +198,13 @@ class Embiggen(Generator):
alphaLayerT.paste(agradientT, (0, 0))
alphaLayerLTC.paste(agradientL, (0, 0))
alphaLayerLTC.paste(agradientT, (0, 0))
alphaLayerLTC.paste(agradientC.resize(
(overlap_size_x, overlap_size_y)), (0, 0))
alphaLayerLTC.paste(agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0))
# make masks with an asymmetric upper-right corner so when the curved transparent corner of the next tile
# to its right is placed it doesn't reveal a hard trailing semi-transparent edge in the overlapping space
alphaLayerTaC = alphaLayerT.copy()
alphaLayerTaC.paste(agradientAsymC.rotate(270).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, 0))
alphaLayerLTaC = alphaLayerLTC.copy()
alphaLayerLTaC.paste(agradientAsymC.rotate(270).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, 0))
if embiggen_tiles:
# Individual unconnected sides
@ -242,7 +282,7 @@ class Embiggen(Generator):
del agradientT
del agradientC
def make_image(x_T):
def make_image():
# Make main tiles -------------------------------------------------
if embiggen_tiles:
print(f'>> Making {len(embiggen_tiles)} Embiggen tiles...')
@ -251,7 +291,20 @@ class Embiggen(Generator):
f'>> Making {(emb_tiles_x * emb_tiles_y)} Embiggen tiles ({emb_tiles_x}x{emb_tiles_y})...')
emb_tile_store = []
# Although we could use the same seed for every tile for determinism, at higher strengths this may
# produce duplicated structures for each tile and make the tiling effect more obvious
# instead track and iterate a local seed we pass to Img2Img
seed = self.seed
seedintlimit = np.iinfo(np.uint32).max - 1 # only retreive this one from numpy
for tile in range(emb_tiles_x * emb_tiles_y):
# Don't iterate on first tile
if tile != 0:
if seed < seedintlimit:
seed += 1
else:
seed = 0
# Determine if this is a re-run and replace
if embiggen_tiles and not tile in embiggen_tiles:
continue
@ -294,21 +347,20 @@ class Embiggen(Generator):
tile_results = gen_img2img.generate(
prompt,
iterations=1,
seed=self.seed,
sampler=sampler,
steps=steps,
cfg_scale=cfg_scale,
conditioning=conditioning,
ddim_eta=ddim_eta,
image_callback=None, # called only after the final image is generated
step_callback=step_callback, # called after each intermediate image is generated
width=width,
height=height,
init_img=init_img, # img2img doesn't need this, but it might in the future
init_image=newinitimage, # notice that init_image is different from init_img
mask_image=None,
strength=strength,
iterations = 1,
seed = seed,
sampler = sampler,
steps = steps,
cfg_scale = cfg_scale,
conditioning = conditioning,
ddim_eta = ddim_eta,
image_callback = None, # called only after the final image is generated
step_callback = step_callback, # called after each intermediate image is generated
width = width,
height = height,
init_image = newinitimage, # notice that init_image is different from init_img
mask_image = None,
strength = strength,
)
emb_tile_store.append(tile_results[0][0])
@ -381,24 +433,24 @@ class Embiggen(Generator):
# bottom of image
elif emb_row_i == emb_tiles_y - 1:
if emb_column_i == 0:
if (tile+1) in embiggen_tiles: # Look-ahead right
intileimage.putalpha(alphaLayerT)
if (tile+1) in embiggen_tiles: # Look-ahead right
intileimage.putalpha(alphaLayerTaC)
else:
intileimage.putalpha(alphaLayerRTC)
elif emb_column_i == emb_tiles_x - 1:
# No tiles to look ahead to
intileimage.putalpha(alphaLayerLTC)
else:
if (tile+1) in embiggen_tiles: # Look-ahead right
intileimage.putalpha(alphaLayerLTC)
if (tile+1) in embiggen_tiles: # Look-ahead right
intileimage.putalpha(alphaLayerLTaC)
else:
intileimage.putalpha(alphaLayerABB)
# vertical middle of image
else:
if emb_column_i == 0:
if (tile+1) in embiggen_tiles: # Look-ahead right
if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down
intileimage.putalpha(alphaLayerT)
if (tile+1) in embiggen_tiles: # Look-ahead right
if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down
intileimage.putalpha(alphaLayerTaC)
else:
intileimage.putalpha(alphaLayerTB)
elif (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down only
@ -411,9 +463,9 @@ class Embiggen(Generator):
else:
intileimage.putalpha(alphaLayerABR)
else:
if (tile+1) in embiggen_tiles: # Look-ahead right
if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down
intileimage.putalpha(alphaLayerLTC)
if (tile+1) in embiggen_tiles: # Look-ahead right
if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down
intileimage.putalpha(alphaLayerLTaC)
else:
intileimage.putalpha(alphaLayerABR)
elif (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down only
@ -425,9 +477,15 @@ class Embiggen(Generator):
if emb_row_i == 0 and emb_column_i >= 1:
intileimage.putalpha(alphaLayerL)
elif emb_row_i >= 1 and emb_column_i == 0:
intileimage.putalpha(alphaLayerT)
if emb_column_i + 1 == emb_tiles_x: # If we don't have anything that can be placed to the right
intileimage.putalpha(alphaLayerT)
else:
intileimage.putalpha(alphaLayerTaC)
else:
intileimage.putalpha(alphaLayerLTC)
if emb_column_i + 1 == emb_tiles_x: # If we don't have anything that can be placed to the right
intileimage.putalpha(alphaLayerLTC)
else:
intileimage.putalpha(alphaLayerLTaC)
# Layer tile onto final image
outputsuperimage.alpha_composite(intileimage, (left, top))
else:

6
ldm/dream/generator/inpaint.py
View File

@ -34,9 +34,9 @@ class Inpaint(Img2Img):
)
sampler = DDIMSampler(self.model, device=self.model.device)
sampler.make_schedule(
ddim_num_steps=steps, ddim_eta=ddim_eta, verbose=False
)
sampler.make_schedule(
ddim_num_steps=steps, ddim_eta=ddim_eta, verbose=False
)
scope = choose_autocast(self.precision)
with scope(self.model.device.type):

8
ldm/dream/generator/txt2img.py
View File

@ -27,6 +27,10 @@ class Txt2Img(Generator):
height // self.downsampling_factor,
width // self.downsampling_factor,
]
if self.free_gpu_mem and self.model.model.device != self.model.device:
self.model.model.to(self.model.device)
samples, _ = sampler.sample(
batch_size = 1,
S = steps,
@ -39,6 +43,10 @@ class Txt2Img(Generator):
eta = ddim_eta,
img_callback = step_callback
)
if self.free_gpu_mem:
self.model.model.to("cpu")
return self.sample_to_image(samples)
return make_image

24
ldm/dream/restoration/base.py
View File

@ -1,34 +1,38 @@
class Restoration():
def __init__(self, gfpgan_dir='./src/gfpgan', gfpgan_model_path='experiments/pretrained_models/GFPGANv1.3.pth', esrgan_bg_tile=400) -> None:
self.gfpgan_dir = gfpgan_dir
self.gfpgan_model_path = gfpgan_model_path
self.esrgan_bg_tile = esrgan_bg_tile
def __init__(self) -> None:
pass
def load_face_restore_models(self):
def load_face_restore_models(self, gfpgan_dir='./src/gfpgan', gfpgan_model_path='experiments/pretrained_models/GFPGANv1.4.pth'):
# Load GFPGAN
gfpgan = self.load_gfpgan()
gfpgan = self.load_gfpgan(gfpgan_dir, gfpgan_model_path)
if gfpgan.gfpgan_model_exists:
print('>> GFPGAN Initialized')
else:
print('>> GFPGAN Disabled')
gfpgan = None
# Load CodeFormer
codeformer = self.load_codeformer()
if codeformer.codeformer_model_exists:
print('>> CodeFormer Initialized')
else:
print('>> CodeFormer Disabled')
codeformer = None
return gfpgan, codeformer
# Face Restore Models
def load_gfpgan(self):
def load_gfpgan(self, gfpgan_dir, gfpgan_model_path):
from ldm.dream.restoration.gfpgan import GFPGAN
return GFPGAN(self.gfpgan_dir, self.gfpgan_model_path)
return GFPGAN(gfpgan_dir, gfpgan_model_path)
def load_codeformer(self):
from ldm.dream.restoration.codeformer import CodeFormerRestoration
return CodeFormerRestoration()
# Upscale Models
def load_esrgan(self):
def load_esrgan(self, esrgan_bg_tile=400):
from ldm.dream.restoration.realesrgan import ESRGAN
esrgan = ESRGAN(self.esrgan_bg_tile)
esrgan = ESRGAN(esrgan_bg_tile)
print('>> ESRGAN Initialized')
return esrgan;

8
ldm/dream/restoration/gfpgan.py
View File

@ -11,14 +11,14 @@ class GFPGAN():
def __init__(
self,
gfpgan_dir='src/gfpgan',
gfpgan_model_path='experiments/pretrained_models/GFPGANv1.3.pth') -> None:
gfpgan_model_path='experiments/pretrained_models/GFPGANv1.4.pth') -> None:
self.model_path = os.path.join(gfpgan_dir, gfpgan_model_path)
self.gfpgan_model_exists = os.path.isfile(self.model_path)
if not self.gfpgan_model_exists:
raise Exception(
'GFPGAN model not found at path ' + self.model_path)
print('## NOT FOUND: GFPGAN model not found at ' + self.model_path)
return None
sys.path.append(os.path.abspath(gfpgan_dir))
def model_exists(self):
@ -50,7 +50,7 @@ class GFPGAN():
f'>> WARNING: GFPGAN not initialized.'
)
print(
f'>> Download https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth to {self.model_path}, \nor change GFPGAN directory with --gfpgan_dir.'
f'>> Download https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth to {self.model_path}, \nor change GFPGAN directory with --gfpgan_dir.'
)
image = image.convert('RGB')

70
ldm/dream/restoration/realesrgan.py
View File

@ -14,73 +14,53 @@ class ESRGAN():
else:
use_half_precision = True
def load_esrgan_bg_upsampler(self, upsampler_scale):
def load_esrgan_bg_upsampler(self):
if not torch.cuda.is_available(): # CPU or MPS on M1
use_half_precision = False
else:
use_half_precision = True
model_path = {
2: 'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.1/RealESRGAN_x2plus.pth',
4: 'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth',
}
from realesrgan.archs.srvgg_arch import SRVGGNetCompact
from realesrgan import RealESRGANer
if upsampler_scale not in model_path:
return None
else:
from basicsr.archs.rrdbnet_arch import RRDBNet
from realesrgan import RealESRGANer
model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=32, upscale=4, act_type='prelu')
model_path = 'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth'
scale = 4
if upsampler_scale == 4:
model = RRDBNet(
num_in_ch=3,
num_out_ch=3,
num_feat=64,
num_block=23,
num_grow_ch=32,
scale=4,
)
if upsampler_scale == 2:
model = RRDBNet(
num_in_ch=3,
num_out_ch=3,
num_feat=64,
num_block=23,
num_grow_ch=32,
scale=2,
)
bg_upsampler = RealESRGANer(
scale=upsampler_scale,
model_path=model_path[upsampler_scale],
model=model,
tile=self.bg_tile_size,
tile_pad=10,
pre_pad=0,
half=use_half_precision,
)
bg_upsampler = RealESRGANer(
scale=scale,
model_path=model_path,
model=model,
tile=self.bg_tile_size,
tile_pad=10,
pre_pad=0,
half=use_half_precision,
)
return bg_upsampler
def process(self, image, strength: float, seed: str = None, upsampler_scale: int = 2):
if seed is not None:
print(
f'>> Real-ESRGAN Upscaling seed:{seed} : scale:{upsampler_scale}x'
)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning)
warnings.filterwarnings('ignore', category=UserWarning)
try:
upsampler = self.load_esrgan_bg_upsampler(upsampler_scale)
upsampler = self.load_esrgan_bg_upsampler()
except Exception:
import traceback
import sys
print('>> Error loading Real-ESRGAN:', file=sys.stderr)
print(traceback.format_exc(), file=sys.stderr)
if upsampler_scale == 0:
print('>> Real-ESRGAN: Invalid scaling option. Image not upscaled.')
return image
if seed is not None:
print(
f'>> Real-ESRGAN Upscaling seed:{seed} : scale:{upsampler_scale}x'
)
output, _ = upsampler.enhance(
np.array(image, dtype=np.uint8),
outscale=upsampler_scale,

2
ldm/dream/server.py
View File

@ -161,7 +161,7 @@ class DreamServer(BaseHTTPRequestHandler):
# is complete. The upscaling replaces the original file, so the second
# entry should not be inserted into the image list.
# LS: This repeats code in dream.py
def image_done(image, seed, upscaled=False):
def image_done(image, seed, upscaled=False, first_seed=None):
name = f'{prefix}.{seed}.png'
iter_opt = copy.copy(opt)
if opt.variation_amount > 0:

86
ldm/generate.py
View File

@ -497,11 +497,8 @@ class Generate:
prompt = None
try:
args = metadata_from_png(image_path)
if len(args) > 1:
print("* Can't postprocess a grid")
return
seed = args[0].seed
prompt = args[0].prompt
seed = args.seed
prompt = args.prompt
print(f'>> retrieved seed {seed} and prompt "{prompt}" from {image_path}')
except:
m = re.search('(\d+)\.png$',image_path)
@ -591,8 +588,8 @@ class Generate:
def _make_images(
self,
img_path,
mask_path,
img,
mask,
width,
height,
fit=False,
@ -600,11 +597,11 @@ class Generate:
):
init_image = None
init_mask = None
if not img_path:
if not img:
return None, None
image = self._load_img(
img_path,
img,
width,
height,
fit=fit
@ -614,7 +611,7 @@ class Generate:
init_image = self._create_init_image(image) # this returns a torch tensor
# if image has a transparent area and no mask was provided, then try to generate mask
if self._has_transparency(image) and not mask_path:
if self._has_transparency(image) and not mask:
print(
'>> Initial image has transparent areas. Will inpaint in these regions.')
if self._check_for_erasure(image):
@ -626,13 +623,19 @@ class Generate:
# this returns a torch tensor
init_mask = self._create_init_mask(image)
if mask_path:
if mask:
mask_image = self._load_img(
mask_path, width, height, fit=fit) # this returns an Image
mask, width, height, fit=fit) # this returns an Image
init_mask = self._create_init_mask(mask_image)
return init_image, init_mask
def _make_base(self):
if not self.generators.get('base'):
from ldm.dream.generator import Generator
self.generators['base'] = Generator(self.model, self.precision)
return self.generators['base']
def _make_img2img(self):
if not self.generators.get('img2img'):
from ldm.dream.generator.img2img import Img2Img
@ -649,6 +652,7 @@ class Generate:
if not self.generators.get('txt2img'):
from ldm.dream.generator.txt2img import Txt2Img
self.generators['txt2img'] = Txt2Img(self.model, self.precision)
self.generators['txt2img'].free_gpu_mem = self.free_gpu_mem
return self.generators['txt2img']
def _make_inpaint(self):
@ -717,6 +721,21 @@ class Generate:
for r in image_list:
image, seed = r
try:
if strength > 0:
if self.gfpgan is not None or self.codeformer is not None:
if facetool == 'gfpgan':
if self.gfpgan is None:
print('>> GFPGAN not found. Face restoration is disabled.')
else:
image = self.gfpgan.process(image, strength, seed)
if facetool == 'codeformer':
if self.codeformer is None:
print('>> CodeFormer not found. Face restoration is disabled.')
else:
cf_device = 'cpu' if str(self.device) == 'mps' else self.device
image = self.codeformer.process(image=image, strength=strength, device=cf_device, seed=seed, fidelity=codeformer_fidelity)
else:
print(">> Face Restoration is disabled.")
if upscale is not None:
if self.esrgan is not None:
if len(upscale) < 2:
@ -725,14 +744,6 @@ class Generate:
image, upscale[1], seed, int(upscale[0]))
else:
print(">> ESRGAN is disabled. Image not upscaled.")
if strength > 0:
if self.gfpgan is not None and self.codeformer is not None:
if facetool == 'codeformer':
image = self.codeformer.process(image=image, strength=strength, device=self.device, seed=seed, fidelity=codeformer_fidelity)
else:
image = self.gfpgan.process(image, strength, seed)
else:
print(">> Face Restoration is disabled.")
except Exception as e:
print(
f'>> Error running RealESRGAN or GFPGAN. Your image was not upscaled.\n{e}'
@ -745,13 +756,7 @@ class Generate:
# to help WebGUI - front end to generator util function
def sample_to_image(self, samples):
return self._sample_to_image(samples)
def _sample_to_image(self, samples):
if not self.base_generator:
from ldm.dream.generator import Generator
self.base_generator = Generator(self.model)
return self.base_generator.sample_to_image(samples)
return self._make_base().sample_to_image(samples)
def _set_sampler(self):
msg = f'>> Setting Sampler to {self.sampler_name}'
@ -828,15 +833,24 @@ class Generate:
return model
def _load_img(self, path, width, height, fit=False):
assert os.path.exists(path), f'>> {path}: File not found'
def _load_img(self, img, width, height, fit=False):
if isinstance(img, Image.Image):
image = img
print(
f'>> using provided input image of size {image.width}x{image.height}'
)
elif isinstance(img, str):
assert os.path.exists(img), f'>> {img}: File not found'
# with Image.open(path) as img:
# image = img.convert('RGBA')
image = Image.open(path)
print(
f'>> loaded input image of size {image.width}x{image.height} from {path}'
)
image = Image.open(img)
print(
f'>> loaded input image of size {image.width}x{image.height} from {img}'
)
else:
image = Image.open(img)
print(
f'>> loaded input image of size {image.width}x{image.height}'
)
if fit:
image = self._fit_image(image, (width, height))
else:
@ -922,7 +936,7 @@ class Generate:
# BUG: We need to use the model's downsample factor rather than hardcoding "8"
from ldm.dream.generator.base import downsampling
image = image.resize((image.width//downsampling, image.height //
downsampling), resample=Image.Resampling.LANCZOS)
downsampling), resample=Image.Resampling.NEAREST)
# print(
# f'>> DEBUG: writing the mask to mask.png'
# )

33
notebooks/Stable-Diffusion-local-Windows.ipynb
View File

@ -13,7 +13,7 @@
"source": [
"Note that you will need NVIDIA drivers, Python 3.10, and Git installed\n",
"beforehand - simplified\n",
"[step-by-step instructions](https://github.com/lstein/stable-diffusion/wiki/Easy-peasy-Windows-install)\n",
"[step-by-step instructions](https://github.com/invoke-ai/InvokeAI/wiki/Easy-peasy-Windows-install)\n",
"are available in the wiki (you'll only need steps 1, 2, & 3 )"
]
},
@ -40,8 +40,8 @@
"outputs": [],
"source": [
"%%cmd\n",
"git clone https://github.com/lstein/stable-diffusion.git\n",
"cd /content/stable-diffusion/\n",
"git clone https://github.com/invoke-ai/InvokeAI.git\n",
"cd /content/InvokeAI/\n",
"git checkout --quiet development"
]
},
@ -52,14 +52,14 @@
"outputs": [],
"source": [
"%%cmd\n",
"pew new --python 3.10 -r requirements-lin-win-colab-CUDA.txt --dont-activate stable-diffusion"
"pew new --python 3.10 -r requirements-lin-win-colab-CUDA.txt --dont-activate invoke-ai"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Switch the notebook kernel to the new 'stable-diffusion' environment!\n",
"# Switch the notebook kernel to the new 'invoke-ai' environment!\n",
"\n",
"## VSCode: restart VSCode and come back to this cell\n",
"\n",
@ -67,7 +67,7 @@
"1. Type \"Select Interpreter\" and select \"Jupyter: Select Interpreter to Start Jupyter Server\"\n",
"1. VSCode will say that it needs to install packages. Click the \"Install\" button.\n",
"1. Once the install is finished, do 1 & 2 again\n",
"1. Pick 'stable-diffusion'\n",
"1. Pick 'invoke-ai'\n",
"1. Run the following cell"
]
},
@ -77,7 +77,7 @@
"metadata": {},
"outputs": [],
"source": [
"%cd stable-diffusion"
"%cd InvokeAI"
]
},
{
@ -88,7 +88,7 @@
"## Jupyter/JupyterLab\n",
"\n",
"1. Run the cell below\n",
"1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'stable-diffusion' from the drop-down.\n"
"1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'invoke-ai' from the drop-down.\n"
]
},
{
@ -106,9 +106,9 @@
"source": [
"# DO NOT RUN THIS CELL IF YOU ARE USING VSCODE!!\n",
"%%cmd\n",
"pew workon stable-diffusion\n",
"pew workon invoke-ai\n",
"pip3 install ipykernel\n",
"python -m ipykernel install --name=stable-diffusion"
"python -m ipykernel install --name=invoke-ai"
]
},
{
@ -182,15 +182,20 @@
"\n",
"Now:\n",
"\n",
"1. `cd` to wherever the 'stable-diffusion' directory is\n",
"1. Run `pew workon stable-diffusion`\n",
"1. `cd` to wherever the 'InvokeAI' directory is\n",
"1. Run `pew workon invoke-ai`\n",
"1. Run `winpty python scripts\\dream.py`"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10.6 ('ldm')",
"display_name": "Python 3.10.6 64-bit",
"language": "python",
"name": "python3"
},
@ -208,7 +213,7 @@
},
"vscode": {
"interpreter": {
"hash": "a05e4574567b7bc2c98f7f9aa579f9ea5b8739b54844ab610ac85881c4be2659"
"hash": "5e164cef426134bf171f386fbddecb52046b6c1479f922ab8dfdd30df05e0e80"
}
}
},

2
requirements.txt
View File

@ -22,7 +22,7 @@ taming-transformers-rom1504
test-tube
torch-fidelity
torchmetrics
transformers
transformers==4.21.*
flask==2.1.3
flask_socketio==5.3.0
flask_cors==3.0.10

32
scripts/dream.py
View File

@ -47,16 +47,19 @@ def main():
# Loading Face Restoration and ESRGAN Modules
try:
gfpgan, codeformer, esrgan = None, None, None
from ldm.dream.restoration import Restoration
restoration = Restoration(opt.gfpgan_dir, opt.gfpgan_model_path, opt.esrgan_bg_tile)
if opt.restore:
gfpgan, codeformer = restoration.load_face_restore_models()
if opt.restore or opt.esrgan:
from ldm.dream.restoration import Restoration
restoration = Restoration()
if opt.restore:
gfpgan, codeformer = restoration.load_face_restore_models(opt.gfpgan_dir, opt.gfpgan_model_path)
else:
print('>> Face restoration disabled')
if opt.esrgan:
esrgan = restoration.load_esrgan(opt.esrgan_bg_tile)
else:
print('>> Upscaling disabled')
else:
print('>> Face restoration disabled')
if opt.esrgan:
esrgan = restoration.load_esrgan()
else:
print('>> Upscaling disabled')
print('>> Face restoration and upscaling disabled')
except (ModuleNotFoundError, ImportError):
import traceback
print(traceback.format_exc(), file=sys.stderr)
@ -105,6 +108,8 @@ def main():
# preload the model
gen.load_model()
#set additional option
gen.free_gpu_mem = opt.free_gpu_mem
if not infile:
print(
@ -170,9 +175,10 @@ def main_loop(gen, opt, infile):
if opt.init_img:
try:
oldargs = metadata_from_png(opt.init_img)
opt.prompt = oldargs.prompt
print(f'>> Retrieved old prompt "{opt.prompt}" from {opt.init_img}')
if not opt.prompt:
oldargs = metadata_from_png(opt.init_img)
opt.prompt = oldargs.prompt
print(f'>> Retrieved old prompt "{opt.prompt}" from {opt.init_img}')
except AttributeError:
pass
except KeyError:
@ -429,7 +435,7 @@ def dream_server_loop(gen, host, port, outdir, gfpgan):
f"Point your browser at http://localhost:{port} or use the host's DNS name or IP address.")
else:
print(">> Default host address now 127.0.0.1 (localhost). Use --host 0.0.0.0 to bind any address.")
print(f">> Point your browser at http://{host}:{port}.")
print(f">> Point your browser at http://{host}:{port}")
try:
dream_server.serve_forever()

30
scripts/preload_models.py
View File

@ -49,33 +49,13 @@ except ModuleNotFoundError:
if gfpgan:
print('Loading models from RealESRGAN and facexlib')
try:
from basicsr.archs.rrdbnet_arch import RRDBNet
from realesrgan.archs.srvgg_arch import SRVGGNetCompact
from facexlib.utils.face_restoration_helper import FaceRestoreHelper
RealESRGANer(
scale=2,
model_path='https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.1/RealESRGAN_x2plus.pth',
model=RRDBNet(
num_in_ch=3,
num_out_ch=3,
num_feat=64,
num_block=23,
num_grow_ch=32,
scale=2,
),
)
RealESRGANer(
scale=4,
model_path='https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth',
model=RRDBNet(
num_in_ch=3,
num_out_ch=3,
num_feat=64,
num_block=23,
num_grow_ch=32,
scale=4,
),
model_path='https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth',
model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=32, upscale=4, act_type='prelu')
)
FaceRestoreHelper(1, det_model='retinaface_resnet50')
@ -87,8 +67,8 @@ if gfpgan:
try:
import urllib.request
model_url = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth'
model_dest = 'src/gfpgan/experiments/pretrained_models/GFPGANv1.3.pth'
model_url = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth'
model_dest = 'src/gfpgan/experiments/pretrained_models/GFPGANv1.4.pth'
if not os.path.exists(model_dest):
print('downloading gfpgan model file...')