From ec2dc24ad7b6b1cc13938ad8a3554eaa80dafb38 Mon Sep 17 00:00:00 2001 From: Matthias Wild <40327258+mauwii@users.noreply.github.com> Date: Thu, 15 Sep 2022 16:53:41 +0200 Subject: [PATCH] Mkdocs-material (#575) * Squashed commit of the following: commit 82d9c25d9a87ea5f6fbdb0445313e48cd8ae3035 Author: mauwii Date: Wed Sep 14 19:29:11 2022 +0200 fix branch name in mkdocs-flow commit 2e276cecc1942050ae8b22f4a417c576c49763a1 Author: mauwii Date: Wed Sep 14 19:28:35 2022 +0200 fix theme name commit 2eb77c1173598e161e8a5a9812b021b831363183 Author: mauwii Date: Wed Sep 14 19:14:42 2022 +0200 fixed some links and formating in main README commit 66a7152e48e8c66c9286f47a6b088920a8fcdcd8 Author: mauwii Date: Wed Sep 14 08:58:58 2022 +0200 trigger mkdocs deployment on main commit 897cc373ce4f9d18d8cb2837b5931c4b674c836b Merge: 89da371 3b5a830 Author: Matthias Wild <40327258+mauwii@users.noreply.github.com> Date: Wed Sep 14 07:51:23 2022 +0200 Merge pull request #1 from mauwii/mkdocs Mkdocs commit 3b5a8308eb6b7ee572ff07f78829f64bcf55b4d9 Author: mauwii Date: Wed Sep 14 07:42:56 2022 +0200 huge update I was pretty busy trying to make the Readmes / docs look good in MkDocs commit 0b4f5a926ff8aad1460e0be8093ee02d719a576d Author: mauwii Date: Wed Sep 14 07:41:45 2022 +0200 update mkdocs config commit 872172ea7008aa1217c1259c7ecab16ddafd1a0e Author: mauwii Date: Wed Sep 14 07:33:49 2022 +0200 added the mkdocs-git-revision-date-plugin commit eac81bf87588ec88b65f3415aa838c330fbd621b Author: mauwii Date: Wed Sep 14 06:46:43 2022 +0200 add prettier config remove markdownlint move and rename requirements-mkdocs.txt commit b36d4cc088a49af56026aa2dd2e85d6bfb181bd1 Author: mauwii Date: Wed Sep 14 02:06:39 2022 +0200 add dark theme commit a14f18fede6febccf1c117cba50a1aff1eac80a9 Author: mauwii Date: Wed Sep 14 01:38:02 2022 +0200 update mkdocs flow and config commit 2764b48693ed382bc01a2cb7aab282dbf95cd9ee Author: mauwii Date: Wed Sep 14 01:15:33 2022 +0200 add mkdocs workflow commit 1bd22523b1a52ac24bc5f950debb450f903c2d51 Author: mauwii Date: Wed Sep 14 00:57:37 2022 +0200 I already begun with formating / revising the sites * change repository in mkdocs config to lstein * adapt changes from repos main README.md Co-authored-by: Lincoln Stein --- .github/workflows/mkdocs-flow.yml | 23 ++ .prettierrc.yaml | 13 + README.md | 164 +++++++------ docs/CHANGELOG.md | 137 ----------- docs/features/CHANGELOG.md | 141 +++++++++++ docs/features/CLI.md | 348 +++++++++++--------------- docs/features/IMG2IMG.md | 37 ++- docs/features/INPAINTING.md | 44 ++-- docs/features/OTHER.md | 84 ++++--- docs/features/TEXTUAL_INVERSION.md | 99 +++++--- docs/features/UPSCALE.md | 128 +++++----- docs/features/VARIATIONS.md | 78 +++--- docs/features/WEB.md | 16 +- docs/help/TROUBLESHOOT.md | 81 +++--- docs/index.md | 183 ++++++++++++++ docs/installation/INSTALL_LINUX.md | 131 +++++----- docs/installation/INSTALL_MAC.md | 353 ++++++++++++++++----------- docs/installation/INSTALL_WINDOWS.md | 141 ++++++----- docs/{ => other}/CONTRIBUTORS.md | 12 +- docs/{ => other}/README-CompViz.md | 150 ++++++++---- mkdocs.yml | 78 ++++++ requirements-mkdocs.txt | 2 + 22 files changed, 1460 insertions(+), 983 deletions(-) create mode 100644 .github/workflows/mkdocs-flow.yml create mode 100644 .prettierrc.yaml delete mode 100644 docs/CHANGELOG.md create mode 100644 docs/features/CHANGELOG.md create mode 100644 docs/index.md rename docs/{ => other}/CONTRIBUTORS.md (93%) rename docs/{ => other}/README-CompViz.md (52%) create mode 100644 mkdocs.yml create mode 100644 requirements-mkdocs.txt diff --git a/.github/workflows/mkdocs-flow.yml b/.github/workflows/mkdocs-flow.yml new file mode 100644 index 0000000000..b34a0c2cda --- /dev/null +++ b/.github/workflows/mkdocs-flow.yml @@ -0,0 +1,23 @@ +name: Deploy +on: + push: + branches: + - main +jobs: + build: + name: Deploy docs to GitHub Pages + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + uses: Tiryoh/actions-mkdocs@v0 + with: + mkdocs_version: 'latest' # option + requirements: '/requirements-mkdocs.txt' # option + configfile: '/mkdocs.yml' # option + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./site diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 0000000000..68eae1ba15 --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,13 @@ +endOfLine: lf +tabWidth: 2 +useTabs: false +singleQuote: true +quoteProps: as-needed +embeddedLanguageFormatting: auto +overrides: + - files: "*.md" + options: + proseWrap: always + printWidth: 100 + parser: markdown + cursorOffset: -1 diff --git a/README.md b/README.md index e1978a4c1a..2dacbc7866 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

Stable Diffusion Dream Script

- +

@@ -12,37 +12,39 @@ pull-requests

-# **Stable Diffusion Dream Script** - -This is a fork of -[CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion), -the open source text-to-image generator. It provides a streamlined -process with various new features and options to aid the image -generation process. It runs on Windows, Mac and Linux machines, -and runs on GPU cards with as little as 4 GB or RAM. +This is a fork of [CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion), the open +source text-to-image generator. It provides a streamlined process with various new features and +options to aid the image generation process. It runs on Windows, Mac and Linux machines, and runs on +GPU cards with as little as 4 GB or RAM. _Note: This fork is rapidly evolving. Please use the -[Issues](https://github.com/lstein/stable-diffusion/issues) tab to -report bugs and make feature requests. Be sure to use the provided -templates. They will help aid diagnose issues faster._ +[Issues](https://github.com/lstein/stable-diffusion/issues) tab to report bugs and make feature +requests. Be sure to use the provided templates. They will help aid diagnose issues faster._ + +**Table of Contents** -# **Table of Contents** 1. [Installation](#installation) -2. [Major Features](#features) -3. [Changelog](#latest-changes) -4. [Troubleshooting](#troubleshooting) -5. [Contributing](#contributing) -6. [Support](#support) +2. [Hardware Requirements](#hardware-requirements) +3. [Features](#features) +4. [Latest Changes](#latest-changes) +5. [Troubleshooting](#troubleshooting) +6. [Contributing](#contributing) +7. [Contributors](#contributors) +8. [Support](#support) +9. [Further Reading](#further-reading) -# Installation +## Installation -This fork is supported across multiple platforms. You can find individual installation instructions below. +This fork is supported across multiple platforms. You can find individual installation instructions +below. -- ## [Linux](docs/installation/INSTALL_LINUX.md) -- ## [Windows](docs/installation/INSTALL_WINDOWS.md) -- ## [Macintosh](docs/installation/INSTALL_MAC.md) +- ### [Linux](docs/installation/INSTALL_LINUX.md) -## **Hardware Requirements** +- ### [Windows](docs/installation/INSTALL_WINDOWS.md) + +- ### [Macintosh](docs/installation/INSTALL_MAC.md) + +## Hardware Requirements **System** @@ -61,109 +63,117 @@ You wil need one of the following: **Note** -If you are have a Nvidia 10xx series card (e.g. the 1080ti), please -run the dream script in full-precision mode as shown below. +If you are have a Nvidia 10xx series card (e.g. the 1080ti), please run the dream script in +full-precision mode as shown below. Similarly, specify full-precision mode on Apple M1 hardware. -To run in full-precision mode, start `dream.py` with the -`--full_precision` flag: +To run in full-precision mode, start `dream.py` with the `--full_precision` flag: -``` +```bash (ldm) ~/stable-diffusion$ python scripts/dream.py --full_precision ``` -# Features +## Features -## **Major Features** +### Major Features -- ## [Interactive Command Line Interface](docs/features/CLI.md) +- #### [Interactive Command Line Interface](docs/features/CLI.md) -- ## [Image To Image](docs/features/IMG2IMG.md) +- #### [Image To Image](docs/features/IMG2IMG.md) -- ## [Inpainting Support](docs/features/INPAINTING.md) +- #### [Inpainting Support](docs/features/INPAINTING.md) -- ## [GFPGAN and Real-ESRGAN Support](docs/features/UPSCALE.md) +- #### [GFPGAN and Real-ESRGAN Support](docs/features/UPSCALE.md) -- ## [Embiggen upscaling](docs/features/EMBIGGEN.md) +- #### [Seamless Tiling](docs/features/OTHER.md#seamless-tiling) -- ## [Seamless Tiling](docs/features/OTHER.md#seamless-tiling) +- #### [Google Colab](docs/features/OTHER.md#google-colab) -- ## [Google Colab](docs/features/OTHER.md#google-colab) +- #### [Web Server](docs/features/WEB.md) -- ## [Web Server](docs/features/WEB.md) +- #### [Reading Prompts From File](docs/features/OTHER.md#reading-prompts-from-a-file) -- ## [Reading Prompts From File](docs/features/OTHER.md#reading-prompts-from-a-file) +- #### [Shortcut: Reusing Seeds](docs/features/OTHER.md#shortcuts-reusing-seeds) -- ## [Shortcut: Reusing Seeds](docs/features/OTHER.md#shortcuts-reusing-seeds) +- #### [Weighted Prompts](docs/features/OTHER.md#weighted-prompts) -- ## [Weighted Prompts](docs/features/OTHER.md#weighted-prompts) +- #### [Variations](docs/features/VARIATIONS.md) -- ## [Variations](docs/features/VARIATIONS.md) +- #### [Personalizing Text-to-Image Generation](docs/features/TEXTUAL_INVERSION.md) -- ## [Personalizing Text-to-Image Generation](docs/features/TEXTUAL_INVERSION.md) +- #### [Simplified API for text to image generation](docs/features/OTHER.md#simplified-api) -- ## [Simplified API for text to image generation](docs/features/OTHER.md#simplified-api) +### Other Features -## **Other Features** +- #### [Creating Transparent Regions for Inpainting](docs/features/INPAINTING.md#creating-transparent-regions-for-inpainting) -- ### [Creating Transparent Regions for Inpainting](docs/features/INPAINTING.md#creating-transparent-regions-for-inpainting) +- #### [Preload Models](docs/features/OTHER.md#preload-models) -- ### [Preload Models](docs/features/OTHER.md#preload-models) - -# Latest Changes +## Latest Changes - v1.14 (11 September 2022) - Memory optimizations for small-RAM cards. 512x512 now possible on 4 GB GPUs. - Full support for Apple hardware with M1 or M2 chips. - - Add "seamless mode" for circular tiling of image. Generates beautiful effects. ([prixt](https://github.com/prixt)). + - Add "seamless mode" for circular tiling of image. Generates beautiful effects. + ([prixt](https://github.com/prixt)). - Inpainting support. - Improved web server GUI. - Lots of code and documentation cleanups. - v1.13 (3 September 2022 - - Support image variations (see [VARIATIONS](docs/features/VARIATIONS.md) ([Kevin Gibbons](https://github.com/bakkot) and many contributors and reviewers) - - Supports a Google Colab notebook for a standalone server running on Google hardware [Arturo Mendivil](https://github.com/artmen1516) - - WebUI supports GFPGAN/ESRGAN facial reconstruction and upscaling [Kevin Gibbons](https://github.com/bakkot) - - WebUI supports incremental display of in-progress images during generation [Kevin Gibbons](https://github.com/bakkot) - - A new configuration file scheme that allows new models (including upcoming stable-diffusion-v1.5) - to be added without altering the code. ([David Wager](https://github.com/maddavid12)) + - Support image variations (see [VARIATIONS](docs/features/VARIATIONS.md) + ([Kevin Gibbons](https://github.com/bakkot) and many contributors and reviewers) + - Supports a Google Colab notebook for a standalone server running on Google hardware + [Arturo Mendivil](https://github.com/artmen1516) + - WebUI supports GFPGAN/ESRGAN facial reconstruction and upscaling + [Kevin Gibbons](https://github.com/bakkot) + - WebUI supports incremental display of in-progress images during generation + [Kevin Gibbons](https://github.com/bakkot) + - A new configuration file scheme that allows new models (including upcoming + stable-diffusion-v1.5) to be added without altering the code. + ([David Wager](https://github.com/maddavid12)) - Can specify --grid on dream.py command line as the default. - Miscellaneous internal bug and stability fixes. - Works on M1 Apple hardware. - Multiple bug fixes. -For older changelogs, please visit **[CHANGELOGS](docs/CHANGELOG.md)**. +For older changelogs, please visit the **[CHANGELOG](docs/features/CHANGELOG.md)**. -# Troubleshooting +## Troubleshooting -Please check out our **[Q&A](docs/help/TROUBLESHOOT.md)** to get solutions for common installation problems and other issues. +Please check out our **[Q&A](docs/help/TROUBLESHOOT.md)** to get solutions for common installation +problems and other issues. -# Contributing +## Contributing -Anyone who wishes to contribute to this project, whether documentation, features, bug fixes, code cleanup, testing, or code reviews, is very much encouraged to do so. If you are unfamiliar with -how to contribute to GitHub projects, here is a [Getting Started Guide](https://opensource.com/article/19/7/create-pull-request-github). +Anyone who wishes to contribute to this project, whether documentation, features, bug fixes, code +cleanup, testing, or code reviews, is very much encouraged to do so. If you are unfamiliar with how +to contribute to GitHub projects, here is a +[Getting Started Guide](https://opensource.com/article/19/7/create-pull-request-github). -A full set of contribution guidelines, along with templates, are in progress, but for now the most important thing is to **make your pull request against the "development" branch**, and not against "main". This will help keep public breakage to a minimum and will allow you to propose more radical changes. +A full set of contribution guidelines, along with templates, are in progress, but for now the most +important thing is to **make your pull request against the "development" branch**, and not against +"main". This will help keep public breakage to a minimum and will allow you to propose more radical +changes. -## **Contributors** +## Contributors -This fork is a combined effort of various people from across the -world. [Check out the list of all these amazing -people](docs/CONTRIBUTORS.md). We thank them for their time, hard work -and effort. +This fork is a combined effort of various people from across the world. +[Check out the list of all these amazing people](docs/other/CONTRIBUTORS.md). We thank them for +their time, hard work and effort. -# Support +## Support -For support, -please use this repository's GitHub Issues tracking service. Feel free -to send me an email if you use and like the script. +For support, please use this repository's GitHub Issues tracking service. Feel free to send me an +email if you use and like the script. -Original portions of the software are Copyright (c) 2020 Lincoln D. Stein (https://github.com/lstein) +Original portions of the software are Copyright (c) 2020 +[Lincoln D. Stein](https://github.com/lstein) -# Further Reading +## Further Reading -Please see the original README for more information on this software -and underlying algorithm, located in the file [README-CompViz.md](docs/README-CompViz.md). +Please see the original README for more information on this software and underlying algorithm, +located in the file [README-CompViz.md](docs/other/README-CompViz.md). diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md deleted file mode 100644 index 31dc8e80db..0000000000 --- a/docs/CHANGELOG.md +++ /dev/null @@ -1,137 +0,0 @@ -# **Changelog** - -## v1.13 (in process) - -- Supports a Google Colab notebook for a standalone server running on Google hardware [Arturo Mendivil](https://github.com/artmen1516) -- WebUI supports GFPGAN/ESRGAN facial reconstruction and upscaling [Kevin Gibbons](https://github.com/bakkot) -- WebUI supports incremental display of in-progress images during generation [Kevin Gibbons](https://github.com/bakkot) -- Output directory can be specified on the dream> command line. -- The grid was displaying duplicated images when not enough images to fill the final row [Muhammad Usama](https://github.com/SMUsamaShah) -- Can specify --grid on dream.py command line as the default. -- Miscellaneous internal bug and stability fixes. - ---- - -## v1.12 (28 August 2022) - -- Improved file handling, including ability to read prompts from standard input. - (kudos to [Yunsaki](https://github.com/yunsaki) -- The web server is now integrated with the dream.py script. Invoke by adding --web to - the dream.py command arguments. -- Face restoration and upscaling via GFPGAN and Real-ESGAN are now automatically - enabled if the GFPGAN directory is located as a sibling to Stable Diffusion. - VRAM requirements are modestly reduced. Thanks to both [Blessedcoolant](https://github.com/blessedcoolant) and - [Oceanswave](https://github.com/oceanswave) for their work on this. -- You can now swap samplers on the dream> command line. [Blessedcoolant](https://github.com/blessedcoolant) - ---- - -## v1.11 (26 August 2022) - -- NEW FEATURE: Support upscaling and face enhancement using the GFPGAN module. (kudos to [Oceanswave](https://github.com/Oceanswave) -- You now can specify a seed of -1 to use the previous image's seed, -2 to use the seed for the image generated before that, etc. - Seed memory only extends back to the previous command, but will work on all images generated with the -n# switch. -- Variant generation support temporarily disabled pending more general solution. -- Created a feature branch named **yunsaki-morphing-dream** which adds experimental support for - iteratively modifying the prompt and its parameters. Please see[ Pull Request #86](https://github.com/lstein/stable-diffusion/pull/86) - for a synopsis of how this works. Note that when this feature is eventually added to the main branch, it will may be modified - significantly. - ---- - -## v1.10 (25 August 2022) - -- A barebones but fully functional interactive web server for online generation of txt2img and img2img. - ---- - -## v1.09 (24 August 2022) - -- A new -v option allows you to generate multiple variants of an initial image - in img2img mode. (kudos to [Oceanswave](https://github.com/Oceanswave). [ - See this discussion in the PR for examples and details on use](https://github.com/lstein/stable-diffusion/pull/71#issuecomment-1226700810)) -- Added ability to personalize text to image generation (kudos to [Oceanswave](https://github.com/Oceanswave) and [nicolai256](https://github.com/nicolai256)) -- Enabled all of the samplers from k_diffusion - ---- - -## v1.08 (24 August 2022) - -- Escape single quotes on the dream> command before trying to parse. This avoids - parse errors. -- Removed instruction to get Python3.8 as first step in Windows install. - Anaconda3 does it for you. -- Added bounds checks for numeric arguments that could cause crashes. -- Cleaned up the copyright and license agreement files. - ---- - -## v1.07 (23 August 2022) - -- Image filenames will now never fill gaps in the sequence, but will be assigned the - next higher name in the chosen directory. This ensures that the alphabetic and chronological - sort orders are the same. - ---- - -## v1.06 (23 August 2022) - -- Added weighted prompt support contributed by [xraxra](https://github.com/xraxra) -- Example of using weighted prompts to tweak a demonic figure contributed by [bmaltais](https://github.com/bmaltais) - ---- - -## v1.05 (22 August 2022 - after the drop) - -- Filenames now use the following formats: - 000010.95183149.png -- Two files produced by the same command (e.g. -n2), - 000010.26742632.png -- distinguished by a different seed. - - 000011.455191342.01.png -- Two files produced by the same command using - 000011.455191342.02.png -- a batch size>1 (e.g. -b2). They have the same seed. - - 000011.4160627868.grid#1-4.png -- a grid of four images (-g); the whole grid can - be regenerated with the indicated key - -- It should no longer be possible for one image to overwrite another -- You can use the "cd" and "pwd" commands at the dream> prompt to set and retrieve - the path of the output directory. - ---- - -## v1.04 (22 August 2022 - after the drop) - -- Updated README to reflect installation of the released weights. -- Suppressed very noisy and inconsequential warning when loading the frozen CLIP - tokenizer. - ---- - -## v1.03 (22 August 2022) - -- The original txt2img and img2img scripts from the CompViz repository have been moved into - a subfolder named "orig_scripts", to reduce confusion. - ---- - -## v1.02 (21 August 2022) - -- A copy of the prompt and all of its switches and options is now stored in the corresponding - image in a tEXt metadata field named "Dream". You can read the prompt using scripts/images2prompt.py, - or an image editor that allows you to explore the full metadata. - **Please run "conda env update -f environment.yaml" to load the k_lms dependencies!!** - ---- - -## v1.01 (21 August 2022) - -- added k_lms sampling. - **Please run "conda env update -f environment.yaml" to load the k_lms dependencies!!** -- use half precision arithmetic by default, resulting in faster execution and lower memory requirements - Pass argument --full_precision to dream.py to get slower but more accurate image generation - ---- - -## Links - -- **[Read Me](../readme.md)** diff --git a/docs/features/CHANGELOG.md b/docs/features/CHANGELOG.md new file mode 100644 index 0000000000..17cf06b4dd --- /dev/null +++ b/docs/features/CHANGELOG.md @@ -0,0 +1,141 @@ +--- +title: Changelog +--- + +## v1.13 (in process) + +- Supports a Google Colab notebook for a standalone server running on Google + hardware [Arturo Mendivil](https://github.com/artmen1516) +- WebUI supports GFPGAN/ESRGAN facial reconstruction and upscaling + [Kevin Gibbons](https://github.com/bakkot) +- WebUI supports incremental display of in-progress images during generation + [Kevin Gibbons](https://github.com/bakkot) +- Output directory can be specified on the dream> command line. +- The grid was displaying duplicated images when not enough images to fill the + final row [Muhammad Usama](https://github.com/SMUsamaShah) +- Can specify --grid on dream.py command line as the default. +- Miscellaneous internal bug and stability fixes. + +--- + +## v1.12 (28 August 2022) + +- Improved file handling, including ability to read prompts from standard input. + (kudos to [Yunsaki](https://github.com/yunsaki) +- The web server is now integrated with the dream.py script. Invoke by adding + --web to the dream.py command arguments. +- Face restoration and upscaling via GFPGAN and Real-ESGAN are now automatically + enabled if the GFPGAN directory is located as a sibling to Stable Diffusion. + VRAM requirements are modestly reduced. Thanks to both + [Blessedcoolant](https://github.com/blessedcoolant) and + [Oceanswave](https://github.com/oceanswave) for their work on this. +- You can now swap samplers on the dream> command line. + [Blessedcoolant](https://github.com/blessedcoolant) + +--- + +## v1.11 (26 August 2022) + +- NEW FEATURE: Support upscaling and face enhancement using the GFPGAN module. + (kudos to [Oceanswave](https://github.com/Oceanswave)) +- You now can specify a seed of -1 to use the previous image's seed, -2 to use + the seed for the image generated before that, etc. Seed memory only extends + back to the previous command, but will work on all images generated with the + -n# switch. +- Variant generation support temporarily disabled pending more general solution. +- Created a feature branch named **yunsaki-morphing-dream** which adds + experimental support for iteratively modifying the prompt and its parameters. + Please + see[ Pull Request #86](https://github.com/lstein/stable-diffusion/pull/86) for + a synopsis of how this works. Note that when this feature is eventually added + to the main branch, it will may be modified significantly. + +--- + +## v1.10 (25 August 2022) + +- A barebones but fully functional interactive web server for online generation + of txt2img and img2img. + +--- + +## v1.09 (24 August 2022) + +- A new -v option allows you to generate multiple variants of an initial image + in img2img mode. (kudos to [Oceanswave](https://github.com/Oceanswave). +- [See this discussion in the PR for examples and details on use](https://github.com/lstein/stable-diffusion/pull/71#issuecomment-1226700810)) +- Added ability to personalize text to image generation (kudos to + [Oceanswave](https://github.com/Oceanswave) and + [nicolai256](https://github.com/nicolai256)) +- Enabled all of the samplers from k_diffusion + +--- + +## v1.08 (24 August 2022) + +- Escape single quotes on the dream> command before trying to parse. This avoids + parse errors. +- Removed instruction to get Python3.8 as first step in Windows install. + Anaconda3 does it for you. +- Added bounds checks for numeric arguments that could cause crashes. +- Cleaned up the copyright and license agreement files. + +--- + +## v1.07 (23 August 2022) + +- Image filenames will now never fill gaps in the sequence, but will be assigned + the next higher name in the chosen directory. This ensures that the alphabetic + and chronological sort orders are the same. + +--- + +## v1.06 (23 August 2022) + +- Added weighted prompt support contributed by + [xraxra](https://github.com/xraxra) +- Example of using weighted prompts to tweak a demonic figure contributed by + [bmaltais](https://github.com/bmaltais) + +--- + +## v1.05 (22 August 2022 - after the drop) + +- Filenames now use the following formats: 000010.95183149.png -- Two files + produced by the same command (e.g. -n2), 000010.26742632.png -- distinguished + by a different seed. + 000011.455191342.01.png -- Two files produced by the same command using + 000011.455191342.02.png -- a batch size>1 (e.g. -b2). They have the same seed. + 000011.4160627868.grid#1-4.png -- a grid of four images (-g); the whole grid + can be regenerated with the indicated key + +- It should no longer be possible for one image to overwrite another +- You can use the "cd" and "pwd" commands at the dream> prompt to set and + retrieve the path of the output directory. + +## v1.04 (22 August 2022 - after the drop) + +- Updated README to reflect installation of the released weights. +- Suppressed very noisy and inconsequential warning when loading the frozen CLIP + tokenizer. + +## v1.03 (22 August 2022) + +- The original txt2img and img2img scripts from the CompViz repository have been + moved into a subfolder named "orig_scripts", to reduce confusion. + +## v1.02 (21 August 2022) + +- A copy of the prompt and all of its switches and options is now stored in the + corresponding image in a tEXt metadata field named "Dream". You can read the + prompt using scripts/images2prompt.py, or an image editor that allows you to + explore the full metadata. **Please run "conda env update -f environment.yaml" + to load the k_lms dependencies!!** + +## v1.01 (21 August 2022) + +- added k_lms sampling. **Please run "conda env update -f environment.yaml" to + load the k_lms dependencies!!** +- use half precision arithmetic by default, resulting in faster execution and + lower memory requirements Pass argument --full_precision to dream.py to get + slower but more accurate image generation diff --git a/docs/features/CLI.md b/docs/features/CLI.md index e50c5833c9..c6013d9ff8 100644 --- a/docs/features/CLI.md +++ b/docs/features/CLI.md @@ -1,32 +1,29 @@ -# **Interactive Command-Line Interface** +--- +title: CLI +--- -The `dream.py` script, located in `scripts/dream.py`, provides an -interactive interface to image generation similar to the "dream -mothership" bot that Stable AI provided on its Discord server. +## **Interactive Command Line Interface** -Unlike the txt2img.py and img2img.py scripts provided in the original -CompViz/stable-diffusion source code repository, the time-consuming -initialization of the AI model initialization only happens once. After -that image generation from the command-line interface is very fast. +The `dream.py` script, located in `scripts/dream.py`, provides an interactive interface to image +generation similar to the "dream mothership" bot that Stable AI provided on its Discord server. -The script uses the readline library to allow for in-line editing, -command history (up and down arrows), autocompletion, and more. To -help keep track of which prompts generated which images, the script -writes a log file of image names and prompts to the selected output -directory. +Unlike the txt2img.py and img2img.py scripts provided in the original CompViz/stable-diffusion +source code repository, the time-consuming initialization of the AI model initialization only +happens once. After that image generation from the command-line interface is very fast. -In addition, as of version 1.02, it also writes the prompt into the -PNG file's metadata where it can be retrieved using -scripts/images2prompt.py +The script uses the readline library to allow for in-line editing, command history (up and down +arrows), autocompletion, and more. To help keep track of which prompts generated which images, the +script writes a log file of image names and prompts to the selected output directory. + +In addition, as of version 1.02, it also writes the prompt into the PNG file's metadata where it can +be retrieved using scripts/images2prompt.py The script is confirmed to work on Linux, Windows and Mac systems. -_Note:_ This script runs from the command-line or can be used as a Web -application. The Web GUI is currently rudimentary, but a much better -replacement is on its way. +_Note:_ This script runs from the command-line or can be used as a Web application. The Web GUI is +currently rudimentary, but a much better replacement is on its way. - -``` +```bash (ldm) ~/stable-diffusion$ python3 ./scripts/dream.py * Initializing, be patient... Loading model from models/ldm/text2img-large/model.ckpt @@ -54,238 +51,175 @@ dream> q

-The `dream>` prompt's arguments are pretty much identical to those -used in the Discord bot, except you don't need to type "!dream" (it -doesn't hurt if you do). A significant change is that creation of -individual images is now the default unless --grid (-g) is given. A -full list is given in [List of prompt arguments] -(#list-of-prompt-arguments). +The `dream>` prompt's arguments are pretty much identical to those used in the Discord bot, except +you don't need to type "!dream" (it doesn't hurt if you do). A significant change is that creation +of individual images is now the default unless --grid (-g) is given. A full list is given in [List +of prompt arguments] (#list-of-prompt-arguments). -# Arguments +## Arguments -The script itself also recognizes a series of command-line switches -that will change important global defaults, such as the directory for -image outputs and the location of the model weight files. +The script itself also recognizes a series of command-line switches that will change important +global defaults, such as the directory for image outputs and the location of the model weight files. -## List of arguments recognized at the command line: +## List of arguments recognized at the command line -These command-line arguments can be passed to dream.py when you first -run it from the Windows, Mac or Linux command line. Some set defaults -that can be overridden on a per-prompt basis (see [List of prompt -arguments] (#list-of-prompt-arguments). Others +These command-line arguments can be passed to dream.py when you first run it from the Windows, Mac +or Linux command line. Some set defaults that can be overridden on a per-prompt basis (see [List of +prompt arguments] (#list-of-prompt-arguments). Others -| Argument | Shortcut | Default | Description | -|--------------------|------------|---------------------|--------------| -| --help | -h | | Print a concise help message. | -| --outdir | -o | outputs/img_samples | Location for generated images. | -| --prompt_as_dir | -p | False | Name output directories using the prompt text. | -| --from_file | | None | Read list of prompts from a file. Use "-" to read from standard input | -| --model | | stable-diffusion-1.4| Loads model specified in configs/models.yaml. Currently one of "stable-diffusion-1.4" or "laion400m"| -| --full_precision | -F | False | Run in slower full-precision mode. Needed for Macintosh M1/M2 hardware and some older video cards. | -| --web | | False | Start in web server mode | -| --host | | localhost | Which network interface web server should listen on. Set to 0.0.0.0 to listen on any. | -| --port | | 9090 | Which port web server should listen for requests on. | -| --config | | configs/models.yaml | Configuration file for models and their weights. | -| --iterations | -n | 1 | How many images to generate per prompt. | -| --grid | -g | False | Save all image series as a grid rather than individually. | -| --sampler | -A| k_lms | Sampler to use. Use -h to get list of available samplers. | -| --seamless | | False | Create interesting effects by tiling elements of the image. | -| --embedding_path | | None | Path to pre-trained embedding manager checkpoints, for custom models | -| --gfpgan_dir | | src/gfpgan | Path to where GFPGAN is installed. | -| --gfpgan_model_path| | experiments/pretrained_models/GFPGANv1.3.pth| Path to GFPGAN model file, relative to --gfpgan_dir. | -| --device | -d| torch.cuda.current_device() | Device to run SD on, e.g. "cuda:0" | +| Argument | Shortcut | Default | Description | +| :---------------------- | :---------: | ------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| --help | -h | | Print a concise help message. | +| --outdir | -o | outputs/img_samples | Location for generated images. | +| --prompt_as_dir | -p | False | Name output directories using the prompt text. | +| --from_file | | None | Read list of prompts from a file. Use "-" to read from standard input | +| --model | | stable-diffusion-1.4 | Loads model specified in configs/models.yaml. Currently one of "stable-diffusion-1.4" or "laion400m" | +| --full_precision | -F | False | Run in slower full-precision mode. Needed for Macintosh M1/M2 hardware and some older video cards. | +| --web | | False | Start in web server mode | +| --host | | localhost | Which network interface web server should listen on. Set to 0.0.0.0 to listen on any. | +| --port | | 9090 | Which port web server should listen for requests on. | +| --config | | configs/models.yaml | Configuration file for models and their weights. | +| --iterations | -n | 1 | How many images to generate per prompt. | +| --grid | -g | False | Save all image series as a grid rather than individually. | +| --sampler | -A | k_lms | Sampler to use. Use -h to get list of available samplers. | +| --seamless | | False | Create interesting effects by tiling elements of the image. | +| --embedding_path | | None | Path to pre-trained embedding manager checkpoints, for custom models | +| --gfpgan_dir | | src/gfpgan | Path to where GFPGAN is installed. | +| --gfpgan_model_path | | experiments/pretrained_models
/GFPGANv1.3.pth | Path to GFPGAN model file, relative to --gfpgan_dir. | +| --device | -d | torch.cuda.current_device() | Device to run SD on, e.g. "cuda:0" | These arguments are deprecated but still work: -| Argument | Shortcut | Default | Description | -|--------------------|------------|---------------------|--------------| -| --weights | | None | Pth to weights file; use `--model stable-diffusion-1.4` instead | -| --laion400m | -l | False | Use older LAION400m weights; use `--model=laion400m` instead | +| Argument | Shortcut | Default | Description | +| ---------------- | -------- | ------- | --------------------------------------------------------------- | +| --weights | | None | Pth to weights file; use `--model stable-diffusion-1.4` instead | +| --laion400m | -l | False | Use older LAION400m weights; use `--model=laion400m` instead | -**A note on path names:** On Windows systems, you may run into - problems when passing the dream script standard backslashed path - names because the Python interpreter treats "\" as an escape. - You can either double your slashes (ick): C:\\\\path\\\\to\\\\my\\\\file, or - use Linux/Mac style forward slashes (better): C:/path/to/my/file. +### **A note on path names:** -## List of prompt arguments +On Windows systems, you may run into problems when passing the dream script standard backslashed +path names because the Python interpreter treats "\" as an escape. You can either double your +slashes (ick): C:\\\\path\\\\to\\\\my\\\\file, or use Linux/Mac style forward slashes (better): +C:/path/to/my/file. -After the dream.py script initializes, it will present you with a -**dream>** prompt. Here you can enter information to generate images -from text (txt2img), to embellish an existing image or sketch -(img2img), or to selectively alter chosen regions of the image -(inpainting). +### List of prompt arguments -### This is an example of txt2img: +After the dream.py script initializes, it will present you with a **dream>** prompt. Here you can +enter information to generate images from text (txt2img), to embellish an existing image or sketch +(img2img), or to selectively alter chosen regions of the image (inpainting). -~~~~ -dream> waterfall and rainbow -W640 -H480 -~~~~ +### This is an example of txt2img -This will create the requested image with the dimensions 640 (width) -and 480 (height). +```bash +dream> "waterfall and rainbow" -W640 -H480 +``` -Here are the dream> command that apply to txt2img: +This will create the requested image with the dimensions 640 (width) and 480 (height). -| Argument | Shortcut | Default | Description | -|--------------------|------------|---------------------|--------------| -| "my prompt" | | | Text prompt to use. The quotation marks are optional. | -| --width | -W | 512 | Width of generated image | -| --height | -H | 512 | Height of generated image | -| --iterations | -n | 1 | How many images to generate from this prompt | -| --steps | -s | 50 | How many steps of refinement to apply | -| --cfg_scale | -C | 7.5 | How hard to try to match the prompt to the generated image; any number greater than 0.0 works, but the useful range is roughly 5.0 to 20.0 | -| --seed | -S | None | Set the random seed for the next series of images. This can be used to recreate an image generated previously.| -| --sampler | -A| k_lms | Sampler to use. Use -h to get list of available samplers. | -| --grid | -g | False | Turn on grid mode to return a single image combining all the images generated by this prompt | -| --individual | -i | True | Turn off grid mode (deprecated; leave off --grid instead) | -| --outdir | -o | outputs/img_samples | Temporarily change the location of these images | -| --seamless | | False | Activate seamless tiling for interesting effects | -| --log_tokenization | -t | False | Display a color-coded list of the parsed tokens derived from the prompt | -| --skip_normalization| -x | False | Weighted subprompts will not be normalized. See [Weighted Prompts](./OTHER.md#weighted-prompts) | -| --upscale | -U | -U 1 0.75| Upscale image by magnification factor (2, 4), and set strength of upscaling (0.0-1.0). If strength not set, will default to 0.75. | -| --gfpgan_strength | -G | -G0 | Fix faces using the GFPGAN algorithm; argument indicates how hard the algorithm should try (0.0-1.0) | -| --save_original | -save_orig| False | When upscaling or fixing faces, this will cause the original image to be saved rather than replaced. | -| --variation |-v| 0.0 | Add a bit of noise (0.0=none, 1.0=high) to the image in order to generate a series of variations. Usually used in combination with -S and -n to generate a series a riffs on a starting image. See [Variations](./VARIATIONS.md). | -| --with_variations | -V| None | Combine two or more variations. See [Variations](./VARIATIONS.md) for now to use this. | +Those are the `dream` commands that apply to txt2img: -Note that the width and height of the image must be multiples of -64. You can provide different values, but they will be rounded down to -the nearest multiple of 64. +| Argument | Shortcut | Default | Description | +| --------------------------- | ---------------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| "my prompt" | | | Text prompt to use. The quotation marks are optional. | +| --width | -W | 512 | Width of generated image | +| --height | -H | 512 | Height of generated image | +| --iterations | -n | 1 | How many images to generate from this prompt | +| --steps | -s | 50 | How many steps of refinement to apply | +| --cfg_scale | -C | 7.5 | How hard to try to match the prompt to the generated image; any number greater than 0.0 works, but the useful range is roughly 5.0 to 20.0 | +| --seed | -S | None | Set the random seed for the next series of images. This can be used to recreate an image generated previously. | +| --sampler | -A | k_lms | Sampler to use. Use -h to get list of available samplers. | +| --grid | -g | False | Turn on grid mode to return a single image combining all the images generated by this prompt | +| --individual | -i | True | Turn off grid mode (deprecated; leave off --grid instead) | +| --outdir | -o | outputs/img_samples | Temporarily change the location of these images | +| --seamless | | False | Activate seamless tiling for interesting effects | +| --log_tokenization | -t | False | Display a color-coded list of the parsed tokens derived from the prompt | +| --skip_normalization | -x | False | Weighted subprompts will not be normalized. See [Weighted Prompts](./OTHER.md#weighted-prompts) | +| --upscale | -U | -U 1 0.75 | Upscale image by magnification factor (2, 4), and set strength of upscaling (0.0-1.0). If strength not set, will default to 0.75. | +| --gfpgan_strength | -G | -G0 | Fix faces using the GFPGAN algorithm; argument indicates how hard the algorithm should try (0.0-1.0) | +| --save_original | -save_orig | False | When upscaling or fixing faces, this will cause the original image to be saved rather than replaced. | +| --variation | -v | 0.0 | Add a bit of noise (0.0=none, 1.0=high) to the image in order to generate a series of variations. Usually used in combination with -S and -n to generate a series a riffs on a starting image. See [Variations](./VARIATIONS.md). | +| --with_variations | -V | None | Combine two or more variations. See [Variations](./VARIATIONS.md) for now to use this. | +Note that the width and height of the image must be multiples of 64. You can provide different +values, but they will be rounded down to the nearest multiple of 64. -### This is an example of img2img: +### This is an example of img2img -~~~~ +```bash dream> waterfall and rainbow -I./vacation-photo.png -W640 -H480 --fit -~~~~ +``` -This will modify the indicated vacation photograph by making it more -like the prompt. Results will vary greatly depending on what is in the -image. We also ask to --fit the image into a box no bigger than -640x480. Otherwise the image size will be identical to the provided -photo and you may run out of memory if it is large. +This will modify the indicated vacation photograph by making it more like the prompt. Results will +vary greatly depending on what is in the image. We also ask to --fit the image into a box no bigger +than 640x480. Otherwise the image size will be identical to the provided photo and you may run out +of memory if it is large. -In addition to the command-line options recognized by txt2img, img2img -accepts additional options: +In addition to the command-line options recognized by txt2img, img2img accepts additional options: -| Argument | Shortcut | Default | Description | -|--------------------|------------|---------------------|--------------| -| --init_img | -I | None | Path to the initialization image | -| --fit | -F | False | Scale the image to fit into the specified -H and -W dimensions | -| --strength | -s | 0.75 | How hard to try to match the prompt to the initial image. Ranges from 0.0-0.99, with higher values replacing the initial image completely.| +| Argument | Shortcut | Default | Description | +| ------------------ | --------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| --init_img | -I | None | Path to the initialization image | +| --fit | -F | False | Scale the image to fit into the specified -H and -W dimensions | +| --strength | -s | 0.75 | How hard to try to match the prompt to the initial image. Ranges from 0.0-0.99, with higher values replacing the initial image completely. | -### This is an example of inpainting: +### This is an example of inpainting -~~~~ -dream> waterfall and rainbow -I./vacation-photo.png -M./vacation-mask.png -W640 -H480 --fit -~~~~ +```bash +dream> "waterfall and rainbow" -I./vacation-photo.png -M./vacation-mask.png -W640 -H480 --fit +``` -This will do the same thing as img2img, but image alterations will -only occur within transparent areas defined by the mask file specified -by -M. You may also supply just a single initial image with the areas -to overpaint made transparent, but you must be careful not to destroy -the pixels underneath when you create the transparent areas. See -[Inpainting](./INPAINTING.md) for details. +This will do the same thing as img2img, but image alterations will only occur within transparent +areas defined by the mask file specified by -M. You may also supply just a single initial image with +the areas to overpaint made transparent, but you must be careful not to destroy the pixels +underneath when you create the transparent areas. See [Inpainting](./INPAINTING.md) for details. -inpainting accepts all the arguments used for txt2img and img2img, as -well as the --mask (-M) argument: +inpainting accepts all the arguments used for txt2img and img2img, as well as the --mask (-M) +argument: -| Argument | Shortcut | Default | Description | -|--------------------|------------|---------------------|--------------| -| --init_mask | -M | None |Path to an image the same size as the initial_image, with areas for inpainting made transparent.| +| Argument | Shortcut | Default | Description | +| ------------------ | -------- | ------- | ------------------------------------------------------------------------------------------------ | +| --init_mask | -M | None | Path to an image the same size as the initial_image, with areas for inpainting made transparent. | +## Command-line editing and completion -# Shortcuts +If you are on a Macintosh or Linux machine, the command-line offers convenient history tracking, +editing, and command completion. -Since one so frequently refers back to a previously-generated seed or -image, dream.py provides an easy shortcut that avoids having to cut -and paste these values. - -Here's how it works. Say you generated 6 images of a man-eating snail: - -~~~~ -dream> man-eating snail -n6 -... ->> Usage stats: ->> 6 image(s) generated in 79.85s ->> Max VRAM used for this generation: 3.36G. Current VRAM utilization:2.21G ->> Max VRAM used since script start: 3.36G -Outputs: -[1] outputs/img-samples/000210.1414805682.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S1414805682 -[2] outputs/img-samples/000210.3312885013.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S3312885013 -[3] outputs/img-samples/000210.1398528919.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S1398528919 -[4] outputs/img-samples/000210.92626031.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S92626031 -[5] outputs/img-samples/000210.1733666373.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S1733666373 -[6] outputs/img-samples/000210.2453524229.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S2453524229 -~~~~ - -The last image generated (with seed 2453524229) looks really good. So let's -pick that one for variation generation. Instead of cutting and pasting -the argument -S2453524229, we can simply refer to the most recent seed as --1, and write: - -~~~~ -dream> man-eating snail -v0.1 -n10 -S-1 ->> Reusing previous seed 2453524229 -...etc... -~~~~ - -You can use -2 to refer to the second to last seed, -3 to the third to -last, etc. It works with both individual images and grids. However, -the numbering system only extends across the last group of images -generated and doesn't reach back to earlier commands. - -The initial image (-I or --init_img) argument works in a similar -way. To use the second-to-most-recent snail image as the initial -image for an img2img render, you could refer to it as -I-2: - -~~~~ -dream> glowing science-fiction snail -I -2 -n4 ->> Reusing previous image outputs/img-samples/000213.2150458613.png -...etc... -~~~~ - -# Command-line editing and completion - -If you are on a Macintosh or Linux machine, the command-line offers -convenient history tracking, editing, and command completion. - -- To scroll through previous commands and potentially edit/reuse them, use the up and down cursor keys. -- To edit the current command, use the left and right cursor keys to position the cursor, and then backspace, delete or insert characters. +- To scroll through previous commands and potentially edit/reuse them, use the up and down cursor + keys. +- To edit the current command, use the left and right cursor keys to position the cursor, and then + backspace, delete or insert characters. - To move to the very beginning of the command, type CTRL-A (or command-A on the Mac) - To move to the end of the command, type CTRL-E. -- To cut a section of the command, position the cursor where you want to start cutting and type CTRL-K. +- To cut a section of the command, position the cursor where you want to start cutting and type + CTRL-K. - To paste a cut section back in, position the cursor where you want to paste, and type CTRL-Y -Windows users can get similar, but more limited, functionality if they -launch dream.py with the "winpty" program: +Windows users can get similar, but more limited, functionality if they launch dream.py with the +"winpty" program: -~~~ +``` > winpty python scripts\dream.py -~~~ +``` -On the Mac and Linux platforms, when you exit dream.py, the last 1000 -lines of your command-line history will be saved. When you restart -dream.py, you can access the saved history using the up-arrow key. +On the Mac and Linux platforms, when you exit dream.py, the last 1000 lines of your command-line +history will be saved. When you restart dream.py, you can access the saved history using the +up-arrow key. -In addition, limited command-line completion is installed. In various -contexts, you can start typing your command and press tab. A list of -potential completions will be presented to you. You can then type a -little more, hit tab again, and eventually autocomplete what you want. +In addition, limited command-line completion is installed. In various contexts, you can start typing +your command and press tab. A list of potential completions will be presented to you. You can then +type a little more, hit tab again, and eventually autocomplete what you want. -When specifying file paths using the one-letter shortcuts, the CLI -will attempt to complete pathnames for you. This is most handy for the --I (init image) and -M (init mask) paths. To initiate completion, start -the path with a slash ("/") or "./". For example: +When specifying file paths using the one-letter shortcuts, the CLI will attempt to complete +pathnames for you. This is most handy for the -I (init image) and -M (init mask) paths. To initiate +completion, start the path with a slash ("/") or "./". For example: -~~~ +``` dream> zebra with a mustache -I./test-pictures -I./test-pictures/Lincoln-and-Parrot.png -I./test-pictures/zebra.jpg -I./test-pictures/madonna.png -I./test-pictures/bad-sketch.png -I./test-pictures/man_with_eagle/ -~~~ +``` You can then type "z", hit tab again, and it will autofill to "zebra.jpg". More text completion features (such as autocompleting seeds) are on their way. - diff --git a/docs/features/IMG2IMG.md b/docs/features/IMG2IMG.md index 3930208e62..eeaecbf981 100644 --- a/docs/features/IMG2IMG.md +++ b/docs/features/IMG2IMG.md @@ -1,30 +1,29 @@ -# **Image-to-Image** +--- +title: Image-to-Image +--- -This script also provides an img2img feature that lets you seed your -creations with an initial drawing or photo. This is a really cool -feature that tells stable diffusion to build the prompt on top of the -image you provide, preserving the original's basic shape and -layout. To use it, provide the `--init_img` option as shown here: +## **IMG2IMG** +This script also provides an `img2img` feature that lets you seed your creations with an initial +drawing or photo. This is a really cool feature that tells stable diffusion to build the prompt on +top of the image you provide, preserving the original's basic shape and layout. To use it, provide +the `--init_img` option as shown here: -``` +```bash dream> "waterfall and rainbow" --init_img=./init-images/crude_drawing.png --strength=0.5 -s100 -n4 ``` -The `--init_img (-I)` option gives the path to the seed -picture. `--strength (-f)` controls how much the original will be -modified, ranging from `0.0` (keep the original intact), to `1.0` -(ignore the original completely). The default is `0.75`, and ranges -from `0.25-0.75` give interesting results. +The `--init_img (-I)` option gives the path to the seed picture. `--strength (-f)` controls how much +the original will be modified, ranging from `0.0` (keep the original intact), to `1.0` (ignore the +original completely). The default is `0.75`, and ranges from `0.25-0.75` give interesting results. You may also pass a `-v` option to generate `-n` count variants on the original image. This is done by passing the first generated image back into img2img the requested number of times. It generates interesting variants. -If the initial image contains transparent regions, then Stable -Diffusion will only draw within the transparent regions, a process -called "inpainting". However, for this to work correctly, the color -information underneath the transparent needs to be preserved, not -erased. See [Creating Transparent Images For -Inpainting](./INPAINTING.md#creating-transparent-regions-for-inpainting) -for details. +If the initial image contains transparent regions, then Stable Diffusion will only draw within the +transparent regions, a process called "inpainting". However, for this to work correctly, the color +information underneath the transparent needs to be preserved, not erased. + +More Details can be found here: +[Creating Transparent Images For Inpainting](./INPAINTING.md#creating-transparent-regions-for-inpainting) diff --git a/docs/features/INPAINTING.md b/docs/features/INPAINTING.md index bf28de31a6..85f631ac12 100644 --- a/docs/features/INPAINTING.md +++ b/docs/features/INPAINTING.md @@ -1,27 +1,27 @@ -# **Creating Transparent Regions for Inpainting** +--- +title: Inpainting +--- -Inpainting is really cool. To do it, you start with an initial image -and use a photoeditor to make one or more regions transparent -(i.e. they have a "hole" in them). You then provide the path to this -image at the dream> command line using the `-I` switch. Stable -Diffusion will only paint within the transparent region. +## **Creating Transparent Regions for Inpainting** -There's a catch. In the current implementation, you have to prepare -the initial image correctly so that the underlying colors are -preserved under the transparent area. Many imaging editing -applications will by default erase the color information under the -transparent pixels and replace them with white or black, which will -lead to suboptimal inpainting. You also must take care to export the -PNG file in such a way that the color information is preserved. +Inpainting is really cool. To do it, you start with an initial image and use a photoeditor to make +one or more regions transparent (i.e. they have a "hole" in them). You then provide the path to this +image at the dream> command line using the `-I` switch. Stable Diffusion will only paint within the +transparent region. -If your photoeditor is erasing the underlying color information, -`dream.py` will give you a big fat warning. If you can't find a way to -coax your photoeditor to retain color values under transparent areas, -then you can combine the `-I` and `-M` switches to provide both the -original unedited image and the masked (partially transparent) image: +There's a catch. In the current implementation, you have to prepare the initial image correctly so +that the underlying colors are preserved under the transparent area. Many imaging editing +applications will by default erase the color information under the transparent pixels and replace +them with white or black, which will lead to suboptimal inpainting. You also must take care to +export the PNG file in such a way that the color information is preserved. -``` -dream> man with cat on shoulder -I./images/man.png -M./images/man-transparent.png +If your photoeditor is erasing the underlying color information, `dream.py` will give you a big fat +warning. If you can't find a way to coax your photoeditor to retain color values under transparent +areas, then you can combine the `-I` and `-M` switches to provide both the original unedited image +and the masked (partially transparent) image: + +```bash +dream> "man with cat on shoulder" -I./images/man.png -M./images/man-transparent.png ``` We are hoping to get rid of the need for this workaround in an upcoming release. @@ -37,5 +37,5 @@ We are hoping to get rid of the need for this workaround in an upcoming release. 5. Open the Layers toolbar (^L) and select "Floating Selection" 6. Set opacity to 0% 7. Export as PNG -8. In the export dialogue, Make sure the "Save colour values from - transparent pixels" checkbox is selected. +8. In the export dialogue, Make sure the "Save colour values from transparent pixels" checkbox is + selected. diff --git a/docs/features/OTHER.md b/docs/features/OTHER.md index 3853b185ed..5bede1f89b 100644 --- a/docs/features/OTHER.md +++ b/docs/features/OTHER.md @@ -1,25 +1,28 @@ +--- +title: Others +--- + ## **Google Colab** Stable Diffusion AI Notebook: +
Open and follow instructions to use an -isolated environment running Dream.
+alt="Open In Colab"/>
Open and follow instructions to use an isolated environment running +Dream.
-Output Example: -![Colab Notebook](../assets/colab_notebook.png) +Output Example: ![Colab Notebook](../assets/colab_notebook.png) --- ## **Seamless Tiling** -The seamless tiling mode causes generated images to seamlessly tile -with itself. To use it, add the `--seamless` option when starting the -script which will result in all generated images to tile, or for each -`dream>` prompt as shown here: +The seamless tiling mode causes generated images to seamlessly tile with itself. To use it, add the +`--seamless` option when starting the script which will result in all generated images to tile, or +for each `dream>` prompt as shown here: -``` +```python dream> "pond garden with lotus by claude monet" --seamless -s100 -n4 ``` @@ -27,12 +30,11 @@ dream> "pond garden with lotus by claude monet" --seamless -s100 -n4 ## **Reading Prompts from a File** -You can automate `dream.py` by providing a text file with the prompts -you want to run, one line per prompt. The text file must be composed -with a text editor (e.g. Notepad) and not a word processor. Each line -should look like what you would type at the dream> prompt: +You can automate `dream.py` by providing a text file with the prompts you want to run, one line per +prompt. The text file must be composed with a text editor (e.g. Notepad) and not a word processor. +Each line should look like what you would type at the dream> prompt: -``` +```bash a beautiful sunny day in the park, children playing -n4 -C10 stormy weather on a mountain top, goats grazing -s100 innovative packaging for a squid's dinner -S137038382 @@ -40,13 +42,13 @@ innovative packaging for a squid's dinner -S137038382 Then pass this file's name to `dream.py` when you invoke it: -``` +```bash (ldm) ~/stable-diffusion$ python3 scripts/dream.py --from_file "path/to/prompts.txt" ``` You may read a series of prompts from standard input by providing a filename of `-`: -``` +```bash (ldm) ~/stable-diffusion$ echo "a beautiful day" | python3 scripts/dream.py --from_file - ``` @@ -54,12 +56,16 @@ You may read a series of prompts from standard input by providing a filename of ## **Shortcuts: Reusing Seeds** -Since it is so common to reuse seeds while refining a prompt, there is now a shortcut as of version 1.11. Provide a `**-S**` (or `**--seed**`) -switch of `-1` to use the seed of the most recent image generated. If you produced multiple images with the `**-n**` switch, then you can go back further using -2, -3, etc. up to the first image generated by the previous command. Sorry, but you can't go back further than one command. +Since it is so common to reuse seeds while refining a prompt, there is now a shortcut as of version +1.11. Provide a `**-S**` (or `**--seed**`) switch of `-1` to use the seed of the most recent image +generated. If you produced multiple images with the `**-n**` switch, then you can go back further +using -2, -3, etc. up to the first image generated by the previous command. Sorry, but you can't go +back further than one command. -Here's an example of using this to do a quick refinement. It also illustrates using the new `**-G**` switch to turn on upscaling and face enhancement (see previous section): +Here's an example of using this to do a quick refinement. It also illustrates using the new `**-G**` +switch to turn on upscaling and face enhancement (see previous section): -``` +```bash dream> a cute child playing hopscotch -G0.5 [...] outputs/img-samples/000039.3498014304.png: "a cute child playing hopscotch" -s50 -W512 -H512 -C7.5 -mk_lms -S3498014304 @@ -76,26 +82,26 @@ outputs/img-samples/000040.3498014304.png: "a cute child playing hopscotch" -G1. ## **Weighted Prompts** You may weight different sections of the prompt to tell the sampler to attach different levels of -priority to them, by adding `:(number)` to the end of the section you wish to up- or downweight. -For example consider this prompt: +priority to them, by adding `:(number)` to the end of the section you wish to up- or downweight. For +example consider this prompt: -``` - tabby cat:0.25 white duck:0.75 hybrid +```bash +tabby cat:0.25 white duck:0.75 hybrid ``` -This will tell the sampler to invest 25% of its effort on the tabby -cat aspect of the image and 75% on the white duck aspect -(surprisingly, this example actually works). The prompt weights can -use any combination of integers and floating point numbers, and they -do not need to add up to 1. +This will tell the sampler to invest 25% of its effort on the tabby cat aspect of the image and 75% +on the white duck aspect (surprisingly, this example actually works). The prompt weights can use any +combination of integers and floating point numbers, and they do not need to add up to 1. --- ## **Simplified API** -For programmers who wish to incorporate stable-diffusion into other products, this repository includes a simplified API for text to image generation, which lets you create images from a prompt in just three lines of code: +For programmers who wish to incorporate stable-diffusion into other products, this repository +includes a simplified API for text to image generation, which lets you create images from a prompt +in just three lines of code: -``` +```bash from ldm.generate import Generate g = Generate() outputs = g.txt2img("a unicorn in manhattan") @@ -109,16 +115,14 @@ Please see ldm/generate.py for more information. A set of example scripts is com ## **Preload Models** -In situations where you have limited internet connectivity or are -blocked behind a firewall, you can use the preload script to preload -the required files for Stable Diffusion to run. +In situations where you have limited internet connectivity or are blocked behind a firewall, you can +use the preload script to preload the required files for Stable Diffusion to run. -The preload script `scripts/preload_models.py` needs to be run once at -least while connected to the internet. In the following runs, it will -load up the cached versions of the required files from the `.cache` -directory of the system. +The preload script `scripts/preload_models.py` needs to be run once at least while connected to the +internet. In the following runs, it will load up the cached versions of the required files from the +`.cache` directory of the system. -``` +```bash (ldm) ~/stable-diffusion$ python3 ./scripts/preload_models.py preloading bert tokenizer... Downloading: 100%|██████████████████████████████████| 28.0/28.0 [00:00<00:00, 49.3kB/s] diff --git a/docs/features/TEXTUAL_INVERSION.md b/docs/features/TEXTUAL_INVERSION.md index 3641732e79..b8dbc21192 100644 --- a/docs/features/TEXTUAL_INVERSION.md +++ b/docs/features/TEXTUAL_INVERSION.md @@ -1,70 +1,91 @@ -# **Personalizing Text-to-Image Generation** +--- +title: TEXTUAL_INVERSION +--- -You may personalize the generated images to provide your own styles or objects by training a new LDM checkpoint and introducing a new vocabulary to the fixed model as a (.pt) embeddings file. Alternatively, you may use or train HuggingFace Concepts embeddings files (.bin) from https://huggingface.co/sd-concepts-library and its associated notebooks. +## **Personalizing Text-to-Image Generation** -**Training** +You may personalize the generated images to provide your own styles or objects +by training a new LDM checkpoint and introducing a new vocabulary to the fixed +model as a (.pt) embeddings file. Alternatively, you may use or train +HuggingFace Concepts embeddings files (.bin) from + and its associated notebooks. -To train, prepare a folder that contains images sized at 512x512 and execute the following: +## **Training** -**WINDOWS**: As the default backend is not available on Windows, if you're using that platform, set the environment variable `PL_TORCH_DISTRIBUTED_BACKEND=gloo` +To train, prepare a folder that contains images sized at 512x512 and execute the +following: -``` -(ldm) ~/stable-diffusion$ python3 ./main.py --base ./configs/stable-diffusion/v1-finetune.yaml \ - -t \ - --actual_resume ./models/ldm/stable-diffusion-v1/model.ckpt \ - -n my_cat \ - --gpus 0, \ - --data_root D:/textual-inversion/my_cat \ - --init_word 'cat' +### WINDOWS + +As the default backend is not available on Windows, if you're using that +platform, set the environment variable `PL_TORCH_DISTRIBUTED_BACKEND` to `gloo` + +```bash +python3 ./main.py --base ./configs/stable-diffusion/v1-finetune.yaml \ + --actual_resume ./models/ldm/stable-diffusion-v1/model.ckpt \ + -t \ + -n my_cat \ + --gpus 0 \ + --data_root D:/textual-inversion/my_cat \ + --init_word 'cat' ``` During the training process, files will be created in -/logs/[project][time][project]/ where you can see the process. +`/logs/[project][time][project]/` where you can see the process. -Conditioning contains the training prompts inputs, reconstruction the -input images for the training epoch samples, samples scaled for a -sample of the prompt and one with the init word provided. +Conditioning contains the training prompts inputs, reconstruction the input +images for the training epoch samples, samples scaled for a sample of the prompt +and one with the init word provided. On a RTX3090, the process for SD will take ~1h @1.6 iterations/sec. -_Note_: According to the associated paper, the optimal number of -images is 3-5. Your model may not converge if you use more images than -that. +!!! Info _Note_ -Training will run indefinitely, but you may wish to stop it (with -ctrl-c) before the heat death of the universe, when you find a low -loss epoch or around ~5000 iterations. Note that you can set a fixed -limit on the number of training steps by decreasing the "max_steps" -option in configs/stable_diffusion/v1-finetune.yaml (currently set to -4000000) + According to the associated paper, the optimal number of + images is 3-5. Your model may not converge if you use more images than + that. -**Running** +Training will run indefinitely, but you may wish to stop it (with ctrl-c) before +the heat death of the universe, when you find a low loss epoch or around ~5000 +iterations. Note that you can set a fixed limit on the number of training steps +by decreasing the "max_steps" option in +configs/stable_diffusion/v1-finetune.yaml (currently set to 4000000) -Once the model is trained, specify the trained .pt or .bin file when -starting dream using +## **Run the Model** -``` -(ldm) ~/stable-diffusion$ python3 ./scripts/dream.py --embedding_path /path/to/embedding.pt --full_precision +Once the model is trained, specify the trained .pt or .bin file when starting +dream using + +```bash +python3 ./scripts/dream.py \ + --embedding_path /path/to/embedding.pt \ + --full_precision ``` Then, to utilize your subject at the dream prompt -``` +```bash dream> "a photo of *" ``` This also works with image2image -``` +```bash dream> "waterfall and rainbow in the style of *" --init_img=./init-images/crude_drawing.png --strength=0.5 -s100 -n4 ``` -For .pt files it's also possible to train multiple tokens (modify the placeholder string in `configs/stable-diffusion/v1-finetune.yaml`) and combine LDM checkpoints using: +For .pt files it's also possible to train multiple tokens (modify the +placeholder string in `configs/stable-diffusion/v1-finetune.yaml`) and combine +LDM checkpoints using: -``` -(ldm) ~/stable-diffusion$ python3 ./scripts/merge_embeddings.py \ - --manager_ckpts /path/to/first/embedding.pt /path/to/second/embedding.pt [...] \ - --output_path /path/to/output/embedding.pt +```bash +python3 ./scripts/merge_embeddings.py \ + --manager_ckpts /path/to/first/embedding.pt \ + [,[...]] \ + --output_path /path/to/output/embedding.pt ``` -Credit goes to rinongal and the repository located at https://github.com/rinongal/textual_inversion Please see the repository and associated paper for details and limitations. +Credit goes to rinongal and the repository + +Please see [the repository](https://github.com/rinongal/textual_inversion) and +associated paper for details and limitations. diff --git a/docs/features/UPSCALE.md b/docs/features/UPSCALE.md index 51f49a70d8..231e6bfbb6 100644 --- a/docs/features/UPSCALE.md +++ b/docs/features/UPSCALE.md @@ -1,105 +1,99 @@ -# **GFPGAN and Real-ESRGAN Support** +--- +title: Upscale +--- -The script also provides the ability to do face restoration and -upscaling with the help of GFPGAN and Real-ESRGAN respectively. +## **GFPGAN and Real-ESRGAN Support** -As of version 1.14, environment.yaml will install the Real-ESRGAN package into the -standard install location for python packages, and will put GFPGAN into a subdirectory of "src" -in the stable-diffusion directory. -(The reason for this is that the standard GFPGAN distribution has a minor bug that adversely affects image -color.) Upscaling with Real-ESRGAN should "just work" without further intervention. Simply pass the --upscale (-U) -option on the dream> command line, or indicate the desired scale on the popup in the Web GUI. +The script also provides the ability to do face restoration and upscaling with the help of GFPGAN +and Real-ESRGAN respectively. -For **GFPGAN** to work, there is one additional step needed. You will need to download and -copy the GFPGAN [models file](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth) -into **src/gfpgan/experiments/pretrained_models**. On Mac and Linux systems, here's how you'd do it using -**wget**: -~~~~ +As of version 1.14, environment.yaml will install the Real-ESRGAN package into the standard install +location for python packages, and will put GFPGAN into a subdirectory of "src" in the +stable-diffusion directory. (The reason for this is that the standard GFPGAN distribution has a +minor bug that adversely affects image color.) Upscaling with Real-ESRGAN should "just work" without +further intervention. Simply pass the --upscale (-U) option on the dream> command line, or indicate +the desired scale on the popup in the Web GUI. + +For **GFPGAN** to work, there is one additional step needed. You will need to download and copy the +GFPGAN [models file](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth) +into **src/gfpgan/experiments/pretrained_models**. On Mac and Linux systems, here's how you'd do it +using **wget**: + +```bash > wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth src/gfpgan/experiments/pretrained_models/ -~~~~ +``` Make sure that you're in the stable-diffusion directory when you do this. -Alternatively, if you have GFPGAN installed elsewhere, or if you are using -an earlier version of this package which asked you to install GFPGAN in a -sibling directory, you may use the `--gfpgan_dir` argument with `dream.py` to set a -custom path to your GFPGAN directory. _There are other GFPGAN related -boot arguments if you wish to customize further._ +Alternatively, if you have GFPGAN installed elsewhere, or if you are using an earlier version of +this package which asked you to install GFPGAN in a sibling directory, you may use the +`--gfpgan_dir` argument with `dream.py` to set a custom path to your GFPGAN directory. _There are +other GFPGAN related boot arguments if you wish to customize further._ -**Note: Internet connection needed:** -Users whose GPU machines are isolated from the Internet (e.g. on a -University cluster) should be aware that the first time you run -dream.py with GFPGAN and Real-ESRGAN turned on, it will try to -download model files from the Internet. To rectify this, you may run -`python3 scripts/preload_models.py` after you have installed GFPGAN -and all its dependencies. +**Note: Internet connection needed:** Users whose GPU machines are isolated from the Internet (e.g. +on a University cluster) should be aware that the first time you run dream.py with GFPGAN and +Real-ESRGAN turned on, it will try to download model files from the Internet. To rectify this, you +may run `python3 scripts/preload_models.py` after you have installed GFPGAN and all its +dependencies. -**Usage** +## **Usage** You will now have access to two new prompt arguments. -**Upscaling** +### **Upscaling** `-U : ` -The upscaling prompt argument takes two values. The first value is a -scaling factor and should be set to either `2` or `4` only. This will -either scale the image 2x or 4x respectively using different models. +The upscaling prompt argument takes two values. The first value is a scaling factor and should be +set to either `2` or `4` only. This will either scale the image 2x or 4x respectively using +different models. -You can set the scaling stength between `0` and `1.0` to control -intensity of the of the scaling. This is handy because AI upscalers -generally tend to smooth out texture details. If you wish to retain -some of those for natural looking results, we recommend using values -between `0.5 to 0.8`. +You can set the scaling stength between `0` and `1.0` to control intensity of the of the scaling. +This is handy because AI upscalers generally tend to smooth out texture details. If you wish to +retain some of those for natural looking results, we recommend using values between `0.5 to 0.8`. -If you do not explicitly specify an upscaling_strength, it will -default to 0.75. +If you do not explicitly specify an upscaling_strength, it will default to 0.75. -**Face Restoration** +### **Face Restoration** `-G : ` -This prompt argument controls the strength of the face restoration -that is being applied. Similar to upscaling, values between `0.5 to 0.8` are recommended. +This prompt argument controls the strength of the face restoration that is being applied. Similar to +upscaling, values between `0.5 to 0.8` are recommended. -You can use either one or both without any conflicts. In cases where -you use both, the image will be first upscaled and then the face -restoration process will be executed to ensure you get the highest +You can use either one or both without any conflicts. In cases where you use both, the image will be +first upscaled and then the face restoration process will be executed to ensure you get the highest quality facial features. `--save_orig` -When you use either `-U` or `-G`, the final result you get is upscaled -or face modified. If you want to save the original Stable Diffusion -generation, you can use the `-save_orig` prompt argument to save the -original unaffected version too. +When you use either `-U` or `-G`, the final result you get is upscaled or face modified. If you want +to save the original Stable Diffusion generation, you can use the `-save_orig` prompt argument to +save the original unaffected version too. -**Example Usage** +### **Example Usage** -``` -dream > superman dancing with a panda bear -U 2 0.6 -G 0.4 +```bash +dream> superman dancing with a panda bear -U 2 0.6 -G 0.4 ``` This also works with img2img: -``` +```bash dream> a man wearing a pineapple hat -I path/to/your/file.png -U 2 0.5 -G 0.6 ``` -**Note** +### **Note** -GFPGAN and Real-ESRGAN are both memory intensive. In order to avoid -crashes and memory overloads during the Stable Diffusion process, -these effects are applied after Stable Diffusion has completed its -work. +GFPGAN and Real-ESRGAN are both memory intensive. In order to avoid crashes and memory overloads +during the Stable Diffusion process, these effects are applied after Stable Diffusion has completed +its work. -In single image generations, you will see the output right away but -when you are using multiple iterations, the images will first be -generated and then upscaled and face restored after that process is -complete. While the image generation is taking place, you will still -be able to preview the base images. +In single image generations, you will see the output right away but when you are using multiple +iterations, the images will first be generated and then upscaled and face restored after that +process is complete. While the image generation is taking place, you will still be able to preview +the base images. -If you wish to stop during the image generation but want to upscale or -face restore a particular generated image, pass it again with the same -prompt and generated seed along with the `-U` and `-G` prompt -arguments to perform those actions. +If you wish to stop during the image generation but want to upscale or face restore a particular +generated image, pass it again with the same prompt and generated seed along with the `-U` and `-G` +prompt arguments to perform those actions. diff --git a/docs/features/VARIATIONS.md b/docs/features/VARIATIONS.md index 99b891576d..23b6cf2917 100644 --- a/docs/features/VARIATIONS.md +++ b/docs/features/VARIATIONS.md @@ -1,26 +1,33 @@ -# **Variations** +--- +title: Variations +--- Release 1.13 of SD-Dream adds support for image variations. You are able to do the following: -1. Generate a series of systematic variations of an image, given a prompt. The amount of variation from one image to the next can be controlled. +1. Generate a series of systematic variations of an image, given a prompt. The + amount of variation from one image to the next can be controlled. -2. Given two or more variations that you like, you can combine them in a weighted fashion. +2. Given two or more variations that you like, you can combine them in a + weighted fashion. --- -This cheat sheet provides a quick guide for how this works in practice, using variations to create the desired image of Xena, Warrior Princess. +This cheat sheet provides a quick guide for how this works in practice, using +variations to create the desired image of Xena, Warrior Princess. --- ## Step 1 -- Find a base image that you like -The prompt we will use throughout is `lucy lawless as xena, warrior princess, character portrait, high resolution.` +The prompt we will use throughout is +`lucy lawless as xena, warrior princess, character portrait, high resolution.` This will be indicated as `prompt` in the examples below. -First we let SD create a series of images in the usual way, in this case requesting six iterations: +First we let SD create a series of images in the usual way, in this case +requesting six iterations: ``` dream> lucy lawless as xena, warrior princess, character portrait, high resolution -n6 @@ -36,17 +43,18 @@ Outputs: The one with seed 3357757885 looks nice: - +![var1](../assets/variation_walkthru/000001.3357757885.png) --- ## Step 2 - Generating Variations -Let's try to generate some variations. Using the same seed, we pass the argument `-v0.1` (or --variant_amount), which generates a series of -variations each differing by a variation amount of 0.2. This number ranges from `0` to `1.0`, with higher numbers being larger amounts of -variation. +Let's try to generate some variations. Using the same seed, we pass the argument +`-v0.1` (or --variant_amount), which generates a series of variations each +differing by a variation amount of 0.2. This number ranges from `0` to `1.0`, +with higher numbers being larger amounts of variation. -``` +```bash dream> "prompt" -n6 -S3357757885 -v0.2 ... Outputs: @@ -60,33 +68,41 @@ Outputs: ### **Variation Sub Seeding** -Note that the output for each image has a `-V` option giving the "variant subseed" for that image, consisting of a seed followed by the -variation amount used to generate it. +Note that the output for each image has a `-V` option giving the "variant +subseed" for that image, consisting of a seed followed by the variation amount +used to generate it. -This gives us a series of closely-related variations, including the two shown here. +This gives us a series of closely-related variations, including the two shown +here. - - +![var2](../assets/variation_walkthru/000002.3647897225.png) -I like the expression on Xena's face in the first one (subseed 3647897225), and the armor on her shoulder in the second one (subseed 1614299449). Can we combine them to get the best of both worlds? +![var3](../assets/variation_walkthru/000002.1614299449.png) -We combine the two variations using `-V` (--with_variations). Again, we must provide the seed for the originally-chosen image in order for -this to work. +I like the expression on Xena's face in the first one (subseed 3647897225), and +the armor on her shoulder in the second one (subseed 1614299449). Can we combine +them to get the best of both worlds? -``` -dream> "prompt" -S3357757885 -V3647897225:0.1,1614299449:0.1 +We combine the two variations using `-V` (--with_variations). Again, we must +provide the seed for the originally-chosen image in order for this to work. + +```bash +dream> "prompt" -S3357757885 -V3647897225,0.1,1614299449,0.1 Outputs: ./outputs/Xena/000003.1614299449.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1 -S3357757885 ``` -Here we are providing equal weights (0.1 and 0.1) for both the subseeds. The resulting image is close, but not exactly what I wanted: +Here we are providing equal weights (0.1 and 0.1) for both the subseeds. The +resulting image is close, but not exactly what I wanted: - +![var4](../assets/variation_walkthru/000003.1614299449.png) -We could either try combining the images with different weights, or we can generate more variations around the almost-but-not-quite image. We do the latter, using both the `-V` (combining) and `-v` (variation strength) options. Note that we use `-n6` to generate 6 variations: +We could either try combining the images with different weights, or we can +generate more variations around the almost-but-not-quite image. We do the +latter, using both the `-V` (combining) and `-v` (variation strength) options. +Note that we use `-n6` to generate 6 variations: -``` -dream> "prompt" -S3357757885 -V3647897225:0.1,1614299449:0.1 -v0.05 -n6 +dream> "prompt" -S3357757885 -V3647897225,0.1,1614299449,0.1 -v0.05 -n6 Outputs: ./outputs/Xena/000004.3279757577.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1,3279757577:0.05 -S3357757885 ./outputs/Xena/000004.2853129515.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1,2853129515:0.05 -S3357757885 @@ -96,9 +112,11 @@ Outputs: ./outputs/Xena/000004.2183375608.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1,2183375608:0.05 -S3357757885 ``` -This produces six images, all slight variations on the combination of the chosen two images. Here's the one I like best: +This produces six images, all slight variations on the combination of the chosen +two images. Here's the one I like best: - +![var5](../assets/variation_walkthru/000004.3747154981.png) -As you can see, this is a very powerful tool, which when combined with subprompt weighting, gives you great control over the content and -quality of your generated images. +As you can see, this is a very powerful tool, which when combined with subprompt +weighting, gives you great control over the content and quality of your +generated images. diff --git a/docs/features/WEB.md b/docs/features/WEB.md index 9eb6aa5bfc..1bbf5f3b55 100644 --- a/docs/features/WEB.md +++ b/docs/features/WEB.md @@ -1,13 +1,19 @@ -# Barebones Web Server +--- +title: Barebones Web Server +--- -As of version 1.10, this distribution comes with a bare bones web server (see screenshot). To use it, run the `dream.py` script by adding the `**--web**` option. +As of version 1.10, this distribution comes with a bare bones web server (see +screenshot). To use it, run the `dream.py` script by adding the `**--web**` +option. -``` +```bash (ldm) ~/stable-diffusion$ python3 scripts/dream.py --web ``` -You can then connect to the server by pointing your web browser at http://localhost:9090, or to the network name or IP address of the server. +You can then connect to the server by pointing your web browser at +http://localhost:9090, or to the network name or IP address of the server. -Kudos to [Tesseract Cat](https://github.com/TesseractCat) for contributing this code, and to [dagf2101](https://github.com/dagf2101) for refining it. +Kudos to [Tesseract Cat](https://github.com/TesseractCat) for contributing this +code, and to [dagf2101](https://github.com/dagf2101) for refining it. ![Dream Web Server](../assets/dream_web_server.png) diff --git a/docs/help/TROUBLESHOOT.md b/docs/help/TROUBLESHOOT.md index cac5dddf23..6458d8ceb9 100644 --- a/docs/help/TROUBLESHOOT.md +++ b/docs/help/TROUBLESHOOT.md @@ -1,68 +1,89 @@ -# **Frequently Asked Questions** +--- +title: F.A.Q. +--- -Here are a few common installation problems and their solutions. Often these are caused by incomplete installations or crashes during the -install process. +## **Frequently-Asked-Questions** + +Here are a few common installation problems and their solutions. Often these are caused by +incomplete installations or crashes during the install process. --- -**QUESTION** +### **QUESTION** During `conda env create -f environment.yaml`, conda hangs indefinitely. -**SOLUTION** +### **SOLUTION** -Enter the stable-diffusion directory and completely remove the `src` directory and all its contents. The safest way to do this is to enter the stable-diffusion directory and give the command `git clean -f`. If this still doesn't fix the problem, try "conda clean -all" and then restart at the `conda env create` step. +Enter the stable-diffusion directory and completely remove the `src` directory and all its contents. +The safest way to do this is to enter the stable-diffusion directory and give the command +`git clean -f`. If this still doesn't fix the problem, try "conda clean -all" and then restart at +the `conda env create` step. --- -**QUESTION** +### **QUESTION** -`dream.py` crashes with the complaint that it can't find `ldm.simplet2i.py`. Or it complains that function is being passed incorrect parameters. +`dream.py` crashes with the complaint that it can't find `ldm.simplet2i.py`. Or it complains that +function is being passed incorrect parameters. -**SOLUTION** +### **SOLUTION** -Reinstall the stable diffusion modules. Enter the `stable-diffusion` directory and give the command `pip install -e .` +Reinstall the stable diffusion modules. Enter the `stable-diffusion` directory and give the command +`pip install -e .` --- -**QUESTION** +### **QUESTION** `dream.py` dies, complaining of various missing modules, none of which starts with `ldm``. -**SOLUTION** +### **SOLUTION** -From within the `stable-diffusion` directory, run `conda env update -f environment.yaml` This is also frequently the solution to -complaints about an unknown function in a module. +From within the `stable-diffusion` directory, run `conda env update -f environment.yaml` This is +also frequently the solution to complaints about an unknown function in a module. --- -**QUESTION** +### **QUESTION** There's a feature or bugfix in the Stable Diffusion GitHub that you want to try out. -**SOLUTION** +### **SOLUTION** -**Main Branch** +#### **Main Branch** -If the fix/feature is on the `main` branch, enter the stable-diffusion directory and do a `git pull`. +If the fix/feature is on the `main` branch, enter the stable-diffusion directory and do a +`git pull`. -Usually this will be sufficient, but if you start to see errors about missing or incorrect modules, use the command `pip install -e .` and/or `conda env update -f environment.yaml` (These commands won't break anything.) +Usually this will be sufficient, but if you start to see errors about missing or incorrect modules, +use the command -**Sub Branch** +`pip install -e .` and/or -If the feature/fix is on a branch (e.g. "_foo-bugfix_"), the recipe is similar, but do a `git pull `. +`conda env update -f environment.yaml` -**Not Committed** +(These commands won't break anything.) -If the feature/fix is in a pull request that has not yet been made part of the main branch or a feature/bugfix branch, then from the page for the desired pull request, look for the line at the top that reads "_xxxx wants to merge xx commits into lstein:main from YYYYYY_". Copy the URL in YYYY. It should have the format `https://github.com//stable-diffusion/tree/` +#### **Sub Branch** -Then **go to the directory above stable-diffusion** and rename the directory to "_stable-diffusion.lstein_", "_stable-diffusion.old_", or anything else. You can then git clone the branch that contains the pull request: +If the feature/fix is on a branch (e.g. "_foo-bugfix_"), the recipe is similar, but do a +`git pull `. -``` -git clone https://github.com//stable-diffusion/tree/ -``` +#### **Not Committed** -You will need to go through the install procedure again, but it should be fast because all the dependencies are already loaded. +If the feature/fix is in a pull request that has not yet been made part of the main branch or a +feature/bugfix branch, then from the page for the desired pull request, look for the line at the top +that reads "_xxxx wants to merge xx commits into lstein:main from YYYYYY_". Copy the URL in YYYY. It +should have the format ---- +`https://github.com//stable-diffusion/tree/` + +Then **go to the directory above stable-diffusion** and rename the directory to +"_stable-diffusion.lstein_", "_stable-diffusion.old_", or anything else. You can then git clone the +branch that contains the pull request: + +`git clone https://github.com//stable-diffusion/tree/` + +You will need to go through the install procedure again, but it should be fast because all the +dependencies are already loaded. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000..c7501cb471 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,183 @@ +--- +title: Home +--- + +

Stable Diffusion Dream Script

+ +

+ +

+ +

+ last-commit + stars +
+ issues + pull-requests +

+ +This is a fork of [CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion), the open +source text-to-image generator. It provides a streamlined process with various new features and +options to aid the image generation process. It runs on Windows, Mac and Linux machines, and runs on +GPU cards with as little as 4 GB or RAM. + +_Note: This fork is rapidly evolving. Please use the +[Issues](https://github.com/lstein/stable-diffusion/issues) tab to report bugs and make feature +requests. Be sure to use the provided templates. They will help aid diagnose issues faster._ + +**Table of Contents** + +1. [Installation](#installation) +2. [Hardware Requirements](#hardware-requirements) +3. [Features](#features) +4. [Latest Changes](#latest-changes) +5. [Troubleshooting](#troubleshooting) +6. [Contributing](#contributing) +7. [Contributors](#contributors) +8. [Support](#support) +9. [Further Reading](#further-reading) + +## Installation + +This fork is supported across multiple platforms. You can find individual installation instructions +below. + +- ### [Linux](installation/INSTALL_LINUX.md) + +- ### [Windows](installation/INSTALL_WINDOWS.md) + +- ### [Macintosh](installation/INSTALL_MAC.md) + +## Hardware Requirements + +**System** + +You wil need one of the following: + +- An NVIDIA-based graphics card with 4 GB or more VRAM memory. +- An Apple computer with an M1 chip. + +**Memory** + +- At least 12 GB Main Memory RAM. + +**Disk** + +- At least 6 GB of free disk space for the machine learning model, Python, and all its dependencies. + +**Note** + +If you are have a Nvidia 10xx series card (e.g. the 1080ti), please run the dream script in +full-precision mode as shown below. + +Similarly, specify full-precision mode on Apple M1 hardware. + +To run in full-precision mode, start `dream.py` with the `--full_precision` flag: + +```bash +(ldm) ~/stable-diffusion$ python scripts/dream.py --full_precision +``` + +## Features + +### Major Features + +- #### [Interactive Command Line Interface](features/CLI.md) + +- #### [Image To Image](features/IMG2IMG.md) + +- #### [Inpainting Support](features/INPAINTING.md) + +- #### [GFPGAN and Real-ESRGAN Support](features/UPSCALE.md) + +- #### [Seamless Tiling](features/OTHER.md#seamless-tiling) + +- #### [Google Colab](features/OTHER.md#google-colab) + +- #### [Web Server](features/WEB.md) + +- #### [Reading Prompts From File](features/OTHER.md#reading-prompts-from-a-file) + +- #### [Shortcut: Reusing Seeds](features/OTHER.md#shortcuts-reusing-seeds) + +- #### [Weighted Prompts](features/OTHER.md#weighted-prompts) + +- #### [Variations](features/VARIATIONS.md) + +- #### [Personalizing Text-to-Image Generation](features/TEXTUAL_INVERSION.md) + +- #### [Simplified API for text to image generation](features/OTHER.md#simplified-api) + +### Other Features + +- #### [Creating Transparent Regions for Inpainting](features/INPAINTING.md#creating-transparent-regions-for-inpainting) + +- #### [Preload Models](features/OTHER.md#preload-models) + +## Latest Changes + +- v1.14 (11 September 2022) + + - Memory optimizations for small-RAM cards. 512x512 now possible on 4 GB GPUs. + - Full support for Apple hardware with M1 or M2 chips. + - Add "seamless mode" for circular tiling of image. Generates beautiful effects. + ([prixt](https://github.com/prixt)). + - Inpainting support. + - Improved web server GUI. + - Lots of code and documentation cleanups. + +- v1.13 (3 September 2022 + + - Support image variations (see [VARIATIONS](features/VARIATIONS.md) + ([Kevin Gibbons](https://github.com/bakkot) and many contributors and reviewers) + - Supports a Google Colab notebook for a standalone server running on Google hardware + [Arturo Mendivil](https://github.com/artmen1516) + - WebUI supports GFPGAN/ESRGAN facial reconstruction and upscaling + [Kevin Gibbons](https://github.com/bakkot) + - WebUI supports incremental display of in-progress images during generation + [Kevin Gibbons](https://github.com/bakkot) + - A new configuration file scheme that allows new models (including upcoming + stable-diffusion-v1.5) to be added without altering the code. + ([David Wager](https://github.com/maddavid12)) + - Can specify --grid on dream.py command line as the default. + - Miscellaneous internal bug and stability fixes. + - Works on M1 Apple hardware. + - Multiple bug fixes. + +For older changelogs, please visit the **[CHANGELOG](features/CHANGELOG.md)**. + +## Troubleshooting + +Please check out our **[Q&A](help/TROUBLESHOOT.md)** to get solutions for common installation +problems and other issues. + +## Contributing + +Anyone who wishes to contribute to this project, whether documentation, features, bug fixes, code +cleanup, testing, or code reviews, is very much encouraged to do so. If you are unfamiliar with how +to contribute to GitHub projects, here is a +[Getting Started Guide](https://opensource.com/article/19/7/create-pull-request-github). + +A full set of contribution guidelines, along with templates, are in progress, but for now the most +important thing is to **make your pull request against the "development" branch**, and not against +"main". This will help keep public breakage to a minimum and will allow you to propose more radical +changes. + +## Contributors + +This fork is a combined effort of various people from across the world. +[Check out the list of all these amazing people](other/CONTRIBUTORS.md). We thank them for their +time, hard work and effort. + +## Support + +For support, please use this repository's GitHub Issues tracking service. Feel free to send me an +email if you use and like the script. + +Original portions of the software are Copyright (c) 2020 +[Lincoln D. Stein](https://github.com/lstein) + +## Further Reading + +Please see the original README for more information on this software and underlying algorithm, +located in the file [README-CompViz.md](other/README-CompViz.md). diff --git a/docs/installation/INSTALL_LINUX.md b/docs/installation/INSTALL_LINUX.md index b7a6cd8ff0..312ab60482 100644 --- a/docs/installation/INSTALL_LINUX.md +++ b/docs/installation/INSTALL_LINUX.md @@ -1,89 +1,110 @@ -# **Linux Installation** +--- +title: Linux +--- -1. You will need to install the following prerequisites if they are not already available. Use your operating system's preferred installer +1. You will need to install the following prerequisites if they are not already + available. Use your operating system's preferred installer. -- Python (version 3.8.5 recommended; higher may work) -- git + - Python (version 3.8.5 recommended; higher may work) + - git 2. Install the Python Anaconda environment manager. -``` -~$ wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh -~$ chmod +x Anaconda3-2022.05-Linux-x86_64.sh -~$ ./Anaconda3-2022.05-Linux-x86_64.sh -``` + ```bash + ~$ wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh + ~$ chmod +x Anaconda3-2022.05-Linux-x86_64.sh + ~$ ./Anaconda3-2022.05-Linux-x86_64.sh + ``` -After installing anaconda, you should log out of your system and log back in. If the installation -worked, your command prompt will be prefixed by the name of the current anaconda environment - `(base)`. + After installing anaconda, you should log out of your system and log back in. If + the installation worked, your command prompt will be prefixed by the name of the + current anaconda environment - `(base)`. 3. Copy the stable-diffusion source code from GitHub: -``` -(base) ~$ git clone https://github.com/lstein/stable-diffusion.git -``` + ```bash + (base) ~$ git clone https://github.com/lstein/stable-diffusion.git + ``` -This will create stable-diffusion folder where you will follow the rest of the steps. + This will create stable-diffusion folder where you will follow the rest of the + steps. -4. Enter the newly-created stable-diffusion folder. From this step forward make sure that you are working in the stable-diffusion directory! +4. Enter the newly-created stable-diffusion folder. From this step forward make + sure that you are working in the stable-diffusion directory! -``` -(base) ~$ cd stable-diffusion -(base) ~/stable-diffusion$ -``` + ```bash + (base) ~$ cd stable-diffusion + (base) ~/stable-diffusion$ + ``` -5. Use anaconda to copy necessary python packages, create a new python environment named `ldm` and activate the environment. +5. Use anaconda to copy necessary python packages, create a new python + environment named `ldm` and activate the environment. -``` -(base) ~/stable-diffusion$ conda env create -f environment.yaml -(base) ~/stable-diffusion$ conda activate ldm -(ldm) ~/stable-diffusion$ -``` + ```bash + (base) ~/stable-diffusion$ conda env create -f environment.yaml + (base) ~/stable-diffusion$ conda activate ldm + (ldm) ~/stable-diffusion$ + ``` -After these steps, your command prompt will be prefixed by `(ldm)` as shown above. + After these steps, your command prompt will be prefixed by `(ldm)` as shown + above. 6. Load a couple of small machine-learning models required by stable diffusion: -``` -(ldm) ~/stable-diffusion$ python3 scripts/preload_models.py -``` + ```bash + (ldm) ~/stable-diffusion$ python3 scripts/preload_models.py + ``` -Note that this step is necessary because I modified the original just-in-time model loading scheme to allow the script to work on GPU machines that are not internet connected. See [Preload Models](../features/OTHER.md#preload-models) + Note that this step is necessary because I modified the original just-in-time + model loading scheme to allow the script to work on GPU machines that are not + internet connected. See [Preload Models](../features/OTHER.md#preload-models) 7. Now you need to install the weights for the stable diffusion model. -- For running with the released weights, you will first need to set up an acount with Hugging Face (https://huggingface.co). -- Use your credentials to log in, and then point your browser at https://huggingface.co/CompVis/stable-diffusion-v-1-4-original. -- You may be asked to sign a license agreement at this point. -- Click on "Files and versions" near the top of the page, and then click on the file named "sd-v1-4.ckpt". You'll be taken to a page that prompts you to click the "download" link. Save the file somewhere safe on your local machine. + - For running with the released weights, you will first need to set up an acount + with [Hugging Face](https://huggingface.co). + - Use your credentials to log in, and then point your browser [here](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original.) + - You may be asked to sign a license agreement at this point. + - Click on "Files and versions" near the top of the page, and then click on the + file named "sd-v1-4.ckpt". You'll be taken to a page that prompts you to click + the "download" link. Save the file somewhere safe on your local machine. -Now run the following commands from within the stable-diffusion directory. This will create a symbolic link from the stable-diffusion model.ckpt file, to the true location of the sd-v1-4.ckpt file. + Now run the following commands from within the stable-diffusion directory. + This will create a symbolic link from the stable-diffusion model.ckpt file, to + the true location of the `sd-v1-4.ckpt` file. -``` -(ldm) ~/stable-diffusion$ mkdir -p models/ldm/stable-diffusion-v1 -(ldm) ~/stable-diffusion$ ln -sf /path/to/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt -``` + ```bash + (ldm) ~/stable-diffusion$ mkdir -p models/ldm/stable-diffusion-v1 + (ldm) ~/stable-diffusion$ ln -sf /path/to/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt + ``` 8. Start generating images! -``` -# for the pre-release weights use the -l or --liaon400m switch -(ldm) ~/stable-diffusion$ python3 scripts/dream.py -l + ```bash + # for the pre-release weights use the -l or --liaon400m switch + (ldm) ~/stable-diffusion$ python3 scripts/dream.py -l -# for the post-release weights do not use the switch -(ldm) ~/stable-diffusion$ python3 scripts/dream.py + # for the post-release weights do not use the switch + (ldm) ~/stable-diffusion$ python3 scripts/dream.py -# for additional configuration switches and arguments, use -h or --help -(ldm) ~/stable-diffusion$ python3 scripts/dream.py -h -``` + # for additional configuration switches and arguments, use -h or --help + (ldm) ~/stable-diffusion$ python3 scripts/dream.py -h + ``` -9. Subsequently, to relaunch the script, be sure to run "conda activate ldm" (step 5, second command), enter the `stable-diffusion` directory, and then launch the dream script (step 8). If you forget to activate the ldm environment, the script will fail with multiple `ModuleNotFound` errors. +9. Subsequently, to relaunch the script, be sure to run "conda activate ldm" + (step 5, second command), enter the `stable-diffusion` directory, and then + launch the dream script (step 8). If you forget to activate the ldm + environment, the script will fail with multiple `ModuleNotFound` errors. -### Updating to newer versions of the script + ### Updating to newer versions of the script -This distribution is changing rapidly. If you used the `git clone` method (step 5) to download the stable-diffusion directory, then to update to the latest and greatest version, launch the Anaconda window, enter `stable-diffusion` and type: + This distribution is changing rapidly. If you used the `git clone` method + (step 5) to download the stable-diffusion directory, then to update to the + latest and greatest version, launch the Anaconda window, enter + `stable-diffusion` and type: -``` -(ldm) ~/stable-diffusion$ git pull -``` + ```bash + (ldm) ~/stable-diffusion$ git pull + ``` -This will bring your local copy into sync with the remote one. + This will bring your local copy into sync with the remote one. diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index c000e818bb..39398c36ac 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -1,37 +1,44 @@ -# **macOS Instructions** +--- +title: macOS +--- -Requirements +## Requirements - macOS 12.3 Monterey or later - Python - Patience - Apple Silicon\* -\*I haven't tested any of this on Intel Macs but I have read that one person got it to work, so Apple Silicon might not be requried. +\*I haven't tested any of this on Intel Macs but I have read that one person got +it to work, so Apple Silicon might not be requried. -Things have moved really fast and so these instructions change often -and are often out-of-date. One of the problems is that there are so -many different ways to run this. +Things have moved really fast and so these instructions change often and are +often out-of-date. One of the problems is that there are so many different ways +to run this. -We are trying to build a testing setup so that when we make changes it -doesn't always break. +We are trying to build a testing setup so that when we make changes it doesn't +always break. How to (this hasn't been 100% tested yet): First get the weights checkpoint download started - it's big: 1. Sign up at https://huggingface.co -2. Go to the [Stable diffusion diffusion model page](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) +2. Go to the + [Stable diffusion diffusion model page](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) 3. Accept the terms and click Access Repository: -4. Download [sd-v1-4.ckpt (4.27 GB)](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/blob/main/sd-v1-4.ckpt) and note where you have saved it (probably the Downloads folder) +4. Download + [sd-v1-4.ckpt (4.27 GB)](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/blob/main/sd-v1-4.ckpt) + and note where you have saved it (probably the Downloads folder) -While that is downloading, open Terminal and run the following commands one at a time. + While that is downloading, open Terminal and run the following commands one + at a time. ```bash # install brew (and Xcode command line tools): + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -# # Now there are two different routes to get the Python (miniconda) environment up and running: # 1. Alongside pyenv # 2. No pyenv @@ -41,11 +48,11 @@ While that is downloading, open Terminal and run the following commands one at a # NOW EITHER DO # 1. Installing alongside pyenv -brew install pyenv-virtualenv # you might have this from before, no problem -pyenv install anaconda3-2022.05 -pyenv virtualenv anaconda3-2022.05 -eval "$(pyenv init -)" -pyenv activate anaconda3-2022.05 + brew install pyenv-virtualenv # you might have this from before, no problem + pyenv install anaconda3-2022.05 + pyenv virtualenv anaconda3-2022.05 + eval "$(pyenv init -)" + pyenv activate anaconda3-2022.05 # OR, # 2. Installing standalone @@ -53,31 +60,31 @@ pyenv activate anaconda3-2022.05 brew install cmake protobuf rust # install miniconda (M1 arm64 version): -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh -/bin/bash Miniconda3-latest-MacOSX-arm64.sh + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh + /bin/bash Miniconda3-latest-MacOSX-arm64.sh # EITHER WAY, # continue from here # clone the repo -git clone https://github.com/lstein/stable-diffusion.git -cd stable-diffusion + git clone https://github.com/lstein/stable-diffusion.git + cd stable-diffusion # # wait until the checkpoint file has downloaded, then proceed # # create symlink to checkpoint -mkdir -p models/ldm/stable-diffusion-v1/ + mkdir -p models/ldm/stable-diffusion-v1/ -PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt + PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt -ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt + ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt # install packages -PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml -conda activate ldm + PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml + conda activate ldm # only need to do this once python scripts/preload_models.py @@ -88,117 +95,172 @@ python scripts/dream.py --full_precision # half-precision requires autocast and The original scripts should work as well. -``` +```bash python scripts/orig_scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms ``` -Note, `export PIP_EXISTS_ACTION=w` is a precaution to fix `conda env -create -f environment-mac.yaml` never finishing in some situations. So -it isn't required but wont hurt. +Note, -After you follow all the instructions and run dream.py you might get several errors. Here's the errors I've seen and found solutions for. +```bash +export PIP_EXISTS_ACTION=w +``` + +is a precaution to fix + +```bash +conda env create -f environment-mac.yaml +``` + +never finishing in some situations. So it isn't required but wont hurt. + +After you follow all the instructions and run dream.py you might get several +errors. Here's the errors I've seen and found solutions for. + +--- ### Is it slow? Be sure to specify 1 sample and 1 iteration. - python ./scripts/orig_scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 +```bash +python ./scripts/orig_scripts/txt2img.py \ + --prompt "ocean" \ + --ddim_steps 5 \ + --n_samples 1 \ + --n_iter 1 +``` + +--- ### Doesn't work anymore? -PyTorch nightly includes support for MPS. Because of this, this setup is inherently unstable. One morning I woke up and it no longer worked no matter what I did until I switched to miniforge. However, I have another Mac that works just fine with Anaconda. If you can't get it to work, please search a little first because many of the errors will get posted and solved. If you can't find a solution please [create an issue](https://github.com/lstein/stable-diffusion/issues). +PyTorch nightly includes support for MPS. Because of this, this setup is +inherently unstable. One morning I woke up and it no longer worked no matter +what I did until I switched to miniforge. However, I have another Mac that works +just fine with Anaconda. If you can't get it to work, please search a little +first because many of the errors will get posted and solved. If you can't find a +solution please +[create an issue](https://github.com/lstein/stable-diffusion/issues). One debugging step is to update to the latest version of PyTorch nightly. - conda install pytorch torchvision torchaudio -c pytorch-nightly +```bash +conda install pytorch torchvision torchaudio -c pytorch-nightly +``` -If `conda env create -f environment-mac.yaml` takes forever run this. +If it takes forever to run - git clean -f +```bash +conda env create -f environment-mac.yaml +``` -And run this. +you could try to run `git clean -f` followed by: - conda clean --yes --all +`conda clean --yes --all` -Or you could reset Anaconda. +Or you could try to completley reset Anaconda: - conda update --force-reinstall -y -n base -c defaults conda +```bash +conda update --force-reinstall -y -n base -c defaults conda +``` -### "No module named cv2", torch, 'ldm', 'transformers', 'taming', etc. +--- + +### "No module named cv2", torch, 'ldm', 'transformers', 'taming', etc There are several causes of these errors. -First, did you remember to `conda activate ldm`? If your terminal prompt -begins with "(ldm)" then you activated it. If it begins with "(base)" -or something else you haven't. +- First, did you remember to `conda activate ldm`? If your terminal prompt + begins with "(ldm)" then you activated it. If it begins with "(base)" or + something else you haven't. -Second, you might've run `./scripts/preload_models.py` or `./scripts/dream.py` -instead of `python ./scripts/preload_models.py` or `python ./scripts/dream.py`. -The cause of this error is long so it's below. +- Second, you might've run `./scripts/preload_models.py` or `./scripts/dream.py` + instead of `python ./scripts/preload_models.py` or + `python ./scripts/dream.py`. The cause of this error is long so it's below. -Third, if it says you're missing taming you need to rebuild your virtual -environment. +- Third, if it says you're missing taming you need to rebuild your virtual + environment. - conda env remove -n ldm - conda env create -f environment-mac.yaml +`conda env remove -n ldm conda env create -f environment-mac.yaml` -Fourth, If you have activated the ldm virtual environment and tried rebuilding it, maybe the problem could be that I have something installed that you don't and you'll just need to manually install it. Make sure you activate the virtual environment so it installs there instead of -globally. +Fourth, If you have activated the ldm virtual environment and tried rebuilding +it, maybe the problem could be that I have something installed that you don't +and you'll just need to manually install it. Make sure you activate the virtual +environment so it installs there instead of globally. - conda activate ldm - pip install *name* +`conda activate ldm pip install _name_` You might also need to install Rust (I mention this again below). +--- + ### How many snakes are living in your computer? You might have multiple Python installations on your system, in which case it's -important to be explicit and consistent about which one to use for a given project. -This is because virtual environments are coupled to the Python that created it (and all -the associated 'system-level' modules). +important to be explicit and consistent about which one to use for a given +project. This is because virtual environments are coupled to the Python that +created it (and all the associated 'system-level' modules). -When you run `python` or `python3`, your shell searches the colon-delimited locations -in the `PATH` environment variable (`echo $PATH` to see that list) in that order - first match wins. -You can ask for the location of the first `python3` found in your `PATH` with the `which` command like this: +When you run `python` or `python3`, your shell searches the colon-delimited +locations in the `PATH` environment variable (`echo $PATH` to see that list) in +that order - first match wins. You can ask for the location of the first +`python3` found in your `PATH` with the `which` command like this: - % which python3 - /usr/bin/python3 +```bash +% which python3 +/usr/bin/python3 +``` -Anything in `/usr/bin` is [part of the OS](https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW6). However, `/usr/bin/python3` is not actually python3, but -rather a stub that offers to install Xcode (which includes python 3). If you have Xcode installed already, -`/usr/bin/python3` will execute `/Library/Developer/CommandLineTools/usr/bin/python3` or +Anything in `/usr/bin` is +[part of the OS](https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW6). +However, `/usr/bin/python3` is not actually python3, but rather a stub that +offers to install Xcode (which includes python 3). If you have Xcode installed +already, `/usr/bin/python3` will execute +`/Library/Developer/CommandLineTools/usr/bin/python3` or `/Applications/Xcode.app/Contents/Developer/usr/bin/python3` (depending on which Xcode you've selected with `xcode-select`). -Note that `/usr/bin/python` is an entirely different python - specifically, python 2. Note: starting in -macOS 12.3, `/usr/bin/python` no longer exists. +Note that `/usr/bin/python` is an entirely different python - specifically, +python 2. Note: starting in macOS 12.3, `/usr/bin/python` no longer exists. - % which python3 - /opt/homebrew/bin/python3 +```bash +% which python3 +/opt/homebrew/bin/python3 +``` If you installed python3 with Homebrew and you've modified your path to search for Homebrew binaries before system ones, you'll see the above path. - % which python - /opt/anaconda3/bin/python +```bash +% which python +/opt/anaconda3/bin/python +``` If you have Anaconda installed, you will see the above path. There is a -`/opt/anaconda3/bin/python3` also. We expect that `/opt/anaconda3/bin/python` -and `/opt/anaconda3/bin/python3` should actually be the *same python*, which you can -verify by comparing the output of `python3 -V` and `python -V`. +`/opt/anaconda3/bin/python3` also. - (ldm) % which python - /Users/name/miniforge3/envs/ldm/bin/python +We expect that `/opt/anaconda3/bin/python` and `/opt/anaconda3/bin/python3` +should actually be the _same python_, which you can verify by comparing the +output of `python3 -V` and `python -V`. -The above is what you'll see if you have miniforge and you've correctly activated -the ldm environment, and you used option 2 in the setup instructions above ("no pyenv"). +```bash +(ldm) % which python +/Users/name/miniforge3/envs/ldm/bin/python +``` + +The above is what you'll see if you have miniforge and you've correctly +activated the ldm environment, and you used option 2 in the setup instructions +above ("no pyenv"). + +```bash +(anaconda3-2022.05) % which python +/Users/name/.pyenv/shims/python +``` - (anaconda3-2022.05) % which python - /Users/name/.pyenv/shims/python - ... and the above is what you'll see if you used option 1 ("Alongside pyenv"). -It's all a mess and you should know [how to modify the path environment variable](https://support.apple.com/guide/terminal/use-environment-variables-apd382cc5fa-4f58-4449-b20a-41c53c006f8f/mac) +It's all a mess and you should know +[how to modify the path environment variable](https://support.apple.com/guide/terminal/use-environment-variables-apd382cc5fa-4f58-4449-b20a-41c53c006f8f/mac) if you want to fix it. Here's a brief hint of all the ways you can modify it (don't really have the time to explain it all here). @@ -211,18 +273,19 @@ if you want to fix it. Here's a brief hint of all the ways you can modify it Which one you use will depend on what you have installed except putting a file in /etc/paths.d is what I prefer to do. -Finally, to answer the question posed by this section's title, it may help to list -all of the `python` / `python3` things found in `$PATH` instead of just the one that -will be executed by default. To do that, add the `-a` switch to `which`: +Finally, to answer the question posed by this section's title, it may help to +list all of the `python` / `python3` things found in `$PATH` instead of just the +one that will be executed by default. To do that, add the `-a` switch to +`which`: % which -a python3 ... ### Debugging? -Tired of waiting for your renders to finish before you can see if it -works? Reduce the steps! The image quality will be horrible but at least you'll -get quick feedback. +Tired of waiting for your renders to finish before you can see if it works? +Reduce the steps! The image quality will be horrible but at least you'll get +quick feedback. python ./scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 @@ -235,15 +298,24 @@ get quick feedback. Example error. ``` -... -NotImplementedError: The operator 'aten::_index_put_impl_' is not current implemented for the MPS device. If you want this op to be added in priority during the prototype phase of this feature, please comment on [https://github.com/pytorch/pytorch/issues/77764](https://github.com/pytorch/pytorch/issues/77764). As a temporary fix, you can set the environment variable `PYTORCH_ENABLE_MPS_FALLBACK=1` to use the CPU as a fallback for this op. WARNING: this will be slower than running natively on MPS. + +... NotImplementedError: The operator 'aten::_index_put_impl_' is not current +implemented for the MPS device. If you want this op to be added in priority +during the prototype phase of this feature, please comment on +[https://github.com/pytorch/pytorch/issues/77764](https://github.com/pytorch/pytorch/issues/77764). +As a temporary fix, you can set the environment variable +`PYTORCH_ENABLE_MPS_FALLBACK=1` to use the CPU as a fallback for this op. +WARNING: this will be slower than running natively on MPS. + ``` -The lstein branch includes this fix in [environment-mac.yaml](https://github.com/lstein/stable-diffusion/blob/main/environment-mac.yaml). +The lstein branch includes this fix in +[environment-mac.yaml](https://github.com/lstein/stable-diffusion/blob/main/environment-mac.yaml). ### "Could not build wheels for tokenizers" -I have not seen this error because I had Rust installed on my computer before I started playing with Stable Diffusion. The fix is to install Rust. +I have not seen this error because I had Rust installed on my computer before I +started playing with Stable Diffusion. The fix is to install Rust. curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh @@ -251,10 +323,9 @@ I have not seen this error because I had Rust installed on my computer before I First this: -> Completely reproducible results are not guaranteed across PyTorch -> releases, individual commits, or different platforms. Furthermore, -> results may not be reproducible between CPU and GPU executions, even -> when using identical seeds. +> Completely reproducible results are not guaranteed across PyTorch releases, +> individual commits, or different platforms. Furthermore, results may not be +> reproducible between CPU and GPU executions, even when using identical seeds. [PyTorch docs](https://pytorch.org/docs/stable/notes/randomness.html) @@ -265,53 +336,56 @@ still working on it. OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized. -You are likely using an Intel package by mistake. Be sure to run conda with -the environment variable `CONDA_SUBDIR=osx-arm64`, like so: +You are likely using an Intel package by mistake. Be sure to run conda with the +environment variable `CONDA_SUBDIR=osx-arm64`, like so: `CONDA_SUBDIR=osx-arm64 conda install ...` -This error happens with Anaconda on Macs when the Intel-only `mkl` is pulled in by -a dependency. [nomkl](https://stackoverflow.com/questions/66224879/what-is-the-nomkl-python-package-used-for) +This error happens with Anaconda on Macs when the Intel-only `mkl` is pulled in +by a dependency. +[nomkl](https://stackoverflow.com/questions/66224879/what-is-the-nomkl-python-package-used-for) is a metapackage designed to prevent this, by making it impossible to install `mkl`, but if your environment is already broken it may not work. Do _not_ use `os.environ['KMP_DUPLICATE_LIB_OK']='True'` or equivalents as this masks the underlying issue of using Intel packages. -### Not enough memory. +### Not enough memory -This seems to be a common problem and is probably the underlying -problem for a lot of symptoms (listed below). The fix is to lower your -image size or to add `model.half()` right after the model is loaded. I -should probably test it out. I've read that the reason this fixes -problems is because it converts the model from 32-bit to 16-bit and -that leaves more RAM for other things. I have no idea how that would -affect the quality of the images though. +This seems to be a common problem and is probably the underlying problem for a +lot of symptoms (listed below). The fix is to lower your image size or to add +`model.half()` right after the model is loaded. I should probably test it out. +I've read that the reason this fixes problems is because it converts the model +from 32-bit to 16-bit and that leaves more RAM for other things. I have no idea +how that would affect the quality of the images though. See [this issue](https://github.com/CompVis/stable-diffusion/issues/71). ### "Error: product of dimension sizes > 2\*\*31'" -This error happens with img2img, which I haven't played with too much -yet. But I know it's because your image is too big or the resolution -isn't a multiple of 32x32. Because the stable-diffusion model was -trained on images that were 512 x 512, it's always best to use that -output size (which is the default). However, if you're using that size -and you get the above error, try 256 x 256 or 512 x 256 or something -as the source image. +This error happens with img2img, which I haven't played with too much yet. But I +know it's because your image is too big or the resolution isn't a multiple of +32x32. Because the stable-diffusion model was trained on images that were 512 x +512, it's always best to use that output size (which is the default). However, +if you're using that size and you get the above error, try 256 x 256 or 512 x +256 or something as the source image. -BTW, 2\*\*31-1 = [2,147,483,647](https://en.wikipedia.org/wiki/2,147,483,647#In_computing), which is also 32-bit signed [LONG_MAX](https://en.wikipedia.org/wiki/C_data_types) in C. +BTW, 2\*\*31-1 = +[2,147,483,647](https://en.wikipedia.org/wiki/2,147,483,647#In_computing), which +is also 32-bit signed [LONG_MAX](https://en.wikipedia.org/wiki/C_data_types) in +C. ### I just got Rickrolled! Do I have a virus? You don't have a virus. It's part of the project. Here's [Rick](https://github.com/lstein/stable-diffusion/blob/main/assets/rick.jpeg) -and here's [the -code](https://github.com/lstein/stable-diffusion/blob/69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc/scripts/txt2img.py#L79) -that swaps him in. It's a NSFW filter, which IMO, doesn't work very -good (and we call this "computer vision", sheesh). +and here's +[the code](https://github.com/lstein/stable-diffusion/blob/69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc/scripts/txt2img.py#L79) +that swaps him in. It's a NSFW filter, which IMO, doesn't work very good (and we +call this "computer vision", sheesh). -Actually, this could be happening because there's not enough RAM. You could try the `model.half()` suggestion or specify smaller output images. +Actually, this could be happening because there's not enough RAM. You could try +the `model.half()` suggestion or specify smaller output images. ### My images come out black @@ -319,31 +393,29 @@ We might have this fixed, we are still testing. There's a [similar issue](https://github.com/CompVis/stable-diffusion/issues/69) on CUDA GPU's where the images come out green. Maybe it's the same issue? -Someone in that issue says to use "--precision full", but this fork -actually disables that flag. I don't know why, someone else provided -that code and I don't know what it does. Maybe the `model.half()` -suggestion above would fix this issue too. I should probably test it. +Someone in that issue says to use "--precision full", but this fork actually +disables that flag. I don't know why, someone else provided that code and I +don't know what it does. Maybe the `model.half()` suggestion above would fix +this issue too. I should probably test it. ### "view size is not compatible with input tensor's size and stride" -``` - File "/opt/anaconda3/envs/ldm/lib/python3.10/site-packages/torch/nn/functional.py", line 2511, in layer_norm - return torch.layer_norm(input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled) +```bash +File "/opt/anaconda3/envs/ldm/lib/python3.10/site-packages/torch/nn/functional.py", line 2511, in layer_norm +return torch.layer_norm(input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled) RuntimeError: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead. ``` -Update to the latest version of lstein/stable-diffusion. We were -patching pytorch but we found a file in stable-diffusion that we could -change instead. This is a 32-bit vs 16-bit problem. +Update to the latest version of lstein/stable-diffusion. We were patching +pytorch but we found a file in stable-diffusion that we could change instead. +This is a 32-bit vs 16-bit problem. + +--- ### The processor must support the Intel bla bla bla What? Intel? On an Apple Silicon? - - Intel MKL FATAL ERROR: This system does not meet the minimum requirements for use of the Intel(R) Math Kernel Library. - The processor must support the Intel(R) Supplemental Streaming SIMD Extensions 3 (Intel(R) SSSE3) instructions. - The processor must support the Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) instructions. - The processor must support the Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions. +`bash Intel MKL FATAL ERROR: This system does not meet the minimum requirements for use of the Intel(R) Math Kernel Library. The processor must support the Intel(R) Supplemental Streaming SIMD Extensions 3 (Intel(R) SSSE3) instructions. The processor must support the Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) instructions. The processor must support the Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions. ` This is due to the Intel `mkl` package getting picked up when you try to install something that depends on it-- Rosetta can translate some Intel instructions but @@ -351,11 +423,13 @@ not the specialized ones here. To avoid this, make sure to use the environment variable `CONDA_SUBDIR=osx-arm64`, which restricts the Conda environment to only use ARM packages, and use `nomkl` as described above. +--- + ### input types 'tensor<2x1280xf32>' and 'tensor<\*xf16>' are not broadcast compatible May appear when just starting to generate, e.g.: -``` +```bash dream> clouds Generating: 0%| | 0/1 [00:004 GB in size, so - downloading may take a while. + - For running with the released weights, you will first need to set up an + acount with Hugging Face (https://huggingface.co). + - Use your credentials to log in, and then point your browser at + https://huggingface.co/CompVis/stable-diffusion-v-1-4-original. + - You may be asked to sign a license agreement at this point. + - Click on "Files and versions" near the top of the page, and then click on + the file named `sd-v1-4.ckpt`. You'll be taken to a page that prompts you + to click the "download" link. Now save the file somewhere safe on your + local machine. + - The weight file is >4 GB in size, so downloading may take a while. -Now run the following commands from **within the stable-diffusion directory** to copy the weights file to the right place: + Now run the following commands from **within the stable-diffusion directory** + to copy the weights file to the right place: -``` -mkdir -p models\ldm\stable-diffusion-v1 -copy C:\path\to\sd-v1-4.ckpt models\ldm\stable-diffusion-v1\model.ckpt -``` + ```bash + mkdir -p models\ldm\stable-diffusion-v1 + copy C:\path\to\sd-v1-4.ckpt models\ldm\stable-diffusion-v1\model.ckpt + ``` -Please replace `C:\path\to\sd-v1.4.ckpt` with the correct path to wherever you stashed this file. If you prefer not to copy or move the .ckpt file, -you may instead create a shortcut to it from within `models\ldm\stable-diffusion-v1\`. + Please replace `C:\path\to\sd-v1.4.ckpt` with the correct path to wherever + you stashed this file. If you prefer not to copy or move the .ckpt file, you + may instead create a shortcut to it from within + `models\ldm\stable-diffusion-v1\`. 9. Start generating images! -``` -# for the pre-release weights -python scripts\dream.py -l + ```bash + # for the pre-release weights + python scripts\dream.py -l -# for the post-release weights -python scripts\dream.py -``` + # for the post-release weights + python scripts\dream.py + ``` -10. Subsequently, to relaunch the script, first activate the Anaconda command window (step 3),enter the stable-diffusion directory (step 5, `cd \path\to\stable-diffusion`), run `conda activate ldm` (step 6b), and then launch the dream script (step 9). +10. Subsequently, to relaunch the script, first activate the Anaconda command + window (step 3),enter the stable-diffusion directory (step 5, + `cd \path\to\stable-diffusion`), run `conda activate ldm` (step 6b), and + then launch the dream script (step 9). -**Note:** Tildebyte has written an alternative ["Easy peasy Windows -install"](https://github.com/lstein/stable-diffusion/wiki/Easy-peasy-Windows-install) -which uses the Windows Powershell and pew. If you are having trouble with Anaconda on Windows, give this a try (or try it first!) + **Note:** Tildebyte has written an alternative + ["Easy peasy Windows install"](https://github.com/lstein/stable-diffusion/wiki/Easy-peasy-Windows-install) + which uses the Windows Powershell and pew. If you are having trouble with + Anaconda on Windows, give this a try (or try it first!) + +--- ### Updating to newer versions of the script -This distribution is changing rapidly. If you used the `git clone` method (step 5) to download the stable-diffusion directory, then to update to the latest and greatest version, launch the Anaconda window, enter `stable-diffusion`, and type: +This distribution is changing rapidly. If you used the `git clone` method +(step 5) to download the stable-diffusion directory, then to update to the +latest and greatest version, launch the Anaconda window, enter +`stable-diffusion`, and type: -``` +```bash git pull conda env update -f environment.yaml ``` diff --git a/docs/CONTRIBUTORS.md b/docs/other/CONTRIBUTORS.md similarity index 93% rename from docs/CONTRIBUTORS.md rename to docs/other/CONTRIBUTORS.md index cca8bade9b..906b0edf33 100644 --- a/docs/CONTRIBUTORS.md +++ b/docs/other/CONTRIBUTORS.md @@ -1,14 +1,16 @@ -# Contributors +--- +title: Contributors +--- The list of all the amazing people who have contributed to the various features that you get to experience in this fork. We thank them for all of their time and hard work. -_Original Author:_ +## __Original Author:__ -- Lincoln D. Stein +- [Lincoln D. Stein](mailto:lincoln.stein@gmail.com) -_Contributions by:_ +## __Contributions by:__ - [Sean McLellan](https://github.com/Oceanswave) - [Kevin Gibbons](https://github.com/bakkot) @@ -49,7 +51,7 @@ _Contributions by:_ - [Any Winter](https://github.com/any-winter-4079) - [Doggettx](https://github.com/doggettx) -_Original CompVis Authors:_ +## __Original CompVis Authors:__ - [Robin Rombach](https://github.com/rromb) - [Patrick von Platen](https://github.com/patrickvonplaten) diff --git a/docs/README-CompViz.md b/docs/other/README-CompViz.md similarity index 52% rename from docs/README-CompViz.md rename to docs/other/README-CompViz.md index ed7df6a4ea..395612092f 100644 --- a/docs/README-CompViz.md +++ b/docs/other/README-CompViz.md @@ -1,6 +1,12 @@ -# Original README from CompViz/stable-diffusion +--- +title: CompViz-Readme +--- -_Stable Diffusion was made possible thanks to a collaboration with [Stability AI](https://stability.ai/) and [Runway](https://runwayml.com/) and builds upon our previous work:_ +# _README from [CompViz/stable-diffusion](https://github.com/CompVis/stable-diffusion)_ + +_Stable Diffusion was made possible thanks to a collaboration with +[Stability AI](https://stability.ai/) and [Runway](https://runwayml.com/) and +builds upon our previous work:_ [**High-Resolution Image Synthesis with Latent Diffusion Models**](https://ommer-lab.com/research/latent-diffusion-models/)
[Robin Rombach](https://github.com/rromb)\*, @@ -9,32 +15,40 @@ _Stable Diffusion was made possible thanks to a collaboration with [Stability AI [Patrick Esser](https://github.com/pesser), [Björn Ommer](https://hci.iwr.uni-heidelberg.de/Staff/bommer)
-**CVPR '22 Oral** +## **CVPR '22 Oral** -which is available on [GitHub](https://github.com/CompVis/latent-diffusion). PDF at [arXiv](https://arxiv.org/abs/2112.10752). Please also visit our [Project page](https://ommer-lab.com/research/latent-diffusion-models/). +which is available on [GitHub](https://github.com/CompVis/latent-diffusion). PDF +at [arXiv](https://arxiv.org/abs/2112.10752). Please also visit our +[Project page](https://ommer-lab.com/research/latent-diffusion-models/). ![txt2img-stable2](../assets/stable-samples/txt2img/merged-0006.png) [Stable Diffusion](#stable-diffusion-v1) is a latent text-to-image diffusion -model. -Thanks to a generous compute donation from [Stability AI](https://stability.ai/) and support from [LAION](https://laion.ai/), we were able to train a Latent Diffusion Model on 512x512 images from a subset of the [LAION-5B](https://laion.ai/blog/laion-5b/) database. -Similar to Google's [Imagen](https://arxiv.org/abs/2205.11487), -this model uses a frozen CLIP ViT-L/14 text encoder to condition the model on text prompts. -With its 860M UNet and 123M text encoder, the model is relatively lightweight and runs on a GPU with at least 10GB VRAM. -See [this section](#stable-diffusion-v1) below and the [model card](https://huggingface.co/CompVis/stable-diffusion). +model. Thanks to a generous compute donation from +[Stability AI](https://stability.ai/) and support from +[LAION](https://laion.ai/), we were able to train a Latent Diffusion Model on +512x512 images from a subset of the [LAION-5B](https://laion.ai/blog/laion-5b/) +database. Similar to Google's [Imagen](https://arxiv.org/abs/2205.11487), this +model uses a frozen CLIP ViT-L/14 text encoder to condition the model on text +prompts. With its 860M UNet and 123M text encoder, the model is relatively +lightweight and runs on a GPU with at least 10GB VRAM. See +[this section](#stable-diffusion-v1) below and the +[model card](https://huggingface.co/CompVis/stable-diffusion). ## Requirements -A suitable [conda](https://conda.io/) environment named `ldm` can be created -and activated with: +A suitable [conda](https://conda.io/) environment named `ldm` can be created and +activated with: -``` +```bash conda env create -f environment.yaml conda activate ldm ``` -You can also update an existing [latent diffusion](https://github.com/CompVis/latent-diffusion) environment by running +You can also update an existing +[latent diffusion](https://github.com/CompVis/latent-diffusion) environment by +running -``` +```bash conda install pytorch torchvision -c pytorch pip install transformers==4.19.2 pip install -e . @@ -42,42 +56,57 @@ pip install -e . ## Stable Diffusion v1 -Stable Diffusion v1 refers to a specific configuration of the model -architecture that uses a downsampling-factor 8 autoencoder with an 860M UNet -and CLIP ViT-L/14 text encoder for the diffusion model. The model was pretrained on 256x256 images and -then finetuned on 512x512 images. +Stable Diffusion v1 refers to a specific configuration of the model architecture +that uses a downsampling-factor 8 autoencoder with an 860M UNet and CLIP +ViT-L/14 text encoder for the diffusion model. The model was pretrained on +256x256 images and then finetuned on 512x512 images. -\*Note: Stable Diffusion v1 is a general text-to-image diffusion model and therefore mirrors biases and (mis-)conceptions that are present -in its training data. -Details on the training procedure and data, as well as the intended use of the model can be found in the corresponding [model card](https://huggingface.co/CompVis/stable-diffusion). -Research into the safe deployment of general text-to-image models is an ongoing effort. To prevent misuse and harm, we currently provide access to the checkpoints only for [academic research purposes upon request](https://stability.ai/academia-access-form). -**This is an experiment in safe and community-driven publication of a capable and general text-to-image model. We are working on a public release with a more permissive license that also incorporates ethical considerations.\*** +\*Note: Stable Diffusion v1 is a general text-to-image diffusion model and +therefore mirrors biases and (mis-)conceptions that are present in its training +data. Details on the training procedure and data, as well as the intended use of +the model can be found in the corresponding +[model card](https://huggingface.co/CompVis/stable-diffusion). Research into the +safe deployment of general text-to-image models is an ongoing effort. To prevent +misuse and harm, we currently provide access to the checkpoints only for +[academic research purposes upon request](https://stability.ai/academia-access-form). +**This is an experiment in safe and community-driven publication of a capable +and general text-to-image model. We are working on a public release with a more +permissive license that also incorporates ethical considerations.\*** [Request access to Stable Diffusion v1 checkpoints for academic research](https://stability.ai/academia-access-form) ### Weights -We currently provide three checkpoints, `sd-v1-1.ckpt`, `sd-v1-2.ckpt` and `sd-v1-3.ckpt`, -which were trained as follows, +We currently provide three checkpoints, `sd-v1-1.ckpt`, `sd-v1-2.ckpt` and +`sd-v1-3.ckpt`, which were trained as follows, -- `sd-v1-1.ckpt`: 237k steps at resolution `256x256` on [laion2B-en](https://huggingface.co/datasets/laion/laion2B-en). - 194k steps at resolution `512x512` on [laion-high-resolution](https://huggingface.co/datasets/laion/laion-high-resolution) (170M examples from LAION-5B with resolution `>= 1024x1024`). -- `sd-v1-2.ckpt`: Resumed from `sd-v1-1.ckpt`. - 515k steps at resolution `512x512` on "laion-improved-aesthetics" (a subset of laion2B-en, - filtered to images with an original size `>= 512x512`, estimated aesthetics score `> 5.0`, and an estimated watermark probability `< 0.5`. The watermark estimate is from the LAION-5B metadata, the aesthetics score is estimated using an [improved aesthetics estimator](https://github.com/christophschuhmann/improved-aesthetic-predictor)). -- `sd-v1-3.ckpt`: Resumed from `sd-v1-2.ckpt`. 195k steps at resolution `512x512` on "laion-improved-aesthetics" and 10\% dropping of the text-conditioning to improve [classifier-free guidance sampling](https://arxiv.org/abs/2207.12598). +- `sd-v1-1.ckpt`: 237k steps at resolution `256x256` on + [laion2B-en](https://huggingface.co/datasets/laion/laion2B-en). 194k steps at + resolution `512x512` on + [laion-high-resolution](https://huggingface.co/datasets/laion/laion-high-resolution) + (170M examples from LAION-5B with resolution `>= 1024x1024`). +- `sd-v1-2.ckpt`: Resumed from `sd-v1-1.ckpt`. 515k steps at resolution + `512x512` on "laion-improved-aesthetics" (a subset of laion2B-en, filtered to + images with an original size `>= 512x512`, estimated aesthetics score `> 5.0`, + and an estimated watermark probability `< 0.5`. The watermark estimate is from + the LAION-5B metadata, the aesthetics score is estimated using an + [improved aesthetics estimator](https://github.com/christophschuhmann/improved-aesthetic-predictor)). +- `sd-v1-3.ckpt`: Resumed from `sd-v1-2.ckpt`. 195k steps at resolution + `512x512` on "laion-improved-aesthetics" and 10\% dropping of the + text-conditioning to improve + [classifier-free guidance sampling](https://arxiv.org/abs/2207.12598). Evaluations with different classifier-free guidance scales (1.5, 2.0, 3.0, 4.0, -5.0, 6.0, 7.0, 8.0) and 50 PLMS sampling -steps show the relative improvements of the checkpoints: -![sd evaluation results](../assets/v1-variants-scores.jpg) +5.0, 6.0, 7.0, 8.0) and 50 PLMS sampling steps show the relative improvements of +the checkpoints: ![sd evaluation results](../assets/v1-variants-scores.jpg) ### Text-to-Image with Stable Diffusion ![txt2img-stable2](../assets/stable-samples/txt2img/merged-0005.png) ![txt2img-stable2](../assets/stable-samples/txt2img/merged-0007.png) -Stable Diffusion is a latent diffusion model conditioned on the (non-pooled) text embeddings of a CLIP ViT-L/14 text encoder. +Stable Diffusion is a latent diffusion model conditioned on the (non-pooled) +text embeddings of a CLIP ViT-L/14 text encoder. #### Sampling Script @@ -94,8 +123,11 @@ and sample with python scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms ``` -By default, this uses a guidance scale of `--scale 7.5`, [Katherine Crowson's implementation](https://github.com/CompVis/latent-diffusion/pull/51) of the [PLMS](https://arxiv.org/abs/2202.09778) sampler, -and renders images of size 512x512 (which it was trained on) in 50 steps. All supported arguments are listed below (type `python scripts/txt2img.py --help`). +By default, this uses a guidance scale of `--scale 7.5`, +[Katherine Crowson's implementation](https://github.com/CompVis/latent-diffusion/pull/51) +of the [PLMS](https://arxiv.org/abs/2202.09778) sampler, and renders images of +size 512x512 (which it was trained on) in 50 steps. All supported arguments are +listed below (type `python scripts/txt2img.py --help`). ```commandline usage: txt2img.py [-h] [--prompt [PROMPT]] [--outdir [OUTDIR]] [--skip_grid] [--skip_save] [--ddim_steps DDIM_STEPS] [--plms] [--laion400m] [--fixed_code] [--ddim_eta DDIM_ETA] [--n_iter N_ITER] [--H H] [--W W] [--C C] [--f F] [--n_samples N_SAMPLES] [--n_rows N_ROWS] @@ -133,14 +165,17 @@ optional arguments: ``` -Note: The inference config for all v1 versions is designed to be used with EMA-only checkpoints. -For this reason `use_ema=False` is set in the configuration, otherwise the code will try to switch from -non-EMA to EMA weights. If you want to examine the effect of EMA vs no EMA, we provide "full" checkpoints -which contain both types of weights. For these, `use_ema=False` will load and use the non-EMA weights. +Note: The inference config for all v1 versions is designed to be used with +EMA-only checkpoints. For this reason `use_ema=False` is set in the +configuration, otherwise the code will try to switch from non-EMA to EMA +weights. If you want to examine the effect of EMA vs no EMA, we provide "full" +checkpoints which contain both types of weights. For these, `use_ema=False` will +load and use the non-EMA weights. #### Diffusers Integration -Another way to download and sample Stable Diffusion is by using the [diffusers library](https://github.com/huggingface/diffusers/tree/main#new--stable-diffusion-is-now-fully-compatible-with-diffusers) +Another way to download and sample Stable Diffusion is by using the +[diffusers library](https://github.com/huggingface/diffusers/tree/main#new--stable-diffusion-is-now-fully-compatible-with-diffusers) ```py # make sure you're logged in with `huggingface-cli login` @@ -161,18 +196,23 @@ image.save("astronaut_rides_horse.png") ### Image Modification with Stable Diffusion -By using a diffusion-denoising mechanism as first proposed by [SDEdit](https://arxiv.org/abs/2108.01073), the model can be used for different -tasks such as text-guided image-to-image translation and upscaling. Similar to the txt2img sampling script, -we provide a script to perform image modification with Stable Diffusion. +By using a diffusion-denoising mechanism as first proposed by +[SDEdit](https://arxiv.org/abs/2108.01073), the model can be used for different +tasks such as text-guided image-to-image translation and upscaling. Similar to +the txt2img sampling script, we provide a script to perform image modification +with Stable Diffusion. -The following describes an example where a rough sketch made in [Pinta](https://www.pinta-project.com/) is converted into a detailed artwork. +The following describes an example where a rough sketch made in +[Pinta](https://www.pinta-project.com/) is converted into a detailed artwork. ``` python scripts/img2img.py --prompt "A fantasy landscape, trending on artstation" --init-img --strength 0.8 ``` -Here, strength is a value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. -Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. See the following example. +Here, strength is a value between 0.0 and 1.0, that controls the amount of noise +that is added to the input image. Values that approach 1.0 allow for lots of +variations but will also produce images that are not semantically consistent +with the input. See the following example. **Input** @@ -183,15 +223,19 @@ Values that approach 1.0 allow for lots of variations but will also produce imag ![out3](../assets/stable-samples/img2img/mountains-3.png) ![out2](../assets/stable-samples/img2img/mountains-2.png) -This procedure can, for example, also be used to upscale samples from the base model. +This procedure can, for example, also be used to upscale samples from the base +model. ## Comments -- Our codebase for the diffusion models builds heavily on [OpenAI's ADM codebase](https://github.com/openai/guided-diffusion) - and [https://github.com/lucidrains/denoising-diffusion-pytorch](https://github.com/lucidrains/denoising-diffusion-pytorch). +- Our codebase for the diffusion models builds heavily on + [OpenAI's ADM codebase](https://github.com/openai/guided-diffusion) and + [https://github.com/lucidrains/denoising-diffusion-pytorch](https://github.com/lucidrains/denoising-diffusion-pytorch). Thanks for open-sourcing! -- The implementation of the transformer encoder is from [x-transformers](https://github.com/lucidrains/x-transformers) by [lucidrains](https://github.com/lucidrains?tab=repositories). +- The implementation of the transformer encoder is from + [x-transformers](https://github.com/lucidrains/x-transformers) by + [lucidrains](https://github.com/lucidrains?tab=repositories). ## BibTeX diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000000..c7c50834d8 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,78 @@ +# General +site_name: Dream Script Docs +site_url: https://lstein.github.io/stable-diffusion/ + +# Repository +repo_name: lstein/stable-diffusion +repo_url: https://github.com/lstein/stable-diffusion +edit_uri: edit/main/docs/ + +# Copyright +copyright: Copyright © 2022 Lincoln D. Stein + +# Configuration +theme: + name: material + features: + - toc.integrate + palette: + - media: '(prefers-color-scheme: light)' + primary: blue + scheme: default + toggle: + icon: material/lightbulb + name: Switch to dark mode + - media: '(prefers-color-scheme: dark)' + primary: dark-blue + accent: white + scheme: slate + toggle: + icon: material/lightbulb-outline + name: Switch to light mode + +# Extensions +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - meta + - md_in_html + - toc: + permalink: '#' + - pymdownx.arithmatex: + generic: true + - pymdownx.betterem: + smart_enable: all + - pymdownx.caret + - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.magiclink: + repo_url_shorthand: true + user: 'Mauwii' + repo: 'stable-diffusion' + - pymdownx.mark + - pymdownx.smartsymbols + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.snippets + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + - tables + +plugins: + - search + - git-revision-date diff --git a/requirements-mkdocs.txt b/requirements-mkdocs.txt new file mode 100644 index 0000000000..7b4de171ed --- /dev/null +++ b/requirements-mkdocs.txt @@ -0,0 +1,2 @@ +mkdocs-material +mkdocs-git-revision-date-plugin>=0.3.2